From b5b8d34182e173180f23092f470f1d0a5263aa9a Mon Sep 17 00:00:00 2001 From: foxnne Date: Thu, 22 Oct 2026 10:49:40 -0500 Subject: [PATCH 1/2] text: replace deprecated std.unicode.utf8Decode --- plugins/text/src/textcore/encoding.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/text/src/textcore/encoding.zig b/plugins/text/src/textcore/encoding.zig index 0d416aae0..722492af4 100644 --- a/plugins/text/src/textcore/encoding.zig +++ b/plugins/text/src/textcore/encoding.zig @@ -21,7 +21,7 @@ pub fn looksBinary(bytes: []const u8) bool { while (i < sample.len) { const n = std.unicode.utf8ByteSequenceLength(sample[i]) catch return true; if (i + n > sample.len) break; - _ = std.unicode.utf8Decode(sample[i..][0..n]) catch return true; + if (!std.unicode.utf8ValidateSlice(sample[i..][0..n])) return true; i += n; } return false; @@ -61,10 +61,10 @@ pub fn appendLossy(list: *std.ArrayList(u8), gpa: std.mem.Allocator, bytes: []co try list.appendSlice(gpa, replacement); break; } - if (std.unicode.utf8Decode(bytes[i..][0..n])) |_| { + if (std.unicode.utf8ValidateSlice(bytes[i..][0..n])) { try list.appendSlice(gpa, bytes[i .. i + n]); i += n; - } else |_| { + } else { try list.appendSlice(gpa, replacement); i += 1; } From e47608f8f70974aa358a7fcd1b9f5d7da4265fab Mon Sep 17 00:00:00 2001 From: "fizzy-linux[bot]" <346940526+fizzy-linux[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:20:00 -0500 Subject: [PATCH 2/2] text: test appendLossy on malformed multi-byte sequences Pins that the utf8ValidateSlice swap replaces bad sequences exactly as utf8Decode did. Agent-Team: linux Worker: claude --- plugins/text/src/textcore/encoding.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/text/src/textcore/encoding.zig b/plugins/text/src/textcore/encoding.zig index 722492af4..3d8b4f51d 100644 --- a/plugins/text/src/textcore/encoding.zig +++ b/plugins/text/src/textcore/encoding.zig @@ -79,4 +79,9 @@ test appendLossy { list.clearRetainingCapacity(); try appendLossy(&list, std.testing.allocator, "a\x00b\xffc"); try std.testing.expectEqualStrings("a\u{FFFD}b\u{FFFD}c", list.items); + list.clearRetainingCapacity(); + // Well-formed multi-byte sequences pass through; a bad continuation byte or a surrogate + // is replaced a byte at a time. + try appendLossy(&list, std.testing.allocator, "\xc3\xa9\xe2\x28\xa1\xed\xa0\x80"); + try std.testing.expectEqualStrings("\u{e9}\u{FFFD}(\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}", list.items); }