authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-08-07 05:47:32-04:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-08-07 06:29:34-04:00
loga805454dea86c3dfc3e3cc9fabaad1b6c0e2b9aa
treee9925073c43151d96ebe4eab50a7aee0ec2051b8
parent8bec737ca22d0f994f606c13cb4a23b6287dad0c

fix partial strings getting dropped when multi-byte codepoints span input buffers

1 files changed, 81 insertions(+), 19 deletions(-)

lib/std/json/scanner.zig+81-19
......@@ -897,7 +897,7 @@ pub const Scanner = struct {
897897 },
898898 .number_post_dot => {
899899 if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
900 switch (try self.expectByte()) {
900 switch (self.input[self.cursor]) {
901901 '0'...'9' => {
902902 self.cursor += 1;
903903 self.state = .number_frac;
......@@ -1032,7 +1032,8 @@ pub const Scanner = struct {
10321032 return error.BufferUnderrun;
10331033 },
10341034 .string_backslash => {
1035 switch (try self.expectByte()) {
1035 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1036 switch (self.input[self.cursor]) {
10361037 '"', '\\', '/' => {
10371038 // Since these characters now represent themselves literally,
10381039 // we can simply begin the next plaintext slice here.
......@@ -1080,7 +1081,8 @@ pub const Scanner = struct {
10801081 }
10811082 },
10821083 .string_backslash_u => {
1083 const c = try self.expectByte();
1084 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1085 const c = self.input[self.cursor];
10841086 switch (c) {
10851087 '0'...'9' => {
10861088 self.utf16_code_units[0] = @as(u16, c - '0') << 12;
......@@ -1098,7 +1100,8 @@ pub const Scanner = struct {
10981100 continue :state_loop;
10991101 },
11001102 .string_backslash_u_1 => {
1101 const c = try self.expectByte();
1103 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1104 const c = self.input[self.cursor];
11021105 switch (c) {
11031106 '0'...'9' => {
11041107 self.utf16_code_units[0] |= @as(u16, c - '0') << 8;
......@@ -1116,7 +1119,8 @@ pub const Scanner = struct {
11161119 continue :state_loop;
11171120 },
11181121 .string_backslash_u_2 => {
1119 const c = try self.expectByte();
1122 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1123 const c = self.input[self.cursor];
11201124 switch (c) {
11211125 '0'...'9' => {
11221126 self.utf16_code_units[0] |= @as(u16, c - '0') << 4;
......@@ -1134,7 +1138,8 @@ pub const Scanner = struct {
11341138 continue :state_loop;
11351139 },
11361140 .string_backslash_u_3 => {
1137 const c = try self.expectByte();
1141 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1142 const c = self.input[self.cursor];
11381143 switch (c) {
11391144 '0'...'9' => {
11401145 self.utf16_code_units[0] |= c - '0';
......@@ -1160,7 +1165,8 @@ pub const Scanner = struct {
11601165 }
11611166 },
11621167 .string_surrogate_half => {
1163 switch (try self.expectByte()) {
1168 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1169 switch (self.input[self.cursor]) {
11641170 '\\' => {
11651171 self.cursor += 1;
11661172 self.state = .string_surrogate_half_backslash;
......@@ -1170,7 +1176,8 @@ pub const Scanner = struct {
11701176 }
11711177 },
11721178 .string_surrogate_half_backslash => {
1173 switch (try self.expectByte()) {
1179 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1180 switch (self.input[self.cursor]) {
11741181 'u' => {
11751182 self.cursor += 1;
11761183 self.state = .string_surrogate_half_backslash_u;
......@@ -1180,7 +1187,8 @@ pub const Scanner = struct {
11801187 }
11811188 },
11821189 .string_surrogate_half_backslash_u => {
1183 switch (try self.expectByte()) {
1190 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1191 switch (self.input[self.cursor]) {
11841192 'D', 'd' => {
11851193 self.cursor += 1;
11861194 self.utf16_code_units[1] = 0xD << 12;
......@@ -1191,7 +1199,8 @@ pub const Scanner = struct {
11911199 }
11921200 },
11931201 .string_surrogate_half_backslash_u_1 => {
1194 const c = try self.expectByte();
1202 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1203 const c = self.input[self.cursor];
11951204 switch (c) {
11961205 'C'...'F' => {
11971206 self.cursor += 1;
......@@ -1209,7 +1218,8 @@ pub const Scanner = struct {
12091218 }
12101219 },
12111220 .string_surrogate_half_backslash_u_2 => {
1212 const c = try self.expectByte();
1221 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1222 const c = self.input[self.cursor];
12131223 switch (c) {
12141224 '0'...'9' => {
12151225 self.cursor += 1;
......@@ -1233,7 +1243,8 @@ pub const Scanner = struct {
12331243 }
12341244 },
12351245 .string_surrogate_half_backslash_u_3 => {
1236 const c = try self.expectByte();
1246 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1247 const c = self.input[self.cursor];
12371248 switch (c) {
12381249 '0'...'9' => {
12391250 self.utf16_code_units[1] |= c - '0';
......@@ -1254,7 +1265,8 @@ pub const Scanner = struct {
12541265 },
12551266
12561267 .string_utf8_last_byte => {
1257 switch (try self.expectByte()) {
1268 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1269 switch (self.input[self.cursor]) {
12581270 0x80...0xBF => {
12591271 self.cursor += 1;
12601272 self.state = .string;
......@@ -1264,7 +1276,8 @@ pub const Scanner = struct {
12641276 }
12651277 },
12661278 .string_utf8_second_to_last_byte => {
1267 switch (try self.expectByte()) {
1279 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1280 switch (self.input[self.cursor]) {
12681281 0x80...0xBF => {
12691282 self.cursor += 1;
12701283 self.state = .string_utf8_last_byte;
......@@ -1274,7 +1287,8 @@ pub const Scanner = struct {
12741287 }
12751288 },
12761289 .string_utf8_second_to_last_byte_guard_against_overlong => {
1277 switch (try self.expectByte()) {
1290 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1291 switch (self.input[self.cursor]) {
12781292 0xA0...0xBF => {
12791293 self.cursor += 1;
12801294 self.state = .string_utf8_last_byte;
......@@ -1284,7 +1298,8 @@ pub const Scanner = struct {
12841298 }
12851299 },
12861300 .string_utf8_second_to_last_byte_guard_against_surrogate_half => {
1287 switch (try self.expectByte()) {
1301 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1302 switch (self.input[self.cursor]) {
12881303 0x80...0x9F => {
12891304 self.cursor += 1;
12901305 self.state = .string_utf8_last_byte;
......@@ -1294,7 +1309,8 @@ pub const Scanner = struct {
12941309 }
12951310 },
12961311 .string_utf8_third_to_last_byte => {
1297 switch (try self.expectByte()) {
1312 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1313 switch (self.input[self.cursor]) {
12981314 0x80...0xBF => {
12991315 self.cursor += 1;
13001316 self.state = .string_utf8_second_to_last_byte;
......@@ -1304,7 +1320,8 @@ pub const Scanner = struct {
13041320 }
13051321 },
13061322 .string_utf8_third_to_last_byte_guard_against_overlong => {
1307 switch (try self.expectByte()) {
1323 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1324 switch (self.input[self.cursor]) {
13081325 0x90...0xBF => {
13091326 self.cursor += 1;
13101327 self.state = .string_utf8_second_to_last_byte;
......@@ -1314,7 +1331,8 @@ pub const Scanner = struct {
13141331 }
13151332 },
13161333 .string_utf8_third_to_last_byte_guard_against_too_large => {
1317 switch (try self.expectByte()) {
1334 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1335 switch (self.input[self.cursor]) {
13181336 0x80...0x8F => {
13191337 self.cursor += 1;
13201338 self.state = .string_utf8_second_to_last_byte;
......@@ -1666,6 +1684,17 @@ pub const Scanner = struct {
16661684 self.value_start = self.cursor;
16671685 return slice;
16681686 }
1687 fn takeValueSliceMinusTrailingOffset(self: *@This(), trailing_negative_offset: usize) []const u8 {
1688 // Check if the escape sequence started before the current input buffer.
1689 // (The algebra here is awkward to avoid unsigned underflow,
1690 // but it's just making sure the slice on the next line isn't UB.)
1691 if (self.cursor <= self.value_start + trailing_negative_offset) return "";
1692 const slice = self.input[self.value_start .. self.cursor - trailing_negative_offset];
1693 // When trailing_negative_offset is non-zero, setting self.value_start doesn't matter,
1694 // because we always set it again while emitting the .partial_string_escaped_*.
1695 self.value_start = self.cursor;
1696 return slice;
1697 }
16691698
16701699 fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token {
16711700 const slice = self.takeValueSlice();
......@@ -1678,6 +1707,39 @@ pub const Scanner = struct {
16781707 return Token{ .partial_number = slice };
16791708 }
16801709
1710 fn endOfBufferInString(self: *@This()) !Token {
1711 if (self.is_end_of_input) return error.UnexpectedEndOfInput;
1712 const slice = self.takeValueSliceMinusTrailingOffset(switch (self.state) {
1713 // Don't include the escape sequence in the partial string.
1714 .string_backslash => 1,
1715 .string_backslash_u => 2,
1716 .string_backslash_u_1 => 3,
1717 .string_backslash_u_2 => 4,
1718 .string_backslash_u_3 => 5,
1719 .string_surrogate_half => 6,
1720 .string_surrogate_half_backslash => 7,
1721 .string_surrogate_half_backslash_u => 8,
1722 .string_surrogate_half_backslash_u_1 => 9,
1723 .string_surrogate_half_backslash_u_2 => 10,
1724 .string_surrogate_half_backslash_u_3 => 11,
1725
1726 // Include everything up to the cursor otherwise.
1727 .string,
1728 .string_utf8_last_byte,
1729 .string_utf8_second_to_last_byte,
1730 .string_utf8_second_to_last_byte_guard_against_overlong,
1731 .string_utf8_second_to_last_byte_guard_against_surrogate_half,
1732 .string_utf8_third_to_last_byte,
1733 .string_utf8_third_to_last_byte_guard_against_overlong,
1734 .string_utf8_third_to_last_byte_guard_against_too_large,
1735 => 0,
1736
1737 else => unreachable,
1738 });
1739 if (slice.len == 0) return error.BufferUnderrun;
1740 return Token{ .partial_string = slice };
1741 }
1742
16811743 fn partialStringCodepoint(code_point: u21) Token {
16821744 var buf: [4]u8 = undefined;
16831745 switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {