authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-08-15 06:11:59-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-08-15 09:11:59-04:00
loga155e35850def120af90993e1b4309b80b97eb85
treed886aa1293c40d8565fae63a168a5c59755d0b17
parentf7b82ed416e32a9164c67a35924b9b99e86707fa
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.json: Fix decoding of UTF-16 surrogate pairs (#16830)

* std.unicode: Add more UTF-16 decoding functions This mostly makes parts of Utf16LeIterator reusable * std.json: Fix decoding of UTF-16 surrogate pairs Before this commit, there were 524,288 codepoints that would get decoded improperly. After this commit, there are 0. Fixes #16828

3 files changed, 95 insertions(+), 46 deletions(-)

lib/std/json/scanner.zig+34-39
......@@ -414,7 +414,7 @@ pub const Scanner = struct {
414414 string_is_object_key: bool = false,
415415 stack: BitStack,
416416 value_start: usize = undefined,
417 unicode_code_point: u21 = undefined,
417 utf16_code_units: [2]u16 = undefined,
418418
419419 input: []const u8 = "",
420420 cursor: usize = 0,
......@@ -1083,13 +1083,13 @@ pub const Scanner = struct {
10831083 const c = try self.expectByte();
10841084 switch (c) {
10851085 '0'...'9' => {
1086 self.unicode_code_point = @as(u21, c - '0') << 12;
1086 self.utf16_code_units[0] = @as(u16, c - '0') << 12;
10871087 },
10881088 'A'...'F' => {
1089 self.unicode_code_point = @as(u21, c - 'A' + 10) << 12;
1089 self.utf16_code_units[0] = @as(u16, c - 'A' + 10) << 12;
10901090 },
10911091 'a'...'f' => {
1092 self.unicode_code_point = @as(u21, c - 'a' + 10) << 12;
1092 self.utf16_code_units[0] = @as(u16, c - 'a' + 10) << 12;
10931093 },
10941094 else => return error.SyntaxError,
10951095 }
......@@ -1101,13 +1101,13 @@ pub const Scanner = struct {
11011101 const c = try self.expectByte();
11021102 switch (c) {
11031103 '0'...'9' => {
1104 self.unicode_code_point |= @as(u21, c - '0') << 8;
1104 self.utf16_code_units[0] |= @as(u16, c - '0') << 8;
11051105 },
11061106 'A'...'F' => {
1107 self.unicode_code_point |= @as(u21, c - 'A' + 10) << 8;
1107 self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 8;
11081108 },
11091109 'a'...'f' => {
1110 self.unicode_code_point |= @as(u21, c - 'a' + 10) << 8;
1110 self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 8;
11111111 },
11121112 else => return error.SyntaxError,
11131113 }
......@@ -1119,13 +1119,13 @@ pub const Scanner = struct {
11191119 const c = try self.expectByte();
11201120 switch (c) {
11211121 '0'...'9' => {
1122 self.unicode_code_point |= @as(u21, c - '0') << 4;
1122 self.utf16_code_units[0] |= @as(u16, c - '0') << 4;
11231123 },
11241124 'A'...'F' => {
1125 self.unicode_code_point |= @as(u21, c - 'A' + 10) << 4;
1125 self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 4;
11261126 },
11271127 'a'...'f' => {
1128 self.unicode_code_point |= @as(u21, c - 'a' + 10) << 4;
1128 self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 4;
11291129 },
11301130 else => return error.SyntaxError,
11311131 }
......@@ -1137,31 +1137,26 @@ pub const Scanner = struct {
11371137 const c = try self.expectByte();
11381138 switch (c) {
11391139 '0'...'9' => {
1140 self.unicode_code_point |= c - '0';
1140 self.utf16_code_units[0] |= c - '0';
11411141 },
11421142 'A'...'F' => {
1143 self.unicode_code_point |= c - 'A' + 10;
1143 self.utf16_code_units[0] |= c - 'A' + 10;
11441144 },
11451145 'a'...'f' => {
1146 self.unicode_code_point |= c - 'a' + 10;
1146 self.utf16_code_units[0] |= c - 'a' + 10;
11471147 },
11481148 else => return error.SyntaxError,
11491149 }
11501150 self.cursor += 1;
1151 switch (self.unicode_code_point) {
1152 0xD800...0xDBFF => {
1153 // High surrogate half.
1154 self.unicode_code_point = 0x10000 | (self.unicode_code_point << 10);
1155 self.state = .string_surrogate_half;
1156 continue :state_loop;
1157 },
1158 0xDC00...0xDFFF => return error.SyntaxError, // Unexpected low surrogate half.
1159 else => {
1160 // Code point from a single UTF-16 code unit.
1161 self.value_start = self.cursor;
1162 self.state = .string;
1163 return self.partialStringCodepoint();
1164 },
1151 if (std.unicode.utf16IsHighSurrogate(self.utf16_code_units[0])) {
1152 self.state = .string_surrogate_half;
1153 continue :state_loop;
1154 } else if (std.unicode.utf16IsLowSurrogate(self.utf16_code_units[0])) {
1155 return error.SyntaxError; // Unexpected low surrogate half.
1156 } else {
1157 self.value_start = self.cursor;
1158 self.state = .string;
1159 return partialStringCodepoint(self.utf16_code_units[0]);
11651160 }
11661161 },
11671162 .string_surrogate_half => {
......@@ -1188,6 +1183,7 @@ pub const Scanner = struct {
11881183 switch (try self.expectByte()) {
11891184 'D', 'd' => {
11901185 self.cursor += 1;
1186 self.utf16_code_units[1] = 0xD << 12;
11911187 self.state = .string_surrogate_half_backslash_u_1;
11921188 continue :state_loop;
11931189 },
......@@ -1199,13 +1195,13 @@ pub const Scanner = struct {
11991195 switch (c) {
12001196 'C'...'F' => {
12011197 self.cursor += 1;
1202 self.unicode_code_point |= @as(u21, c - 'C') << 8;
1198 self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 8;
12031199 self.state = .string_surrogate_half_backslash_u_2;
12041200 continue :state_loop;
12051201 },
12061202 'c'...'f' => {
12071203 self.cursor += 1;
1208 self.unicode_code_point |= @as(u21, c - 'c') << 8;
1204 self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 8;
12091205 self.state = .string_surrogate_half_backslash_u_2;
12101206 continue :state_loop;
12111207 },
......@@ -1217,19 +1213,19 @@ pub const Scanner = struct {
12171213 switch (c) {
12181214 '0'...'9' => {
12191215 self.cursor += 1;
1220 self.unicode_code_point |= @as(u21, c - '0') << 4;
1216 self.utf16_code_units[1] |= @as(u16, c - '0') << 4;
12211217 self.state = .string_surrogate_half_backslash_u_3;
12221218 continue :state_loop;
12231219 },
12241220 'A'...'F' => {
12251221 self.cursor += 1;
1226 self.unicode_code_point |= @as(u21, c - 'A' + 10) << 4;
1222 self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 4;
12271223 self.state = .string_surrogate_half_backslash_u_3;
12281224 continue :state_loop;
12291225 },
12301226 'a'...'f' => {
12311227 self.cursor += 1;
1232 self.unicode_code_point |= @as(u21, c - 'a' + 10) << 4;
1228 self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 4;
12331229 self.state = .string_surrogate_half_backslash_u_3;
12341230 continue :state_loop;
12351231 },
......@@ -1240,20 +1236,21 @@ pub const Scanner = struct {
12401236 const c = try self.expectByte();
12411237 switch (c) {
12421238 '0'...'9' => {
1243 self.unicode_code_point |= c - '0';
1239 self.utf16_code_units[1] |= c - '0';
12441240 },
12451241 'A'...'F' => {
1246 self.unicode_code_point |= c - 'A' + 10;
1242 self.utf16_code_units[1] |= c - 'A' + 10;
12471243 },
12481244 'a'...'f' => {
1249 self.unicode_code_point |= c - 'a' + 10;
1245 self.utf16_code_units[1] |= c - 'a' + 10;
12501246 },
12511247 else => return error.SyntaxError,
12521248 }
12531249 self.cursor += 1;
12541250 self.value_start = self.cursor;
12551251 self.state = .string;
1256 return self.partialStringCodepoint();
1252 const code_point = std.unicode.utf16DecodeSurrogatePair(&self.utf16_code_units) catch unreachable;
1253 return partialStringCodepoint(code_point);
12571254 },
12581255
12591256 .string_utf8_last_byte => {
......@@ -1681,9 +1678,7 @@ pub const Scanner = struct {
16811678 return Token{ .partial_number = slice };
16821679 }
16831680
1684 fn partialStringCodepoint(self: *@This()) Token {
1685 const code_point = self.unicode_code_point;
1686 self.unicode_code_point = undefined;
1681 fn partialStringCodepoint(code_point: u21) Token {
16871682 var buf: [4]u8 = undefined;
16881683 switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {
16891684 1 => return Token{ .partial_string_escaped_1 = buf[0..1].* },
lib/std/json/scanner_test.zig+1
......@@ -236,6 +236,7 @@ const string_test_cases = .{
236236 .{ "\\u000a", "\n" },
237237 .{ "𝄞", "\u{1D11E}" },
238238 .{ "\\uD834\\uDD1E", "\u{1D11E}" },
239 .{ "\\uD87F\\uDFFE", "\u{2FFFE}" },
239240 .{ "\\uff20", "＠" },
240241};
241242
lib/std/unicode.zig+60-7
......@@ -293,6 +293,58 @@ pub const Utf8Iterator = struct {
293293 }
294294};
295295
296pub fn utf16IsHighSurrogate(c: u16) bool {
297 return c & ~@as(u16, 0x03ff) == 0xd800;
298}
299
300pub fn utf16IsLowSurrogate(c: u16) bool {
301 return c & ~@as(u16, 0x03ff) == 0xdc00;
302}
303
304/// Returns how many code units the UTF-16 representation would require
305/// for the given codepoint.
306pub fn utf16CodepointSequenceLength(c: u21) !u2 {
307 if (c <= 0xFFFF) return 1;
308 if (c <= 0x10FFFF) return 2;
309 return error.CodepointTooLarge;
310}
311
312test utf16CodepointSequenceLength {
313 try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength('a'));
314 try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength(0xFFFF));
315 try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10000));
316 try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10FFFF));
317 try testing.expectError(error.CodepointTooLarge, utf16CodepointSequenceLength(0x110000));
318}
319
320/// Given the first code unit of a UTF-16 codepoint, returns a number 1-2
321/// indicating the total length of the codepoint in UTF-16 code units.
322/// If this code unit does not match the form of a UTF-16 start code unit, returns Utf16InvalidStartCodeUnit.
323pub fn utf16CodeUnitSequenceLength(first_code_unit: u16) !u2 {
324 if (utf16IsHighSurrogate(first_code_unit)) return 2;
325 if (utf16IsLowSurrogate(first_code_unit)) return error.Utf16InvalidStartCodeUnit;
326 return 1;
327}
328
329test utf16CodeUnitSequenceLength {
330 try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength('a'));
331 try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength(0xFFFF));
332 try testing.expectEqual(@as(u2, 2), try utf16CodeUnitSequenceLength(0xDBFF));
333 try testing.expectError(error.Utf16InvalidStartCodeUnit, utf16CodeUnitSequenceLength(0xDFFF));
334}
335
336/// Decodes the codepoint encoded in the given pair of UTF-16 code units.
337/// Asserts that `surrogate_pair.len >= 2` and that the first code unit is a high surrogate.
338/// If the second code unit is not a low surrogate, error.ExpectedSecondSurrogateHalf is returned.
339pub fn utf16DecodeSurrogatePair(surrogate_pair: []const u16) !u21 {
340 assert(surrogate_pair.len >= 2);
341 assert(utf16IsHighSurrogate(surrogate_pair[0]));
342 const high_half: u21 = surrogate_pair[0];
343 const low_half = surrogate_pair[1];
344 if (!utf16IsLowSurrogate(low_half)) return error.ExpectedSecondSurrogateHalf;
345 return 0x10000 + ((high_half & 0x03ff) << 10) | (low_half & 0x03ff);
346}
347
296348pub const Utf16LeIterator = struct {
297349 bytes: []const u8,
298350 i: usize,
......@@ -307,19 +359,20 @@ pub const Utf16LeIterator = struct {
307359 pub fn nextCodepoint(it: *Utf16LeIterator) !?u21 {
308360 assert(it.i <= it.bytes.len);
309361 if (it.i == it.bytes.len) return null;
310 const c0: u21 = mem.readIntLittle(u16, it.bytes[it.i..][0..2]);
362 var code_units: [2]u16 = undefined;
363 code_units[0] = mem.readIntLittle(u16, it.bytes[it.i..][0..2]);
311364 it.i += 2;
312 if (c0 & ~@as(u21, 0x03ff) == 0xd800) {
365 if (utf16IsHighSurrogate(code_units[0])) {
313366 // surrogate pair
314367 if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;
315 const c1: u21 = mem.readIntLittle(u16, it.bytes[it.i..][0..2]);
316 if (c1 & ~@as(u21, 0x03ff) != 0xdc00) return error.ExpectedSecondSurrogateHalf;
368 code_units[1] = mem.readIntLittle(u16, it.bytes[it.i..][0..2]);
369 const codepoint = try utf16DecodeSurrogatePair(&code_units);
317370 it.i += 2;
318 return 0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff));
319 } else if (c0 & ~@as(u21, 0x03ff) == 0xdc00) {
371 return codepoint;
372 } else if (utf16IsLowSurrogate(code_units[0])) {
320373 return error.UnexpectedSecondSurrogateHalf;
321374 } else {
322 return c0;
375 return code_units[0];
323376 }
324377 }
325378};