authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-11-02 11:46:50+01:00
committergravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-11-02 11:46:50+01:00
log02efc2236a8223872f57a1a7a5aec756b5a23a76
tree30c48ad99e9cdedcb89858dd2f918662e86b670b
parent4fd1ec78e93d496811534c6e3aa9af126650f5de

std: Fix json utf{8,16} decoding on BE targets

Byteswap some values when LE ordering is required.

2 files changed, 11 insertions(+), 82 deletions(-)

lib/std/json.zig+11-13
......@@ -2125,7 +2125,11 @@ fn unescapeString(output: []u8, input: []const u8) !void {
21252125
21262126 const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex + 8 .. inIndex + 12], 16) catch unreachable;
21272127
2128 if (std.unicode.utf16leToUtf8(output[outIndex..], &[2]u16{ firstCodeUnit, secondCodeUnit })) |byteCount| {
2128 const utf16le_seq = [2]u16{
2129 mem.littleToNative(u16, firstCodeUnit),
2130 mem.littleToNative(u16, secondCodeUnit),
2131 };
2132 if (std.unicode.utf16leToUtf8(output[outIndex..], &utf16le_seq)) |byteCount| {
21292133 outIndex += byteCount;
21302134 inIndex += 12;
21312135 } else |_| {
......@@ -2265,9 +2269,6 @@ test "integer after float has proper type" {
22652269}
22662270
22672271test "escaped characters" {
2268 // https://github.com/ziglang/zig/issues/5127
2269 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
2270
22712272 var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
22722273 defer arena_allocator.deinit();
22732274 const input =
......@@ -2300,9 +2301,6 @@ test "escaped characters" {
23002301}
23012302
23022303test "string copy option" {
2303 // https://github.com/ziglang/zig/issues/5127
2304 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
2305
23062304 const input =
23072305 \\{
23082306 \\ "noescape": "aą😂",
......@@ -2721,9 +2719,9 @@ test "stringify struct with indentation" {
27212719 \\}
27222720 ,
27232721 struct {
2724 foo: u32,
2725 bar: [3]u32,
2726 }{
2722 foo: u32,
2723 bar: [3]u32,
2724 }{
27272725 .foo = 42,
27282726 .bar = .{ 1, 2, 3 },
27292727 },
......@@ -2734,9 +2732,9 @@ test "stringify struct with indentation" {
27342732 try teststringify(
27352733 "{\n\t\"foo\":42,\n\t\"bar\":[\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}",
27362734 struct {
2737 foo: u32,
2738 bar: [3]u32,
2739 }{
2735 foo: u32,
2736 bar: [3]u32,
2737 }{
27402738 .foo = 42,
27412739 .bar = .{ 1, 2, 3 },
27422740 },
lib/std/json/test.zig-69
......@@ -337,18 +337,12 @@ test "y_string_1_2_3_bytes_UTF-8_sequences" {
337337}
338338
339339test "y_string_accepted_surrogate_pair" {
340 // https://github.com/ziglang/zig/issues/5127
341 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
342
343340 ok(
344341 \\["\uD801\udc37"]
345342 );
346343}
347344
348345test "y_string_accepted_surrogate_pairs" {
349 // https://github.com/ziglang/zig/issues/5127
350 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
351
352346 ok(
353347 \\["\ud83d\ude39\ud83d\udc8d"]
354348 );
......@@ -415,9 +409,6 @@ test "y_string_in_array_with_leading_space" {
415409}
416410
417411test "y_string_last_surrogates_1_and_2" {
418 // https://github.com/ziglang/zig/issues/5127
419 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
420
421412 ok(
422413 \\["\uDBFF\uDFFF"]
423414 );
......@@ -478,9 +469,6 @@ test "y_string_space" {
478469}
479470
480471test "y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF" {
481 // https://github.com/ziglang/zig/issues/5127
482 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
483
484472 ok(
485473 \\["\uD834\uDd1e"]
486474 );
......@@ -523,90 +511,60 @@ test "y_string_unescaped_char_delete" {
523511}
524512
525513test "y_string_unicode_2" {
526 // https://github.com/ziglang/zig/issues/5127
527 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
528
529514 ok(
530515 \\["⍂㈴⍂"]
531516 );
532517}
533518
534519test "y_string_unicodeEscapedBackslash" {
535 // https://github.com/ziglang/zig/issues/5127
536 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
537
538520 ok(
539521 \\["\u005C"]
540522 );
541523}
542524
543525test "y_string_unicode_escaped_double_quote" {
544 // https://github.com/ziglang/zig/issues/5127
545 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
546
547526 ok(
548527 \\["\u0022"]
549528 );
550529}
551530
552531test "y_string_unicode" {
553 // https://github.com/ziglang/zig/issues/5127
554 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
555
556532 ok(
557533 \\["\uA66D"]
558534 );
559535}
560536
561537test "y_string_unicode_U+10FFFE_nonchar" {
562 // https://github.com/ziglang/zig/issues/5127
563 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
564
565538 ok(
566539 \\["\uDBFF\uDFFE"]
567540 );
568541}
569542
570543test "y_string_unicode_U+1FFFE_nonchar" {
571 // https://github.com/ziglang/zig/issues/5127
572 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
573
574544 ok(
575545 \\["\uD83F\uDFFE"]
576546 );
577547}
578548
579549test "y_string_unicode_U+200B_ZERO_WIDTH_SPACE" {
580 // https://github.com/ziglang/zig/issues/5127
581 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
582
583550 ok(
584551 \\["\u200B"]
585552 );
586553}
587554
588555test "y_string_unicode_U+2064_invisible_plus" {
589 // https://github.com/ziglang/zig/issues/5127
590 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
591
592556 ok(
593557 \\["\u2064"]
594558 );
595559}
596560
597561test "y_string_unicode_U+FDD0_nonchar" {
598 // https://github.com/ziglang/zig/issues/5127
599 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
600
601562 ok(
602563 \\["\uFDD0"]
603564 );
604565}
605566
606567test "y_string_unicode_U+FFFE_nonchar" {
607 // https://github.com/ziglang/zig/issues/5127
608 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
609
610568 ok(
611569 \\["\uFFFE"]
612570 );
......@@ -1858,63 +1816,42 @@ test "i_object_key_lone_2nd_surrogate" {
18581816}
18591817
18601818test "i_string_1st_surrogate_but_2nd_missing" {
1861 // https://github.com/ziglang/zig/issues/5127
1862 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1863
18641819 anyStreamingErrNonStreaming(
18651820 \\["\uDADA"]
18661821 );
18671822}
18681823
18691824test "i_string_1st_valid_surrogate_2nd_invalid" {
1870 // https://github.com/ziglang/zig/issues/5127
1871 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1872
18731825 anyStreamingErrNonStreaming(
18741826 \\["\uD888\u1234"]
18751827 );
18761828}
18771829
18781830test "i_string_incomplete_surrogate_and_escape_valid" {
1879 // https://github.com/ziglang/zig/issues/5127
1880 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1881
18821831 anyStreamingErrNonStreaming(
18831832 \\["\uD800\n"]
18841833 );
18851834}
18861835
18871836test "i_string_incomplete_surrogate_pair" {
1888 // https://github.com/ziglang/zig/issues/5127
1889 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1890
18911837 anyStreamingErrNonStreaming(
18921838 \\["\uDd1ea"]
18931839 );
18941840}
18951841
18961842test "i_string_incomplete_surrogates_escape_valid" {
1897 // https://github.com/ziglang/zig/issues/5127
1898 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1899
19001843 anyStreamingErrNonStreaming(
19011844 \\["\uD800\uD800\n"]
19021845 );
19031846}
19041847
19051848test "i_string_invalid_lonely_surrogate" {
1906 // https://github.com/ziglang/zig/issues/5127
1907 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1908
19091849 anyStreamingErrNonStreaming(
19101850 \\["\ud800"]
19111851 );
19121852}
19131853
19141854test "i_string_invalid_surrogate" {
1915 // https://github.com/ziglang/zig/issues/5127
1916 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1917
19181855 anyStreamingErrNonStreaming(
19191856 \\["\ud800abc"]
19201857 );
......@@ -1927,9 +1864,6 @@ test "i_string_invalid_utf-8" {
19271864}
19281865
19291866test "i_string_inverted_surrogates_U+1D11E" {
1930 // https://github.com/ziglang/zig/issues/5127
1931 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1932
19331867 anyStreamingErrNonStreaming(
19341868 \\["\uDd1e\uD834"]
19351869 );
......@@ -1942,9 +1876,6 @@ test "i_string_iso_latin_1" {
19421876}
19431877
19441878test "i_string_lone_second_surrogate" {
1945 // https://github.com/ziglang/zig/issues/5127
1946 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
1947
19481879 anyStreamingErrNonStreaming(
19491880 \\["\uDFAA"]
19501881 );