From 02efc2236a8223872f57a1a7a5aec756b5a23a76 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Mon, 2 Nov 2020 11:46:50 +0100 Subject: [PATCH] std: Fix json utf{8,16} decoding on BE targets Byteswap some values when LE ordering is required. --- lib/std/json.zig | 24 +++++++-------- lib/std/json/test.zig | 69 ------------------------------------------- 2 files changed, 11 insertions(+), 82 deletions(-) diff --git a/lib/std/json.zig b/lib/std/json.zig index f3594bc3ebd84f594c049d994f348d2f38b3f306..b64216b2d419ff56f2f0f51c38b5c3eab71fdce9 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -2125,7 +2125,11 @@ fn unescapeString(output: []u8, input: []const u8) !void { const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex + 8 .. inIndex + 12], 16) catch unreachable; - if (std.unicode.utf16leToUtf8(output[outIndex..], &[2]u16{ firstCodeUnit, secondCodeUnit })) |byteCount| { + const utf16le_seq = [2]u16{ + mem.littleToNative(u16, firstCodeUnit), + mem.littleToNative(u16, secondCodeUnit), + }; + if (std.unicode.utf16leToUtf8(output[outIndex..], &utf16le_seq)) |byteCount| { outIndex += byteCount; inIndex += 12; } else |_| { @@ -2265,9 +2269,6 @@ test "integer after float has proper type" { } test "escaped characters" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_allocator.deinit(); const input = @@ -2300,9 +2301,6 @@ test "escaped characters" { } test "string copy option" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - const input = \\{ \\ "noescape": "aą😂", @@ -2721,9 +2719,9 @@ test "stringify struct with indentation" { \\} , struct { - foo: u32, - bar: [3]u32, - }{ + foo: u32, + bar: [3]u32, + }{ .foo = 42, .bar = .{ 1, 2, 3 }, }, @@ -2734,9 +2732,9 @@ test "stringify struct with indentation" { try teststringify( "{\n\t\"foo\":42,\n\t\"bar\":[\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}", struct { - foo: u32, - bar: [3]u32, - }{ + foo: u32, + bar: [3]u32, + }{ .foo = 42, .bar = .{ 1, 2, 3 }, }, diff --git a/lib/std/json/test.zig b/lib/std/json/test.zig index 3a7c25e3834a8b59562df2a74b810c6ee26f58a8..bce9ed448d30ffd28d1a4cdc52cd4f8f66337cc3 100644 --- a/lib/std/json/test.zig +++ b/lib/std/json/test.zig @@ -337,18 +337,12 @@ test "y_string_1_2_3_bytes_UTF-8_sequences" { } test "y_string_accepted_surrogate_pair" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\uD801\udc37"] ); } test "y_string_accepted_surrogate_pairs" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\ud83d\ude39\ud83d\udc8d"] ); @@ -415,9 +409,6 @@ test "y_string_in_array_with_leading_space" { } test "y_string_last_surrogates_1_and_2" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\uDBFF\uDFFF"] ); @@ -478,9 +469,6 @@ test "y_string_space" { } test "y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\uD834\uDd1e"] ); @@ -523,90 +511,60 @@ test "y_string_unescaped_char_delete" { } test "y_string_unicode_2" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["⍂㈴⍂"] ); } test "y_string_unicodeEscapedBackslash" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\u005C"] ); } test "y_string_unicode_escaped_double_quote" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\u0022"] ); } test "y_string_unicode" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\uA66D"] ); } test "y_string_unicode_U+10FFFE_nonchar" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\uDBFF\uDFFE"] ); } test "y_string_unicode_U+1FFFE_nonchar" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\uD83F\uDFFE"] ); } test "y_string_unicode_U+200B_ZERO_WIDTH_SPACE" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\u200B"] ); } test "y_string_unicode_U+2064_invisible_plus" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\u2064"] ); } test "y_string_unicode_U+FDD0_nonchar" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\uFDD0"] ); } test "y_string_unicode_U+FFFE_nonchar" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - ok( \\["\uFFFE"] ); @@ -1858,63 +1816,42 @@ test "i_object_key_lone_2nd_surrogate" { } test "i_string_1st_surrogate_but_2nd_missing" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\uDADA"] ); } test "i_string_1st_valid_surrogate_2nd_invalid" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\uD888\u1234"] ); } test "i_string_incomplete_surrogate_and_escape_valid" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\uD800\n"] ); } test "i_string_incomplete_surrogate_pair" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\uDd1ea"] ); } test "i_string_incomplete_surrogates_escape_valid" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\uD800\uD800\n"] ); } test "i_string_invalid_lonely_surrogate" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\ud800"] ); } test "i_string_invalid_surrogate" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\ud800abc"] ); @@ -1927,9 +1864,6 @@ test "i_string_invalid_utf-8" { } test "i_string_inverted_surrogates_U+1D11E" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\uDd1e\uD834"] ); @@ -1942,9 +1876,6 @@ test "i_string_iso_latin_1" { } test "i_string_lone_second_surrogate" { - // https://github.com/ziglang/zig/issues/5127 - if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest; - anyStreamingErrNonStreaming( \\["\uDFAA"] ); -- 2.54.0