authorgravatar for metaleap@users.noreply.github.comPhil Schumann <metaleap@users.noreply.github.com> 2020-04-08 02:27:18+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-04-07 20:27:18-04:00
logb109186dd5e11e3da2f0f49de370e2b8447e92a0
treecced2835d93f8c351c3d91e4dbd835428d92ec1f
parent66b2477ab668447c6cbff68770de14ec8db991be
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std/zig/parse_string_literal.zig: add hex+unicode escapes (#4678)

1 files changed, 56 insertions(+), 7 deletions(-)

lib/std/zig/parse_string_literal.zig+56-7
...@@ -19,17 +19,19 @@ pub fn parseStringLiteral(...@@ -19,17 +19,19 @@ pub fn parseStringLiteral(
19 bytes: []const u8,19 bytes: []const u8,
20 bad_index: *usize, // populated if error.InvalidCharacter is returned20 bad_index: *usize, // populated if error.InvalidCharacter is returned
21) ParseStringLiteralError![]u8 {21) ParseStringLiteralError![]u8 {
22 const first_index = if (bytes[0] == 'c') @as(usize, 2) else @as(usize, 1);22 assert(bytes.len >= 2 and bytes[0] == '"' and bytes[bytes.len - 1] == '"');
23 assert(bytes[bytes.len - 1] == '"');
2423
25 var list = std.ArrayList(u8).init(allocator);24 var list = std.ArrayList(u8).init(allocator);
26 errdefer list.deinit();25 errdefer list.deinit();
2726
28 const slice = bytes[first_index..];27 const slice = bytes[1..];
29 try list.ensureCapacity(slice.len - 1);28 try list.ensureCapacity(slice.len - 1);
3029
31 var state = State.Start;30 var state = State.Start;
32 for (slice) |b, index| {31 var index: usize = 0;
32 while (index < slice.len) : (index += 1) {
33 const b = slice[index];
34
33 switch (state) {35 switch (state) {
34 State.Start => switch (b) {36 State.Start => switch (b) {
35 '\\' => state = State.Backslash,37 '\\' => state = State.Backslash,
...@@ -41,9 +43,6 @@ pub fn parseStringLiteral(...@@ -41,9 +43,6 @@ pub fn parseStringLiteral(
41 else => try list.append(b),43 else => try list.append(b),
42 },44 },
43 State.Backslash => switch (b) {45 State.Backslash => switch (b) {
44 'x' => @panic("TODO"),
45 'u' => @panic("TODO"),
46 'U' => @panic("TODO"),
47 'n' => {46 'n' => {
48 try list.append('\n');47 try list.append('\n');
49 state = State.Start;48 state = State.Start;
...@@ -60,10 +59,46 @@ pub fn parseStringLiteral(...@@ -60,10 +59,46 @@ pub fn parseStringLiteral(
60 try list.append('\t');59 try list.append('\t');
61 state = State.Start;60 state = State.Start;
62 },61 },
62 '\'' => {
63 try list.append('\'');
64 state = State.Start;
65 },
63 '"' => {66 '"' => {
64 try list.append('"');67 try list.append('"');
65 state = State.Start;68 state = State.Start;
66 },69 },
70 'x' => {
71 // TODO: add more/better/broader tests for this.
72 const index_continue = index + 3;
73 if (slice.len >= index_continue)
74 if (std.fmt.parseUnsigned(u8, slice[index + 1 .. index_continue], 16)) |char| {
75 try list.append(char);
76 state = State.Start;
77 index = index_continue - 1; // loop-header increments again
78 continue;
79 } else |_| {};
80
81 bad_index.* = index;
82 return error.InvalidCharacter;
83 },
84 'u' => {
85 // TODO: add more/better/broader tests for this.
86 if (slice.len > index + 2 and slice[index + 1] == '{')
87 if (std.mem.indexOfScalarPos(u8, slice[0..std.math.min(index + 9, slice.len)], index + 3, '}')) |index_end| {
88 const hex_str = slice[index + 2 .. index_end];
89 if (std.fmt.parseUnsigned(u32, hex_str, 16)) |uint| {
90 if (uint <= 0x10ffff) {
91 try list.appendSlice(std.mem.toBytes(uint)[0..]);
92 state = State.Start;
93 index = index_end; // loop-header increments
94 continue;
95 }
96 } else |_| {}
97 };
98
99 bad_index.* = index;
100 return error.InvalidCharacter;
101 },
67 else => {102 else => {
68 bad_index.* = index;103 bad_index.* = index;
69 return error.InvalidCharacter;104 return error.InvalidCharacter;
...@@ -74,3 +109,17 @@ pub fn parseStringLiteral(...@@ -74,3 +109,17 @@ pub fn parseStringLiteral(
74 }109 }
75 unreachable;110 unreachable;
76}111}
112
113test "parseStringLiteral" {
114 const expect = std.testing.expect;
115 const eql = std.mem.eql;
116
117 var fixed_buf_mem: [32]u8 = undefined;
118 var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(fixed_buf_mem[0..]);
119 var alloc = &fixed_buf_alloc.allocator;
120 var bad_index: usize = undefined;
121
122 expect(eql(u8, "foo", try parseStringLiteral(alloc, "\"foo\"", &bad_index)));
123 expect(eql(u8, "foo", try parseStringLiteral(alloc, "\"f\x6f\x6f\"", &bad_index)));
124 expect(eql(u8, "f💯", try parseStringLiteral(alloc, "\"f\u{1f4af}\"", &bad_index)));
125}