authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-04-19 20:33:15-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-04-19 20:33:15-04:00
log91ca0e4b02ff8f67e7e18a21fdcd1168f1f5a675
treef2b7023446657ff141fa3791e80cb9c81d16f5ce
parentded6e0326d8965de8763806593b008c9c28d5508

implement rendering escaped zig string literals

3 files changed, 157 insertions(+), 126 deletions(-)

lib/std/zig.zig+2-1
...@@ -2,8 +2,9 @@ const tokenizer = @import("zig/tokenizer.zig");...@@ -2,8 +2,9 @@ const tokenizer = @import("zig/tokenizer.zig");
2pub const Token = tokenizer.Token;2pub const Token = tokenizer.Token;
3pub const Tokenizer = tokenizer.Tokenizer;3pub const Tokenizer = tokenizer.Tokenizer;
4pub const parse = @import("zig/parse.zig").parse;4pub const parse = @import("zig/parse.zig").parse;
5pub const parseStringLiteral = @import("zig/parse_string_literal.zig").parseStringLiteral;5pub const parseStringLiteral = @import("zig/string_literal.zig").parse;
6pub const render = @import("zig/render.zig").render;6pub const render = @import("zig/render.zig").render;
7pub const renderStringLiteral = @import("zig/string_literal.zig").render;
7pub const ast = @import("zig/ast.zig");8pub const ast = @import("zig/ast.zig");
8pub const system = @import("zig/system.zig");9pub const system = @import("zig/system.zig");
9pub const CrossTarget = @import("zig/cross_target.zig").CrossTarget;10pub const CrossTarget = @import("zig/cross_target.zig").CrossTarget;
lib/std/zig/parse_string_literal.zig deleted-125
...@@ -1,125 +0,0 @@
1const std = @import("../std.zig");
2const assert = std.debug.assert;
3
4const State = enum {
5 Start,
6 Backslash,
7};
8
9pub const ParseStringLiteralError = error{
10 OutOfMemory,
11
12 /// When this is returned, index will be the position of the character.
13 InvalidCharacter,
14};
15
16/// caller owns returned memory
17pub fn parseStringLiteral(
18 allocator: *std.mem.Allocator,
19 bytes: []const u8,
20 bad_index: *usize, // populated if error.InvalidCharacter is returned
21) ParseStringLiteralError![]u8 {
22 assert(bytes.len >= 2 and bytes[0] == '"' and bytes[bytes.len - 1] == '"');
23
24 var list = std.ArrayList(u8).init(allocator);
25 errdefer list.deinit();
26
27 const slice = bytes[1..];
28 try list.ensureCapacity(slice.len - 1);
29
30 var state = State.Start;
31 var index: usize = 0;
32 while (index < slice.len) : (index += 1) {
33 const b = slice[index];
34
35 switch (state) {
36 State.Start => switch (b) {
37 '\\' => state = State.Backslash,
38 '\n' => {
39 bad_index.* = index;
40 return error.InvalidCharacter;
41 },
42 '"' => return list.toOwnedSlice(),
43 else => try list.append(b),
44 },
45 State.Backslash => switch (b) {
46 'n' => {
47 try list.append('\n');
48 state = State.Start;
49 },
50 'r' => {
51 try list.append('\r');
52 state = State.Start;
53 },
54 '\\' => {
55 try list.append('\\');
56 state = State.Start;
57 },
58 't' => {
59 try list.append('\t');
60 state = State.Start;
61 },
62 '\'' => {
63 try list.append('\'');
64 state = State.Start;
65 },
66 '"' => {
67 try list.append('"');
68 state = State.Start;
69 },
70 'x' => {
71 // TODO: add more/better/broader tests for this.
72 const index_continue = index + 3;
73 if (slice.len >= index_continue)
74 if (std.fmt.parseUnsigned(u8, slice[index + 1 .. index_continue], 16)) |char| {
75 try list.append(char);
76 state = State.Start;
77 index = index_continue - 1; // loop-header increments again
78 continue;
79 } else |_| {};
80
81 bad_index.* = index;
82 return error.InvalidCharacter;
83 },
84 'u' => {
85 // TODO: add more/better/broader tests for this.
86 if (slice.len > index + 2 and slice[index + 1] == '{')
87 if (std.mem.indexOfScalarPos(u8, slice[0..std.math.min(index + 9, slice.len)], index + 3, '}')) |index_end| {
88 const hex_str = slice[index + 2 .. index_end];
89 if (std.fmt.parseUnsigned(u32, hex_str, 16)) |uint| {
90 if (uint <= 0x10ffff) {
91 try list.appendSlice(std.mem.toBytes(uint)[0..]);
92 state = State.Start;
93 index = index_end; // loop-header increments
94 continue;
95 }
96 } else |_| {}
97 };
98
99 bad_index.* = index;
100 return error.InvalidCharacter;
101 },
102 else => {
103 bad_index.* = index;
104 return error.InvalidCharacter;
105 },
106 },
107 else => unreachable,
108 }
109 }
110 unreachable;
111}
112
113test "parseStringLiteral" {
114 const expect = std.testing.expect;
115 const eql = std.mem.eql;
116
117 var fixed_buf_mem: [32]u8 = undefined;
118 var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(fixed_buf_mem[0..]);
119 var alloc = &fixed_buf_alloc.allocator;
120 var bad_index: usize = undefined;
121
122 expect(eql(u8, "foo", try parseStringLiteral(alloc, "\"foo\"", &bad_index)));
123 expect(eql(u8, "foo", try parseStringLiteral(alloc, "\"f\x6f\x6f\"", &bad_index)));
124 expect(eql(u8, "f💯", try parseStringLiteral(alloc, "\"f\u{1f4af}\"", &bad_index)));
125}
lib/std/zig/string_literal.zig created+155
...@@ -0,0 +1,155 @@
1const std = @import("../std.zig");
2const assert = std.debug.assert;
3
4const State = enum {
5 Start,
6 Backslash,
7};
8
9pub const ParseError = error{
10 OutOfMemory,
11
12 /// When this is returned, index will be the position of the character.
13 InvalidCharacter,
14};
15
16/// caller owns returned memory
17pub fn parse(
18 allocator: *std.mem.Allocator,
19 bytes: []const u8,
20 bad_index: *usize, // populated if error.InvalidCharacter is returned
21) ParseError![]u8 {
22 assert(bytes.len >= 2 and bytes[0] == '"' and bytes[bytes.len - 1] == '"');
23
24 var list = std.ArrayList(u8).init(allocator);
25 errdefer list.deinit();
26
27 const slice = bytes[1..];
28 try list.ensureCapacity(slice.len - 1);
29
30 var state = State.Start;
31 var index: usize = 0;
32 while (index < slice.len) : (index += 1) {
33 const b = slice[index];
34
35 switch (state) {
36 State.Start => switch (b) {
37 '\\' => state = State.Backslash,
38 '\n' => {
39 bad_index.* = index;
40 return error.InvalidCharacter;
41 },
42 '"' => return list.toOwnedSlice(),
43 else => try list.append(b),
44 },
45 State.Backslash => switch (b) {
46 'n' => {
47 try list.append('\n');
48 state = State.Start;
49 },
50 'r' => {
51 try list.append('\r');
52 state = State.Start;
53 },
54 '\\' => {
55 try list.append('\\');
56 state = State.Start;
57 },
58 't' => {
59 try list.append('\t');
60 state = State.Start;
61 },
62 '\'' => {
63 try list.append('\'');
64 state = State.Start;
65 },
66 '"' => {
67 try list.append('"');
68 state = State.Start;
69 },
70 'x' => {
71 // TODO: add more/better/broader tests for this.
72 const index_continue = index + 3;
73 if (slice.len >= index_continue)
74 if (std.fmt.parseUnsigned(u8, slice[index + 1 .. index_continue], 16)) |char| {
75 try list.append(char);
76 state = State.Start;
77 index = index_continue - 1; // loop-header increments again
78 continue;
79 } else |_| {};
80
81 bad_index.* = index;
82 return error.InvalidCharacter;
83 },
84 'u' => {
85 // TODO: add more/better/broader tests for this.
86 if (slice.len > index + 2 and slice[index + 1] == '{')
87 if (std.mem.indexOfScalarPos(u8, slice[0..std.math.min(index + 9, slice.len)], index + 3, '}')) |index_end| {
88 const hex_str = slice[index + 2 .. index_end];
89 if (std.fmt.parseUnsigned(u32, hex_str, 16)) |uint| {
90 if (uint <= 0x10ffff) {
91 try list.appendSlice(std.mem.toBytes(uint)[0..]);
92 state = State.Start;
93 index = index_end; // loop-header increments
94 continue;
95 }
96 } else |_| {}
97 };
98
99 bad_index.* = index;
100 return error.InvalidCharacter;
101 },
102 else => {
103 bad_index.* = index;
104 return error.InvalidCharacter;
105 },
106 },
107 else => unreachable,
108 }
109 }
110 unreachable;
111}
112
113test "parse" {
114 const expect = std.testing.expect;
115 const eql = std.mem.eql;
116
117 var fixed_buf_mem: [32]u8 = undefined;
118 var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(fixed_buf_mem[0..]);
119 var alloc = &fixed_buf_alloc.allocator;
120 var bad_index: usize = undefined;
121
122 expect(eql(u8, "foo", try parse(alloc, "\"foo\"", &bad_index)));
123 expect(eql(u8, "foo", try parse(alloc, "\"f\x6f\x6f\"", &bad_index)));
124 expect(eql(u8, "f💯", try parse(alloc, "\"f\u{1f4af}\"", &bad_index)));
125}
126
127/// Writes a Zig-syntax escaped string literal to the stream. Includes the double quotes.
128pub fn render(utf8: []const u8, out_stream: var) !void {
129 try out_stream.writeByte('"');
130 for (utf8) |byte| switch (byte) {
131 '\n' => try out_stream.writeAll("\\n"),
132 '\r' => try out_stream.writeAll("\\r"),
133 '\t' => try out_stream.writeAll("\\t"),
134 '\\' => try out_stream.writeAll("\\\\"),
135 '"' => try out_stream.writeAll("\\\""),
136 ' ', '!', '#'...'[', ']'...'~' => try out_stream.writeByte(byte),
137 else => try out_stream.print("\\x{x:0>2}", .{byte}),
138 };
139 try out_stream.writeByte('"');
140}
141
142test "render" {
143 const expect = std.testing.expect;
144 const eql = std.mem.eql;
145
146 var fixed_buf_mem: [32]u8 = undefined;
147
148 {
149 var fbs = std.io.fixedBufferStream(&fixed_buf_mem);
150 try render(" \\ hi \x07 \x11 \" derp", fbs.outStream());
151 expect(eql(u8,
152 \\" \\ hi \x07 \x11 \" derp"
153 , fbs.getWritten()));
154 }
155}