authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-03-01 20:51:01-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-03-02 14:45:19-05:00
log5c8a507e7a8e2e58a0ca855689bcd2edd2ab6ab8
tree97fc20a6da874c0d808c31e92609e95bb854c973
parentaa867c7dbe6576f61f957667fef769030aff7c69

stage2 parser: UTF-8 encode \u{NNNNNN} escape sequences

The core of this change is to re-use the escape sequence parsing logic for parsing both string and character literals. The actual fix is that UTF-8 encoding was missing for string literals with \u{...} escape sequences.

4 files changed, 311 insertions(+), 390 deletions(-)

lib/std/zig.zig+4-199
......@@ -14,6 +14,10 @@ pub const Ast = @import("zig/Ast.zig");
1414pub const system = @import("zig/system.zig");
1515pub const CrossTarget = @import("zig/CrossTarget.zig");
1616
17// Character literal parsing
18pub const ParsedCharLiteral = string_literal.ParsedCharLiteral;
19pub const parseCharLiteral = string_literal.parseCharLiteral;
20
1721// Files needed by translate-c.
1822pub const c_builtins = @import("zig/c_builtins.zig");
1923pub const c_translation = @import("zig/c_translation.zig");
......@@ -185,205 +189,6 @@ pub fn binNameAlloc(allocator: std.mem.Allocator, options: BinNameOptions) error
185189 }
186190}
187191
188pub const ParsedCharLiteral = union(enum) {
189 success: u32,
190 /// The character after backslash is not recognized.
191 invalid_escape_character: usize,
192 /// Expected hex digit at this index.
193 expected_hex_digit: usize,
194 /// Unicode escape sequence had no digits with rbrace at this index.
195 empty_unicode_escape_sequence: usize,
196 /// Expected hex digit or '}' at this index.
197 expected_hex_digit_or_rbrace: usize,
198 /// The unicode point is outside the range of Unicode codepoints.
199 unicode_escape_overflow: usize,
200 /// Expected '{' at this index.
201 expected_lbrace: usize,
202 /// Expected the terminating single quote at this index.
203 expected_end: usize,
204 /// The character at this index cannot be represented without an escape sequence.
205 invalid_character: usize,
206};
207
208/// Only validates escape sequence characters.
209/// Slice must be valid utf8 starting and ending with "'" and exactly one codepoint in between.
210pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
211 assert(slice.len >= 3 and slice[0] == '\'' and slice[slice.len - 1] == '\'');
212
213 switch (slice[1]) {
214 0 => return .{ .invalid_character = 1 },
215 '\\' => switch (slice[2]) {
216 'n' => return .{ .success = '\n' },
217 'r' => return .{ .success = '\r' },
218 '\\' => return .{ .success = '\\' },
219 't' => return .{ .success = '\t' },
220 '\'' => return .{ .success = '\'' },
221 '"' => return .{ .success = '"' },
222 'x' => {
223 if (slice.len < 4) {
224 return .{ .expected_hex_digit = 3 };
225 }
226 var value: u32 = 0;
227 var i: usize = 3;
228 while (i < 5) : (i += 1) {
229 const c = slice[i];
230 switch (c) {
231 '0'...'9' => {
232 value *= 16;
233 value += c - '0';
234 },
235 'a'...'f' => {
236 value *= 16;
237 value += c - 'a' + 10;
238 },
239 'A'...'F' => {
240 value *= 16;
241 value += c - 'A' + 10;
242 },
243 else => {
244 return .{ .expected_hex_digit = i };
245 },
246 }
247 }
248 if (slice[i] != '\'') {
249 return .{ .expected_end = i };
250 }
251 return .{ .success = value };
252 },
253 'u' => {
254 var i: usize = 3;
255 if (slice[i] != '{') {
256 return .{ .expected_lbrace = i };
257 }
258 i += 1;
259 if (slice[i] == '}') {
260 return .{ .empty_unicode_escape_sequence = i };
261 }
262
263 var value: u32 = 0;
264 while (i < slice.len) : (i += 1) {
265 const c = slice[i];
266 switch (c) {
267 '0'...'9' => {
268 value *= 16;
269 value += c - '0';
270 },
271 'a'...'f' => {
272 value *= 16;
273 value += c - 'a' + 10;
274 },
275 'A'...'F' => {
276 value *= 16;
277 value += c - 'A' + 10;
278 },
279 '}' => {
280 i += 1;
281 break;
282 },
283 else => return .{ .expected_hex_digit_or_rbrace = i },
284 }
285 if (value > 0x10ffff) {
286 return .{ .unicode_escape_overflow = i };
287 }
288 }
289 if (slice[i] != '\'') {
290 return .{ .expected_end = i };
291 }
292 return .{ .success = value };
293 },
294 else => return .{ .invalid_escape_character = 2 },
295 },
296 else => {
297 const codepoint = std.unicode.utf8Decode(slice[1 .. slice.len - 1]) catch unreachable;
298 return .{ .success = codepoint };
299 },
300 }
301}
302
303test "parseCharLiteral" {
304 try std.testing.expectEqual(
305 ParsedCharLiteral{ .success = 'a' },
306 parseCharLiteral("'a'"),
307 );
308 try std.testing.expectEqual(
309 ParsedCharLiteral{ .success = 'ä' },
310 parseCharLiteral("'ä'"),
311 );
312 try std.testing.expectEqual(
313 ParsedCharLiteral{ .success = 0 },
314 parseCharLiteral("'\\x00'"),
315 );
316 try std.testing.expectEqual(
317 ParsedCharLiteral{ .success = 0x4f },
318 parseCharLiteral("'\\x4f'"),
319 );
320 try std.testing.expectEqual(
321 ParsedCharLiteral{ .success = 0x4f },
322 parseCharLiteral("'\\x4F'"),
323 );
324 try std.testing.expectEqual(
325 ParsedCharLiteral{ .success = 0x3041 },
326 parseCharLiteral("'ぁ'"),
327 );
328 try std.testing.expectEqual(
329 ParsedCharLiteral{ .success = 0 },
330 parseCharLiteral("'\\u{0}'"),
331 );
332 try std.testing.expectEqual(
333 ParsedCharLiteral{ .success = 0x3041 },
334 parseCharLiteral("'\\u{3041}'"),
335 );
336 try std.testing.expectEqual(
337 ParsedCharLiteral{ .success = 0x7f },
338 parseCharLiteral("'\\u{7f}'"),
339 );
340 try std.testing.expectEqual(
341 ParsedCharLiteral{ .success = 0x7fff },
342 parseCharLiteral("'\\u{7FFF}'"),
343 );
344
345 try std.testing.expectEqual(
346 ParsedCharLiteral{ .expected_hex_digit = 4 },
347 parseCharLiteral("'\\x0'"),
348 );
349 try std.testing.expectEqual(
350 ParsedCharLiteral{ .expected_end = 5 },
351 parseCharLiteral("'\\x000'"),
352 );
353 try std.testing.expectEqual(
354 ParsedCharLiteral{ .invalid_escape_character = 2 },
355 parseCharLiteral("'\\y'"),
356 );
357 try std.testing.expectEqual(
358 ParsedCharLiteral{ .expected_lbrace = 3 },
359 parseCharLiteral("'\\u'"),
360 );
361 try std.testing.expectEqual(
362 ParsedCharLiteral{ .expected_lbrace = 3 },
363 parseCharLiteral("'\\uFFFF'"),
364 );
365 try std.testing.expectEqual(
366 ParsedCharLiteral{ .empty_unicode_escape_sequence = 4 },
367 parseCharLiteral("'\\u{}'"),
368 );
369 try std.testing.expectEqual(
370 ParsedCharLiteral{ .unicode_escape_overflow = 9 },
371 parseCharLiteral("'\\u{FFFFFF}'"),
372 );
373 try std.testing.expectEqual(
374 ParsedCharLiteral{ .expected_hex_digit_or_rbrace = 8 },
375 parseCharLiteral("'\\u{FFFF'"),
376 );
377 try std.testing.expectEqual(
378 ParsedCharLiteral{ .expected_end = 9 },
379 parseCharLiteral("'\\u{FFFF}x'"),
380 );
381 try std.testing.expectEqual(
382 ParsedCharLiteral{ .invalid_character = 1 },
383 parseCharLiteral("'\x00'"),
384 );
385}
386
387192test {
388193 @import("std").testing.refAllDecls(@This());
389194}
lib/std/zig/string_literal.zig+255-111
......@@ -1,129 +1,268 @@
11const std = @import("../std.zig");
22const assert = std.debug.assert;
3const utf8Decode = std.unicode.utf8Decode;
4const utf8Encode = std.unicode.utf8Encode;
35
46pub const ParseError = error{
57 OutOfMemory,
6 InvalidStringLiteral,
8 InvalidLiteral,
9};
10
11pub const ParsedCharLiteral = union(enum) {
12 success: u21,
13 failure: Error,
714};
815
916pub const Result = union(enum) {
1017 success,
11 /// Found an invalid character at this index.
18 failure: Error,
19};
20
21pub const Error = union(enum) {
22 /// The character after backslash is missing or not recognized.
23 invalid_escape_character: usize,
24 /// Expected hex digit at this index.
25 expected_hex_digit: usize,
26 /// Unicode escape sequence had no digits with rbrace at this index.
27 empty_unicode_escape_sequence: usize,
28 /// Expected hex digit or '}' at this index.
29 expected_hex_digit_or_rbrace: usize,
30 /// Invalid unicode codepoint at this index.
31 invalid_unicode_codepoint: usize,
32 /// Expected '{' at this index.
33 expected_lbrace: usize,
34 /// Expected '}' at this index.
35 expected_rbrace: usize,
36 /// Expected '\'' at this index.
37 expected_single_quote: usize,
38 /// The character at this index cannot be represented without an escape sequence.
1239 invalid_character: usize,
13 /// Expected hex digits at this index.
14 expected_hex_digits: usize,
15 /// Invalid hex digits at this index.
16 invalid_hex_escape: usize,
17 /// Invalid unicode escape at this index.
18 invalid_unicode_escape: usize,
19 /// The left brace at this index is missing a matching right brace.
20 missing_matching_rbrace: usize,
21 /// Expected unicode digits at this index.
22 expected_unicode_digits: usize,
2340};
2441
42/// Only validates escape sequence characters.
43/// Slice must be valid utf8 starting and ending with "'" and exactly one codepoint in between.
44pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
45 assert(slice.len >= 3 and slice[0] == '\'' and slice[slice.len - 1] == '\'');
46
47 switch (slice[1]) {
48 '\\' => {
49 var offset: usize = 1;
50 const result = parseEscapeSequence(slice, &offset);
51 if (result == .success and (offset + 1 != slice.len or slice[offset] != '\''))
52 return .{ .failure = .{ .expected_single_quote = offset } };
53
54 return result;
55 },
56 0 => return .{ .failure = .{ .invalid_character = 1 } },
57 else => {
58 const codepoint = utf8Decode(slice[1 .. slice.len - 1]) catch unreachable;
59 return .{ .success = codepoint };
60 },
61 }
62}
63
64/// Parse an escape sequence from `slice[offset..]`. If parsing is successful,
65/// offset is updated to reflect the characters consumed.
66fn parseEscapeSequence(slice: []const u8, offset: *usize) ParsedCharLiteral {
67 assert(slice.len > offset.*);
68 assert(slice[offset.*] == '\\');
69
70 if (slice.len == offset.* + 1)
71 return .{ .failure = .{ .invalid_escape_character = offset.* + 1 } };
72
73 offset.* += 2;
74 switch (slice[offset.* - 1]) {
75 'n' => return .{ .success = '\n' },
76 'r' => return .{ .success = '\r' },
77 '\\' => return .{ .success = '\\' },
78 't' => return .{ .success = '\t' },
79 '\'' => return .{ .success = '\'' },
80 '"' => return .{ .success = '"' },
81 'x' => {
82 var value: u8 = 0;
83 var i: usize = offset.*;
84 while (i < offset.* + 2) : (i += 1) {
85 if (i == slice.len) return .{ .failure = .{ .expected_hex_digit = i } };
86
87 const c = slice[i];
88 switch (c) {
89 '0'...'9' => {
90 value *= 16;
91 value += c - '0';
92 },
93 'a'...'f' => {
94 value *= 16;
95 value += c - 'a' + 10;
96 },
97 'A'...'F' => {
98 value *= 16;
99 value += c - 'A' + 10;
100 },
101 else => {
102 return .{ .failure = .{ .expected_hex_digit = i } };
103 },
104 }
105 }
106 offset.* = i;
107 return .{ .success = value };
108 },
109 'u' => {
110 var i: usize = offset.*;
111 if (i >= slice.len or slice[i] != '{') return .{ .failure = .{ .expected_lbrace = i } };
112 i += 1;
113 if (i >= slice.len) return .{ .failure = .{ .expected_hex_digit_or_rbrace = i } };
114 if (slice[i] == '}') return .{ .failure = .{ .empty_unicode_escape_sequence = i } };
115
116 var value: u32 = 0;
117 while (i < slice.len) : (i += 1) {
118 const c = slice[i];
119 switch (c) {
120 '0'...'9' => {
121 value *= 16;
122 value += c - '0';
123 },
124 'a'...'f' => {
125 value *= 16;
126 value += c - 'a' + 10;
127 },
128 'A'...'F' => {
129 value *= 16;
130 value += c - 'A' + 10;
131 },
132 '}' => {
133 i += 1;
134 break;
135 },
136 else => return .{ .failure = .{ .expected_hex_digit_or_rbrace = i } },
137 }
138 if (value > 0x10ffff) {
139 return .{ .failure = .{ .invalid_unicode_codepoint = i } };
140 }
141 } else {
142 return .{ .failure = .{ .expected_rbrace = i } };
143 }
144 offset.* = i;
145 return .{ .success = @intCast(u21, value) };
146 },
147 else => return .{ .failure = .{ .invalid_escape_character = offset.* - 1 } },
148 }
149}
150
151test "parseCharLiteral" {
152 try std.testing.expectEqual(
153 ParsedCharLiteral{ .success = 'a' },
154 parseCharLiteral("'a'"),
155 );
156 try std.testing.expectEqual(
157 ParsedCharLiteral{ .success = 'ä' },
158 parseCharLiteral("'ä'"),
159 );
160 try std.testing.expectEqual(
161 ParsedCharLiteral{ .success = 0 },
162 parseCharLiteral("'\\x00'"),
163 );
164 try std.testing.expectEqual(
165 ParsedCharLiteral{ .success = 0x4f },
166 parseCharLiteral("'\\x4f'"),
167 );
168 try std.testing.expectEqual(
169 ParsedCharLiteral{ .success = 0x4f },
170 parseCharLiteral("'\\x4F'"),
171 );
172 try std.testing.expectEqual(
173 ParsedCharLiteral{ .success = 0x3041 },
174 parseCharLiteral("'ぁ'"),
175 );
176 try std.testing.expectEqual(
177 ParsedCharLiteral{ .success = 0 },
178 parseCharLiteral("'\\u{0}'"),
179 );
180 try std.testing.expectEqual(
181 ParsedCharLiteral{ .success = 0x3041 },
182 parseCharLiteral("'\\u{3041}'"),
183 );
184 try std.testing.expectEqual(
185 ParsedCharLiteral{ .success = 0x7f },
186 parseCharLiteral("'\\u{7f}'"),
187 );
188 try std.testing.expectEqual(
189 ParsedCharLiteral{ .success = 0x7fff },
190 parseCharLiteral("'\\u{7FFF}'"),
191 );
192 try std.testing.expectEqual(
193 ParsedCharLiteral{ .failure = .{ .expected_hex_digit = 4 } },
194 parseCharLiteral("'\\x0'"),
195 );
196 try std.testing.expectEqual(
197 ParsedCharLiteral{ .failure = .{ .expected_single_quote = 5 } },
198 parseCharLiteral("'\\x000'"),
199 );
200 try std.testing.expectEqual(
201 ParsedCharLiteral{ .failure = .{ .invalid_escape_character = 2 } },
202 parseCharLiteral("'\\y'"),
203 );
204 try std.testing.expectEqual(
205 ParsedCharLiteral{ .failure = .{ .expected_lbrace = 3 } },
206 parseCharLiteral("'\\u'"),
207 );
208 try std.testing.expectEqual(
209 ParsedCharLiteral{ .failure = .{ .expected_lbrace = 3 } },
210 parseCharLiteral("'\\uFFFF'"),
211 );
212 try std.testing.expectEqual(
213 ParsedCharLiteral{ .failure = .{ .empty_unicode_escape_sequence = 4 } },
214 parseCharLiteral("'\\u{}'"),
215 );
216 try std.testing.expectEqual(
217 ParsedCharLiteral{ .failure = .{ .invalid_unicode_codepoint = 9 } },
218 parseCharLiteral("'\\u{FFFFFF}'"),
219 );
220 try std.testing.expectEqual(
221 ParsedCharLiteral{ .failure = .{ .expected_hex_digit_or_rbrace = 8 } },
222 parseCharLiteral("'\\u{FFFF'"),
223 );
224 try std.testing.expectEqual(
225 ParsedCharLiteral{ .failure = .{ .expected_single_quote = 9 } },
226 parseCharLiteral("'\\u{FFFF}x'"),
227 );
228 try std.testing.expectEqual(
229 ParsedCharLiteral{ .failure = .{ .invalid_character = 1 } },
230 parseCharLiteral("'\x00'"),
231 );
232}
233
25234/// Parses `bytes` as a Zig string literal and appends the result to `buf`.
26235/// Asserts `bytes` has '"' at beginning and end.
27236pub fn parseAppend(buf: *std.ArrayList(u8), bytes: []const u8) error{OutOfMemory}!Result {
28237 assert(bytes.len >= 2 and bytes[0] == '"' and bytes[bytes.len - 1] == '"');
29 const slice = bytes[1..];
30
31 const prev_len = buf.items.len;
32 try buf.ensureUnusedCapacity(slice.len - 1);
33 errdefer buf.shrinkRetainingCapacity(prev_len);
34
35 const State = enum {
36 Start,
37 Backslash,
38 };
39
40 var state = State.Start;
41 var index: usize = 0;
42 while (true) : (index += 1) {
43 const b = slice[index];
44
45 switch (state) {
46 State.Start => switch (b) {
47 '\\' => state = State.Backslash,
48 '\n' => {
49 return Result{ .invalid_character = index };
50 },
51 '"' => return Result.success,
52 else => try buf.append(b),
53 },
54 State.Backslash => switch (b) {
55 'n' => {
56 try buf.append('\n');
57 state = State.Start;
58 },
59 'r' => {
60 try buf.append('\r');
61 state = State.Start;
62 },
63 '\\' => {
64 try buf.append('\\');
65 state = State.Start;
66 },
67 't' => {
68 try buf.append('\t');
69 state = State.Start;
70 },
71 '\'' => {
72 try buf.append('\'');
73 state = State.Start;
74 },
75 '"' => {
76 try buf.append('"');
77 state = State.Start;
78 },
79 'x' => {
80 // TODO: add more/better/broader tests for this.
81 const index_continue = index + 3;
82 if (slice.len < index_continue) {
83 return Result{ .expected_hex_digits = index };
84 }
85 if (std.fmt.parseUnsigned(u8, slice[index + 1 .. index_continue], 16)) |byte| {
86 try buf.append(byte);
87 state = State.Start;
88 index = index_continue - 1; // loop-header increments again
89 } else |err| switch (err) {
90 error.Overflow => unreachable, // 2 digits base 16 fits in a u8.
91 error.InvalidCharacter => {
92 return Result{ .invalid_hex_escape = index + 1 };
93 },
94 }
95 },
96 'u' => {
97 // TODO: add more/better/broader tests for this.
98 // TODO: we are already inside a nice, clean state machine... use it
99 // instead of this hacky code.
100 if (slice.len > index + 2 and slice[index + 1] == '{') {
101 if (std.mem.indexOfScalarPos(u8, slice[0..std.math.min(index + 9, slice.len)], index + 3, '}')) |index_end| {
102 const hex_str = slice[index + 2 .. index_end];
103 if (std.fmt.parseUnsigned(u32, hex_str, 16)) |uint| {
104 if (uint <= 0x10ffff) {
105 // TODO this incorrectly depends on endianness
106 try buf.appendSlice(std.mem.toBytes(uint)[0..]);
107 state = State.Start;
108 index = index_end; // loop-header increments
109 continue;
110 }
111 } else |err| switch (err) {
112 error.Overflow => unreachable,
113 error.InvalidCharacter => {
114 return Result{ .invalid_unicode_escape = index + 1 };
115 },
116 }
238 try buf.ensureUnusedCapacity(bytes.len - 2);
239
240 var index: usize = 1;
241 while (true) {
242 const b = bytes[index];
243
244 switch (b) {
245 '\\' => {
246 const escape_char_index = index + 1;
247 const result = parseEscapeSequence(bytes, &index);
248 switch (result) {
249 .success => |codepoint| {
250 if (bytes[escape_char_index] == 'u') {
251 buf.items.len += utf8Encode(codepoint, buf.unusedCapacitySlice()) catch {
252 return Result{ .failure = .{ .invalid_unicode_codepoint = escape_char_index + 1 } };
253 };
117254 } else {
118 return Result{ .missing_matching_rbrace = index + 1 };
255 buf.appendAssumeCapacity(@intCast(u8, codepoint));
119256 }
120 } else {
121 return Result{ .expected_unicode_digits = index };
122 }
123 },
124 else => {
125 return Result{ .invalid_character = index };
126 },
257 },
258 .failure => |err| return Result{ .failure = err },
259 }
260 },
261 '\n' => return Result{ .failure = .{ .invalid_character = index } },
262 '"' => return Result.success,
263 else => {
264 try buf.append(b);
265 index += 1;
127266 },
128267 }
129268 } else unreachable; // TODO should not need else unreachable on while(true)
......@@ -137,18 +276,23 @@ pub fn parseAlloc(allocator: std.mem.Allocator, bytes: []const u8) ParseError![]
137276
138277 switch (try parseAppend(&buf, bytes)) {
139278 .success => return buf.toOwnedSlice(),
140 else => return error.InvalidStringLiteral,
279 .failure => return error.InvalidLiteral,
141280 }
142281}
143282
144283test "parse" {
145284 const expect = std.testing.expect;
285 const expectError = std.testing.expectError;
146286 const eql = std.mem.eql;
147287
148 var fixed_buf_mem: [32]u8 = undefined;
149 var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(fixed_buf_mem[0..]);
288 var fixed_buf_mem: [64]u8 = undefined;
289 var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(&fixed_buf_mem);
150290 var alloc = fixed_buf_alloc.allocator();
151291
292 try expectError(error.InvalidLiteral, parseAlloc(alloc, "\"\\x6\""));
293 try expect(eql(u8, "foo\nbar", try parseAlloc(alloc, "\"foo\\nbar\"")));
294 try expect(eql(u8, "\x12foo", try parseAlloc(alloc, "\"\\x12foo\"")));
295 try expect(eql(u8, "bytes\u{1234}foo", try parseAlloc(alloc, "\"bytes\\u{1234}foo\"")));
152296 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"foo\"")));
153297 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"f\x6f\x6f\"")));
154298 try expect(eql(u8, "f💯", try parseAlloc(alloc, "\"f\u{1f4af}\"")));
src/AstGen.zig+47-79
......@@ -6447,7 +6447,7 @@ fn multilineStringLiteral(
64476447 return rvalue(gz, rl, result, node);
64486448}
64496449
6450fn charLiteral(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) !Zir.Inst.Ref {
6450fn charLiteral(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
64516451 const astgen = gz.astgen;
64526452 const tree = astgen.tree;
64536453 const main_tokens = tree.nodes.items(.main_token);
......@@ -6459,70 +6459,7 @@ fn charLiteral(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) !Zir.Inst.Ref {
64596459 const result = try gz.addInt(codepoint);
64606460 return rvalue(gz, rl, result, node);
64616461 },
6462 .invalid_escape_character => |bad_index| {
6463 return astgen.failOff(
6464 main_token,
6465 @intCast(u32, bad_index),
6466 "invalid escape character: '{c}'",
6467 .{slice[bad_index]},
6468 );
6469 },
6470 .expected_hex_digit => |bad_index| {
6471 return astgen.failOff(
6472 main_token,
6473 @intCast(u32, bad_index),
6474 "expected hex digit, found '{c}'",
6475 .{slice[bad_index]},
6476 );
6477 },
6478 .empty_unicode_escape_sequence => |bad_index| {
6479 return astgen.failOff(
6480 main_token,
6481 @intCast(u32, bad_index),
6482 "empty unicode escape sequence",
6483 .{},
6484 );
6485 },
6486 .expected_hex_digit_or_rbrace => |bad_index| {
6487 return astgen.failOff(
6488 main_token,
6489 @intCast(u32, bad_index),
6490 "expected hex digit or '}}', found '{c}'",
6491 .{slice[bad_index]},
6492 );
6493 },
6494 .unicode_escape_overflow => |bad_index| {
6495 return astgen.failOff(
6496 main_token,
6497 @intCast(u32, bad_index),
6498 "unicode escape too large to be a valid codepoint",
6499 .{},
6500 );
6501 },
6502 .expected_lbrace => |bad_index| {
6503 return astgen.failOff(
6504 main_token,
6505 @intCast(u32, bad_index),
6506 "expected '{{', found '{c}",
6507 .{slice[bad_index]},
6508 );
6509 },
6510 .expected_end => |bad_index| {
6511 return astgen.failOff(
6512 main_token,
6513 @intCast(u32, bad_index),
6514 "expected ending single quote ('), found '{c}",
6515 .{slice[bad_index]},
6516 );
6517 },
6518 .invalid_character => |bad_index| {
6519 return astgen.failOff(
6520 main_token,
6521 @intCast(u32, bad_index),
6522 "invalid byte in character literal: '{c}'",
6523 .{slice[bad_index]},
6524 );
6525 },
6462 .failure => |err| return astgen.failWithStrLitError(err, main_token, slice, 0),
65266463 }
65276464}
65286465
......@@ -8958,52 +8895,83 @@ fn parseStrLit(
89588895 buf.* = buf_managed.moveToUnmanaged();
89598896 switch (try result) {
89608897 .success => return,
8961 .invalid_character => |bad_index| {
8898 .failure => |err| return astgen.failWithStrLitError(err, token, bytes, offset),
8899 }
8900}
8901
8902fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token: Ast.TokenIndex, bytes: []const u8, offset: u32) InnerError {
8903 const raw_string = bytes[offset..];
8904 switch (err) {
8905 .invalid_escape_character => |bad_index| {
89628906 return astgen.failOff(
89638907 token,
89648908 offset + @intCast(u32, bad_index),
8965 "invalid string literal character: '{c}'",
8909 "invalid escape character: '{c}'",
89668910 .{raw_string[bad_index]},
89678911 );
89688912 },
8969 .expected_hex_digits => |bad_index| {
8913 .expected_hex_digit => |bad_index| {
89708914 return astgen.failOff(
89718915 token,
89728916 offset + @intCast(u32, bad_index),
8973 "expected hex digits after '\\x'",
8917 "expected hex digit, found '{c}'",
8918 .{raw_string[bad_index]},
8919 );
8920 },
8921 .empty_unicode_escape_sequence => |bad_index| {
8922 return astgen.failOff(
8923 token,
8924 offset + @intCast(u32, bad_index),
8925 "empty unicode escape sequence",
89748926 .{},
89758927 );
89768928 },
8977 .invalid_hex_escape => |bad_index| {
8929 .expected_hex_digit_or_rbrace => |bad_index| {
89788930 return astgen.failOff(
89798931 token,
89808932 offset + @intCast(u32, bad_index),
8981 "invalid hex digit: '{c}'",
8933 "expected hex digit or '}}', found '{c}'",
89828934 .{raw_string[bad_index]},
89838935 );
89848936 },
8985 .invalid_unicode_escape => |bad_index| {
8937 .invalid_unicode_codepoint => |bad_index| {
89868938 return astgen.failOff(
89878939 token,
89888940 offset + @intCast(u32, bad_index),
8989 "invalid unicode digit: '{c}'",
8941 "unicode escape does not correspond to a valid codepoint",
8942 .{},
8943 );
8944 },
8945 .expected_lbrace => |bad_index| {
8946 return astgen.failOff(
8947 token,
8948 offset + @intCast(u32, bad_index),
8949 "expected '{{', found '{c}",
89908950 .{raw_string[bad_index]},
89918951 );
89928952 },
8993 .missing_matching_rbrace => |bad_index| {
8953 .expected_rbrace => |bad_index| {
89948954 return astgen.failOff(
89958955 token,
89968956 offset + @intCast(u32, bad_index),
8997 "missing matching '}}' character",
8998 .{},
8957 "expected '}}', found '{c}",
8958 .{raw_string[bad_index]},
89998959 );
90008960 },
9001 .expected_unicode_digits => |bad_index| {
8961 .expected_single_quote => |bad_index| {
90028962 return astgen.failOff(
90038963 token,
90048964 offset + @intCast(u32, bad_index),
9005 "expected unicode digits after '\\u'",
9006 .{},
8965 "expected single quote ('), found '{c}",
8966 .{raw_string[bad_index]},
8967 );
8968 },
8969 .invalid_character => |bad_index| {
8970 return astgen.failOff(
8971 token,
8972 offset + @intCast(u32, bad_index),
8973 "invalid byte in string or character literal: '{c}'",
8974 .{raw_string[bad_index]},
90078975 );
90088976 },
90098977 }
test/behavior/basic.zig+5-1
......@@ -662,7 +662,11 @@ test "multiline string literal is null terminated" {
662662}
663663
664664test "string escapes" {
665 if (builtin.zig_backend != .stage1) return error.SkipZigTest; // TODO
665 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
666 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
667 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
668 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
669 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
666670
667671 try expectEqualStrings("\"", "\x22");
668672 try expectEqualStrings("\'", "\x27");