Commit 377e8579f9539c56fc5988c9a452a01438af89f3 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-07-31 11:51:19-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-07-31 16:57:42-07:00
log	377e8579f9539c56fc5988c9a452a01438af89f3
tree	542f9ac9cf63e7b083e9a606e6c8a77eade2e6ce
parent	c08effc20abe2595ba5c83e25b78c274ec9c58ec

std.zig.tokenizer: simplify

I pointed a fuzzer at the tokenizer and it crashed immediately. Upon
inspection, I was dissatisfied with the implementation. This commit
removes several mechanisms:
* Removes the "invalid byte" compile error note.
* Dramatically simplifies tokenizer recovery by making recovery always
  occur at newlines, and never otherwise.
* Removes UTF-8 validation.
* Moves some character validation logic to `std.zig.parseCharLiteral`.

Removing UTF-8 validation is a regression of #663, however, the existing
implementation was already buggy. When adding this functionality back,
it must be fuzz-tested while checking the property that it matches an
independent Unicode validation implementation on the same file. While
we're at it, fuzzing should check the other properties of that proposal,
such as no ASCII control characters existing inside the source code.

Other changes included in this commit:

* Deprecate `std.unicode.utf8Decode` and its WTF-8 counterpart. This
  function has an awkward API that is too easy to misuse.
* Make `utf8Decode2` and friends use arrays as parameters, eliminating a
  runtime assertion in favor of using the type system.

After this commit, the crash found by fuzzing, which was
"\x07\xd5\x80\xc3=o\xda|a\xfc{\x9a\xec\x91\xdf\x0f\\\x1a^\xbe;\x8c\xbf\xee\xea"
no longer causes a crash. However, I did not feel the need to add this
test case because the simplified logic eradicates most crashes of this
nature.

12 files changed, 234 insertions(+), 392 deletions(-)

lib
- std
  - unicode.zig
  - zig
    Ast.zig
    AstGen.zig
    parser_test.zig
    string_literal.zig
    tokenizer.zig
src
- Package
  - Manifest.zig
test
- cases
  - compile_errors
    empty_char_lit.zig
    invalid_legacy_unicode_escape.zig
    invalid_unicode_escape.zig
    normal_string_with_newline.zig
- compile_errors.zig

lib/std/unicode.zig+14-19

 @@ -95,16 +95,13 @@ pub inline fn utf8EncodeComptime(comptime c: u21) [
 const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error;
-/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
+/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
-/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.
-/// If you already know the length at comptime, you can call one of
-/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.
 pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {
     return switch (bytes.len) {
-        1 => @as(u21, bytes[0]),
+        1 => bytes[0],
-        2 => utf8Decode2(bytes),
+        2 => utf8Decode2(bytes[0..2].*),
-        3 => utf8Decode3(bytes),
+        3 => utf8Decode3(bytes[0..3].*),
-        4 => utf8Decode4(bytes),
+        4 => utf8Decode4(bytes[0..4].*),
         else => unreachable,
     };
 }
 @@ -113,8 +110,7 @@ const Utf8Decode2Error = error{
     Utf8ExpectedContinuation,
     Utf8OverlongEncoding,
 };
-pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
+pub fn utf8Decode2(bytes: [2]u8) Utf8Decode2Error!u21 {
-    assert(bytes.len == 2);
     assert(bytes[0] & 0b11100000 == 0b11000000);
     var value: u21 = bytes[0] & 0b00011111;
 @@ -130,7 +126,7 @@ pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
 const Utf8Decode3Error = Utf8Decode3AllowSurrogateHalfError || error{
     Utf8EncodesSurrogateHalf,
 };
-pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {
+pub fn utf8Decode3(bytes: [3]u8) Utf8Decode3Error!u21 {
     const value = try utf8Decode3AllowSurrogateHalf(bytes);
     if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
 @@ -142,8 +138,7 @@ const Utf8Decode3AllowSurrogateHalfError = error{
     Utf8ExpectedContinuation,
     Utf8OverlongEncoding,
 };
-pub fn utf8Decode3AllowSurrogateHalf(bytes: []const u8) Utf8Decode3AllowSurrogateHalfError!u21 {
+pub fn utf8Decode3AllowSurrogateHalf(bytes: [3]u8) Utf8Decode3AllowSurrogateHalfError!u21 {
-    assert(bytes.len == 3);
     assert(bytes[0] & 0b11110000 == 0b11100000);
     var value: u21 = bytes[0] & 0b00001111;
 @@ -165,8 +160,7 @@ const Utf8Decode4Error = error{
     Utf8OverlongEncoding,
     Utf8CodepointTooLarge,
 };
-pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
+pub fn utf8Decode4(bytes: [4]u8) Utf8Decode4Error!u21 {
-    assert(bytes.len == 4);
     assert(bytes[0] & 0b11111000 == 0b11110000);
     var value: u21 = bytes[0] & 0b00000111;
 @@ -1637,12 +1631,13 @@ pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3 {
 const Wtf8DecodeError = Utf8Decode2Error || Utf8Decode3AllowSurrogateHalfError || Utf8Decode4Error;
+/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
 pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21 {
     return switch (bytes.len) {
-        1 => @as(u21, bytes[0]),
+        1 => bytes[0],
-        2 => utf8Decode2(bytes),
+        2 => utf8Decode2(bytes[0..2].*),
-        3 => utf8Decode3AllowSurrogateHalf(bytes),
+        3 => utf8Decode3AllowSurrogateHalf(bytes[0..3].*),
-        4 => utf8Decode4(bytes),
+        4 => utf8Decode4(bytes[0..4].*),
         else => unreachable,
     };
 }

lib/std/zig/Ast.zig+1-1

...	@@ -69,7 +69,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!A	...	@@ -69,7 +69,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!A
69	const token = tokenizer.next();	69	const token = tokenizer.next();
70	try tokens.append(gpa, .{	70	try tokens.append(gpa, .{
71	.tag = token.tag,	71	.tag = token.tag,
72	.start = @as(u32, @intCast(token.loc.start)),	72	.start = @intCast(token.loc.start),
73	});	73	});
74	if (token.tag == .eof) break;	74	if (token.tag == .eof) break;
75	}	75	}

lib/std/zig/AstGen.zig+3-12

 @@ -11351,6 +11351,9 @@ fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token
                 .{raw_string[bad_index]},
             );
         },
+        .empty_char_literal => {
+            return astgen.failOff(token, offset, "empty character literal", .{});
+        },
     }
 }
 @@ -13820,21 +13823,9 @@ fn lowerAstErrors(astgen: *AstGen) !void {
     var msg: std.ArrayListUnmanaged(u8) = .{};
     defer msg.deinit(gpa);
-    const token_starts = tree.tokens.items(.start);
-    const token_tags = tree.tokens.items(.tag);
     var notes: std.ArrayListUnmanaged(u32) = .{};
     defer notes.deinit(gpa);
-    const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
-    if (token_tags[tok] == .invalid) {
-        const bad_off: u32 = @intCast(tree.tokenSlice(tok).len);
-        const byte_abs = token_starts[tok] + bad_off;
-        try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
-            std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
-        }));
-    }
     for (tree.errors[1..]) |note| {
         if (!note.is_note) break;

lib/std/zig/parser_test.zig-1

...	@@ -6061,7 +6061,6 @@ test "recovery: invalid container members" {	...	@@ -6061,7 +6061,6 @@ test "recovery: invalid container members" {
6061	, &[_]Error{	6061	, &[_]Error{
6062	.expected_expr,	6062	.expected_expr,
6063	.expected_comma_after_field,	6063	.expected_comma_after_field,
6064	.expected_type_expr,
6065	.expected_semi_after_stmt,	6064	.expected_semi_after_stmt,
6066	});	6065	});
6067	}	6066	}

lib/std/zig/string_literal.zig+19-5

 @@ -1,6 +1,5 @@
 const std = @import("../std.zig");
 const assert = std.debug.assert;
-const utf8Decode = std.unicode.utf8Decode;
 const utf8Encode = std.unicode.utf8Encode;
 pub const ParseError = error{
 @@ -37,12 +36,16 @@ pub const Error = union(enum) {
     expected_single_quote: usize,
     /// The character at this index cannot be represented without an escape sequence.
     invalid_character: usize,
+    /// `''`. Not returned for string literals.
+    empty_char_literal,
 };
-/// Only validates escape sequence characters.
+/// Asserts the slice starts and ends with single-quotes.
-/// Slice must be valid utf8 starting and ending with "'" and exactly one codepoint in between.
+/// Returns an error if there is not exactly one UTF-8 codepoint in between.
 pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
-    assert(slice.len >= 3 and slice[0] == '\'' and slice[slice.len - 1] == '\'');
+    if (slice.len < 3) return .{ .failure = .empty_char_literal };
+    assert(slice[0] == '\'');
+    assert(slice[slice.len - 1] == '\'');
     switch (slice[1]) {
         '\\' => {
 @@ -55,7 +58,18 @@ pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
         },
         0 => return .{ .failure = .{ .invalid_character = 1 } },
         else => {
-            const codepoint = utf8Decode(slice[1 .. slice.len - 1]) catch unreachable;
+            const inner = slice[1 .. slice.len - 1];
+            const n = std.unicode.utf8ByteSequenceLength(inner[0]) catch return .{
+                .failure = .{ .invalid_unicode_codepoint = 1 },
+            };
+            if (inner.len > n) return .{ .failure = .{ .expected_single_quote = 1 + n } };
+            const codepoint = switch (n) {
+                1 => inner[0],
+                2 => std.unicode.utf8Decode2(inner[0..2].*),
+                3 => std.unicode.utf8Decode3(inner[0..3].*),
+                4 => std.unicode.utf8Decode4(inner[0..4].*),
+                else => unreachable,
+            } catch return .{ .failure = .{ .invalid_unicode_codepoint = 1 } };
             return .{ .success = codepoint };
         },
     }

lib/std/zig/tokenizer.zig+176-336

...	@@ -320,7 +320,7 @@ pub const Token = struct {	...	@@ -320,7 +320,7 @@ pub const Token = struct {
320		320
321	pub fn symbol(tag: Tag) []const u8 {	321	pub fn symbol(tag: Tag) []const u8 {
322	return tag.lexeme() orelse switch (tag) {	322	return tag.lexeme() orelse switch (tag) {
323	.invalid => "invalid bytes",	323	.invalid => "invalid token",
324	.identifier => "an identifier",	324	.identifier => "an identifier",
325	.string_literal, .multiline_string_literal_line => "a string literal",	325	.string_literal, .multiline_string_literal_line => "a string literal",
326	.char_literal => "a character literal",	326	.char_literal => "a character literal",
...	@@ -338,22 +338,22 @@ pub const Tokenizer = struct {	...	@@ -338,22 +338,22 @@ pub const Tokenizer = struct {
338	buffer: [:0]const u8,	338	buffer: [:0]const u8,
339	index: usize,	339	index: usize,
340		340
341	/// For debugging purposes	341	/// For debugging purposes.
342	pub fn dump(self: Tokenizer, token: const Token) void {	342	pub fn dump(self: Tokenizer, token: const Token) void {
343	std.debug.print("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] });	343	std.debug.print("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] });
344	}	344	}
345		345
346	pub fn init(buffer: [:0]const u8) Tokenizer {	346	pub fn init(buffer: [:0]const u8) Tokenizer {
347	// Skip the UTF-8 BOM if present	347	// Skip the UTF-8 BOM if present.
348	const src_start: usize = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0;	348	return .{
349	return Tokenizer{
350	.buffer = buffer,	349	.buffer = buffer,
351	.index = src_start,	350	.index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0,
352	};	351	};
353	}	352	}
354		353
355	const State = enum {	354	const State = enum {
356	start,	355	start,
		356	expect_newline,
357	identifier,	357	identifier,
358	builtin,	358	builtin,
359	string_literal,	359	string_literal,
...	@@ -361,10 +361,6 @@ pub const Tokenizer = struct {	...	@@ -361,10 +361,6 @@ pub const Tokenizer = struct {
361	multiline_string_literal_line,	361	multiline_string_literal_line,
362	char_literal,	362	char_literal,
363	char_literal_backslash,	363	char_literal_backslash,
364	char_literal_hex_escape,
365	char_literal_unicode_escape_saw_u,
366	char_literal_unicode_escape,
367	char_literal_end,
368	backslash,	364	backslash,
369	equal,	365	equal,
370	bang,	366	bang,
...	@@ -400,32 +396,38 @@ pub const Tokenizer = struct {	...	@@ -400,32 +396,38 @@ pub const Tokenizer = struct {
400	period_2,	396	period_2,
401	period_asterisk,	397	period_asterisk,
402	saw_at_sign,	398	saw_at_sign,
		399	invalid,
403	};	400	};
404		401
		402	/// After this returns invalid, it will reset on the next newline, returning tokens starting from there.
		403	/// An eof token will always be returned at the end.
405	pub fn next(self: *Tokenizer) Token {	404	pub fn next(self: *Tokenizer) Token {
406	var state: State = .start;	405	var state: State = .start;
407	var result = Token{	406	var result: Token = .{
408	.tag = .eof,	407	.tag = undefined,
409	.loc = .{	408	.loc = .{
410	.start = self.index,	409	.start = self.index,
411	.end = undefined,	410	.end = undefined,
412	},	411	},
413	};	412	};
414	var seen_escape_digits: usize = undefined;
415	while (true) : (self.index += 1) {	413	while (true) : (self.index += 1) {
416	const c = self.buffer[self.index];	414	const c = self.buffer[self.index];
417	switch (state) {	415	switch (state) {
418	.start => switch (c) {	416	.start => switch (c) {
419	0 => {	417	0 => {
420	if (self.index != self.buffer.len) {	418	if (self.index == self.buffer.len) return .{
421	result.tag = .invalid;	419	.tag = .eof,
422	result.loc.end = self.index;	420	.loc = .{
423	self.index += 1;	421	.start = self.index,
424	return result;	422	.end = self.index,
425	}	423	},
426	break;	424	};
427	},	425	state = .invalid;
428	' ', '\n', '\t', '\r' => {	426	},
		427	'\r' => {
		428	state = .expect_newline;
		429	},
		430	' ', '\n', '\t' => {
429	result.loc.start = self.index + 1;	431	result.loc.start = self.index + 1;
430	},	432	},
431	'"' => {	433	'"' => {
...	@@ -434,6 +436,7 @@ pub const Tokenizer = struct {	...	@@ -434,6 +436,7 @@ pub const Tokenizer = struct {
434	},	436	},
435	'\'' => {	437	'\'' => {
436	state = .char_literal;	438	state = .char_literal;
		439	result.tag = .char_literal;
437	},	440	},
438	'a'...'z', 'A'...'Z', '_' => {	441	'a'...'z', 'A'...'Z', '_' => {
439	state = .identifier;	442	state = .identifier;
...	@@ -545,14 +548,37 @@ pub const Tokenizer = struct {	...	@@ -545,14 +548,37 @@ pub const Tokenizer = struct {
545	result.tag = .number_literal;	548	result.tag = .number_literal;
546	},	549	},
547	else => {	550	else => {
		551	state = .invalid;
		552	},
		553	},
		554
		555	.expect_newline => switch (c) {
		556	'\n' => {
		557	result.loc.start = self.index + 1;
		558	state = .start;
		559	},
		560	else => {
		561	state = .invalid;
		562	},
		563	},
		564
		565	.invalid => switch (c) {
		566	0 => if (self.index == self.buffer.len) {
		567	result.tag = .invalid;
		568	break;
		569	},
		570	'\n' => {
548	result.tag = .invalid;	571	result.tag = .invalid;
549	result.loc.end = self.index;	572	break;
550	self.index += std.unicode.utf8ByteSequenceLength(c) catch 1;
551	return result;
552	},	573	},
		574	else => continue,
553	},	575	},
554		576
555	.saw_at_sign => switch (c) {	577	.saw_at_sign => switch (c) {
		578	0, '\n' => {
		579	result.tag = .invalid;
		580	break;
		581	},
556	'"' => {	582	'"' => {
557	result.tag = .identifier;	583	result.tag = .identifier;
558	state = .string_literal;	584	state = .string_literal;
...	@@ -562,8 +588,7 @@ pub const Tokenizer = struct {	...	@@ -562,8 +588,7 @@ pub const Tokenizer = struct {
562	result.tag = .builtin;	588	result.tag = .builtin;
563	},	589	},
564	else => {	590	else => {
565	result.tag = .invalid;	591	state = .invalid;
566	break;
567	},	592	},
568	},	593	},
569		594
...	@@ -698,7 +723,7 @@ pub const Tokenizer = struct {	...	@@ -698,7 +723,7 @@ pub const Tokenizer = struct {
698	},	723	},
699		724
700	.identifier => switch (c) {	725	.identifier => switch (c) {
701	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},	726	'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
702	else => {	727	else => {
703	if (Token.getKeyword(self.buffer[result.loc.start..self.index])) \|tag\| {	728	if (Token.getKeyword(self.buffer[result.loc.start..self.index])) \|tag\| {
704	result.tag = tag;	729	result.tag = tag;
...	@@ -707,26 +732,37 @@ pub const Tokenizer = struct {	...	@@ -707,26 +732,37 @@ pub const Tokenizer = struct {
707	},	732	},
708	},	733	},
709	.builtin => switch (c) {	734	.builtin => switch (c) {
710	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},	735	'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
711	else => break,	736	else => break,
712	},	737	},
713	.backslash => switch (c) {	738	.backslash => switch (c) {
		739	0 => {
		740	result.tag = .invalid;
		741	break;
		742	},
714	'\\' => {	743	'\\' => {
715	state = .multiline_string_literal_line;	744	state = .multiline_string_literal_line;
716	},	745	},
717	else => {	746	'\n' => {
718	result.tag = .invalid;	747	result.tag = .invalid;
719	break;	748	break;
720	},	749	},
		750	else => {
		751	state = .invalid;
		752	},
721	},	753	},
722	.string_literal => switch (c) {	754	.string_literal => switch (c) {
723	0, '\n' => {	755	0 => {
724	result.tag = .invalid;
725	result.loc.end = self.index;
726	if (self.index != self.buffer.len) {	756	if (self.index != self.buffer.len) {
727	self.index += 1;	757	state = .invalid;
		758	continue;
728	}	759	}
729	return result;	760	result.tag = .invalid;
		761	break;
		762	},
		763	'\n' => {
		764	result.tag = .invalid;
		765	break;
730	},	766	},
731	'\\' => {	767	'\\' => {
732	state = .string_literal_backslash;	768	state = .string_literal_backslash;
...	@@ -735,150 +771,74 @@ pub const Tokenizer = struct {	...	@@ -735,150 +771,74 @@ pub const Tokenizer = struct {
735	self.index += 1;	771	self.index += 1;
736	break;	772	break;
737	},	773	},
738	else => {	774	0x01...0x09, 0x0b...0x1f, 0x7f => {
739	if (self.invalidCharacterLength()) \|len\| {	775	state = .invalid;
740	result.tag = .invalid;
741	result.loc.end = self.index;
742	self.index += len;
743	return result;
744	}
745
746	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
747	},	776	},
		777	else => continue,
748	},	778	},
749		779
750	.string_literal_backslash => switch (c) {	780	.string_literal_backslash => switch (c) {
751	0, '\n' => {	781	0, '\n' => {
752	result.tag = .invalid;	782	result.tag = .invalid;
753	result.loc.end = self.index;	783	break;
754	if (self.index != self.buffer.len) {
755	self.index += 1;
756	}
757	return result;
758	},	784	},
759	else => {	785	else => {
760	state = .string_literal;	786	state = .string_literal;
761
762	if (self.invalidCharacterLength()) \|len\| {
763	result.tag = .invalid;
764	result.loc.end = self.index;
765	self.index += len;
766	return result;
767	}
768
769	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
770	},	787	},
771	},	788	},
772		789
773	.char_literal => switch (c) {	790	.char_literal => switch (c) {
774	0, '\n', '\'' => {	791	0 => {
775	result.tag = .invalid;
776	result.loc.end = self.index;
777	if (self.index != self.buffer.len) {	792	if (self.index != self.buffer.len) {
778	self.index += 1;	793	state = .invalid;
		794	continue;
779	}	795	}
780	return result;	796	result.tag = .invalid;
		797	break;
		798	},
		799	'\n' => {
		800	result.tag = .invalid;
		801	break;
781	},	802	},
782	'\\' => {	803	'\\' => {
783	state = .char_literal_backslash;	804	state = .char_literal_backslash;
784	},	805	},
785	else => {	806	'\'' => {
786	state = .char_literal_end;	807	self.index += 1;
787		808	break;
788	if (self.invalidCharacterLength()) \|len\| {
789	result.tag = .invalid;
790	result.loc.end = self.index;
791	self.index += len;
792	return result;
793	}
794
795	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
796	},	809	},
		810	0x01...0x09, 0x0b...0x1f, 0x7f => {
		811	state = .invalid;
		812	},
		813	else => continue,
797	},	814	},
798		815
799	.char_literal_backslash => switch (c) {	816	.char_literal_backslash => switch (c) {
800	0, '\n' => {	817	0 => {
801	result.tag = .invalid;
802	result.loc.end = self.index;
803	if (self.index != self.buffer.len) {	818	if (self.index != self.buffer.len) {
804	self.index += 1;	819	state = .invalid;
805	}	820	continue;
806	return result;
807	},
808	'x' => {
809	state = .char_literal_hex_escape;
810	seen_escape_digits = 0;
811	},
812	'u' => {
813	state = .char_literal_unicode_escape_saw_u;
814	},
815	else => {
816	state = .char_literal_end;
817
818	if (self.invalidCharacterLength()) \|len\| {
819	result.tag = .invalid;
820	result.loc.end = self.index;
821	self.index += len;
822	return result;
823	}
824
825	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
826	},
827	},
828
829	.char_literal_hex_escape => switch (c) {
830	'0'...'9', 'a'...'f', 'A'...'F' => {
831	seen_escape_digits += 1;
832	if (seen_escape_digits == 2) {
833	state = .char_literal_end;
834	}	821	}
835	},
836	else => {
837	result.tag = .invalid;	822	result.tag = .invalid;
838	break;	823	break;
839	},	824	},
840	},	825	'\n' => {
841
842	.char_literal_unicode_escape_saw_u => switch (c) {
843	'{' => {
844	state = .char_literal_unicode_escape;
845	},
846	else => {
847	result.tag = .invalid;
848	break;
849	},
850	},
851
852	.char_literal_unicode_escape => switch (c) {
853	'0'...'9', 'a'...'f', 'A'...'F' => {},
854	'}' => {
855	state = .char_literal_end; // too many/few digits handled later
856	},
857	else => {
858	result.tag = .invalid;	826	result.tag = .invalid;
859	break;	827	break;
860	},	828	},
861	},	829	0x01...0x09, 0x0b...0x1f, 0x7f => {
862		830	state = .invalid;
863	.char_literal_end => switch (c) {
864	'\'' => {
865	result.tag = .char_literal;
866	self.index += 1;
867	break;
868	},	831	},
869	else => {	832	else => {
870	result.tag = .invalid;	833	state = .char_literal;
871	break;
872	},	834	},
873	},	835	},
874		836
875	.multiline_string_literal_line => switch (c) {	837	.multiline_string_literal_line => switch (c) {
876	0 => {	838	0 => {
877	if (self.index != self.buffer.len) {	839	if (self.index != self.buffer.len) {
878	result.tag = .invalid;	840	state = .invalid;
879	result.loc.end = self.index;	841	continue;
880	self.index += 1;
881	return result;
882	}	842	}
883	break;	843	break;
884	},	844	},
...	@@ -886,17 +846,10 @@ pub const Tokenizer = struct {	...	@@ -886,17 +846,10 @@ pub const Tokenizer = struct {
886	self.index += 1;	846	self.index += 1;
887	break;	847	break;
888	},	848	},
889	'\t' => {},	849	0x01...0x08, 0x0b...0x1f, 0x7f => {
890	else => {	850	state = .invalid;
891	if (self.invalidCharacterLength()) \|len\| {
892	result.tag = .invalid;
893	result.loc.end = self.index;
894	self.index += len;
895	return result;
896	}
897
898	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
899	},	851	},
		852	else => continue,
900	},	853	},
901		854
902	.bang => switch (c) {	855	.bang => switch (c) {
...	@@ -1113,12 +1066,16 @@ pub const Tokenizer = struct {	...	@@ -1113,12 +1066,16 @@ pub const Tokenizer = struct {
1113	.line_comment_start => switch (c) {	1066	.line_comment_start => switch (c) {
1114	0 => {	1067	0 => {
1115	if (self.index != self.buffer.len) {	1068	if (self.index != self.buffer.len) {
1116	result.tag = .invalid;	1069	state = .invalid;
1117	result.loc.end = self.index;	1070	continue;
1118	self.index += 1;
1119	return result;
1120	}	1071	}
1121	break;	1072	return .{
		1073	.tag = .eof,
		1074	.loc = .{
		1075	.start = self.index,
		1076	.end = self.index,
		1077	},
		1078	};
1122	},	1079	},
1123	'/' => {	1080	'/' => {
1124	state = .doc_comment_start;	1081	state = .doc_comment_start;
...	@@ -1127,105 +1084,74 @@ pub const Tokenizer = struct {	...	@@ -1127,105 +1084,74 @@ pub const Tokenizer = struct {
1127	result.tag = .container_doc_comment;	1084	result.tag = .container_doc_comment;
1128	state = .doc_comment;	1085	state = .doc_comment;
1129	},	1086	},
		1087	'\r' => {
		1088	state = .expect_newline;
		1089	},
1130	'\n' => {	1090	'\n' => {
1131	state = .start;	1091	state = .start;
1132	result.loc.start = self.index + 1;	1092	result.loc.start = self.index + 1;
1133	},	1093	},
1134	'\t' => {	1094	0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1135	state = .line_comment;	1095	state = .invalid;
1136	},	1096	},
1137	else => {	1097	else => {
1138	state = .line_comment;	1098	state = .line_comment;
1139
1140	if (self.invalidCharacterLength()) \|len\| {
1141	result.tag = .invalid;
1142	result.loc.end = self.index;
1143	self.index += len;
1144	return result;
1145	}
1146
1147	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1148	},	1099	},
1149	},	1100	},
1150	.doc_comment_start => switch (c) {	1101	.doc_comment_start => switch (c) {
1151	'/' => {	1102	0, '\n', '\r' => {
1152	state = .line_comment;
1153	},
1154	0 => {
1155	if (self.index != self.buffer.len) {
1156	result.tag = .invalid;
1157	result.loc.end = self.index;
1158	self.index += 1;
1159	return result;
1160	}
1161	result.tag = .doc_comment;	1103	result.tag = .doc_comment;
1162	break;	1104	break;
1163	},	1105	},
1164	'\n' => {	1106	'/' => {
1165	result.tag = .doc_comment;	1107	state = .line_comment;
1166	break;
1167	},	1108	},
1168	'\t' => {	1109	0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1169	state = .doc_comment;	1110	state = .invalid;
1170	result.tag = .doc_comment;
1171	},	1111	},
1172	else => {	1112	else => {
1173	state = .doc_comment;	1113	state = .doc_comment;
1174	result.tag = .doc_comment;	1114	result.tag = .doc_comment;
1175
1176	if (self.invalidCharacterLength()) \|len\| {
1177	result.tag = .invalid;
1178	result.loc.end = self.index;
1179	self.index += len;
1180	return result;
1181	}
1182
1183	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1184	},	1115	},
1185	},	1116	},
1186	.line_comment => switch (c) {	1117	.line_comment => switch (c) {
1187	0 => {	1118	0 => {
1188	if (self.index != self.buffer.len) {	1119	if (self.index != self.buffer.len) {
1189	result.tag = .invalid;	1120	state = .invalid;
1190	result.loc.end = self.index;	1121	continue;
1191	self.index += 1;
1192	return result;
1193	}	1122	}
1194	break;	1123	return .{
		1124	.tag = .eof,
		1125	.loc = .{
		1126	.start = self.index,
		1127	.end = self.index,
		1128	},
		1129	};
		1130	},
		1131	'\r' => {
		1132	state = .expect_newline;
1195	},	1133	},
1196	'\n' => {	1134	'\n' => {
1197	state = .start;	1135	state = .start;
1198	result.loc.start = self.index + 1;	1136	result.loc.start = self.index + 1;
1199	},	1137	},
1200	'\t' => {},	1138	0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1201	else => {	1139	state = .invalid;
1202	if (self.invalidCharacterLength()) \|len\| {
1203	result.tag = .invalid;
1204	result.loc.end = self.index;
1205	self.index += len;
1206	return result;
1207	}
1208
1209	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1210	},	1140	},
		1141	else => continue,
1211	},	1142	},
1212	.doc_comment => switch (c) {	1143	.doc_comment => switch (c) {
1213	0, '\n' => break,	1144	0, '\n', '\r' => {
1214	'\t' => {},	1145	break;
1215	else => {
1216	if (self.invalidCharacterLength()) \|len\| {
1217	result.tag = .invalid;
1218	result.loc.end = self.index;
1219	self.index += len;
1220	return result;
1221	}
1222
1223	self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1224	},	1146	},
		1147	0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
		1148	state = .invalid;
		1149	},
		1150	else => continue,
1225	},	1151	},
1226	.int => switch (c) {	1152	.int => switch (c) {
1227	'.' => state = .int_period,	1153	'.' => state = .int_period,
1228	'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},	1154	'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1229	'e', 'E', 'p', 'P' => state = .int_exponent,	1155	'e', 'E', 'p', 'P' => state = .int_exponent,
1230	else => break,	1156	else => break,
1231	},	1157	},
...	@@ -1249,7 +1175,7 @@ pub const Tokenizer = struct {	...	@@ -1249,7 +1175,7 @@ pub const Tokenizer = struct {
1249	},	1175	},
1250	},	1176	},
1251	.float => switch (c) {	1177	.float => switch (c) {
1252	'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},	1178	'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1253	'e', 'E', 'p', 'P' => state = .float_exponent,	1179	'e', 'E', 'p', 'P' => state = .float_exponent,
1254	else => break,	1180	else => break,
1255	},	1181	},
...	@@ -1263,57 +1189,9 @@ pub const Tokenizer = struct {	...	@@ -1263,57 +1189,9 @@ pub const Tokenizer = struct {
1263	}	1189	}
1264	}	1190	}
1265		1191
1266	if (result.tag == .eof) {
1267	result.loc.start = self.index;
1268	}
1269
1270	result.loc.end = self.index;	1192	result.loc.end = self.index;
1271	return result;	1193	return result;
1272	}	1194	}
1273
1274	fn invalidCharacterLength(self: *Tokenizer) ?u3 {
1275	const c0 = self.buffer[self.index];
1276	if (std.ascii.isAscii(c0)) {
1277	if (c0 == '\r') {
1278	if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {
1279	// Carriage returns are only allowed just before a linefeed as part of a CRLF pair, otherwise
1280	// they constitute an illegal byte!
1281	return null;
1282	} else {
1283	return 1;
1284	}
1285	} else if (std.ascii.isControl(c0)) {
1286	// ascii control codes are never allowed
1287	// (note that \n was checked before we got here)
1288	return 1;
1289	}
1290	// looks fine to me.
1291	return null;
1292	} else {
1293	// check utf8-encoded character.
1294	const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
1295	if (self.index + length > self.buffer.len) {
1296	return @as(u3, @intCast(self.buffer.len - self.index));
1297	}
1298	const bytes = self.buffer[self.index .. self.index + length];
1299	switch (length) {
1300	2 => {
1301	const value = std.unicode.utf8Decode2(bytes) catch return length;
1302	if (value == 0x85) return length; // U+0085 (NEL)
1303	},
1304	3 => {
1305	const value = std.unicode.utf8Decode3(bytes) catch return length;
1306	if (value == 0x2028) return length; // U+2028 (LS)
1307	if (value == 0x2029) return length; // U+2029 (PS)
1308	},
1309	4 => {
1310	_ = std.unicode.utf8Decode4(bytes) catch return length;
1311	},
1312	else => unreachable,
1313	}
1314	return null;
1315	}
1316	}
1317	};	1195	};
1318		1196
1319	test "keywords" {	1197	test "keywords" {
...	@@ -1355,7 +1233,7 @@ test "code point literal with hex escape" {	...	@@ -1355,7 +1233,7 @@ test "code point literal with hex escape" {
1355	, &.{.char_literal});	1233	, &.{.char_literal});
1356	try testTokenize(	1234	try testTokenize(
1357	\\'\x1'	1235	\\'\x1'
1358	, &.{ .invalid, .invalid });	1236	, &.{.char_literal});
1359	}	1237	}
1360		1238
1361	test "newline in char literal" {	1239	test "newline in char literal" {
...	@@ -1396,40 +1274,30 @@ test "code point literal with unicode escapes" {	...	@@ -1396,40 +1274,30 @@ test "code point literal with unicode escapes" {
1396	// Invalid unicode escapes	1274	// Invalid unicode escapes
1397	try testTokenize(	1275	try testTokenize(
1398	\\'\u'	1276	\\'\u'
1399	, &.{ .invalid, .invalid });	1277	, &.{.char_literal});
1400	try testTokenize(	1278	try testTokenize(
1401	\\'\u{{'	1279	\\'\u{{'
1402	, &.{ .invalid, .l_brace, .invalid });	1280	, &.{.char_literal});
1403	try testTokenize(	1281	try testTokenize(
1404	\\'\u{}'	1282	\\'\u{}'
1405	, &.{.char_literal});	1283	, &.{.char_literal});
1406	try testTokenize(	1284	try testTokenize(
1407	\\'\u{s}'	1285	\\'\u{s}'
1408	, &.{	1286	, &.{.char_literal});
1409	.invalid,
1410	.identifier,
1411	.r_brace,
1412	.invalid,
1413	});
1414	try testTokenize(	1287	try testTokenize(
1415	\\'\u{2z}'	1288	\\'\u{2z}'
1416	, &.{	1289	, &.{.char_literal});
1417	.invalid,
1418	.identifier,
1419	.r_brace,
1420	.invalid,
1421	});
1422	try testTokenize(	1290	try testTokenize(
1423	\\'\u{4a'	1291	\\'\u{4a'
1424	, &.{ .invalid, .invalid }); // 4a is valid	1292	, &.{.char_literal});
1425		1293
1426	// Test old-style unicode literals	1294	// Test old-style unicode literals
1427	try testTokenize(	1295	try testTokenize(
1428	\\'\u0333'	1296	\\'\u0333'
1429	, &.{ .invalid, .number_literal, .invalid });	1297	, &.{.char_literal});
1430	try testTokenize(	1298	try testTokenize(
1431	\\'\U0333'	1299	\\'\U0333'
1432	, &.{ .invalid, .number_literal, .invalid });	1300	, &.{.char_literal});
1433	}	1301	}
1434		1302
1435	test "code point literal with unicode code point" {	1303	test "code point literal with unicode code point" {
...	@@ -1465,24 +1333,15 @@ test "invalid token characters" {	...	@@ -1465,24 +1333,15 @@ test "invalid token characters" {
1465	try testTokenize("`", &.{.invalid});	1333	try testTokenize("`", &.{.invalid});
1466	try testTokenize("'c", &.{.invalid});	1334	try testTokenize("'c", &.{.invalid});
1467	try testTokenize("'", &.{.invalid});	1335	try testTokenize("'", &.{.invalid});
1468	try testTokenize("''", &.{.invalid});	1336	try testTokenize("''", &.{.char_literal});
1469	try testTokenize("'\n'", &.{ .invalid, .invalid });	1337	try testTokenize("'\n'", &.{ .invalid, .invalid });
1470	}	1338	}
1471		1339
1472	test "invalid literal/comment characters" {	1340	test "invalid literal/comment characters" {
1473	try testTokenize("\"\x00\"", &.{	1341	try testTokenize("\"\x00\"", &.{.invalid});
1474	.invalid,	1342	try testTokenize("//\x00", &.{.invalid});
1475	.invalid, // Incomplete string literal starting after invalid	1343	try testTokenize("//\x1f", &.{.invalid});
1476	});	1344	try testTokenize("//\x7f", &.{.invalid});
1477	try testTokenize("//\x00", &.{
1478	.invalid,
1479	});
1480	try testTokenize("//\x1f", &.{
1481	.invalid,
1482	});
1483	try testTokenize("//\x7f", &.{
1484	.invalid,
1485	});
1486	}	1345	}
1487		1346
1488	test "utf8" {	1347	test "utf8" {
...	@@ -1491,46 +1350,24 @@ test "utf8" {	...	@@ -1491,46 +1350,24 @@ test "utf8" {
1491	}	1350	}
1492		1351
1493	test "invalid utf8" {	1352	test "invalid utf8" {
1494	try testTokenize("//\x80", &.{	1353	try testTokenize("//\x80", &.{});
1495	.invalid,	1354	try testTokenize("//\xbf", &.{});
1496	});	1355	try testTokenize("//\xf8", &.{});
1497	try testTokenize("//\xbf", &.{	1356	try testTokenize("//\xff", &.{});
1498	.invalid,	1357	try testTokenize("//\xc2\xc0", &.{});
1499	});	1358	try testTokenize("//\xe0", &.{});
1500	try testTokenize("//\xf8", &.{	1359	try testTokenize("//\xf0", &.{});
1501	.invalid,	1360	try testTokenize("//\xf0\x90\x80\xc0", &.{});
1502	});
1503	try testTokenize("//\xff", &.{
1504	.invalid,
1505	});
1506	try testTokenize("//\xc2\xc0", &.{
1507	.invalid,
1508	});
1509	try testTokenize("//\xe0", &.{
1510	.invalid,
1511	});
1512	try testTokenize("//\xf0", &.{
1513	.invalid,
1514	});
1515	try testTokenize("//\xf0\x90\x80\xc0", &.{
1516	.invalid,
1517	});
1518	}	1361	}
1519		1362
1520	test "illegal unicode codepoints" {	1363	test "illegal unicode codepoints" {
1521	// unicode newline characters.U+0085, U+2028, U+2029	1364	// unicode newline characters.U+0085, U+2028, U+2029
1522	try testTokenize("//\xc2\x84", &.{});	1365	try testTokenize("//\xc2\x84", &.{});
1523	try testTokenize("//\xc2\x85", &.{	1366	try testTokenize("//\xc2\x85", &.{});
1524	.invalid,
1525	});
1526	try testTokenize("//\xc2\x86", &.{});	1367	try testTokenize("//\xc2\x86", &.{});
1527	try testTokenize("//\xe2\x80\xa7", &.{});	1368	try testTokenize("//\xe2\x80\xa7", &.{});
1528	try testTokenize("//\xe2\x80\xa8", &.{	1369	try testTokenize("//\xe2\x80\xa8", &.{});
1529	.invalid,	1370	try testTokenize("//\xe2\x80\xa9", &.{});
1530	});
1531	try testTokenize("//\xe2\x80\xa9", &.{
1532	.invalid,
1533	});
1534	try testTokenize("//\xe2\x80\xaa", &.{});	1371	try testTokenize("//\xe2\x80\xaa", &.{});
1535	}	1372	}
1536		1373
...	@@ -1892,8 +1729,8 @@ test "multi line string literal with only 1 backslash" {	...	@@ -1892,8 +1729,8 @@ test "multi line string literal with only 1 backslash" {
1892	}	1729	}
1893		1730
1894	test "invalid builtin identifiers" {	1731	test "invalid builtin identifiers" {
1895	try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });	1732	try testTokenize("@()", &.{.invalid});
1896	try testTokenize("@0()", &.{ .invalid, .number_literal, .l_paren, .r_paren });	1733	try testTokenize("@0()", &.{.invalid});
1897	}	1734	}
1898		1735
1899	test "invalid token with unfinished escape right before eof" {	1736	test "invalid token with unfinished escape right before eof" {
...	@@ -1921,12 +1758,12 @@ test "saturating operators" {	...	@@ -1921,12 +1758,12 @@ test "saturating operators" {
1921	}	1758	}
1922		1759
1923	test "null byte before eof" {	1760	test "null byte before eof" {
1924	try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });	1761	try testTokenize("123 \x00 456", &.{ .number_literal, .invalid });
1925	try testTokenize("//\x00", &.{.invalid});	1762	try testTokenize("//\x00", &.{.invalid});
1926	try testTokenize("\\\\\x00", &.{.invalid});	1763	try testTokenize("\\\\\x00", &.{.invalid});
1927	try testTokenize("\x00", &.{.invalid});	1764	try testTokenize("\x00", &.{.invalid});
1928	try testTokenize("// NUL\x00\n", &.{.invalid});	1765	try testTokenize("// NUL\x00\n", &.{.invalid});
1929	try testTokenize("///\x00\n", &.{.invalid});	1766	try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
1930	try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });	1767	try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
1931	}	1768	}
1932		1769
...	@@ -1936,6 +1773,9 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v	...	@@ -1936,6 +1773,9 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v
1936	const token = tokenizer.next();	1773	const token = tokenizer.next();
1937	try std.testing.expectEqual(expected_token_tag, token.tag);	1774	try std.testing.expectEqual(expected_token_tag, token.tag);
1938	}	1775	}
		1776	// Last token should always be eof, even when the last token was invalid,
		1777	// in which case the tokenizer is in an invalid state, which can only be
		1778	// recovered by opinionated means outside the scope of this implementation.
1939	const last_token = tokenizer.next();	1779	const last_token = tokenizer.next();
1940	try std.testing.expectEqual(Token.Tag.eof, last_token.tag);	1780	try std.testing.expectEqual(Token.Tag.eof, last_token.tag);
1941	try std.testing.expectEqual(source.len, last_token.loc.start);	1781	try std.testing.expectEqual(source.len, last_token.loc.start);

src/Package/Manifest.zig+3

 @@ -549,6 +549,9 @@ const Parse = struct {
                     .{raw_string[bad_index]},
                 );
             },
+            .empty_char_literal => {
+                try p.appendErrorOff(token, offset, "empty character literal", .{});
+            },
         }
     }

test/cases/compile_errors/empty_char_lit.zig created+9

...	@@ -0,0 +1,9 @@
1	export fn entry() u8 {
2	return '';
3	}
4
5	// error
6	// backend=stage2
7	// target=native
8	//
9	// :2:12: error: empty character literal

test/cases/compile_errors/invalid_legacy_unicode_escape.zig+1-2

...	@@ -6,5 +6,4 @@ export fn entry() void {	...	@@ -6,5 +6,4 @@ export fn entry() void {
6	// backend=stage2	6	// backend=stage2
7	// target=native	7	// target=native
8	//	8	//
9	// :2:15: error: expected expression, found 'invalid bytes'	9	// :2:17: error: invalid escape character: 'U'
10	// :2:18: note: invalid byte: '1'

test/cases/compile_errors/invalid_unicode_escape.zig+1-2

...	@@ -6,6 +6,5 @@ export fn entry() void {	...	@@ -6,6 +6,5 @@ export fn entry() void {
6	// backend=stage2	6	// backend=stage2
7	// target=native	7	// target=native
8	//	8	//
9	// :2:15: error: expected expression, found 'invalid bytes'	9	// :2:21: error: expected hex digit or '}', found 'z'
10	// :2:21: note: invalid byte: 'z'
11		10

test/cases/compile_errors/normal_string_with_newline.zig+1-2

...	@@ -5,5 +5,4 @@ b";	...	@@ -5,5 +5,4 @@ b";
5	// backend=stage2	5	// backend=stage2
6	// target=native	6	// target=native
7	//	7	//
8	// :1:13: error: expected expression, found 'invalid bytes'	8	// :1:13: error: expected expression, found 'invalid token'
9	// :1:15: note: invalid byte: '\n'

test/compile_errors.zig+6-12

...	@@ -42,8 +42,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {	...	@@ -42,8 +42,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
42	const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);	42	const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);
43		43
44	case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{	44	case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{
45	":1:13: error: expected expression, found 'invalid bytes'",	45	":1:13: error: expected expression, found 'invalid token'",
46	":1:19: note: invalid byte: '\\r'",
47	});	46	});
48	}	47	}
49		48
...	@@ -179,8 +178,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {	...	@@ -179,8 +178,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
179	\\ return true;	178	\\ return true;
180	\\}	179	\\}
181	, &[_][]const u8{	180	, &[_][]const u8{
182	":1:1: error: expected type expression, found 'invalid bytes'",	181	":1:1: error: expected type expression, found 'invalid token'",
183	":1:1: note: invalid byte: '\\xff'",
184	});	182	});
185	}	183	}
186		184
...	@@ -222,8 +220,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {	...	@@ -222,8 +220,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
222	const case = ctx.obj("invalid byte in string", b.graph.host);	220	const case = ctx.obj("invalid byte in string", b.graph.host);
223		221
224	case.addError("_ = \"\x01Q\";", &[_][]const u8{	222	case.addError("_ = \"\x01Q\";", &[_][]const u8{
225	":1:5: error: expected expression, found 'invalid bytes'",	223	":1:5: error: expected expression, found 'invalid token'",
226	":1:6: note: invalid byte: '\\x01'",
227	});	224	});
228	}	225	}
229		226
...	@@ -231,8 +228,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {	...	@@ -231,8 +228,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
231	const case = ctx.obj("invalid byte in comment", b.graph.host);	228	const case = ctx.obj("invalid byte in comment", b.graph.host);
232		229
233	case.addError("//\x01Q", &[_][]const u8{	230	case.addError("//\x01Q", &[_][]const u8{
234	":1:1: error: expected type expression, found 'invalid bytes'",	231	":1:1: error: expected type expression, found 'invalid token'",
235	":1:3: note: invalid byte: '\\x01'",
236	});	232	});
237	}	233	}
238		234
...	@@ -240,8 +236,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {	...	@@ -240,8 +236,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
240	const case = ctx.obj("control character in character literal", b.graph.host);	236	const case = ctx.obj("control character in character literal", b.graph.host);
241		237
242	case.addError("const c = '\x01';", &[_][]const u8{	238	case.addError("const c = '\x01';", &[_][]const u8{
243	":1:11: error: expected expression, found 'invalid bytes'",	239	":1:11: error: expected expression, found 'invalid token'",
244	":1:12: note: invalid byte: '\\x01'",
245	});	240	});
246	}	241	}
247		242
...	@@ -249,8 +244,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {	...	@@ -249,8 +244,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
249	const case = ctx.obj("invalid byte at start of token", b.graph.host);	244	const case = ctx.obj("invalid byte at start of token", b.graph.host);
250		245
251	case.addError("x = \x00Q", &[_][]const u8{	246	case.addError("x = \x00Q", &[_][]const u8{
252	":1:5: error: expected expression, found 'invalid bytes'",	247	":1:5: error: expected expression, found 'invalid token'",
253	":1:5: note: invalid byte: '\\x00'",
254	});	248	});
255	}	249	}
256	}	250	}