Commit 377e8579f9539c56fc5988c9a452a01438af89f3 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-07-31 11:51:19-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-07-31 16:57:42-07:00
log	377e8579f9539c56fc5988c9a452a01438af89f3
tree	542f9ac9cf63e7b083e9a606e6c8a77eade2e6ce
parent	c08effc20abe2595ba5c83e25b78c274ec9c58ec

std.zig.tokenizer: simplify

I pointed a fuzzer at the tokenizer and it crashed immediately. Upon
inspection, I was dissatisfied with the implementation. This commit
removes several mechanisms:
* Removes the "invalid byte" compile error note.
* Dramatically simplifies tokenizer recovery by making recovery always
  occur at newlines, and never otherwise.
* Removes UTF-8 validation.
* Moves some character validation logic to `std.zig.parseCharLiteral`.

Removing UTF-8 validation is a regression of #663, however, the existing
implementation was already buggy. When adding this functionality back,
it must be fuzz-tested while checking the property that it matches an
independent Unicode validation implementation on the same file. While
we're at it, fuzzing should check the other properties of that proposal,
such as no ASCII control characters existing inside the source code.

Other changes included in this commit:

* Deprecate `std.unicode.utf8Decode` and its WTF-8 counterpart. This
  function has an awkward API that is too easy to misuse.
* Make `utf8Decode2` and friends use arrays as parameters, eliminating a
  runtime assertion in favor of using the type system.

After this commit, the crash found by fuzzing, which was
"\x07\xd5\x80\xc3=o\xda|a\xfc{\x9a\xec\x91\xdf\x0f\\\x1a^\xbe;\x8c\xbf\xee\xea"
no longer causes a crash. However, I did not feel the need to add this
test case because the simplified logic eradicates most crashes of this
nature.

12 files changed, 234 insertions(+), 392 deletions(-)

lib
- std
  - unicode.zig
  - zig
    Ast.zig
    AstGen.zig
    parser_test.zig
    string_literal.zig
    tokenizer.zig
src
- Package
  - Manifest.zig
test
- cases
  - compile_errors
    empty_char_lit.zig
    invalid_legacy_unicode_escape.zig
    invalid_unicode_escape.zig
    normal_string_with_newline.zig
- compile_errors.zig

lib/std/unicode.zig+14-19

...	...	@@ -95,16 +95,13 @@ pub inline fn utf8EncodeComptime(comptime c: u21) [
95	95
96	96	const Utf8DecodeError = Utf8Decode2Error \|\| Utf8Decode3Error \|\| Utf8Decode4Error;
97	97
98		/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
99		/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.
100		/// If you already know the length at comptime, you can call one of
101		/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.
	98	/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
102	99	pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {
103	100	return switch (bytes.len) {
104		1 => @as(u21, bytes[0]),
105		2 => utf8Decode2(bytes),
106		3 => utf8Decode3(bytes),
107		4 => utf8Decode4(bytes),
	101	1 => bytes[0],
	102	2 => utf8Decode2(bytes[0..2].*),
	103	3 => utf8Decode3(bytes[0..3].*),
	104	4 => utf8Decode4(bytes[0..4].*),
108	105	else => unreachable,
109	106	};
110	107	}
...	...	@@ -113,8 +110,7 @@ const Utf8Decode2Error = error{
113	110	Utf8ExpectedContinuation,
114	111	Utf8OverlongEncoding,
115	112	};
116		pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
117		assert(bytes.len == 2);
	113	pub fn utf8Decode2(bytes: [2]u8) Utf8Decode2Error!u21 {
118	114	assert(bytes[0] & 0b11100000 == 0b11000000);
119	115	var value: u21 = bytes[0] & 0b00011111;
120	116
...	...	@@ -130,7 +126,7 @@ pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
130	126	const Utf8Decode3Error = Utf8Decode3AllowSurrogateHalfError \|\| error{
131	127	Utf8EncodesSurrogateHalf,
132	128	};
133		pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {
	129	pub fn utf8Decode3(bytes: [3]u8) Utf8Decode3Error!u21 {
134	130	const value = try utf8Decode3AllowSurrogateHalf(bytes);
135	131
136	132	if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
...	...	@@ -142,8 +138,7 @@ const Utf8Decode3AllowSurrogateHalfError = error{
142	138	Utf8ExpectedContinuation,
143	139	Utf8OverlongEncoding,
144	140	};
145		pub fn utf8Decode3AllowSurrogateHalf(bytes: []const u8) Utf8Decode3AllowSurrogateHalfError!u21 {
146		assert(bytes.len == 3);
	141	pub fn utf8Decode3AllowSurrogateHalf(bytes: [3]u8) Utf8Decode3AllowSurrogateHalfError!u21 {
147	142	assert(bytes[0] & 0b11110000 == 0b11100000);
148	143	var value: u21 = bytes[0] & 0b00001111;
149	144
...	...	@@ -165,8 +160,7 @@ const Utf8Decode4Error = error{
165	160	Utf8OverlongEncoding,
166	161	Utf8CodepointTooLarge,
167	162	};
168		pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
169		assert(bytes.len == 4);
	163	pub fn utf8Decode4(bytes: [4]u8) Utf8Decode4Error!u21 {
170	164	assert(bytes[0] & 0b11111000 == 0b11110000);
171	165	var value: u21 = bytes[0] & 0b00000111;
172	166
...	...	@@ -1637,12 +1631,13 @@ pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3 {
1637	1631
1638	1632	const Wtf8DecodeError = Utf8Decode2Error \|\| Utf8Decode3AllowSurrogateHalfError \|\| Utf8Decode4Error;
1639	1633
	1634	/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
1640	1635	pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21 {
1641	1636	return switch (bytes.len) {
1642		1 => @as(u21, bytes[0]),
1643		2 => utf8Decode2(bytes),
1644		3 => utf8Decode3AllowSurrogateHalf(bytes),
1645		4 => utf8Decode4(bytes),
	1637	1 => bytes[0],
	1638	2 => utf8Decode2(bytes[0..2].*),
	1639	3 => utf8Decode3AllowSurrogateHalf(bytes[0..3].*),
	1640	4 => utf8Decode4(bytes[0..4].*),
1646	1641	else => unreachable,
1647	1642	};
1648	1643	}

lib/std/zig/Ast.zig+1-1

...	...	@@ -69,7 +69,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!A
69	69	const token = tokenizer.next();
70	70	try tokens.append(gpa, .{
71	71	.tag = token.tag,
72		.start = @as(u32, @intCast(token.loc.start)),
	72	.start = @intCast(token.loc.start),
73	73	});
74	74	if (token.tag == .eof) break;
75	75	}

lib/std/zig/AstGen.zig+3-12

...	...	@@ -11351,6 +11351,9 @@ fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token
11351	11351	.{raw_string[bad_index]},
11352	11352	);
11353	11353	},
	11354	.empty_char_literal => {
	11355	return astgen.failOff(token, offset, "empty character literal", .{});
	11356	},
11354	11357	}
11355	11358	}
11356	11359
...	...	@@ -13820,21 +13823,9 @@ fn lowerAstErrors(astgen: *AstGen) !void {
13820	13823	var msg: std.ArrayListUnmanaged(u8) = .{};
13821	13824	defer msg.deinit(gpa);
13822	13825
13823		const token_starts = tree.tokens.items(.start);
13824		const token_tags = tree.tokens.items(.tag);
13825
13826	13826	var notes: std.ArrayListUnmanaged(u32) = .{};
13827	13827	defer notes.deinit(gpa);
13828	13828
13829		const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
13830		if (token_tags[tok] == .invalid) {
13831		const bad_off: u32 = @intCast(tree.tokenSlice(tok).len);
13832		const byte_abs = token_starts[tok] + bad_off;
13833		try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
13834		std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
13835		}));
13836		}
13837
13838	13829	for (tree.errors[1..]) \|note\| {
13839	13830	if (!note.is_note) break;
13840	13831

lib/std/zig/parser_test.zig-1

...	...	@@ -6061,7 +6061,6 @@ test "recovery: invalid container members" {
6061	6061	, &[_]Error{
6062	6062	.expected_expr,
6063	6063	.expected_comma_after_field,
6064		.expected_type_expr,
6065	6064	.expected_semi_after_stmt,
6066	6065	});
6067	6066	}

lib/std/zig/string_literal.zig+19-5

...	...	@@ -1,6 +1,5 @@
1	1	const std = @import("../std.zig");
2	2	const assert = std.debug.assert;
3		const utf8Decode = std.unicode.utf8Decode;
4	3	const utf8Encode = std.unicode.utf8Encode;
5	4
6	5	pub const ParseError = error{
...	...	@@ -37,12 +36,16 @@ pub const Error = union(enum) {
37	36	expected_single_quote: usize,
38	37	/// The character at this index cannot be represented without an escape sequence.
39	38	invalid_character: usize,
	39	/// `''`. Not returned for string literals.
	40	empty_char_literal,
40	41	};
41	42
42		/// Only validates escape sequence characters.
43		/// Slice must be valid utf8 starting and ending with "'" and exactly one codepoint in between.
	43	/// Asserts the slice starts and ends with single-quotes.
	44	/// Returns an error if there is not exactly one UTF-8 codepoint in between.
44	45	pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
45		assert(slice.len >= 3 and slice[0] == '\'' and slice[slice.len - 1] == '\'');
	46	if (slice.len < 3) return .{ .failure = .empty_char_literal };
	47	assert(slice[0] == '\'');
	48	assert(slice[slice.len - 1] == '\'');
46	49
47	50	switch (slice[1]) {
48	51	'\\' => {
...	...	@@ -55,7 +58,18 @@ pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
55	58	},
56	59	0 => return .{ .failure = .{ .invalid_character = 1 } },
57	60	else => {
58		const codepoint = utf8Decode(slice[1 .. slice.len - 1]) catch unreachable;
	61	const inner = slice[1 .. slice.len - 1];
	62	const n = std.unicode.utf8ByteSequenceLength(inner[0]) catch return .{
	63	.failure = .{ .invalid_unicode_codepoint = 1 },
	64	};
	65	if (inner.len > n) return .{ .failure = .{ .expected_single_quote = 1 + n } };
	66	const codepoint = switch (n) {
	67	1 => inner[0],
	68	2 => std.unicode.utf8Decode2(inner[0..2].*),
	69	3 => std.unicode.utf8Decode3(inner[0..3].*),
	70	4 => std.unicode.utf8Decode4(inner[0..4].*),
	71	else => unreachable,
	72	} catch return .{ .failure = .{ .invalid_unicode_codepoint = 1 } };
59	73	return .{ .success = codepoint };
60	74	},
61	75	}

lib/std/zig/tokenizer.zig+176-336

...	...	@@ -320,7 +320,7 @@ pub const Token = struct {
320	320
321	321	pub fn symbol(tag: Tag) []const u8 {
322	322	return tag.lexeme() orelse switch (tag) {
323		.invalid => "invalid bytes",
	323	.invalid => "invalid token",
324	324	.identifier => "an identifier",
325	325	.string_literal, .multiline_string_literal_line => "a string literal",
326	326	.char_literal => "a character literal",
...	...	@@ -338,22 +338,22 @@ pub const Tokenizer = struct {
338	338	buffer: [:0]const u8,
339	339	index: usize,
340	340
341		/// For debugging purposes
	341	/// For debugging purposes.
342	342	pub fn dump(self: Tokenizer, token: const Token) void {
343	343	std.debug.print("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] });
344	344	}
345	345
346	346	pub fn init(buffer: [:0]const u8) Tokenizer {
347		// Skip the UTF-8 BOM if present
348		const src_start: usize = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0;
349		return Tokenizer{
	347	// Skip the UTF-8 BOM if present.
	348	return .{
350	349	.buffer = buffer,
351		.index = src_start,
	350	.index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0,
352	351	};
353	352	}
354	353
355	354	const State = enum {
356	355	start,
	356	expect_newline,
357	357	identifier,
358	358	builtin,
359	359	string_literal,
...	...	@@ -361,10 +361,6 @@ pub const Tokenizer = struct {
361	361	multiline_string_literal_line,
362	362	char_literal,
363	363	char_literal_backslash,
364		char_literal_hex_escape,
365		char_literal_unicode_escape_saw_u,
366		char_literal_unicode_escape,
367		char_literal_end,
368	364	backslash,
369	365	equal,
370	366	bang,
...	...	@@ -400,32 +396,38 @@ pub const Tokenizer = struct {
400	396	period_2,
401	397	period_asterisk,
402	398	saw_at_sign,
	399	invalid,
403	400	};
404	401
	402	/// After this returns invalid, it will reset on the next newline, returning tokens starting from there.
	403	/// An eof token will always be returned at the end.
405	404	pub fn next(self: *Tokenizer) Token {
406	405	var state: State = .start;
407		var result = Token{
408		.tag = .eof,
	406	var result: Token = .{
	407	.tag = undefined,
409	408	.loc = .{
410	409	.start = self.index,
411	410	.end = undefined,
412	411	},
413	412	};
414		var seen_escape_digits: usize = undefined;
415	413	while (true) : (self.index += 1) {
416	414	const c = self.buffer[self.index];
417	415	switch (state) {
418	416	.start => switch (c) {
419	417	0 => {
420		if (self.index != self.buffer.len) {
421		result.tag = .invalid;
422		result.loc.end = self.index;
423		self.index += 1;
424		return result;
425		}
426		break;
427		},
428		' ', '\n', '\t', '\r' => {
	418	if (self.index == self.buffer.len) return .{
	419	.tag = .eof,
	420	.loc = .{
	421	.start = self.index,
	422	.end = self.index,
	423	},
	424	};
	425	state = .invalid;
	426	},
	427	'\r' => {
	428	state = .expect_newline;
	429	},
	430	' ', '\n', '\t' => {
429	431	result.loc.start = self.index + 1;
430	432	},
431	433	'"' => {
...	...	@@ -434,6 +436,7 @@ pub const Tokenizer = struct {
434	436	},
435	437	'\'' => {
436	438	state = .char_literal;
	439	result.tag = .char_literal;
437	440	},
438	441	'a'...'z', 'A'...'Z', '_' => {
439	442	state = .identifier;
...	...	@@ -545,14 +548,37 @@ pub const Tokenizer = struct {
545	548	result.tag = .number_literal;
546	549	},
547	550	else => {
	551	state = .invalid;
	552	},
	553	},
	554
	555	.expect_newline => switch (c) {
	556	'\n' => {
	557	result.loc.start = self.index + 1;
	558	state = .start;
	559	},
	560	else => {
	561	state = .invalid;
	562	},
	563	},
	564
	565	.invalid => switch (c) {
	566	0 => if (self.index == self.buffer.len) {
	567	result.tag = .invalid;
	568	break;
	569	},
	570	'\n' => {
548	571	result.tag = .invalid;
549		result.loc.end = self.index;
550		self.index += std.unicode.utf8ByteSequenceLength(c) catch 1;
551		return result;
	572	break;
552	573	},
	574	else => continue,
553	575	},
554	576
555	577	.saw_at_sign => switch (c) {
	578	0, '\n' => {
	579	result.tag = .invalid;
	580	break;
	581	},
556	582	'"' => {
557	583	result.tag = .identifier;
558	584	state = .string_literal;
...	...	@@ -562,8 +588,7 @@ pub const Tokenizer = struct {
562	588	result.tag = .builtin;
563	589	},
564	590	else => {
565		result.tag = .invalid;
566		break;
	591	state = .invalid;
567	592	},
568	593	},
569	594
...	...	@@ -698,7 +723,7 @@ pub const Tokenizer = struct {
698	723	},
699	724
700	725	.identifier => switch (c) {
701		'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	726	'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
702	727	else => {
703	728	if (Token.getKeyword(self.buffer[result.loc.start..self.index])) \|tag\| {
704	729	result.tag = tag;
...	...	@@ -707,26 +732,37 @@ pub const Tokenizer = struct {
707	732	},
708	733	},
709	734	.builtin => switch (c) {
710		'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	735	'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
711	736	else => break,
712	737	},
713	738	.backslash => switch (c) {
	739	0 => {
	740	result.tag = .invalid;
	741	break;
	742	},
714	743	'\\' => {
715	744	state = .multiline_string_literal_line;
716	745	},
717		else => {
	746	'\n' => {
718	747	result.tag = .invalid;
719	748	break;
720	749	},
	750	else => {
	751	state = .invalid;
	752	},
721	753	},
722	754	.string_literal => switch (c) {
723		0, '\n' => {
724		result.tag = .invalid;
725		result.loc.end = self.index;
	755	0 => {
726	756	if (self.index != self.buffer.len) {
727		self.index += 1;
	757	state = .invalid;
	758	continue;
728	759	}
729		return result;
	760	result.tag = .invalid;
	761	break;
	762	},
	763	'\n' => {
	764	result.tag = .invalid;
	765	break;
730	766	},
731	767	'\\' => {
732	768	state = .string_literal_backslash;
...	...	@@ -735,150 +771,74 @@ pub const Tokenizer = struct {
735	771	self.index += 1;
736	772	break;
737	773	},
738		else => {
739		if (self.invalidCharacterLength()) \|len\| {
740		result.tag = .invalid;
741		result.loc.end = self.index;
742		self.index += len;
743		return result;
744		}
745
746		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
	774	0x01...0x09, 0x0b...0x1f, 0x7f => {
	775	state = .invalid;
747	776	},
	777	else => continue,
748	778	},
749	779
750	780	.string_literal_backslash => switch (c) {
751	781	0, '\n' => {
752	782	result.tag = .invalid;
753		result.loc.end = self.index;
754		if (self.index != self.buffer.len) {
755		self.index += 1;
756		}
757		return result;
	783	break;
758	784	},
759	785	else => {
760	786	state = .string_literal;
761
762		if (self.invalidCharacterLength()) \|len\| {
763		result.tag = .invalid;
764		result.loc.end = self.index;
765		self.index += len;
766		return result;
767		}
768
769		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
770	787	},
771	788	},
772	789
773	790	.char_literal => switch (c) {
774		0, '\n', '\'' => {
775		result.tag = .invalid;
776		result.loc.end = self.index;
	791	0 => {
777	792	if (self.index != self.buffer.len) {
778		self.index += 1;
	793	state = .invalid;
	794	continue;
779	795	}
780		return result;
	796	result.tag = .invalid;
	797	break;
	798	},
	799	'\n' => {
	800	result.tag = .invalid;
	801	break;
781	802	},
782	803	'\\' => {
783	804	state = .char_literal_backslash;
784	805	},
785		else => {
786		state = .char_literal_end;
787
788		if (self.invalidCharacterLength()) \|len\| {
789		result.tag = .invalid;
790		result.loc.end = self.index;
791		self.index += len;
792		return result;
793		}
794
795		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
	806	'\'' => {
	807	self.index += 1;
	808	break;
796	809	},
	810	0x01...0x09, 0x0b...0x1f, 0x7f => {
	811	state = .invalid;
	812	},
	813	else => continue,
797	814	},
798	815
799	816	.char_literal_backslash => switch (c) {
800		0, '\n' => {
801		result.tag = .invalid;
802		result.loc.end = self.index;
	817	0 => {
803	818	if (self.index != self.buffer.len) {
804		self.index += 1;
805		}
806		return result;
807		},
808		'x' => {
809		state = .char_literal_hex_escape;
810		seen_escape_digits = 0;
811		},
812		'u' => {
813		state = .char_literal_unicode_escape_saw_u;
814		},
815		else => {
816		state = .char_literal_end;
817
818		if (self.invalidCharacterLength()) \|len\| {
819		result.tag = .invalid;
820		result.loc.end = self.index;
821		self.index += len;
822		return result;
823		}
824
825		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
826		},
827		},
828
829		.char_literal_hex_escape => switch (c) {
830		'0'...'9', 'a'...'f', 'A'...'F' => {
831		seen_escape_digits += 1;
832		if (seen_escape_digits == 2) {
833		state = .char_literal_end;
	819	state = .invalid;
	820	continue;
834	821	}
835		},
836		else => {
837	822	result.tag = .invalid;
838	823	break;
839	824	},
840		},
841
842		.char_literal_unicode_escape_saw_u => switch (c) {
843		'{' => {
844		state = .char_literal_unicode_escape;
845		},
846		else => {
847		result.tag = .invalid;
848		break;
849		},
850		},
851
852		.char_literal_unicode_escape => switch (c) {
853		'0'...'9', 'a'...'f', 'A'...'F' => {},
854		'}' => {
855		state = .char_literal_end; // too many/few digits handled later
856		},
857		else => {
	825	'\n' => {
858	826	result.tag = .invalid;
859	827	break;
860	828	},
861		},
862
863		.char_literal_end => switch (c) {
864		'\'' => {
865		result.tag = .char_literal;
866		self.index += 1;
867		break;
	829	0x01...0x09, 0x0b...0x1f, 0x7f => {
	830	state = .invalid;
868	831	},
869	832	else => {
870		result.tag = .invalid;
871		break;
	833	state = .char_literal;
872	834	},
873	835	},
874	836
875	837	.multiline_string_literal_line => switch (c) {
876	838	0 => {
877	839	if (self.index != self.buffer.len) {
878		result.tag = .invalid;
879		result.loc.end = self.index;
880		self.index += 1;
881		return result;
	840	state = .invalid;
	841	continue;
882	842	}
883	843	break;
884	844	},
...	...	@@ -886,17 +846,10 @@ pub const Tokenizer = struct {
886	846	self.index += 1;
887	847	break;
888	848	},
889		'\t' => {},
890		else => {
891		if (self.invalidCharacterLength()) \|len\| {
892		result.tag = .invalid;
893		result.loc.end = self.index;
894		self.index += len;
895		return result;
896		}
897
898		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
	849	0x01...0x08, 0x0b...0x1f, 0x7f => {
	850	state = .invalid;
899	851	},
	852	else => continue,
900	853	},
901	854
902	855	.bang => switch (c) {
...	...	@@ -1113,12 +1066,16 @@ pub const Tokenizer = struct {
1113	1066	.line_comment_start => switch (c) {
1114	1067	0 => {
1115	1068	if (self.index != self.buffer.len) {
1116		result.tag = .invalid;
1117		result.loc.end = self.index;
1118		self.index += 1;
1119		return result;
	1069	state = .invalid;
	1070	continue;
1120	1071	}
1121		break;
	1072	return .{
	1073	.tag = .eof,
	1074	.loc = .{
	1075	.start = self.index,
	1076	.end = self.index,
	1077	},
	1078	};
1122	1079	},
1123	1080	'/' => {
1124	1081	state = .doc_comment_start;
...	...	@@ -1127,105 +1084,74 @@ pub const Tokenizer = struct {
1127	1084	result.tag = .container_doc_comment;
1128	1085	state = .doc_comment;
1129	1086	},
	1087	'\r' => {
	1088	state = .expect_newline;
	1089	},
1130	1090	'\n' => {
1131	1091	state = .start;
1132	1092	result.loc.start = self.index + 1;
1133	1093	},
1134		'\t' => {
1135		state = .line_comment;
	1094	0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	1095	state = .invalid;
1136	1096	},
1137	1097	else => {
1138	1098	state = .line_comment;
1139
1140		if (self.invalidCharacterLength()) \|len\| {
1141		result.tag = .invalid;
1142		result.loc.end = self.index;
1143		self.index += len;
1144		return result;
1145		}
1146
1147		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1148	1099	},
1149	1100	},
1150	1101	.doc_comment_start => switch (c) {
1151		'/' => {
1152		state = .line_comment;
1153		},
1154		0 => {
1155		if (self.index != self.buffer.len) {
1156		result.tag = .invalid;
1157		result.loc.end = self.index;
1158		self.index += 1;
1159		return result;
1160		}
	1102	0, '\n', '\r' => {
1161	1103	result.tag = .doc_comment;
1162	1104	break;
1163	1105	},
1164		'\n' => {
1165		result.tag = .doc_comment;
1166		break;
	1106	'/' => {
	1107	state = .line_comment;
1167	1108	},
1168		'\t' => {
1169		state = .doc_comment;
1170		result.tag = .doc_comment;
	1109	0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	1110	state = .invalid;
1171	1111	},
1172	1112	else => {
1173	1113	state = .doc_comment;
1174	1114	result.tag = .doc_comment;
1175
1176		if (self.invalidCharacterLength()) \|len\| {
1177		result.tag = .invalid;
1178		result.loc.end = self.index;
1179		self.index += len;
1180		return result;
1181		}
1182
1183		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1184	1115	},
1185	1116	},
1186	1117	.line_comment => switch (c) {
1187	1118	0 => {
1188	1119	if (self.index != self.buffer.len) {
1189		result.tag = .invalid;
1190		result.loc.end = self.index;
1191		self.index += 1;
1192		return result;
	1120	state = .invalid;
	1121	continue;
1193	1122	}
1194		break;
	1123	return .{
	1124	.tag = .eof,
	1125	.loc = .{
	1126	.start = self.index,
	1127	.end = self.index,
	1128	},
	1129	};
	1130	},
	1131	'\r' => {
	1132	state = .expect_newline;
1195	1133	},
1196	1134	'\n' => {
1197	1135	state = .start;
1198	1136	result.loc.start = self.index + 1;
1199	1137	},
1200		'\t' => {},
1201		else => {
1202		if (self.invalidCharacterLength()) \|len\| {
1203		result.tag = .invalid;
1204		result.loc.end = self.index;
1205		self.index += len;
1206		return result;
1207		}
1208
1209		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
	1138	0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	1139	state = .invalid;
1210	1140	},
	1141	else => continue,
1211	1142	},
1212	1143	.doc_comment => switch (c) {
1213		0, '\n' => break,
1214		'\t' => {},
1215		else => {
1216		if (self.invalidCharacterLength()) \|len\| {
1217		result.tag = .invalid;
1218		result.loc.end = self.index;
1219		self.index += len;
1220		return result;
1221		}
1222
1223		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
	1144	0, '\n', '\r' => {
	1145	break;
1224	1146	},
	1147	0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	1148	state = .invalid;
	1149	},
	1150	else => continue,
1225	1151	},
1226	1152	.int => switch (c) {
1227	1153	'.' => state = .int_period,
1228		'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},
	1154	'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1229	1155	'e', 'E', 'p', 'P' => state = .int_exponent,
1230	1156	else => break,
1231	1157	},
...	...	@@ -1249,7 +1175,7 @@ pub const Tokenizer = struct {
1249	1175	},
1250	1176	},
1251	1177	.float => switch (c) {
1252		'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},
	1178	'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1253	1179	'e', 'E', 'p', 'P' => state = .float_exponent,
1254	1180	else => break,
1255	1181	},
...	...	@@ -1263,57 +1189,9 @@ pub const Tokenizer = struct {
1263	1189	}
1264	1190	}
1265	1191
1266		if (result.tag == .eof) {
1267		result.loc.start = self.index;
1268		}
1269
1270	1192	result.loc.end = self.index;
1271	1193	return result;
1272	1194	}
1273
1274		fn invalidCharacterLength(self: *Tokenizer) ?u3 {
1275		const c0 = self.buffer[self.index];
1276		if (std.ascii.isAscii(c0)) {
1277		if (c0 == '\r') {
1278		if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {
1279		// Carriage returns are only allowed just before a linefeed as part of a CRLF pair, otherwise
1280		// they constitute an illegal byte!
1281		return null;
1282		} else {
1283		return 1;
1284		}
1285		} else if (std.ascii.isControl(c0)) {
1286		// ascii control codes are never allowed
1287		// (note that \n was checked before we got here)
1288		return 1;
1289		}
1290		// looks fine to me.
1291		return null;
1292		} else {
1293		// check utf8-encoded character.
1294		const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
1295		if (self.index + length > self.buffer.len) {
1296		return @as(u3, @intCast(self.buffer.len - self.index));
1297		}
1298		const bytes = self.buffer[self.index .. self.index + length];
1299		switch (length) {
1300		2 => {
1301		const value = std.unicode.utf8Decode2(bytes) catch return length;
1302		if (value == 0x85) return length; // U+0085 (NEL)
1303		},
1304		3 => {
1305		const value = std.unicode.utf8Decode3(bytes) catch return length;
1306		if (value == 0x2028) return length; // U+2028 (LS)
1307		if (value == 0x2029) return length; // U+2029 (PS)
1308		},
1309		4 => {
1310		_ = std.unicode.utf8Decode4(bytes) catch return length;
1311		},
1312		else => unreachable,
1313		}
1314		return null;
1315		}
1316		}
1317	1195	};
1318	1196
1319	1197	test "keywords" {
...	...	@@ -1355,7 +1233,7 @@ test "code point literal with hex escape" {
1355	1233	, &.{.char_literal});
1356	1234	try testTokenize(
1357	1235	\\'\x1'
1358		, &.{ .invalid, .invalid });
	1236	, &.{.char_literal});
1359	1237	}
1360	1238
1361	1239	test "newline in char literal" {
...	...	@@ -1396,40 +1274,30 @@ test "code point literal with unicode escapes" {
1396	1274	// Invalid unicode escapes
1397	1275	try testTokenize(
1398	1276	\\'\u'
1399		, &.{ .invalid, .invalid });
	1277	, &.{.char_literal});
1400	1278	try testTokenize(
1401	1279	\\'\u{{'
1402		, &.{ .invalid, .l_brace, .invalid });
	1280	, &.{.char_literal});
1403	1281	try testTokenize(
1404	1282	\\'\u{}'
1405	1283	, &.{.char_literal});
1406	1284	try testTokenize(
1407	1285	\\'\u{s}'
1408		, &.{
1409		.invalid,
1410		.identifier,
1411		.r_brace,
1412		.invalid,
1413		});
	1286	, &.{.char_literal});
1414	1287	try testTokenize(
1415	1288	\\'\u{2z}'
1416		, &.{
1417		.invalid,
1418		.identifier,
1419		.r_brace,
1420		.invalid,
1421		});
	1289	, &.{.char_literal});
1422	1290	try testTokenize(
1423	1291	\\'\u{4a'
1424		, &.{ .invalid, .invalid }); // 4a is valid
	1292	, &.{.char_literal});
1425	1293
1426	1294	// Test old-style unicode literals
1427	1295	try testTokenize(
1428	1296	\\'\u0333'
1429		, &.{ .invalid, .number_literal, .invalid });
	1297	, &.{.char_literal});
1430	1298	try testTokenize(
1431	1299	\\'\U0333'
1432		, &.{ .invalid, .number_literal, .invalid });
	1300	, &.{.char_literal});
1433	1301	}
1434	1302
1435	1303	test "code point literal with unicode code point" {
...	...	@@ -1465,24 +1333,15 @@ test "invalid token characters" {
1465	1333	try testTokenize("`", &.{.invalid});
1466	1334	try testTokenize("'c", &.{.invalid});
1467	1335	try testTokenize("'", &.{.invalid});
1468		try testTokenize("''", &.{.invalid});
	1336	try testTokenize("''", &.{.char_literal});
1469	1337	try testTokenize("'\n'", &.{ .invalid, .invalid });
1470	1338	}
1471	1339
1472	1340	test "invalid literal/comment characters" {
1473		try testTokenize("\"\x00\"", &.{
1474		.invalid,
1475		.invalid, // Incomplete string literal starting after invalid
1476		});
1477		try testTokenize("//\x00", &.{
1478		.invalid,
1479		});
1480		try testTokenize("//\x1f", &.{
1481		.invalid,
1482		});
1483		try testTokenize("//\x7f", &.{
1484		.invalid,
1485		});
	1341	try testTokenize("\"\x00\"", &.{.invalid});
	1342	try testTokenize("//\x00", &.{.invalid});
	1343	try testTokenize("//\x1f", &.{.invalid});
	1344	try testTokenize("//\x7f", &.{.invalid});
1486	1345	}
1487	1346
1488	1347	test "utf8" {
...	...	@@ -1491,46 +1350,24 @@ test "utf8" {
1491	1350	}
1492	1351
1493	1352	test "invalid utf8" {
1494		try testTokenize("//\x80", &.{
1495		.invalid,
1496		});
1497		try testTokenize("//\xbf", &.{
1498		.invalid,
1499		});
1500		try testTokenize("//\xf8", &.{
1501		.invalid,
1502		});
1503		try testTokenize("//\xff", &.{
1504		.invalid,
1505		});
1506		try testTokenize("//\xc2\xc0", &.{
1507		.invalid,
1508		});
1509		try testTokenize("//\xe0", &.{
1510		.invalid,
1511		});
1512		try testTokenize("//\xf0", &.{
1513		.invalid,
1514		});
1515		try testTokenize("//\xf0\x90\x80\xc0", &.{
1516		.invalid,
1517		});
	1353	try testTokenize("//\x80", &.{});
	1354	try testTokenize("//\xbf", &.{});
	1355	try testTokenize("//\xf8", &.{});
	1356	try testTokenize("//\xff", &.{});
	1357	try testTokenize("//\xc2\xc0", &.{});
	1358	try testTokenize("//\xe0", &.{});
	1359	try testTokenize("//\xf0", &.{});
	1360	try testTokenize("//\xf0\x90\x80\xc0", &.{});
1518	1361	}
1519	1362
1520	1363	test "illegal unicode codepoints" {
1521	1364	// unicode newline characters.U+0085, U+2028, U+2029
1522	1365	try testTokenize("//\xc2\x84", &.{});
1523		try testTokenize("//\xc2\x85", &.{
1524		.invalid,
1525		});
	1366	try testTokenize("//\xc2\x85", &.{});
1526	1367	try testTokenize("//\xc2\x86", &.{});
1527	1368	try testTokenize("//\xe2\x80\xa7", &.{});
1528		try testTokenize("//\xe2\x80\xa8", &.{
1529		.invalid,
1530		});
1531		try testTokenize("//\xe2\x80\xa9", &.{
1532		.invalid,
1533		});
	1369	try testTokenize("//\xe2\x80\xa8", &.{});
	1370	try testTokenize("//\xe2\x80\xa9", &.{});
1534	1371	try testTokenize("//\xe2\x80\xaa", &.{});
1535	1372	}
1536	1373
...	...	@@ -1892,8 +1729,8 @@ test "multi line string literal with only 1 backslash" {
1892	1729	}
1893	1730
1894	1731	test "invalid builtin identifiers" {
1895		try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });
1896		try testTokenize("@0()", &.{ .invalid, .number_literal, .l_paren, .r_paren });
	1732	try testTokenize("@()", &.{.invalid});
	1733	try testTokenize("@0()", &.{.invalid});
1897	1734	}
1898	1735
1899	1736	test "invalid token with unfinished escape right before eof" {
...	...	@@ -1921,12 +1758,12 @@ test "saturating operators" {
1921	1758	}
1922	1759
1923	1760	test "null byte before eof" {
1924		try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });
	1761	try testTokenize("123 \x00 456", &.{ .number_literal, .invalid });
1925	1762	try testTokenize("//\x00", &.{.invalid});
1926	1763	try testTokenize("\\\\\x00", &.{.invalid});
1927	1764	try testTokenize("\x00", &.{.invalid});
1928	1765	try testTokenize("// NUL\x00\n", &.{.invalid});
1929		try testTokenize("///\x00\n", &.{.invalid});
	1766	try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
1930	1767	try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
1931	1768	}
1932	1769
...	...	@@ -1936,6 +1773,9 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v
1936	1773	const token = tokenizer.next();
1937	1774	try std.testing.expectEqual(expected_token_tag, token.tag);
1938	1775	}
	1776	// Last token should always be eof, even when the last token was invalid,
	1777	// in which case the tokenizer is in an invalid state, which can only be
	1778	// recovered by opinionated means outside the scope of this implementation.
1939	1779	const last_token = tokenizer.next();
1940	1780	try std.testing.expectEqual(Token.Tag.eof, last_token.tag);
1941	1781	try std.testing.expectEqual(source.len, last_token.loc.start);

src/Package/Manifest.zig+3

...	...	@@ -549,6 +549,9 @@ const Parse = struct {
549	549	.{raw_string[bad_index]},
550	550	);
551	551	},
	552	.empty_char_literal => {
	553	try p.appendErrorOff(token, offset, "empty character literal", .{});
	554	},
552	555	}
553	556	}
554	557

test/cases/compile_errors/empty_char_lit.zig created+9

...	...	@@ -0,0 +1,9 @@
	1	export fn entry() u8 {
	2	return '';
	3	}
	4
	5	// error
	6	// backend=stage2
	7	// target=native
	8	//
	9	// :2:12: error: empty character literal

test/cases/compile_errors/invalid_legacy_unicode_escape.zig+1-2

...	...	@@ -6,5 +6,4 @@ export fn entry() void {
6	6	// backend=stage2
7	7	// target=native
8	8	//
9		// :2:15: error: expected expression, found 'invalid bytes'
10		// :2:18: note: invalid byte: '1'
	9	// :2:17: error: invalid escape character: 'U'

test/cases/compile_errors/invalid_unicode_escape.zig+1-2

...	...	@@ -6,6 +6,5 @@ export fn entry() void {
6	6	// backend=stage2
7	7	// target=native
8	8	//
9		// :2:15: error: expected expression, found 'invalid bytes'
10		// :2:21: note: invalid byte: 'z'
	9	// :2:21: error: expected hex digit or '}', found 'z'
11	10

test/cases/compile_errors/normal_string_with_newline.zig+1-2

...	...	@@ -5,5 +5,4 @@ b";
5	5	// backend=stage2
6	6	// target=native
7	7	//
8		// :1:13: error: expected expression, found 'invalid bytes'
9		// :1:15: note: invalid byte: '\n'
	8	// :1:13: error: expected expression, found 'invalid token'

test/compile_errors.zig+6-12

...	...	@@ -42,8 +42,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
42	42	const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);
43	43
44	44	case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{
45		":1:13: error: expected expression, found 'invalid bytes'",
46		":1:19: note: invalid byte: '\\r'",
	45	":1:13: error: expected expression, found 'invalid token'",
47	46	});
48	47	}
49	48
...	...	@@ -179,8 +178,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
179	178	\\ return true;
180	179	\\}
181	180	, &[_][]const u8{
182		":1:1: error: expected type expression, found 'invalid bytes'",
183		":1:1: note: invalid byte: '\\xff'",
	181	":1:1: error: expected type expression, found 'invalid token'",
184	182	});
185	183	}
186	184
...	...	@@ -222,8 +220,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
222	220	const case = ctx.obj("invalid byte in string", b.graph.host);
223	221
224	222	case.addError("_ = \"\x01Q\";", &[_][]const u8{
225		":1:5: error: expected expression, found 'invalid bytes'",
226		":1:6: note: invalid byte: '\\x01'",
	223	":1:5: error: expected expression, found 'invalid token'",
227	224	});
228	225	}
229	226
...	...	@@ -231,8 +228,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
231	228	const case = ctx.obj("invalid byte in comment", b.graph.host);
232	229
233	230	case.addError("//\x01Q", &[_][]const u8{
234		":1:1: error: expected type expression, found 'invalid bytes'",
235		":1:3: note: invalid byte: '\\x01'",
	231	":1:1: error: expected type expression, found 'invalid token'",
236	232	});
237	233	}
238	234
...	...	@@ -240,8 +236,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
240	236	const case = ctx.obj("control character in character literal", b.graph.host);
241	237
242	238	case.addError("const c = '\x01';", &[_][]const u8{
243		":1:11: error: expected expression, found 'invalid bytes'",
244		":1:12: note: invalid byte: '\\x01'",
	239	":1:11: error: expected expression, found 'invalid token'",
245	240	});
246	241	}
247	242
...	...	@@ -249,8 +244,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
249	244	const case = ctx.obj("invalid byte at start of token", b.graph.host);
250	245
251	246	case.addError("x = \x00Q", &[_][]const u8{
252		":1:5: error: expected expression, found 'invalid bytes'",
253		":1:5: note: invalid byte: '\\x00'",
	247	":1:5: error: expected expression, found 'invalid token'",
254	248	});
255	249	}
256	250	}