Commit eb1a199dff2b54271bd275c2528bdd898bf1d4eb | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-07-31 19:52:34-07:00
committer	GitHub <noreply@github.com> 2024-07-31 19:52:34-07:00
log	eb1a199dff2b54271bd275c2528bdd898bf1d4eb
tree	0cef5904cef4e638dc67248384663c6779b5e293
parent	059856acfc9f87d723a90af6a4214e128b8cae2e
parent	c2b8afcac9e427102370dc5bac8c3d9621eee6d8
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #20885 from ziglang/simplify-tokenizer

std.zig.tokenizer: simplification and spec conformance

19 files changed, 418 insertions(+), 529 deletions(-)

lib
- std
  - crypto
    ml_kem.zig
  - macho.zig
  - unicode.zig
  - zig
    Ast.zig
    AstGen.zig
    parser_test.zig
    string_literal.zig
    system
    darwin
    macos.zig
    linux.zig
    tokenizer.zig
src
- Package
  - Manifest.zig
- link
  - tapi
    yaml
    test.zig
test

lib/std/crypto/ml_kem.zig+3-3

...	...	@@ -677,10 +677,10 @@ fn montReduce(x: i32) i16 {
677	677	// Note gcd(2¹⁶, q) = 1 as q is prime. Write q' := 62209 = q⁻¹ mod R.
678	678	// First we compute
679	679	//
680		// m := ((x mod R) q') mod R
	680	// m := ((x mod R) q') mod R
681	681	// = x q' mod R
682		// = int16(x q')
683		// = int16(int32(x) * int32(q'))
	682	// = int16(x q')
	683	// = int16(int32(x) * int32(q'))
684	684	//
685	685	// Note that x q' might be as big as 2³² and could overflow the int32
686	686	// multiplication in the last line. However for any int32s a and b,

lib/std/macho.zig+4-5

...	...	@@ -203,8 +203,7 @@ pub const symtab_command = extern struct {
203	203	/// local symbols (static and debugging symbols) - grouped by module
204	204	/// defined external symbols - grouped by module (sorted by name if not lib)
205	205	/// undefined external symbols (sorted by name if MH_BINDATLOAD is not set,
206		/// and in order the were seen by the static
207		/// linker if MH_BINDATLOAD is set)
	206	/// and in order the were seen by the static linker if MH_BINDATLOAD is set)
208	207	/// In this load command there are offsets and counts to each of the three groups
209	208	/// of symbols.
210	209	///
...	...	@@ -219,9 +218,9 @@ pub const symtab_command = extern struct {
219	218	/// shared library. For executable and object modules, which are files
220	219	/// containing only one module, the information that would be in these three
221	220	/// tables is determined as follows:
222		/// table of contents - the defined external symbols are sorted by name
223		/// module table - the file contains only one module so everything in the
224		/// file is part of the module.
	221	/// table of contents - the defined external symbols are sorted by name
	222	/// module table - the file contains only one module so everything in the file
	223	/// is part of the module.
225	224	/// reference symbol table - is the defined and undefined external symbols
226	225	///
227	226	/// For dynamically linked shared library files this load command also contains

lib/std/unicode.zig+14-19

...	...	@@ -95,16 +95,13 @@ pub inline fn utf8EncodeComptime(comptime c: u21) [
95	95
96	96	const Utf8DecodeError = Utf8Decode2Error \|\| Utf8Decode3Error \|\| Utf8Decode4Error;
97	97
98		/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
99		/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.
100		/// If you already know the length at comptime, you can call one of
101		/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.
	98	/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
102	99	pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {
103	100	return switch (bytes.len) {
104		1 => @as(u21, bytes[0]),
105		2 => utf8Decode2(bytes),
106		3 => utf8Decode3(bytes),
107		4 => utf8Decode4(bytes),
	101	1 => bytes[0],
	102	2 => utf8Decode2(bytes[0..2].*),
	103	3 => utf8Decode3(bytes[0..3].*),
	104	4 => utf8Decode4(bytes[0..4].*),
108	105	else => unreachable,
109	106	};
110	107	}
...	...	@@ -113,8 +110,7 @@ const Utf8Decode2Error = error{
113	110	Utf8ExpectedContinuation,
114	111	Utf8OverlongEncoding,
115	112	};
116		pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
117		assert(bytes.len == 2);
	113	pub fn utf8Decode2(bytes: [2]u8) Utf8Decode2Error!u21 {
118	114	assert(bytes[0] & 0b11100000 == 0b11000000);
119	115	var value: u21 = bytes[0] & 0b00011111;
120	116
...	...	@@ -130,7 +126,7 @@ pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
130	126	const Utf8Decode3Error = Utf8Decode3AllowSurrogateHalfError \|\| error{
131	127	Utf8EncodesSurrogateHalf,
132	128	};
133		pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {
	129	pub fn utf8Decode3(bytes: [3]u8) Utf8Decode3Error!u21 {
134	130	const value = try utf8Decode3AllowSurrogateHalf(bytes);
135	131
136	132	if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
...	...	@@ -142,8 +138,7 @@ const Utf8Decode3AllowSurrogateHalfError = error{
142	138	Utf8ExpectedContinuation,
143	139	Utf8OverlongEncoding,
144	140	};
145		pub fn utf8Decode3AllowSurrogateHalf(bytes: []const u8) Utf8Decode3AllowSurrogateHalfError!u21 {
146		assert(bytes.len == 3);
	141	pub fn utf8Decode3AllowSurrogateHalf(bytes: [3]u8) Utf8Decode3AllowSurrogateHalfError!u21 {
147	142	assert(bytes[0] & 0b11110000 == 0b11100000);
148	143	var value: u21 = bytes[0] & 0b00001111;
149	144
...	...	@@ -165,8 +160,7 @@ const Utf8Decode4Error = error{
165	160	Utf8OverlongEncoding,
166	161	Utf8CodepointTooLarge,
167	162	};
168		pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
169		assert(bytes.len == 4);
	163	pub fn utf8Decode4(bytes: [4]u8) Utf8Decode4Error!u21 {
170	164	assert(bytes[0] & 0b11111000 == 0b11110000);
171	165	var value: u21 = bytes[0] & 0b00000111;
172	166
...	...	@@ -1637,12 +1631,13 @@ pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3 {
1637	1631
1638	1632	const Wtf8DecodeError = Utf8Decode2Error \|\| Utf8Decode3AllowSurrogateHalfError \|\| Utf8Decode4Error;
1639	1633
	1634	/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
1640	1635	pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21 {
1641	1636	return switch (bytes.len) {
1642		1 => @as(u21, bytes[0]),
1643		2 => utf8Decode2(bytes),
1644		3 => utf8Decode3AllowSurrogateHalf(bytes),
1645		4 => utf8Decode4(bytes),
	1637	1 => bytes[0],
	1638	2 => utf8Decode2(bytes[0..2].*),
	1639	3 => utf8Decode3AllowSurrogateHalf(bytes[0..3].*),
	1640	4 => utf8Decode4(bytes[0..4].*),
1646	1641	else => unreachable,
1647	1642	};
1648	1643	}

lib/std/zig/Ast.zig+1-1

...	...	@@ -69,7 +69,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!A
69	69	const token = tokenizer.next();
70	70	try tokens.append(gpa, .{
71	71	.tag = token.tag,
72		.start = @as(u32, @intCast(token.loc.start)),
	72	.start = @intCast(token.loc.start),
73	73	});
74	74	if (token.tag == .eof) break;
75	75	}

lib/std/zig/AstGen.zig+3-12

...	...	@@ -11351,6 +11351,9 @@ fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token
11351	11351	.{raw_string[bad_index]},
11352	11352	);
11353	11353	},
	11354	.empty_char_literal => {
	11355	return astgen.failOff(token, offset, "empty character literal", .{});
	11356	},
11354	11357	}
11355	11358	}
11356	11359
...	...	@@ -13820,21 +13823,9 @@ fn lowerAstErrors(astgen: *AstGen) !void {
13820	13823	var msg: std.ArrayListUnmanaged(u8) = .{};
13821	13824	defer msg.deinit(gpa);
13822	13825
13823		const token_starts = tree.tokens.items(.start);
13824		const token_tags = tree.tokens.items(.tag);
13825
13826	13826	var notes: std.ArrayListUnmanaged(u32) = .{};
13827	13827	defer notes.deinit(gpa);
13828	13828
13829		const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
13830		if (token_tags[tok] == .invalid) {
13831		const bad_off: u32 = @intCast(tree.tokenSlice(tok).len);
13832		const byte_abs = token_starts[tok] + bad_off;
13833		try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
13834		std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
13835		}));
13836		}
13837
13838	13829	for (tree.errors[1..]) \|note\| {
13839	13830	if (!note.is_note) break;
13840	13831

lib/std/zig/parser_test.zig-1

...	...	@@ -6061,7 +6061,6 @@ test "recovery: invalid container members" {
6061	6061	, &[_]Error{
6062	6062	.expected_expr,
6063	6063	.expected_comma_after_field,
6064		.expected_type_expr,
6065	6064	.expected_semi_after_stmt,
6066	6065	});
6067	6066	}

lib/std/zig/string_literal.zig+19-5

...	...	@@ -1,6 +1,5 @@
1	1	const std = @import("../std.zig");
2	2	const assert = std.debug.assert;
3		const utf8Decode = std.unicode.utf8Decode;
4	3	const utf8Encode = std.unicode.utf8Encode;
5	4
6	5	pub const ParseError = error{
...	...	@@ -37,12 +36,16 @@ pub const Error = union(enum) {
37	36	expected_single_quote: usize,
38	37	/// The character at this index cannot be represented without an escape sequence.
39	38	invalid_character: usize,
	39	/// `''`. Not returned for string literals.
	40	empty_char_literal,
40	41	};
41	42
42		/// Only validates escape sequence characters.
43		/// Slice must be valid utf8 starting and ending with "'" and exactly one codepoint in between.
	43	/// Asserts the slice starts and ends with single-quotes.
	44	/// Returns an error if there is not exactly one UTF-8 codepoint in between.
44	45	pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
45		assert(slice.len >= 3 and slice[0] == '\'' and slice[slice.len - 1] == '\'');
	46	if (slice.len < 3) return .{ .failure = .empty_char_literal };
	47	assert(slice[0] == '\'');
	48	assert(slice[slice.len - 1] == '\'');
46	49
47	50	switch (slice[1]) {
48	51	'\\' => {
...	...	@@ -55,7 +58,18 @@ pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
55	58	},
56	59	0 => return .{ .failure = .{ .invalid_character = 1 } },
57	60	else => {
58		const codepoint = utf8Decode(slice[1 .. slice.len - 1]) catch unreachable;
	61	const inner = slice[1 .. slice.len - 1];
	62	const n = std.unicode.utf8ByteSequenceLength(inner[0]) catch return .{
	63	.failure = .{ .invalid_unicode_codepoint = 1 },
	64	};
	65	if (inner.len > n) return .{ .failure = .{ .expected_single_quote = 1 + n } };
	66	const codepoint = switch (n) {
	67	1 => inner[0],
	68	2 => std.unicode.utf8Decode2(inner[0..2].*),
	69	3 => std.unicode.utf8Decode3(inner[0..3].*),
	70	4 => std.unicode.utf8Decode4(inner[0..4].*),
	71	else => unreachable,
	72	} catch return .{ .failure = .{ .invalid_unicode_codepoint = 1 } };
59	73	return .{ .success = codepoint };
60	74	},
61	75	}

lib/std/zig/system/darwin/macos.zig+46-46

...	...	@@ -303,16 +303,16 @@ test "detect" {
303	303	\\<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
304	304	\\<plist version="1.0">
305	305	\\<dict>
306		\\ <key>ProductBuildVersion</key>
307		\\ <string>7W98</string>
308		\\ <key>ProductCopyright</key>
309		\\ <string>Apple Computer, Inc. 1983-2004</string>
310		\\ <key>ProductName</key>
311		\\ <string>Mac OS X</string>
312		\\ <key>ProductUserVisibleVersion</key>
313		\\ <string>10.3.9</string>
314		\\ <key>ProductVersion</key>
315		\\ <string>10.3.9</string>
	306	\\ <key>ProductBuildVersion</key>
	307	\\ <string>7W98</string>
	308	\\ <key>ProductCopyright</key>
	309	\\ <string>Apple Computer, Inc. 1983-2004</string>
	310	\\ <key>ProductName</key>
	311	\\ <string>Mac OS X</string>
	312	\\ <key>ProductUserVisibleVersion</key>
	313	\\ <string>10.3.9</string>
	314	\\ <key>ProductVersion</key>
	315	\\ <string>10.3.9</string>
316	316	\\</dict>
317	317	\\</plist>
318	318	,
...	...	@@ -323,18 +323,18 @@ test "detect" {
323	323	\\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
324	324	\\<plist version="1.0">
325	325	\\<dict>
326		\\ <key>ProductBuildVersion</key>
327		\\ <string>19G68</string>
328		\\ <key>ProductCopyright</key>
329		\\ <string>1983-2020 Apple Inc.</string>
330		\\ <key>ProductName</key>
331		\\ <string>Mac OS X</string>
332		\\ <key>ProductUserVisibleVersion</key>
333		\\ <string>10.15.6</string>
334		\\ <key>ProductVersion</key>
335		\\ <string>10.15.6</string>
336		\\ <key>iOSSupportVersion</key>
337		\\ <string>13.6</string>
	326	\\ <key>ProductBuildVersion</key>
	327	\\ <string>19G68</string>
	328	\\ <key>ProductCopyright</key>
	329	\\ <string>1983-2020 Apple Inc.</string>
	330	\\ <key>ProductName</key>
	331	\\ <string>Mac OS X</string>
	332	\\ <key>ProductUserVisibleVersion</key>
	333	\\ <string>10.15.6</string>
	334	\\ <key>ProductVersion</key>
	335	\\ <string>10.15.6</string>
	336	\\ <key>iOSSupportVersion</key>
	337	\\ <string>13.6</string>
338	338	\\</dict>
339	339	\\</plist>
340	340	,
...	...	@@ -345,18 +345,18 @@ test "detect" {
345	345	\\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
346	346	\\<plist version="1.0">
347	347	\\<dict>
348		\\ <key>ProductBuildVersion</key>
349		\\ <string>20A2408</string>
350		\\ <key>ProductCopyright</key>
351		\\ <string>1983-2020 Apple Inc.</string>
352		\\ <key>ProductName</key>
353		\\ <string>macOS</string>
354		\\ <key>ProductUserVisibleVersion</key>
355		\\ <string>11.0</string>
356		\\ <key>ProductVersion</key>
357		\\ <string>11.0</string>
358		\\ <key>iOSSupportVersion</key>
359		\\ <string>14.2</string>
	348	\\ <key>ProductBuildVersion</key>
	349	\\ <string>20A2408</string>
	350	\\ <key>ProductCopyright</key>
	351	\\ <string>1983-2020 Apple Inc.</string>
	352	\\ <key>ProductName</key>
	353	\\ <string>macOS</string>
	354	\\ <key>ProductUserVisibleVersion</key>
	355	\\ <string>11.0</string>
	356	\\ <key>ProductVersion</key>
	357	\\ <string>11.0</string>
	358	\\ <key>iOSSupportVersion</key>
	359	\\ <string>14.2</string>
360	360	\\</dict>
361	361	\\</plist>
362	362	,
...	...	@@ -367,18 +367,18 @@ test "detect" {
367	367	\\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
368	368	\\<plist version="1.0">
369	369	\\<dict>
370		\\ <key>ProductBuildVersion</key>
371		\\ <string>20C63</string>
372		\\ <key>ProductCopyright</key>
373		\\ <string>1983-2020 Apple Inc.</string>
374		\\ <key>ProductName</key>
375		\\ <string>macOS</string>
376		\\ <key>ProductUserVisibleVersion</key>
377		\\ <string>11.1</string>
378		\\ <key>ProductVersion</key>
379		\\ <string>11.1</string>
380		\\ <key>iOSSupportVersion</key>
381		\\ <string>14.3</string>
	370	\\ <key>ProductBuildVersion</key>
	371	\\ <string>20C63</string>
	372	\\ <key>ProductCopyright</key>
	373	\\ <string>1983-2020 Apple Inc.</string>
	374	\\ <key>ProductName</key>
	375	\\ <string>macOS</string>
	376	\\ <key>ProductUserVisibleVersion</key>
	377	\\ <string>11.1</string>
	378	\\ <key>ProductVersion</key>
	379	\\ <string>11.1</string>
	380	\\ <key>iOSSupportVersion</key>
	381	\\ <string>14.3</string>
382	382	\\</dict>
383	383	\\</plist>
384	384	,

lib/std/zig/system/linux.zig+30-30

...	...	@@ -109,12 +109,12 @@ const RiscvCpuinfoParser = CpuinfoParser(RiscvCpuinfoImpl);
109	109
110	110	test "cpuinfo: RISC-V" {
111	111	try testParser(RiscvCpuinfoParser, .riscv64, &Target.riscv.cpu.sifive_u74,
112		\\processor : 0
113		\\hart : 1
114		\\isa : rv64imafdc
115		\\mmu : sv39
116		\\isa-ext :
117		\\uarch : sifive,u74-mc
	112	\\processor : 0
	113	\\hart : 1
	114	\\isa : rv64imafdc
	115	\\mmu : sv39
	116	\\isa-ext :
	117	\\uarch : sifive,u74-mc
118	118	);
119	119	}
120	120
...	...	@@ -177,16 +177,16 @@ const PowerpcCpuinfoParser = CpuinfoParser(PowerpcCpuinfoImpl);
177	177
178	178	test "cpuinfo: PowerPC" {
179	179	try testParser(PowerpcCpuinfoParser, .powerpc, &Target.powerpc.cpu.@"970",
180		\\processor : 0
181		\\cpu : PPC970MP, altivec supported
182		\\clock : 1250.000000MHz
183		\\revision : 1.1 (pvr 0044 0101)
	180	\\processor : 0
	181	\\cpu : PPC970MP, altivec supported
	182	\\clock : 1250.000000MHz
	183	\\revision : 1.1 (pvr 0044 0101)
184	184	);
185	185	try testParser(PowerpcCpuinfoParser, .powerpc64le, &Target.powerpc.cpu.pwr8,
186		\\processor : 0
187		\\cpu : POWER8 (raw), altivec supported
188		\\clock : 2926.000000MHz
189		\\revision : 2.0 (pvr 004d 0200)
	186	\\processor : 0
	187	\\cpu : POWER8 (raw), altivec supported
	188	\\clock : 2926.000000MHz
	189	\\revision : 2.0 (pvr 004d 0200)
190	190	);
191	191	}
192	192
...	...	@@ -304,25 +304,25 @@ test "cpuinfo: ARM" {
304	304	\\CPU revision : 7
305	305	);
306	306	try testParser(ArmCpuinfoParser, .arm, &Target.arm.cpu.cortex_a7,
307		\\processor : 0
308		\\model name : ARMv7 Processor rev 3 (v7l)
309		\\BogoMIPS : 18.00
310		\\Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae
311		\\CPU implementer : 0x41
	307	\\processor : 0
	308	\\model name : ARMv7 Processor rev 3 (v7l)
	309	\\BogoMIPS : 18.00
	310	\\Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae
	311	\\CPU implementer : 0x41
312	312	\\CPU architecture: 7
313		\\CPU variant : 0x0
314		\\CPU part : 0xc07
315		\\CPU revision : 3
	313	\\CPU variant : 0x0
	314	\\CPU part : 0xc07
	315	\\CPU revision : 3
316	316	\\
317		\\processor : 4
318		\\model name : ARMv7 Processor rev 3 (v7l)
319		\\BogoMIPS : 90.00
320		\\Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae
321		\\CPU implementer : 0x41
	317	\\processor : 4
	318	\\model name : ARMv7 Processor rev 3 (v7l)
	319	\\BogoMIPS : 90.00
	320	\\Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae
	321	\\CPU implementer : 0x41
322	322	\\CPU architecture: 7
323		\\CPU variant : 0x2
324		\\CPU part : 0xc0f
325		\\CPU revision : 3
	323	\\CPU variant : 0x2
	324	\\CPU part : 0xc0f
	325	\\CPU revision : 3
326	326	);
327	327	try testParser(ArmCpuinfoParser, .aarch64, &Target.aarch64.cpu.cortex_a72,
328	328	\\processor : 0

lib/std/zig/tokenizer.zig+254-355

...	...	@@ -320,7 +320,7 @@ pub const Token = struct {
320	320
321	321	pub fn symbol(tag: Tag) []const u8 {
322	322	return tag.lexeme() orelse switch (tag) {
323		.invalid => "invalid bytes",
	323	.invalid => "invalid token",
324	324	.identifier => "an identifier",
325	325	.string_literal, .multiline_string_literal_line => "a string literal",
326	326	.char_literal => "a character literal",
...	...	@@ -338,22 +338,22 @@ pub const Tokenizer = struct {
338	338	buffer: [:0]const u8,
339	339	index: usize,
340	340
341		/// For debugging purposes
	341	/// For debugging purposes.
342	342	pub fn dump(self: Tokenizer, token: const Token) void {
343	343	std.debug.print("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] });
344	344	}
345	345
346	346	pub fn init(buffer: [:0]const u8) Tokenizer {
347		// Skip the UTF-8 BOM if present
348		const src_start: usize = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0;
349		return Tokenizer{
	347	// Skip the UTF-8 BOM if present.
	348	return .{
350	349	.buffer = buffer,
351		.index = src_start,
	350	.index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0,
352	351	};
353	352	}
354	353
355	354	const State = enum {
356	355	start,
	356	expect_newline,
357	357	identifier,
358	358	builtin,
359	359	string_literal,
...	...	@@ -361,10 +361,6 @@ pub const Tokenizer = struct {
361	361	multiline_string_literal_line,
362	362	char_literal,
363	363	char_literal_backslash,
364		char_literal_hex_escape,
365		char_literal_unicode_escape_saw_u,
366		char_literal_unicode_escape,
367		char_literal_end,
368	364	backslash,
369	365	equal,
370	366	bang,
...	...	@@ -400,30 +396,33 @@ pub const Tokenizer = struct {
400	396	period_2,
401	397	period_asterisk,
402	398	saw_at_sign,
	399	invalid,
403	400	};
404	401
	402	/// After this returns invalid, it will reset on the next newline, returning tokens starting from there.
	403	/// An eof token will always be returned at the end.
405	404	pub fn next(self: *Tokenizer) Token {
406	405	var state: State = .start;
407		var result = Token{
408		.tag = .eof,
	406	var result: Token = .{
	407	.tag = undefined,
409	408	.loc = .{
410	409	.start = self.index,
411	410	.end = undefined,
412	411	},
413	412	};
414		var seen_escape_digits: usize = undefined;
415	413	while (true) : (self.index += 1) {
416	414	const c = self.buffer[self.index];
417	415	switch (state) {
418	416	.start => switch (c) {
419	417	0 => {
420		if (self.index != self.buffer.len) {
421		result.tag = .invalid;
422		result.loc.end = self.index;
423		self.index += 1;
424		return result;
425		}
426		break;
	418	if (self.index == self.buffer.len) return .{
	419	.tag = .eof,
	420	.loc = .{
	421	.start = self.index,
	422	.end = self.index,
	423	},
	424	};
	425	state = .invalid;
427	426	},
428	427	' ', '\n', '\t', '\r' => {
429	428	result.loc.start = self.index + 1;
...	...	@@ -434,6 +433,7 @@ pub const Tokenizer = struct {
434	433	},
435	434	'\'' => {
436	435	state = .char_literal;
	436	result.tag = .char_literal;
437	437	},
438	438	'a'...'z', 'A'...'Z', '_' => {
439	439	state = .identifier;
...	...	@@ -545,14 +545,44 @@ pub const Tokenizer = struct {
545	545	result.tag = .number_literal;
546	546	},
547	547	else => {
	548	state = .invalid;
	549	},
	550	},
	551
	552	.expect_newline => switch (c) {
	553	0 => {
	554	if (self.index == self.buffer.len) {
	555	result.tag = .invalid;
	556	break;
	557	}
	558	state = .invalid;
	559	},
	560	'\n' => {
	561	result.loc.start = self.index + 1;
	562	state = .start;
	563	},
	564	else => {
	565	state = .invalid;
	566	},
	567	},
	568
	569	.invalid => switch (c) {
	570	0 => if (self.index == self.buffer.len) {
548	571	result.tag = .invalid;
549		result.loc.end = self.index;
550		self.index += std.unicode.utf8ByteSequenceLength(c) catch 1;
551		return result;
	572	break;
552	573	},
	574	'\n' => {
	575	result.tag = .invalid;
	576	break;
	577	},
	578	else => continue,
553	579	},
554	580
555	581	.saw_at_sign => switch (c) {
	582	0, '\n' => {
	583	result.tag = .invalid;
	584	break;
	585	},
556	586	'"' => {
557	587	result.tag = .identifier;
558	588	state = .string_literal;
...	...	@@ -562,8 +592,7 @@ pub const Tokenizer = struct {
562	592	result.tag = .builtin;
563	593	},
564	594	else => {
565		result.tag = .invalid;
566		break;
	595	state = .invalid;
567	596	},
568	597	},
569	598
...	...	@@ -698,7 +727,7 @@ pub const Tokenizer = struct {
698	727	},
699	728
700	729	.identifier => switch (c) {
701		'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	730	'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
702	731	else => {
703	732	if (Token.getKeyword(self.buffer[result.loc.start..self.index])) \|tag\| {
704	733	result.tag = tag;
...	...	@@ -707,26 +736,37 @@ pub const Tokenizer = struct {
707	736	},
708	737	},
709	738	.builtin => switch (c) {
710		'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	739	'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
711	740	else => break,
712	741	},
713	742	.backslash => switch (c) {
	743	0 => {
	744	result.tag = .invalid;
	745	break;
	746	},
714	747	'\\' => {
715	748	state = .multiline_string_literal_line;
716	749	},
717		else => {
	750	'\n' => {
718	751	result.tag = .invalid;
719	752	break;
720	753	},
	754	else => {
	755	state = .invalid;
	756	},
721	757	},
722	758	.string_literal => switch (c) {
723		0, '\n' => {
724		result.tag = .invalid;
725		result.loc.end = self.index;
	759	0 => {
726	760	if (self.index != self.buffer.len) {
727		self.index += 1;
	761	state = .invalid;
	762	continue;
728	763	}
729		return result;
	764	result.tag = .invalid;
	765	break;
	766	},
	767	'\n' => {
	768	result.tag = .invalid;
	769	break;
730	770	},
731	771	'\\' => {
732	772	state = .string_literal_backslash;
...	...	@@ -735,150 +775,74 @@ pub const Tokenizer = struct {
735	775	self.index += 1;
736	776	break;
737	777	},
738		else => {
739		if (self.invalidCharacterLength()) \|len\| {
740		result.tag = .invalid;
741		result.loc.end = self.index;
742		self.index += len;
743		return result;
744		}
745
746		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
	778	0x01...0x09, 0x0b...0x1f, 0x7f => {
	779	state = .invalid;
747	780	},
	781	else => continue,
748	782	},
749	783
750	784	.string_literal_backslash => switch (c) {
751	785	0, '\n' => {
752	786	result.tag = .invalid;
753		result.loc.end = self.index;
754		if (self.index != self.buffer.len) {
755		self.index += 1;
756		}
757		return result;
	787	break;
758	788	},
759	789	else => {
760	790	state = .string_literal;
761
762		if (self.invalidCharacterLength()) \|len\| {
763		result.tag = .invalid;
764		result.loc.end = self.index;
765		self.index += len;
766		return result;
767		}
768
769		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
770	791	},
771	792	},
772	793
773	794	.char_literal => switch (c) {
774		0, '\n', '\'' => {
775		result.tag = .invalid;
776		result.loc.end = self.index;
	795	0 => {
777	796	if (self.index != self.buffer.len) {
778		self.index += 1;
	797	state = .invalid;
	798	continue;
779	799	}
780		return result;
	800	result.tag = .invalid;
	801	break;
	802	},
	803	'\n' => {
	804	result.tag = .invalid;
	805	break;
781	806	},
782	807	'\\' => {
783	808	state = .char_literal_backslash;
784	809	},
785		else => {
786		state = .char_literal_end;
787
788		if (self.invalidCharacterLength()) \|len\| {
789		result.tag = .invalid;
790		result.loc.end = self.index;
791		self.index += len;
792		return result;
793		}
794
795		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
	810	'\'' => {
	811	self.index += 1;
	812	break;
796	813	},
	814	0x01...0x09, 0x0b...0x1f, 0x7f => {
	815	state = .invalid;
	816	},
	817	else => continue,
797	818	},
798	819
799	820	.char_literal_backslash => switch (c) {
800		0, '\n' => {
801		result.tag = .invalid;
802		result.loc.end = self.index;
	821	0 => {
803	822	if (self.index != self.buffer.len) {
804		self.index += 1;
805		}
806		return result;
807		},
808		'x' => {
809		state = .char_literal_hex_escape;
810		seen_escape_digits = 0;
811		},
812		'u' => {
813		state = .char_literal_unicode_escape_saw_u;
814		},
815		else => {
816		state = .char_literal_end;
817
818		if (self.invalidCharacterLength()) \|len\| {
819		result.tag = .invalid;
820		result.loc.end = self.index;
821		self.index += len;
822		return result;
	823	state = .invalid;
	824	continue;
823	825	}
824
825		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
826		},
827		},
828
829		.char_literal_hex_escape => switch (c) {
830		'0'...'9', 'a'...'f', 'A'...'F' => {
831		seen_escape_digits += 1;
832		if (seen_escape_digits == 2) {
833		state = .char_literal_end;
834		}
835		},
836		else => {
837	826	result.tag = .invalid;
838	827	break;
839	828	},
840		},
841
842		.char_literal_unicode_escape_saw_u => switch (c) {
843		'{' => {
844		state = .char_literal_unicode_escape;
845		},
846		else => {
847		result.tag = .invalid;
848		break;
849		},
850		},
851
852		.char_literal_unicode_escape => switch (c) {
853		'0'...'9', 'a'...'f', 'A'...'F' => {},
854		'}' => {
855		state = .char_literal_end; // too many/few digits handled later
856		},
857		else => {
	829	'\n' => {
858	830	result.tag = .invalid;
859	831	break;
860	832	},
861		},
862
863		.char_literal_end => switch (c) {
864		'\'' => {
865		result.tag = .char_literal;
866		self.index += 1;
867		break;
	833	0x01...0x09, 0x0b...0x1f, 0x7f => {
	834	state = .invalid;
868	835	},
869	836	else => {
870		result.tag = .invalid;
871		break;
	837	state = .char_literal;
872	838	},
873	839	},
874	840
875	841	.multiline_string_literal_line => switch (c) {
876	842	0 => {
877	843	if (self.index != self.buffer.len) {
878		result.tag = .invalid;
879		result.loc.end = self.index;
880		self.index += 1;
881		return result;
	844	state = .invalid;
	845	continue;
882	846	}
883	847	break;
884	848	},
...	...	@@ -886,17 +850,18 @@ pub const Tokenizer = struct {
886	850	self.index += 1;
887	851	break;
888	852	},
889		'\t' => {},
890		else => {
891		if (self.invalidCharacterLength()) \|len\| {
892		result.tag = .invalid;
893		result.loc.end = self.index;
894		self.index += len;
895		return result;
	853	'\r' => {
	854	if (self.buffer[self.index + 1] == '\n') {
	855	self.index += 2;
	856	break;
	857	} else {
	858	state = .invalid;
896	859	}
897
898		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
899	860	},
	861	0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	862	state = .invalid;
	863	},
	864	else => continue,
900	865	},
901	866
902	867	.bang => switch (c) {
...	...	@@ -1113,12 +1078,16 @@ pub const Tokenizer = struct {
1113	1078	.line_comment_start => switch (c) {
1114	1079	0 => {
1115	1080	if (self.index != self.buffer.len) {
1116		result.tag = .invalid;
1117		result.loc.end = self.index;
1118		self.index += 1;
1119		return result;
	1081	state = .invalid;
	1082	continue;
1120	1083	}
1121		break;
	1084	return .{
	1085	.tag = .eof,
	1086	.loc = .{
	1087	.start = self.index,
	1088	.end = self.index,
	1089	},
	1090	};
1122	1091	},
1123	1092	'/' => {
1124	1093	state = .doc_comment_start;
...	...	@@ -1127,105 +1096,91 @@ pub const Tokenizer = struct {
1127	1096	result.tag = .container_doc_comment;
1128	1097	state = .doc_comment;
1129	1098	},
	1099	'\r' => {
	1100	state = .expect_newline;
	1101	},
1130	1102	'\n' => {
1131	1103	state = .start;
1132	1104	result.loc.start = self.index + 1;
1133	1105	},
1134		'\t' => {
1135		state = .line_comment;
	1106	0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	1107	state = .invalid;
1136	1108	},
1137	1109	else => {
1138	1110	state = .line_comment;
1139
1140		if (self.invalidCharacterLength()) \|len\| {
1141		result.tag = .invalid;
1142		result.loc.end = self.index;
1143		self.index += len;
1144		return result;
1145		}
1146
1147		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1148	1111	},
1149	1112	},
1150	1113	.doc_comment_start => switch (c) {
1151		'/' => {
1152		state = .line_comment;
	1114	0, '\n' => {
	1115	result.tag = .doc_comment;
	1116	break;
1153	1117	},
1154		0 => {
1155		if (self.index != self.buffer.len) {
1156		result.tag = .invalid;
1157		result.loc.end = self.index;
	1118	'\r' => {
	1119	if (self.buffer[self.index + 1] == '\n') {
1158	1120	self.index += 1;
1159		return result;
	1121	result.tag = .doc_comment;
	1122	break;
	1123	} else {
	1124	state = .invalid;
1160	1125	}
1161		result.tag = .doc_comment;
1162		break;
1163	1126	},
1164		'\n' => {
1165		result.tag = .doc_comment;
1166		break;
	1127	'/' => {
	1128	state = .line_comment;
1167	1129	},
1168		'\t' => {
1169		state = .doc_comment;
1170		result.tag = .doc_comment;
	1130	0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	1131	state = .invalid;
1171	1132	},
1172	1133	else => {
1173	1134	state = .doc_comment;
1174	1135	result.tag = .doc_comment;
1175
1176		if (self.invalidCharacterLength()) \|len\| {
1177		result.tag = .invalid;
1178		result.loc.end = self.index;
1179		self.index += len;
1180		return result;
1181		}
1182
1183		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1184	1136	},
1185	1137	},
1186	1138	.line_comment => switch (c) {
1187	1139	0 => {
1188	1140	if (self.index != self.buffer.len) {
1189		result.tag = .invalid;
1190		result.loc.end = self.index;
1191		self.index += 1;
1192		return result;
	1141	state = .invalid;
	1142	continue;
1193	1143	}
1194		break;
	1144	return .{
	1145	.tag = .eof,
	1146	.loc = .{
	1147	.start = self.index,
	1148	.end = self.index,
	1149	},
	1150	};
	1151	},
	1152	'\r' => {
	1153	state = .expect_newline;
1195	1154	},
1196	1155	'\n' => {
1197	1156	state = .start;
1198	1157	result.loc.start = self.index + 1;
1199	1158	},
1200		'\t' => {},
1201		else => {
1202		if (self.invalidCharacterLength()) \|len\| {
1203		result.tag = .invalid;
1204		result.loc.end = self.index;
1205		self.index += len;
1206		return result;
1207		}
1208
1209		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
	1159	0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	1160	state = .invalid;
1210	1161	},
	1162	else => continue,
1211	1163	},
1212	1164	.doc_comment => switch (c) {
1213		0, '\n' => break,
1214		'\t' => {},
1215		else => {
1216		if (self.invalidCharacterLength()) \|len\| {
1217		result.tag = .invalid;
1218		result.loc.end = self.index;
1219		self.index += len;
1220		return result;
	1165	0, '\n' => {
	1166	break;
	1167	},
	1168	'\r' => {
	1169	if (self.buffer[self.index + 1] == '\n') {
	1170	self.index += 1;
	1171	break;
	1172	} else {
	1173	state = .invalid;
1221	1174	}
1222
1223		self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1224	1175	},
	1176	0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
	1177	state = .invalid;
	1178	},
	1179	else => continue,
1225	1180	},
1226	1181	.int => switch (c) {
1227	1182	'.' => state = .int_period,
1228		'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},
	1183	'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1229	1184	'e', 'E', 'p', 'P' => state = .int_exponent,
1230	1185	else => break,
1231	1186	},
...	...	@@ -1249,7 +1204,7 @@ pub const Tokenizer = struct {
1249	1204	},
1250	1205	},
1251	1206	.float => switch (c) {
1252		'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},
	1207	'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1253	1208	'e', 'E', 'p', 'P' => state = .float_exponent,
1254	1209	else => break,
1255	1210	},
...	...	@@ -1263,57 +1218,9 @@ pub const Tokenizer = struct {
1263	1218	}
1264	1219	}
1265	1220
1266		if (result.tag == .eof) {
1267		result.loc.start = self.index;
1268		}
1269
1270	1221	result.loc.end = self.index;
1271	1222	return result;
1272	1223	}
1273
1274		fn invalidCharacterLength(self: *Tokenizer) ?u3 {
1275		const c0 = self.buffer[self.index];
1276		if (std.ascii.isAscii(c0)) {
1277		if (c0 == '\r') {
1278		if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {
1279		// Carriage returns are only allowed just before a linefeed as part of a CRLF pair, otherwise
1280		// they constitute an illegal byte!
1281		return null;
1282		} else {
1283		return 1;
1284		}
1285		} else if (std.ascii.isControl(c0)) {
1286		// ascii control codes are never allowed
1287		// (note that \n was checked before we got here)
1288		return 1;
1289		}
1290		// looks fine to me.
1291		return null;
1292		} else {
1293		// check utf8-encoded character.
1294		const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
1295		if (self.index + length > self.buffer.len) {
1296		return @as(u3, @intCast(self.buffer.len - self.index));
1297		}
1298		const bytes = self.buffer[self.index .. self.index + length];
1299		switch (length) {
1300		2 => {
1301		const value = std.unicode.utf8Decode2(bytes) catch return length;
1302		if (value == 0x85) return length; // U+0085 (NEL)
1303		},
1304		3 => {
1305		const value = std.unicode.utf8Decode3(bytes) catch return length;
1306		if (value == 0x2028) return length; // U+2028 (LS)
1307		if (value == 0x2029) return length; // U+2029 (PS)
1308		},
1309		4 => {
1310		_ = std.unicode.utf8Decode4(bytes) catch return length;
1311		},
1312		else => unreachable,
1313		}
1314		return null;
1315		}
1316		}
1317	1224	};
1318	1225
1319	1226	test "keywords" {
...	...	@@ -1355,7 +1262,7 @@ test "code point literal with hex escape" {
1355	1262	, &.{.char_literal});
1356	1263	try testTokenize(
1357	1264	\\'\x1'
1358		, &.{ .invalid, .invalid });
	1265	, &.{.char_literal});
1359	1266	}
1360	1267
1361	1268	test "newline in char literal" {
...	...	@@ -1396,40 +1303,30 @@ test "code point literal with unicode escapes" {
1396	1303	// Invalid unicode escapes
1397	1304	try testTokenize(
1398	1305	\\'\u'
1399		, &.{ .invalid, .invalid });
	1306	, &.{.char_literal});
1400	1307	try testTokenize(
1401	1308	\\'\u{{'
1402		, &.{ .invalid, .l_brace, .invalid });
	1309	, &.{.char_literal});
1403	1310	try testTokenize(
1404	1311	\\'\u{}'
1405	1312	, &.{.char_literal});
1406	1313	try testTokenize(
1407	1314	\\'\u{s}'
1408		, &.{
1409		.invalid,
1410		.identifier,
1411		.r_brace,
1412		.invalid,
1413		});
	1315	, &.{.char_literal});
1414	1316	try testTokenize(
1415	1317	\\'\u{2z}'
1416		, &.{
1417		.invalid,
1418		.identifier,
1419		.r_brace,
1420		.invalid,
1421		});
	1318	, &.{.char_literal});
1422	1319	try testTokenize(
1423	1320	\\'\u{4a'
1424		, &.{ .invalid, .invalid }); // 4a is valid
	1321	, &.{.char_literal});
1425	1322
1426	1323	// Test old-style unicode literals
1427	1324	try testTokenize(
1428	1325	\\'\u0333'
1429		, &.{ .invalid, .number_literal, .invalid });
	1326	, &.{.char_literal});
1430	1327	try testTokenize(
1431	1328	\\'\U0333'
1432		, &.{ .invalid, .number_literal, .invalid });
	1329	, &.{.char_literal});
1433	1330	}
1434	1331
1435	1332	test "code point literal with unicode code point" {
...	...	@@ -1465,24 +1362,15 @@ test "invalid token characters" {
1465	1362	try testTokenize("`", &.{.invalid});
1466	1363	try testTokenize("'c", &.{.invalid});
1467	1364	try testTokenize("'", &.{.invalid});
1468		try testTokenize("''", &.{.invalid});
	1365	try testTokenize("''", &.{.char_literal});
1469	1366	try testTokenize("'\n'", &.{ .invalid, .invalid });
1470	1367	}
1471	1368
1472	1369	test "invalid literal/comment characters" {
1473		try testTokenize("\"\x00\"", &.{
1474		.invalid,
1475		.invalid, // Incomplete string literal starting after invalid
1476		});
1477		try testTokenize("//\x00", &.{
1478		.invalid,
1479		});
1480		try testTokenize("//\x1f", &.{
1481		.invalid,
1482		});
1483		try testTokenize("//\x7f", &.{
1484		.invalid,
1485		});
	1370	try testTokenize("\"\x00\"", &.{.invalid});
	1371	try testTokenize("//\x00", &.{.invalid});
	1372	try testTokenize("//\x1f", &.{.invalid});
	1373	try testTokenize("//\x7f", &.{.invalid});
1486	1374	}
1487	1375
1488	1376	test "utf8" {
...	...	@@ -1491,46 +1379,24 @@ test "utf8" {
1491	1379	}
1492	1380
1493	1381	test "invalid utf8" {
1494		try testTokenize("//\x80", &.{
1495		.invalid,
1496		});
1497		try testTokenize("//\xbf", &.{
1498		.invalid,
1499		});
1500		try testTokenize("//\xf8", &.{
1501		.invalid,
1502		});
1503		try testTokenize("//\xff", &.{
1504		.invalid,
1505		});
1506		try testTokenize("//\xc2\xc0", &.{
1507		.invalid,
1508		});
1509		try testTokenize("//\xe0", &.{
1510		.invalid,
1511		});
1512		try testTokenize("//\xf0", &.{
1513		.invalid,
1514		});
1515		try testTokenize("//\xf0\x90\x80\xc0", &.{
1516		.invalid,
1517		});
	1382	try testTokenize("//\x80", &.{});
	1383	try testTokenize("//\xbf", &.{});
	1384	try testTokenize("//\xf8", &.{});
	1385	try testTokenize("//\xff", &.{});
	1386	try testTokenize("//\xc2\xc0", &.{});
	1387	try testTokenize("//\xe0", &.{});
	1388	try testTokenize("//\xf0", &.{});
	1389	try testTokenize("//\xf0\x90\x80\xc0", &.{});
1518	1390	}
1519	1391
1520	1392	test "illegal unicode codepoints" {
1521	1393	// unicode newline characters.U+0085, U+2028, U+2029
1522	1394	try testTokenize("//\xc2\x84", &.{});
1523		try testTokenize("//\xc2\x85", &.{
1524		.invalid,
1525		});
	1395	try testTokenize("//\xc2\x85", &.{});
1526	1396	try testTokenize("//\xc2\x86", &.{});
1527	1397	try testTokenize("//\xe2\x80\xa7", &.{});
1528		try testTokenize("//\xe2\x80\xa8", &.{
1529		.invalid,
1530		});
1531		try testTokenize("//\xe2\x80\xa9", &.{
1532		.invalid,
1533		});
	1398	try testTokenize("//\xe2\x80\xa8", &.{});
	1399	try testTokenize("//\xe2\x80\xa9", &.{});
1534	1400	try testTokenize("//\xe2\x80\xaa", &.{});
1535	1401	}
1536	1402
...	...	@@ -1549,30 +1415,6 @@ test "string identifier and builtin fns" {
1549	1415	});
1550	1416	}
1551	1417
1552		test "multiline string literal with literal tab" {
1553		try testTokenize(
1554		\\\\foo bar
1555		, &.{
1556		.multiline_string_literal_line,
1557		});
1558		}
1559
1560		test "comments with literal tab" {
1561		try testTokenize(
1562		\\//foo bar
1563		\\//!foo bar
1564		\\///foo bar
1565		\\// foo
1566		\\/// foo
1567		\\/// /foo
1568		, &.{
1569		.container_doc_comment,
1570		.doc_comment,
1571		.doc_comment,
1572		.doc_comment,
1573		});
1574		}
1575
1576	1418	test "pipe and then invalid" {
1577	1419	try testTokenize("\|\|=", &.{
1578	1420	.pipe_pipe,
...	...	@@ -1892,8 +1734,8 @@ test "multi line string literal with only 1 backslash" {
1892	1734	}
1893	1735
1894	1736	test "invalid builtin identifiers" {
1895		try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });
1896		try testTokenize("@0()", &.{ .invalid, .number_literal, .l_paren, .r_paren });
	1737	try testTokenize("@()", &.{.invalid});
	1738	try testTokenize("@0()", &.{.invalid});
1897	1739	}
1898	1740
1899	1741	test "invalid token with unfinished escape right before eof" {
...	...	@@ -1921,21 +1763,78 @@ test "saturating operators" {
1921	1763	}
1922	1764
1923	1765	test "null byte before eof" {
1924		try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });
	1766	try testTokenize("123 \x00 456", &.{ .number_literal, .invalid });
1925	1767	try testTokenize("//\x00", &.{.invalid});
1926	1768	try testTokenize("\\\\\x00", &.{.invalid});
1927	1769	try testTokenize("\x00", &.{.invalid});
1928	1770	try testTokenize("// NUL\x00\n", &.{.invalid});
1929		try testTokenize("///\x00\n", &.{.invalid});
	1771	try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
1930	1772	try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
1931	1773	}
1932	1774
	1775	test "invalid tabs and carriage returns" {
	1776	// "Inside Line Comments and Documentation Comments, Any TAB is rejected by
	1777	// the grammar since it is ambiguous how it should be rendered."
	1778	// https://github.com/ziglang/zig-spec/issues/38
	1779	try testTokenize("//\t", &.{.invalid});
	1780	try testTokenize("// \t", &.{.invalid});
	1781	try testTokenize("///\t", &.{.invalid});
	1782	try testTokenize("/// \t", &.{.invalid});
	1783	try testTokenize("//!\t", &.{.invalid});
	1784	try testTokenize("//! \t", &.{.invalid});
	1785
	1786	// "Inside Line Comments and Documentation Comments, CR directly preceding
	1787	// NL is unambiguously part of the newline sequence. It is accepted by the
	1788	// grammar and removed by zig fmt, leaving only NL. CR anywhere else is
	1789	// rejected by the grammar."
	1790	// https://github.com/ziglang/zig-spec/issues/38
	1791	try testTokenize("//\r", &.{.invalid});
	1792	try testTokenize("// \r", &.{.invalid});
	1793	try testTokenize("///\r", &.{.invalid});
	1794	try testTokenize("/// \r", &.{.invalid});
	1795	try testTokenize("//\r ", &.{.invalid});
	1796	try testTokenize("// \r ", &.{.invalid});
	1797	try testTokenize("///\r ", &.{.invalid});
	1798	try testTokenize("/// \r ", &.{.invalid});
	1799	try testTokenize("//\r\n", &.{});
	1800	try testTokenize("// \r\n", &.{});
	1801	try testTokenize("///\r\n", &.{.doc_comment});
	1802	try testTokenize("/// \r\n", &.{.doc_comment});
	1803	try testTokenize("//!\r", &.{.invalid});
	1804	try testTokenize("//! \r", &.{.invalid});
	1805	try testTokenize("//!\r ", &.{.invalid});
	1806	try testTokenize("//! \r ", &.{.invalid});
	1807	try testTokenize("//!\r\n", &.{.container_doc_comment});
	1808	try testTokenize("//! \r\n", &.{.container_doc_comment});
	1809
	1810	// The control characters TAB and CR are rejected by the grammar inside multi-line string literals,
	1811	// except if CR is directly before NL.
	1812	// https://github.com/ziglang/zig-spec/issues/38
	1813	try testTokenize("\\\\\r", &.{.invalid});
	1814	try testTokenize("\\\\\r ", &.{.invalid});
	1815	try testTokenize("\\\\ \r", &.{.invalid});
	1816	try testTokenize("\\\\\t", &.{.invalid});
	1817	try testTokenize("\\\\\t ", &.{.invalid});
	1818	try testTokenize("\\\\ \t", &.{.invalid});
	1819	try testTokenize("\\\\\r\n", &.{.multiline_string_literal_line});
	1820
	1821	// "TAB used as whitespace is...accepted by the grammar. CR used as
	1822	// whitespace, whether directly preceding NL or stray, is...accepted by the
	1823	// grammar."
	1824	// https://github.com/ziglang/zig-spec/issues/38
	1825	try testTokenize("\tpub\tswitch\t", &.{ .keyword_pub, .keyword_switch });
	1826	try testTokenize("\rpub\rswitch\r", &.{ .keyword_pub, .keyword_switch });
	1827	}
	1828
1933	1829	fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
1934	1830	var tokenizer = Tokenizer.init(source);
1935	1831	for (expected_token_tags) \|expected_token_tag\| {
1936	1832	const token = tokenizer.next();
1937	1833	try std.testing.expectEqual(expected_token_tag, token.tag);
1938	1834	}
	1835	// Last token should always be eof, even when the last token was invalid,
	1836	// in which case the tokenizer is in an invalid state, which can only be
	1837	// recovered by opinionated means outside the scope of this implementation.
1939	1838	const last_token = tokenizer.next();
1940	1839	try std.testing.expectEqual(Token.Tag.eof, last_token.tag);
1941	1840	try std.testing.expectEqual(source.len, last_token.loc.start);

src/Package/Manifest.zig+3

...	...	@@ -549,6 +549,9 @@ const Parse = struct {
549	549	.{raw_string[bad_index]},
550	550	);
551	551	},
	552	.empty_char_literal => {
	553	try p.appendErrorOff(token, offset, "empty character literal", .{});
	554	},
552	555	}
553	556	}
554	557

src/link/tapi/yaml/test.zig+1-4

...	...	@@ -237,10 +237,7 @@ test "double quoted string" {
237	237	try testing.expectEqualStrings(
238	238	\\"here" are some escaped quotes
239	239	, arr[1]);
240		try testing.expectEqualStrings(
241		\\newlines and tabs
242		\\are supported
243		, arr[2]);
	240	try testing.expectEqualStrings("newlines and tabs\nare\tsupported", arr[2]);
244	241	try testing.expectEqualStrings(
245	242	\\let's have
246	243	\\some fun!

test/cases/compile_errors/empty_char_lit.zig created+9

...	...	@@ -0,0 +1,9 @@
	1	export fn entry() u8 {
	2	return '';
	3	}
	4
	5	// error
	6	// backend=stage2
	7	// target=native
	8	//
	9	// :2:12: error: empty character literal

test/cases/compile_errors/invalid_legacy_unicode_escape.zig+1-2

...	...	@@ -6,5 +6,4 @@ export fn entry() void {
6	6	// backend=stage2
7	7	// target=native
8	8	//
9		// :2:15: error: expected expression, found 'invalid bytes'
10		// :2:18: note: invalid byte: '1'
	9	// :2:17: error: invalid escape character: 'U'

test/cases/compile_errors/invalid_unicode_escape.zig+1-2

...	...	@@ -6,6 +6,5 @@ export fn entry() void {
6	6	// backend=stage2
7	7	// target=native
8	8	//
9		// :2:15: error: expected expression, found 'invalid bytes'
10		// :2:21: note: invalid byte: 'z'
	9	// :2:21: error: expected hex digit or '}', found 'z'
11	10

test/cases/compile_errors/normal_string_with_newline.zig+1-2

...	...	@@ -5,5 +5,4 @@ b";
5	5	// backend=stage2
6	6	// target=native
7	7	//
8		// :1:13: error: expected expression, found 'invalid bytes'
9		// :1:15: note: invalid byte: '\n'
	8	// :1:13: error: expected expression, found 'invalid token'

test/compile_errors.zig+5-19

...	...	@@ -38,15 +38,6 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
38	38	});
39	39	}
40	40
41		{
42		const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);
43
44		case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{
45		":1:13: error: expected expression, found 'invalid bytes'",
46		":1:19: note: invalid byte: '\\r'",
47		});
48		}
49
50	41	{
51	42	const case = ctx.obj("missing semicolon at EOF", b.graph.host);
52	43	case.addError(
...	...	@@ -179,8 +170,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
179	170	\\ return true;
180	171	\\}
181	172	, &[_][]const u8{
182		":1:1: error: expected type expression, found 'invalid bytes'",
183		":1:1: note: invalid byte: '\\xff'",
	173	":1:1: error: expected type expression, found 'invalid token'",
184	174	});
185	175	}
186	176
...	...	@@ -222,8 +212,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
222	212	const case = ctx.obj("invalid byte in string", b.graph.host);
223	213
224	214	case.addError("_ = \"\x01Q\";", &[_][]const u8{
225		":1:5: error: expected expression, found 'invalid bytes'",
226		":1:6: note: invalid byte: '\\x01'",
	215	":1:5: error: expected expression, found 'invalid token'",
227	216	});
228	217	}
229	218
...	...	@@ -231,8 +220,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
231	220	const case = ctx.obj("invalid byte in comment", b.graph.host);
232	221
233	222	case.addError("//\x01Q", &[_][]const u8{
234		":1:1: error: expected type expression, found 'invalid bytes'",
235		":1:3: note: invalid byte: '\\x01'",
	223	":1:1: error: expected type expression, found 'invalid token'",
236	224	});
237	225	}
238	226
...	...	@@ -240,8 +228,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
240	228	const case = ctx.obj("control character in character literal", b.graph.host);
241	229
242	230	case.addError("const c = '\x01';", &[_][]const u8{
243		":1:11: error: expected expression, found 'invalid bytes'",
244		":1:12: note: invalid byte: '\\x01'",
	231	":1:11: error: expected expression, found 'invalid token'",
245	232	});
246	233	}
247	234
...	...	@@ -249,8 +236,7 @@ pub fn addCases(ctx: Cases, b: std.Build) !void {
249	236	const case = ctx.obj("invalid byte at start of token", b.graph.host);
250	237
251	238	case.addError("x = \x00Q", &[_][]const u8{
252		":1:5: error: expected expression, found 'invalid bytes'",
253		":1:5: note: invalid byte: '\\x00'",
	239	":1:5: error: expected expression, found 'invalid token'",
254	240	});
255	241	}
256	242	}

test/run_translated_c.zig+6-6

...	...	@@ -26,17 +26,17 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {
26	26	\\void baz(void);
27	27	\\struct foo { int x; };
28	28	\\void bar() {
29		\\ struct foo tmp;
	29	\\ struct foo tmp;
30	30	\\}
31	31	\\
32	32	\\void baz() {
33		\\ struct foo tmp;
	33	\\ struct foo tmp;
34	34	\\}
35	35	\\
36	36	\\int main(void) {
37		\\ bar();
38		\\ baz();
39		\\ return 0;
	37	\\ bar();
	38	\\ baz();
	39	\\ return 0;
40	40	\\}
41	41	, "");
42	42
...	...	@@ -53,7 +53,7 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {
53	53	cases.add("parenthesized string literal",
54	54	\\void foo(const char *s) {}
55	55	\\int main(void) {
56		\\ foo(("bar"));
	56	\\ foo(("bar"));
57	57	\\}
58	58	, "");
59	59

test/translate_c.zig+17-17

...	...	@@ -133,20 +133,20 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
133	133
134	134	cases.add("scoped typedef",
135	135	\\void foo() {
136		\\ typedef union {
137		\\ int A;
138		\\ int B;
139		\\ int C;
140		\\ } Foo;
141		\\ Foo a = {0};
142		\\ {
143		\\ typedef union {
144		\\ int A;
145		\\ int B;
146		\\ int C;
147		\\ } Foo;
148		\\ Foo a = {0};
149		\\ }
	136	\\ typedef union {
	137	\\ int A;
	138	\\ int B;
	139	\\ int C;
	140	\\ } Foo;
	141	\\ Foo a = {0};
	142	\\ {
	143	\\ typedef union {
	144	\\ int A;
	145	\\ int B;
	146	\\ int C;
	147	\\ } Foo;
	148	\\ Foo a = {0};
	149	\\ }
150	150	\\}
151	151	, &[_][]const u8{
152	152	\\pub export fn foo() void {
...	...	@@ -2004,18 +2004,18 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
2004	2004	\\ break;
2005	2005	\\ }
2006	2006	\\ case 4:
2007		\\ case 5:
	2007	\\ case 5:
2008	2008	\\ res = 69;
2009	2009	\\ {
2010	2010	\\ res = 5;
2011		\\ return;
	2011	\\ return;
2012	2012	\\ }
2013	2013	\\ case 6:
2014	2014	\\ switch (res) {
2015	2015	\\ case 9: break;
2016	2016	\\ }
2017	2017	\\ res = 1;
2018		\\ return;
	2018	\\ return;
2019	2019	\\ }
2020	2020	\\}
2021	2021	, &[_][]const u8{