Commit 99a2fc2cde4c193d11322b2b22086fb4bc99f9fc | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-09-15 18:02:42-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-09-15 18:02:42-07:00
log	99a2fc2cde4c193d11322b2b22086fb4bc99f9fc
tree	0a23520aa45f2e15c8130e9b59caa3bb59255bd0
parent	a0b43ff3b3c22cb5c57e6728d6cb35d722f22a3b

stage2: implement .d file parsing for C objects

5 files changed, 1088 insertions(+), 1054 deletions(-)

BRANCH_TODO
src-self-hosted

BRANCH_TODO-1

...	...	@@ -1,4 +1,3 @@
1		* handle .d files from c objects
2	1	* glibc .so files
3	2	* support rpaths in ELF linker code
4	3	* build & link against compiler-rt

src-self-hosted/Cache.zig+39

...	...	@@ -444,6 +444,45 @@ pub const CacheHash = struct {
444	444	try self.populateFileHash(new_ch_file);
445	445	}
446	446
	447	pub fn addDepFilePost(self: *CacheHash, dir: fs.Dir, dep_file_basename: []const u8) !void {
	448	assert(self.manifest_file != null);
	449
	450	const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, MANIFEST_FILE_SIZE_MAX);
	451	defer self.cache.gpa.free(dep_file_contents);
	452
	453	const DepTokenizer = @import("DepTokenizer.zig");
	454	var it = DepTokenizer.init(self.cache.gpa, dep_file_contents);
	455	defer it.deinit();
	456
	457	// Skip first token: target.
	458	{
	459	const opt_result = it.next() catch \|err\| switch (err) {
	460	error.OutOfMemory => return error.OutOfMemory,
	461	error.InvalidInput => {
	462	std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text });
	463	return error.InvalidDepFile;
	464	},
	465	};
	466	_ = opt_result orelse return; // Empty dep file OK.
	467	}
	468	// Process 0+ preqreqs.
	469	// Clang is invoked in single-source mode so we never get more targets.
	470	while (true) {
	471	const opt_result = it.next() catch \|err\| switch (err) {
	472	error.OutOfMemory => return error.OutOfMemory,
	473	error.InvalidInput => {
	474	std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text });
	475	return error.InvalidDepFile;
	476	},
	477	};
	478	const result = opt_result orelse return;
	479	switch (result.id) {
	480	.target => return,
	481	.prereq => try self.addFilePost(result.bytes),
	482	}
	483	}
	484	}
	485
447	486	/// Returns a base64 encoded hash of the inputs.
448	487	pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 {
449	488	assert(self.manifest_file != null);

src-self-hosted/Compilation.zig+30-14

...	...	@@ -1016,8 +1016,11 @@ fn updateCObject(comp: Compilation, c_object: CObject) !void {
1016	1016	try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" });
1017	1017
1018	1018	const ext = classifyFileExt(c_object.src.src_path);
1019		// TODO capture the .d file and deal with caching stuff
1020		try comp.addCCArgs(arena, &argv, ext, false, null);
	1019	const out_dep_path: ?[]const u8 = if (comp.disable_c_depfile or !ext.clangSupportsDepFile())
	1020	null
	1021	else
	1022	try std.fmt.allocPrint(arena, "{}.d", .{out_obj_path});
	1023	try comp.addCCArgs(arena, &argv, ext, false, out_dep_path);
1021	1024
1022	1025	try argv.append("-o");
1023	1026	try argv.append(out_obj_path);
...	...	@@ -1086,7 +1089,15 @@ fn updateCObject(comp: Compilation, c_object: CObject) !void {
1086	1089	}
1087	1090	}
1088	1091
1089		// TODO handle .d files
	1092	if (out_dep_path) \|dep_file_path\| {
	1093	const dep_basename = std.fs.path.basename(dep_file_path);
	1094	// Add the files depended on to the cache system.
	1095	try ch.addDepFilePost(zig_cache_tmp_dir, dep_basename);
	1096	// Just to save disk space, we delete the file because it is never needed again.
	1097	zig_cache_tmp_dir.deleteFile(dep_basename) catch \|err\| {
	1098	std.log.warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) });
	1099	};
	1100	}
1090	1101
1091	1102	// Rename into place.
1092	1103	const digest = ch.final();
...	...	@@ -1118,11 +1129,12 @@ fn updateCObject(comp: Compilation, c_object: CObject) !void {
1118	1129
1119	1130	fn tmpFilePath(comp: Compilation, arena: Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 {
1120	1131	const s = std.fs.path.sep_str;
1121		return std.fmt.allocPrint(
1122		arena,
1123		"{}" ++ s ++ "tmp" ++ s ++ "{x}-{}",
1124		.{ comp.zig_cache_directory.path.?, comp.rand.int(u64), suffix },
1125		);
	1132	const rand_int = comp.rand.int(u64);
	1133	if (comp.zig_cache_directory.path) \|p\| {
	1134	return std.fmt.allocPrint(arena, "{}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix });
	1135	} else {
	1136	return std.fmt.allocPrint(arena, "tmp" ++ s ++ "{x}-{s}", .{ rand_int, suffix });
	1137	}
1126	1138	}
1127	1139
1128	1140	/// Add common C compiler args between translate-c and C object compilation.
...	...	@@ -1233,15 +1245,12 @@ fn addCCArgs(
1233	1245	try argv.append("-Xclang");
1234	1246	try argv.append("-detailed-preprocessing-record");
1235	1247	}
1236		if (out_dep_path) \|p\| {
1237		try argv.append("-MD");
1238		try argv.append("-MV");
1239		try argv.append("-MF");
1240		try argv.append(p);
1241		}
1242	1248	},
1243	1249	.so, .assembly, .ll, .bc, .unknown => {},
1244	1250	}
	1251	if (out_dep_path) \|p\| {
	1252	try argv.appendSlice(&[_][]const u8{ "-MD", "-MV", "-MF", p });
	1253	}
1245	1254	// Argh, why doesn't the assembler accept the list of CPU features?!
1246	1255	// I don't see a way to do this other than hard coding everything.
1247	1256	switch (target.cpu.arch) {
...	...	@@ -1388,6 +1397,13 @@ pub const FileExt = enum {
1388	1397	assembly,
1389	1398	so,
1390	1399	unknown,
	1400
	1401	pub fn clangSupportsDepFile(ext: FileExt) bool {
	1402	return switch (ext) {
	1403	.c, .cpp, .h => true,
	1404	.ll, .bc, .assembly, .so, .unknown => false,
	1405	};
	1406	}
1391	1407	};
1392	1408
1393	1409	pub fn hasCExt(filename: []const u8) bool {

src-self-hosted/DepTokenizer.zig created+1019

...	...	@@ -0,0 +1,1019 @@
	1	const Tokenizer = @This();
	2
	3	arena: std.heap.ArenaAllocator,
	4	index: usize,
	5	bytes: []const u8,
	6	error_text: []const u8,
	7	state: State,
	8
	9	const std = @import("std");
	10	const testing = std.testing;
	11	const assert = std.debug.assert;
	12
	13	pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer {
	14	return Tokenizer{
	15	.arena = std.heap.ArenaAllocator.init(allocator),
	16	.index = 0,
	17	.bytes = bytes,
	18	.error_text = "",
	19	.state = State{ .lhs = {} },
	20	};
	21	}
	22
	23	pub fn deinit(self: *Tokenizer) void {
	24	self.arena.deinit();
	25	}
	26
	27	pub fn next(self: *Tokenizer) Error!?Token {
	28	while (self.index < self.bytes.len) {
	29	const char = self.bytes[self.index];
	30	while (true) {
	31	switch (self.state) {
	32	.lhs => switch (char) {
	33	'\t', '\n', '\r', ' ' => {
	34	// silently ignore whitespace
	35	break; // advance
	36	},
	37	else => {
	38	self.state = State{ .target = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
	39	},
	40	},
	41	.target => \|*target\| switch (char) {
	42	'\t', '\n', '\r', ' ' => {
	43	return self.errorIllegalChar(self.index, char, "invalid target", .{});
	44	},
	45	'$' => {
	46	self.state = State{ .target_dollar_sign = target.* };
	47	break; // advance
	48	},
	49	'\\' => {
	50	self.state = State{ .target_reverse_solidus = target.* };
	51	break; // advance
	52	},
	53	':' => {
	54	self.state = State{ .target_colon = target.* };
	55	break; // advance
	56	},
	57	else => {
	58	try target.append(char);
	59	break; // advance
	60	},
	61	},
	62	.target_reverse_solidus => \|*target\| switch (char) {
	63	'\t', '\n', '\r' => {
	64	return self.errorIllegalChar(self.index, char, "bad target escape", .{});
	65	},
	66	' ', '#', '\\' => {
	67	try target.append(char);
	68	self.state = State{ .target = target.* };
	69	break; // advance
	70	},
	71	'$' => {
	72	try target.appendSlice(self.bytes[self.index - 1 .. self.index]);
	73	self.state = State{ .target_dollar_sign = target.* };
	74	break; // advance
	75	},
	76	else => {
	77	try target.appendSlice(self.bytes[self.index - 1 .. self.index + 1]);
	78	self.state = State{ .target = target.* };
	79	break; // advance
	80	},
	81	},
	82	.target_dollar_sign => \|*target\| switch (char) {
	83	'$' => {
	84	try target.append(char);
	85	self.state = State{ .target = target.* };
	86	break; // advance
	87	},
	88	else => {
	89	return self.errorIllegalChar(self.index, char, "expecting '$'", .{});
	90	},
	91	},
	92	.target_colon => \|*target\| switch (char) {
	93	'\n', '\r' => {
	94	const bytes = target.span();
	95	if (bytes.len != 0) {
	96	self.state = State{ .lhs = {} };
	97	return Token{ .id = .target, .bytes = bytes };
	98	}
	99	// silently ignore null target
	100	self.state = State{ .lhs = {} };
	101	continue;
	102	},
	103	'\\' => {
	104	self.state = State{ .target_colon_reverse_solidus = target.* };
	105	break; // advance
	106	},
	107	else => {
	108	const bytes = target.span();
	109	if (bytes.len != 0) {
	110	self.state = State{ .rhs = {} };
	111	return Token{ .id = .target, .bytes = bytes };
	112	}
	113	// silently ignore null target
	114	self.state = State{ .lhs = {} };
	115	continue;
	116	},
	117	},
	118	.target_colon_reverse_solidus => \|*target\| switch (char) {
	119	'\n', '\r' => {
	120	const bytes = target.span();
	121	if (bytes.len != 0) {
	122	self.state = State{ .lhs = {} };
	123	return Token{ .id = .target, .bytes = bytes };
	124	}
	125	// silently ignore null target
	126	self.state = State{ .lhs = {} };
	127	continue;
	128	},
	129	else => {
	130	try target.appendSlice(self.bytes[self.index - 2 .. self.index + 1]);
	131	self.state = State{ .target = target.* };
	132	break;
	133	},
	134	},
	135	.rhs => switch (char) {
	136	'\t', ' ' => {
	137	// silently ignore horizontal whitespace
	138	break; // advance
	139	},
	140	'\n', '\r' => {
	141	self.state = State{ .lhs = {} };
	142	continue;
	143	},
	144	'\\' => {
	145	self.state = State{ .rhs_continuation = {} };
	146	break; // advance
	147	},
	148	'"' => {
	149	self.state = State{ .prereq_quote = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
	150	break; // advance
	151	},
	152	else => {
	153	self.state = State{ .prereq = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
	154	},
	155	},
	156	.rhs_continuation => switch (char) {
	157	'\n' => {
	158	self.state = State{ .rhs = {} };
	159	break; // advance
	160	},
	161	'\r' => {
	162	self.state = State{ .rhs_continuation_linefeed = {} };
	163	break; // advance
	164	},
	165	else => {
	166	return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
	167	},
	168	},
	169	.rhs_continuation_linefeed => switch (char) {
	170	'\n' => {
	171	self.state = State{ .rhs = {} };
	172	break; // advance
	173	},
	174	else => {
	175	return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
	176	},
	177	},
	178	.prereq_quote => \|*prereq\| switch (char) {
	179	'"' => {
	180	const bytes = prereq.span();
	181	self.index += 1;
	182	self.state = State{ .rhs = {} };
	183	return Token{ .id = .prereq, .bytes = bytes };
	184	},
	185	else => {
	186	try prereq.append(char);
	187	break; // advance
	188	},
	189	},
	190	.prereq => \|*prereq\| switch (char) {
	191	'\t', ' ' => {
	192	const bytes = prereq.span();
	193	self.state = State{ .rhs = {} };
	194	return Token{ .id = .prereq, .bytes = bytes };
	195	},
	196	'\n', '\r' => {
	197	const bytes = prereq.span();
	198	self.state = State{ .lhs = {} };
	199	return Token{ .id = .prereq, .bytes = bytes };
	200	},
	201	'\\' => {
	202	self.state = State{ .prereq_continuation = prereq.* };
	203	break; // advance
	204	},
	205	else => {
	206	try prereq.append(char);
	207	break; // advance
	208	},
	209	},
	210	.prereq_continuation => \|*prereq\| switch (char) {
	211	'\n' => {
	212	const bytes = prereq.span();
	213	self.index += 1;
	214	self.state = State{ .rhs = {} };
	215	return Token{ .id = .prereq, .bytes = bytes };
	216	},
	217	'\r' => {
	218	self.state = State{ .prereq_continuation_linefeed = prereq.* };
	219	break; // advance
	220	},
	221	else => {
	222	// not continuation
	223	try prereq.appendSlice(self.bytes[self.index - 1 .. self.index + 1]);
	224	self.state = State{ .prereq = prereq.* };
	225	break; // advance
	226	},
	227	},
	228	.prereq_continuation_linefeed => \|prereq\| switch (char) {
	229	'\n' => {
	230	const bytes = prereq.span();
	231	self.index += 1;
	232	self.state = State{ .rhs = {} };
	233	return Token{ .id = .prereq, .bytes = bytes };
	234	},
	235	else => {
	236	return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
	237	},
	238	},
	239	}
	240	}
	241	self.index += 1;
	242	}
	243
	244	// eof, handle maybe incomplete token
	245	if (self.index == 0) return null;
	246	const idx = self.index - 1;
	247	switch (self.state) {
	248	.lhs,
	249	.rhs,
	250	.rhs_continuation,
	251	.rhs_continuation_linefeed,
	252	=> {},
	253	.target => \|target\| {
	254	return self.errorPosition(idx, target.span(), "incomplete target", .{});
	255	},
	256	.target_reverse_solidus,
	257	.target_dollar_sign,
	258	=> {
	259	const index = self.index - 1;
	260	return self.errorIllegalChar(idx, self.bytes[idx], "incomplete escape", .{});
	261	},
	262	.target_colon => \|target\| {
	263	const bytes = target.span();
	264	if (bytes.len != 0) {
	265	self.index += 1;
	266	self.state = State{ .rhs = {} };
	267	return Token{ .id = .target, .bytes = bytes };
	268	}
	269	// silently ignore null target
	270	self.state = State{ .lhs = {} };
	271	},
	272	.target_colon_reverse_solidus => \|target\| {
	273	const bytes = target.span();
	274	if (bytes.len != 0) {
	275	self.index += 1;
	276	self.state = State{ .rhs = {} };
	277	return Token{ .id = .target, .bytes = bytes };
	278	}
	279	// silently ignore null target
	280	self.state = State{ .lhs = {} };
	281	},
	282	.prereq_quote => \|prereq\| {
	283	return self.errorPosition(idx, prereq.span(), "incomplete quoted prerequisite", .{});
	284	},
	285	.prereq => \|prereq\| {
	286	const bytes = prereq.span();
	287	self.state = State{ .lhs = {} };
	288	return Token{ .id = .prereq, .bytes = bytes };
	289	},
	290	.prereq_continuation => \|prereq\| {
	291	const bytes = prereq.span();
	292	self.state = State{ .lhs = {} };
	293	return Token{ .id = .prereq, .bytes = bytes };
	294	},
	295	.prereq_continuation_linefeed => \|prereq\| {
	296	const bytes = prereq.span();
	297	self.state = State{ .lhs = {} };
	298	return Token{ .id = .prereq, .bytes = bytes };
	299	},
	300	}
	301	return null;
	302	}
	303
	304	fn errorf(self: *Tokenizer, comptime fmt: []const u8, args: anytype) Error {
	305	self.error_text = try std.fmt.allocPrintZ(&self.arena.allocator, fmt, args);
	306	return Error.InvalidInput;
	307	}
	308
	309	fn errorPosition(self: *Tokenizer, position: usize, bytes: []const u8, comptime fmt: []const u8, args: anytype) Error {
	310	var buffer = std.ArrayList(u8).init(&self.arena.allocator);
	311	try buffer.outStream().print(fmt, args);
	312	try buffer.appendSlice(" '");
	313	const out = buffer.writer();
	314	try printCharValues(out, bytes);
	315	try buffer.appendSlice("'");
	316	try buffer.outStream().print(" at position {}", .{position - (bytes.len - 1)});
	317	try buffer.append(0);
	318	self.error_text = buffer.items[0 .. buffer.items.len - 1 :0];
	319	return Error.InvalidInput;
	320	}
	321
	322	fn errorIllegalChar(self: *Tokenizer, position: usize, char: u8, comptime fmt: []const u8, args: anytype) Error {
	323	var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0);
	324	try buffer.appendSlice("illegal char ");
	325	try printUnderstandableChar(&buffer, char);
	326	try buffer.outStream().print(" at position {}", .{position});
	327	if (fmt.len != 0) try buffer.outStream().print(": " ++ fmt, args);
	328	self.error_text = buffer.span();
	329	return Error.InvalidInput;
	330	}
	331
	332	const Error = error{
	333	OutOfMemory,
	334	InvalidInput,
	335	};
	336
	337	const State = union(enum) {
	338	lhs: void,
	339	target: std.ArrayListSentineled(u8, 0),
	340	target_reverse_solidus: std.ArrayListSentineled(u8, 0),
	341	target_dollar_sign: std.ArrayListSentineled(u8, 0),
	342	target_colon: std.ArrayListSentineled(u8, 0),
	343	target_colon_reverse_solidus: std.ArrayListSentineled(u8, 0),
	344	rhs: void,
	345	rhs_continuation: void,
	346	rhs_continuation_linefeed: void,
	347	prereq_quote: std.ArrayListSentineled(u8, 0),
	348	prereq: std.ArrayListSentineled(u8, 0),
	349	prereq_continuation: std.ArrayListSentineled(u8, 0),
	350	prereq_continuation_linefeed: std.ArrayListSentineled(u8, 0),
	351	};
	352
	353	pub const Token = struct {
	354	id: ID,
	355	bytes: []const u8,
	356
	357	pub const ID = enum {
	358	target,
	359	prereq,
	360	};
	361	};
	362
	363	test "empty file" {
	364	try depTokenizer("", "");
	365	}
	366
	367	test "empty whitespace" {
	368	try depTokenizer("\n", "");
	369	try depTokenizer("\r", "");
	370	try depTokenizer("\r\n", "");
	371	try depTokenizer(" ", "");
	372	}
	373
	374	test "empty colon" {
	375	try depTokenizer(":", "");
	376	try depTokenizer("\n:", "");
	377	try depTokenizer("\r:", "");
	378	try depTokenizer("\r\n:", "");
	379	try depTokenizer(" :", "");
	380	}
	381
	382	test "empty target" {
	383	try depTokenizer("foo.o:", "target = {foo.o}");
	384	try depTokenizer(
	385	\\foo.o:
	386	\\bar.o:
	387	\\abcd.o:
	388	,
	389	\\target = {foo.o}
	390	\\target = {bar.o}
	391	\\target = {abcd.o}
	392	);
	393	}
	394
	395	test "whitespace empty target" {
	396	try depTokenizer("\nfoo.o:", "target = {foo.o}");
	397	try depTokenizer("\rfoo.o:", "target = {foo.o}");
	398	try depTokenizer("\r\nfoo.o:", "target = {foo.o}");
	399	try depTokenizer(" foo.o:", "target = {foo.o}");
	400	}
	401
	402	test "escape empty target" {
	403	try depTokenizer("\\ foo.o:", "target = { foo.o}");
	404	try depTokenizer("\\#foo.o:", "target = {#foo.o}");
	405	try depTokenizer("\\\\foo.o:", "target = {\\foo.o}");
	406	try depTokenizer("$$foo.o:", "target = {$foo.o}");
	407	}
	408
	409	test "empty target linefeeds" {
	410	try depTokenizer("\n", "");
	411	try depTokenizer("\r\n", "");
	412
	413	const expect = "target = {foo.o}";
	414	try depTokenizer(
	415	\\foo.o:
	416	, expect);
	417	try depTokenizer(
	418	\\foo.o:
	419	\\
	420	, expect);
	421	try depTokenizer(
	422	\\foo.o:
	423	, expect);
	424	try depTokenizer(
	425	\\foo.o:
	426	\\
	427	, expect);
	428	}
	429
	430	test "empty target linefeeds + continuations" {
	431	const expect = "target = {foo.o}";
	432	try depTokenizer(
	433	\\foo.o:\
	434	, expect);
	435	try depTokenizer(
	436	\\foo.o:\
	437	\\
	438	, expect);
	439	try depTokenizer(
	440	\\foo.o:\
	441	, expect);
	442	try depTokenizer(
	443	\\foo.o:\
	444	\\
	445	, expect);
	446	}
	447
	448	test "empty target linefeeds + hspace + continuations" {
	449	const expect = "target = {foo.o}";
	450	try depTokenizer(
	451	\\foo.o: \
	452	, expect);
	453	try depTokenizer(
	454	\\foo.o: \
	455	\\
	456	, expect);
	457	try depTokenizer(
	458	\\foo.o: \
	459	, expect);
	460	try depTokenizer(
	461	\\foo.o: \
	462	\\
	463	, expect);
	464	}
	465
	466	test "prereq" {
	467	const expect =
	468	\\target = {foo.o}
	469	\\prereq = {foo.c}
	470	;
	471	try depTokenizer("foo.o: foo.c", expect);
	472	try depTokenizer(
	473	\\foo.o: \
	474	\\foo.c
	475	, expect);
	476	try depTokenizer(
	477	\\foo.o: \
	478	\\ foo.c
	479	, expect);
	480	try depTokenizer(
	481	\\foo.o: \
	482	\\ foo.c
	483	, expect);
	484	}
	485
	486	test "prereq continuation" {
	487	const expect =
	488	\\target = {foo.o}
	489	\\prereq = {foo.h}
	490	\\prereq = {bar.h}
	491	;
	492	try depTokenizer(
	493	\\foo.o: foo.h\
	494	\\bar.h
	495	, expect);
	496	try depTokenizer(
	497	\\foo.o: foo.h\
	498	\\bar.h
	499	, expect);
	500	}
	501
	502	test "multiple prereqs" {
	503	const expect =
	504	\\target = {foo.o}
	505	\\prereq = {foo.c}
	506	\\prereq = {foo.h}
	507	\\prereq = {bar.h}
	508	;
	509	try depTokenizer("foo.o: foo.c foo.h bar.h", expect);
	510	try depTokenizer(
	511	\\foo.o: \
	512	\\foo.c foo.h bar.h
	513	, expect);
	514	try depTokenizer(
	515	\\foo.o: foo.c foo.h bar.h\
	516	, expect);
	517	try depTokenizer(
	518	\\foo.o: foo.c foo.h bar.h\
	519	\\
	520	, expect);
	521	try depTokenizer(
	522	\\foo.o: \
	523	\\foo.c \
	524	\\ foo.h\
	525	\\bar.h
	526	\\
	527	, expect);
	528	try depTokenizer(
	529	\\foo.o: \
	530	\\foo.c \
	531	\\ foo.h\
	532	\\bar.h\
	533	\\
	534	, expect);
	535	try depTokenizer(
	536	\\foo.o: \
	537	\\foo.c \
	538	\\ foo.h\
	539	\\bar.h\
	540	, expect);
	541	}
	542
	543	test "multiple targets and prereqs" {
	544	try depTokenizer(
	545	\\foo.o: foo.c
	546	\\bar.o: bar.c a.h b.h c.h
	547	\\abc.o: abc.c \
	548	\\ one.h two.h \
	549	\\ three.h four.h
	550	,
	551	\\target = {foo.o}
	552	\\prereq = {foo.c}
	553	\\target = {bar.o}
	554	\\prereq = {bar.c}
	555	\\prereq = {a.h}
	556	\\prereq = {b.h}
	557	\\prereq = {c.h}
	558	\\target = {abc.o}
	559	\\prereq = {abc.c}
	560	\\prereq = {one.h}
	561	\\prereq = {two.h}
	562	\\prereq = {three.h}
	563	\\prereq = {four.h}
	564	);
	565	try depTokenizer(
	566	\\ascii.o: ascii.c
	567	\\base64.o: base64.c stdio.h
	568	\\elf.o: elf.c a.h b.h c.h
	569	\\macho.o: \
	570	\\ macho.c\
	571	\\ a.h b.h c.h
	572	,
	573	\\target = {ascii.o}
	574	\\prereq = {ascii.c}
	575	\\target = {base64.o}
	576	\\prereq = {base64.c}
	577	\\prereq = {stdio.h}
	578	\\target = {elf.o}
	579	\\prereq = {elf.c}
	580	\\prereq = {a.h}
	581	\\prereq = {b.h}
	582	\\prereq = {c.h}
	583	\\target = {macho.o}
	584	\\prereq = {macho.c}
	585	\\prereq = {a.h}
	586	\\prereq = {b.h}
	587	\\prereq = {c.h}
	588	);
	589	try depTokenizer(
	590	\\a$$scii.o: ascii.c
	591	\\\\base64.o: "\base64.c" "s t#dio.h"
	592	\\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$"
	593	\\macho.o: \
	594	\\ "macho!.c" \
	595	\\ a.h b.h c.h
	596	,
	597	\\target = {a$scii.o}
	598	\\prereq = {ascii.c}
	599	\\target = {\base64.o}
	600	\\prereq = {\base64.c}
	601	\\prereq = {s t#dio.h}
	602	\\target = {e\lf.o}
	603	\\prereq = {e\lf.c}
	604	\\prereq = {a.h$$}
	605	\\prereq = {$$b.h c.h$$}
	606	\\target = {macho.o}
	607	\\prereq = {macho!.c}
	608	\\prereq = {a.h}
	609	\\prereq = {b.h}
	610	\\prereq = {c.h}
	611	);
	612	}
	613
	614	test "windows quoted prereqs" {
	615	try depTokenizer(
	616	\\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c"
	617	\\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \
	618	\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \
	619	\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h"
	620	,
	621	\\target = {c:\foo.o}
	622	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c}
	623	\\target = {c:\foo2.o}
	624	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c}
	625	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h}
	626	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h}
	627	);
	628	}
	629
	630	test "windows mixed prereqs" {
	631	try depTokenizer(
	632	\\cimport.o: \
	633	\\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \
	634	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \
	635	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \
	636	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \
	637	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \
	638	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \
	639	\\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \
	640	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \
	641	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \
	642	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \
	643	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \
	644	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \
	645	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \
	646	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \
	647	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \
	648	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h"
	649	,
	650	\\target = {cimport.o}
	651	\\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h}
	652	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h}
	653	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h}
	654	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h}
	655	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h}
	656	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h}
	657	\\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h}
	658	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h}
	659	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h}
	660	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h}
	661	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h}
	662	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h}
	663	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h}
	664	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h}
	665	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h}
	666	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h}
	667	);
	668	}
	669
	670	test "funky targets" {
	671	try depTokenizer(
	672	\\C:\Users\anon\foo.o:
	673	\\C:\Users\anon\foo\ .o:
	674	\\C:\Users\anon\foo\#.o:
	675	\\C:\Users\anon\foo$$.o:
	676	\\C:\Users\anon\\\ foo.o:
	677	\\C:\Users\anon\\#foo.o:
	678	\\C:\Users\anon\$$foo.o:
	679	\\C:\Users\anon\\\ \ \ \ \ foo.o:
	680	,
	681	\\target = {C:\Users\anon\foo.o}
	682	\\target = {C:\Users\anon\foo .o}
	683	\\target = {C:\Users\anon\foo#.o}
	684	\\target = {C:\Users\anon\foo$.o}
	685	\\target = {C:\Users\anon\ foo.o}
	686	\\target = {C:\Users\anon\#foo.o}
	687	\\target = {C:\Users\anon\$foo.o}
	688	\\target = {C:\Users\anon\ foo.o}
	689	);
	690	}
	691
	692	test "error incomplete escape - reverse_solidus" {
	693	try depTokenizer("\\",
	694	\\ERROR: illegal char '\' at position 0: incomplete escape
	695	);
	696	try depTokenizer("\t\\",
	697	\\ERROR: illegal char '\' at position 1: incomplete escape
	698	);
	699	try depTokenizer("\n\\",
	700	\\ERROR: illegal char '\' at position 1: incomplete escape
	701	);
	702	try depTokenizer("\r\\",
	703	\\ERROR: illegal char '\' at position 1: incomplete escape
	704	);
	705	try depTokenizer("\r\n\\",
	706	\\ERROR: illegal char '\' at position 2: incomplete escape
	707	);
	708	try depTokenizer(" \\",
	709	\\ERROR: illegal char '\' at position 1: incomplete escape
	710	);
	711	}
	712
	713	test "error incomplete escape - dollar_sign" {
	714	try depTokenizer("$",
	715	\\ERROR: illegal char '$' at position 0: incomplete escape
	716	);
	717	try depTokenizer("\t$",
	718	\\ERROR: illegal char '$' at position 1: incomplete escape
	719	);
	720	try depTokenizer("\n$",
	721	\\ERROR: illegal char '$' at position 1: incomplete escape
	722	);
	723	try depTokenizer("\r$",
	724	\\ERROR: illegal char '$' at position 1: incomplete escape
	725	);
	726	try depTokenizer("\r\n$",
	727	\\ERROR: illegal char '$' at position 2: incomplete escape
	728	);
	729	try depTokenizer(" $",
	730	\\ERROR: illegal char '$' at position 1: incomplete escape
	731	);
	732	}
	733
	734	test "error incomplete target" {
	735	try depTokenizer("foo.o",
	736	\\ERROR: incomplete target 'foo.o' at position 0
	737	);
	738	try depTokenizer("\tfoo.o",
	739	\\ERROR: incomplete target 'foo.o' at position 1
	740	);
	741	try depTokenizer("\nfoo.o",
	742	\\ERROR: incomplete target 'foo.o' at position 1
	743	);
	744	try depTokenizer("\rfoo.o",
	745	\\ERROR: incomplete target 'foo.o' at position 1
	746	);
	747	try depTokenizer("\r\nfoo.o",
	748	\\ERROR: incomplete target 'foo.o' at position 2
	749	);
	750	try depTokenizer(" foo.o",
	751	\\ERROR: incomplete target 'foo.o' at position 1
	752	);
	753
	754	try depTokenizer("\\ foo.o",
	755	\\ERROR: incomplete target ' foo.o' at position 1
	756	);
	757	try depTokenizer("\\#foo.o",
	758	\\ERROR: incomplete target '#foo.o' at position 1
	759	);
	760	try depTokenizer("\\\\foo.o",
	761	\\ERROR: incomplete target '\foo.o' at position 1
	762	);
	763	try depTokenizer("$$foo.o",
	764	\\ERROR: incomplete target '$foo.o' at position 1
	765	);
	766	}
	767
	768	test "error illegal char at position - bad target escape" {
	769	try depTokenizer("\\\t",
	770	\\ERROR: illegal char \x09 at position 1: bad target escape
	771	);
	772	try depTokenizer("\\\n",
	773	\\ERROR: illegal char \x0A at position 1: bad target escape
	774	);
	775	try depTokenizer("\\\r",
	776	\\ERROR: illegal char \x0D at position 1: bad target escape
	777	);
	778	try depTokenizer("\\\r\n",
	779	\\ERROR: illegal char \x0D at position 1: bad target escape
	780	);
	781	}
	782
	783	test "error illegal char at position - execting dollar_sign" {
	784	try depTokenizer("$\t",
	785	\\ERROR: illegal char \x09 at position 1: expecting '$'
	786	);
	787	try depTokenizer("$\n",
	788	\\ERROR: illegal char \x0A at position 1: expecting '$'
	789	);
	790	try depTokenizer("$\r",
	791	\\ERROR: illegal char \x0D at position 1: expecting '$'
	792	);
	793	try depTokenizer("$\r\n",
	794	\\ERROR: illegal char \x0D at position 1: expecting '$'
	795	);
	796	}
	797
	798	test "error illegal char at position - invalid target" {
	799	try depTokenizer("foo\t.o",
	800	\\ERROR: illegal char \x09 at position 3: invalid target
	801	);
	802	try depTokenizer("foo\n.o",
	803	\\ERROR: illegal char \x0A at position 3: invalid target
	804	);
	805	try depTokenizer("foo\r.o",
	806	\\ERROR: illegal char \x0D at position 3: invalid target
	807	);
	808	try depTokenizer("foo\r\n.o",
	809	\\ERROR: illegal char \x0D at position 3: invalid target
	810	);
	811	}
	812
	813	test "error target - continuation expecting end-of-line" {
	814	try depTokenizer("foo.o: \\\t",
	815	\\target = {foo.o}
	816	\\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line
	817	);
	818	try depTokenizer("foo.o: \\ ",
	819	\\target = {foo.o}
	820	\\ERROR: illegal char \x20 at position 8: continuation expecting end-of-line
	821	);
	822	try depTokenizer("foo.o: \\x",
	823	\\target = {foo.o}
	824	\\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
	825	);
	826	try depTokenizer("foo.o: \\\x0dx",
	827	\\target = {foo.o}
	828	\\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
	829	);
	830	}
	831
	832	test "error prereq - continuation expecting end-of-line" {
	833	try depTokenizer("foo.o: foo.h\\\x0dx",
	834	\\target = {foo.o}
	835	\\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
	836	);
	837	}
	838
	839	// - tokenize input, emit textual representation, and compare to expect
	840	fn depTokenizer(input: []const u8, expect: []const u8) !void {
	841	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	842	const arena = &arena_allocator.allocator;
	843	defer arena_allocator.deinit();
	844
	845	var it = Tokenizer.init(arena, input);
	846	var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);
	847	var i: usize = 0;
	848	while (true) {
	849	const r = it.next() catch \|err\| {
	850	switch (err) {
	851	Tokenizer.Error.InvalidInput => {
	852	if (i != 0) try buffer.appendSlice("\n");
	853	try buffer.appendSlice("ERROR: ");
	854	try buffer.appendSlice(it.error_text);
	855	},
	856	else => return err,
	857	}
	858	break;
	859	};
	860	const token = r orelse break;
	861	if (i != 0) try buffer.appendSlice("\n");
	862	try buffer.appendSlice(@tagName(token.id));
	863	try buffer.appendSlice(" = {");
	864	for (token.bytes) \|b\| {
	865	try buffer.append(printable_char_tab[b]);
	866	}
	867	try buffer.appendSlice("}");
	868	i += 1;
	869	}
	870	const got: []const u8 = buffer.span();
	871
	872	if (std.mem.eql(u8, expect, got)) {
	873	testing.expect(true);
	874	return;
	875	}
	876
	877	const out = std.io.getStdErr().writer();
	878
	879	try out.writeAll("\n");
	880	try printSection(out, "<<<< input", input);
	881	try printSection(out, "==== expect", expect);
	882	try printSection(out, ">>>> got", got);
	883	try printRuler(out);
	884
	885	testing.expect(false);
	886	}
	887
	888	fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
	889	try printLabel(out, label, bytes);
	890	try hexDump(out, bytes);
	891	try printRuler(out);
	892	try out.writeAll(bytes);
	893	try out.writeAll("\n");
	894	}
	895
	896	fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
	897	var buf: [80]u8 = undefined;
	898	var text = try std.fmt.bufPrint(buf[0..], "{} {} bytes ", .{ label, bytes.len });
	899	try out.writeAll(text);
	900	var i: usize = text.len;
	901	const end = 79;
	902	while (i < 79) : (i += 1) {
	903	try out.writeAll(&[_]u8{label[0]});
	904	}
	905	try out.writeAll("\n");
	906	}
	907
	908	fn printRuler(out: anytype) !void {
	909	var i: usize = 0;
	910	const end = 79;
	911	while (i < 79) : (i += 1) {
	912	try out.writeAll("-");
	913	}
	914	try out.writeAll("\n");
	915	}
	916
	917	fn hexDump(out: anytype, bytes: []const u8) !void {
	918	const n16 = bytes.len >> 4;
	919	var line: usize = 0;
	920	var offset: usize = 0;
	921	while (line < n16) : (line += 1) {
	922	try hexDump16(out, offset, bytes[offset .. offset + 16]);
	923	offset += 16;
	924	}
	925
	926	const n = bytes.len & 0x0f;
	927	if (n > 0) {
	928	try printDecValue(out, offset, 8);
	929	try out.writeAll(":");
	930	try out.writeAll(" ");
	931	var end1 = std.math.min(offset + n, offset + 8);
	932	for (bytes[offset..end1]) \|b\| {
	933	try out.writeAll(" ");
	934	try printHexValue(out, b, 2);
	935	}
	936	var end2 = offset + n;
	937	if (end2 > end1) {
	938	try out.writeAll(" ");
	939	for (bytes[end1..end2]) \|b\| {
	940	try out.writeAll(" ");
	941	try printHexValue(out, b, 2);
	942	}
	943	}
	944	const short = 16 - n;
	945	var i: usize = 0;
	946	while (i < short) : (i += 1) {
	947	try out.writeAll(" ");
	948	}
	949	if (end2 > end1) {
	950	try out.writeAll(" \|");
	951	} else {
	952	try out.writeAll(" \|");
	953	}
	954	try printCharValues(out, bytes[offset..end2]);
	955	try out.writeAll("\|\n");
	956	offset += n;
	957	}
	958
	959	try printDecValue(out, offset, 8);
	960	try out.writeAll(":");
	961	try out.writeAll("\n");
	962	}
	963
	964	fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void {
	965	try printDecValue(out, offset, 8);
	966	try out.writeAll(":");
	967	try out.writeAll(" ");
	968	for (bytes[0..8]) \|b\| {
	969	try out.writeAll(" ");
	970	try printHexValue(out, b, 2);
	971	}
	972	try out.writeAll(" ");
	973	for (bytes[8..16]) \|b\| {
	974	try out.writeAll(" ");
	975	try printHexValue(out, b, 2);
	976	}
	977	try out.writeAll(" \|");
	978	try printCharValues(out, bytes);
	979	try out.writeAll("\|\n");
	980	}
	981
	982	fn printDecValue(out: anytype, value: u64, width: u8) !void {
	983	var buffer: [20]u8 = undefined;
	984	const len = std.fmt.formatIntBuf(buffer[0..], value, 10, false, .{ .width = width, .fill = '0' });
	985	try out.writeAll(buffer[0..len]);
	986	}
	987
	988	fn printHexValue(out: anytype, value: u64, width: u8) !void {
	989	var buffer: [16]u8 = undefined;
	990	const len = std.fmt.formatIntBuf(buffer[0..], value, 16, false, .{ .width = width, .fill = '0' });
	991	try out.writeAll(buffer[0..len]);
	992	}
	993
	994	fn printCharValues(out: anytype, bytes: []const u8) !void {
	995	for (bytes) \|b\| {
	996	try out.writeAll(&[_]u8{printable_char_tab[b]});
	997	}
	998	}
	999
	1000	fn printUnderstandableChar(buffer: *std.ArrayListSentineled(u8, 0), char: u8) !void {
	1001	if (!std.ascii.isPrint(char) or char == ' ') {
	1002	try buffer.outStream().print("\\x{X:0>2}", .{char});
	1003	} else {
	1004	try buffer.appendSlice("'");
	1005	try buffer.append(printable_char_tab[char]);
	1006	try buffer.appendSlice("'");
	1007	}
	1008	}
	1009
	1010	// zig fmt: off
	1011	const printable_char_tab: []const u8 =
	1012	"................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
	1013	"@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{\|}~." ++
	1014	"................................................................" ++
	1015	"................................................................";
	1016	// zig fmt: on
	1017	comptime {
	1018	assert(printable_char_tab.len == 256);
	1019	}

src-self-hosted/dep_tokenizer.zig deleted-1039

...	...	@@ -1,1039 +0,0 @@
1		const std = @import("std");
2		const testing = std.testing;
3
4		pub const Tokenizer = struct {
5		arena: std.heap.ArenaAllocator,
6		index: usize,
7		bytes: []const u8,
8		error_text: []const u8,
9		state: State,
10
11		pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer {
12		return Tokenizer{
13		.arena = std.heap.ArenaAllocator.init(allocator),
14		.index = 0,
15		.bytes = bytes,
16		.error_text = "",
17		.state = State{ .lhs = {} },
18		};
19		}
20
21		pub fn deinit(self: *Tokenizer) void {
22		self.arena.deinit();
23		}
24
25		pub fn next(self: *Tokenizer) Error!?Token {
26		while (self.index < self.bytes.len) {
27		const char = self.bytes[self.index];
28		while (true) {
29		switch (self.state) {
30		.lhs => switch (char) {
31		'\t', '\n', '\r', ' ' => {
32		// silently ignore whitespace
33		break; // advance
34		},
35		else => {
36		self.state = State{ .target = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
37		},
38		},
39		.target => \|*target\| switch (char) {
40		'\t', '\n', '\r', ' ' => {
41		return self.errorIllegalChar(self.index, char, "invalid target", .{});
42		},
43		'$' => {
44		self.state = State{ .target_dollar_sign = target.* };
45		break; // advance
46		},
47		'\\' => {
48		self.state = State{ .target_reverse_solidus = target.* };
49		break; // advance
50		},
51		':' => {
52		self.state = State{ .target_colon = target.* };
53		break; // advance
54		},
55		else => {
56		try target.append(char);
57		break; // advance
58		},
59		},
60		.target_reverse_solidus => \|*target\| switch (char) {
61		'\t', '\n', '\r' => {
62		return self.errorIllegalChar(self.index, char, "bad target escape", .{});
63		},
64		' ', '#', '\\' => {
65		try target.append(char);
66		self.state = State{ .target = target.* };
67		break; // advance
68		},
69		'$' => {
70		try target.appendSlice(self.bytes[self.index - 1 .. self.index]);
71		self.state = State{ .target_dollar_sign = target.* };
72		break; // advance
73		},
74		else => {
75		try target.appendSlice(self.bytes[self.index - 1 .. self.index + 1]);
76		self.state = State{ .target = target.* };
77		break; // advance
78		},
79		},
80		.target_dollar_sign => \|*target\| switch (char) {
81		'$' => {
82		try target.append(char);
83		self.state = State{ .target = target.* };
84		break; // advance
85		},
86		else => {
87		return self.errorIllegalChar(self.index, char, "expecting '$'", .{});
88		},
89		},
90		.target_colon => \|*target\| switch (char) {
91		'\n', '\r' => {
92		const bytes = target.span();
93		if (bytes.len != 0) {
94		self.state = State{ .lhs = {} };
95		return Token{ .id = .target, .bytes = bytes };
96		}
97		// silently ignore null target
98		self.state = State{ .lhs = {} };
99		continue;
100		},
101		'\\' => {
102		self.state = State{ .target_colon_reverse_solidus = target.* };
103		break; // advance
104		},
105		else => {
106		const bytes = target.span();
107		if (bytes.len != 0) {
108		self.state = State{ .rhs = {} };
109		return Token{ .id = .target, .bytes = bytes };
110		}
111		// silently ignore null target
112		self.state = State{ .lhs = {} };
113		continue;
114		},
115		},
116		.target_colon_reverse_solidus => \|*target\| switch (char) {
117		'\n', '\r' => {
118		const bytes = target.span();
119		if (bytes.len != 0) {
120		self.state = State{ .lhs = {} };
121		return Token{ .id = .target, .bytes = bytes };
122		}
123		// silently ignore null target
124		self.state = State{ .lhs = {} };
125		continue;
126		},
127		else => {
128		try target.appendSlice(self.bytes[self.index - 2 .. self.index + 1]);
129		self.state = State{ .target = target.* };
130		break;
131		},
132		},
133		.rhs => switch (char) {
134		'\t', ' ' => {
135		// silently ignore horizontal whitespace
136		break; // advance
137		},
138		'\n', '\r' => {
139		self.state = State{ .lhs = {} };
140		continue;
141		},
142		'\\' => {
143		self.state = State{ .rhs_continuation = {} };
144		break; // advance
145		},
146		'"' => {
147		self.state = State{ .prereq_quote = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
148		break; // advance
149		},
150		else => {
151		self.state = State{ .prereq = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
152		},
153		},
154		.rhs_continuation => switch (char) {
155		'\n' => {
156		self.state = State{ .rhs = {} };
157		break; // advance
158		},
159		'\r' => {
160		self.state = State{ .rhs_continuation_linefeed = {} };
161		break; // advance
162		},
163		else => {
164		return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
165		},
166		},
167		.rhs_continuation_linefeed => switch (char) {
168		'\n' => {
169		self.state = State{ .rhs = {} };
170		break; // advance
171		},
172		else => {
173		return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
174		},
175		},
176		.prereq_quote => \|*prereq\| switch (char) {
177		'"' => {
178		const bytes = prereq.span();
179		self.index += 1;
180		self.state = State{ .rhs = {} };
181		return Token{ .id = .prereq, .bytes = bytes };
182		},
183		else => {
184		try prereq.append(char);
185		break; // advance
186		},
187		},
188		.prereq => \|*prereq\| switch (char) {
189		'\t', ' ' => {
190		const bytes = prereq.span();
191		self.state = State{ .rhs = {} };
192		return Token{ .id = .prereq, .bytes = bytes };
193		},
194		'\n', '\r' => {
195		const bytes = prereq.span();
196		self.state = State{ .lhs = {} };
197		return Token{ .id = .prereq, .bytes = bytes };
198		},
199		'\\' => {
200		self.state = State{ .prereq_continuation = prereq.* };
201		break; // advance
202		},
203		else => {
204		try prereq.append(char);
205		break; // advance
206		},
207		},
208		.prereq_continuation => \|*prereq\| switch (char) {
209		'\n' => {
210		const bytes = prereq.span();
211		self.index += 1;
212		self.state = State{ .rhs = {} };
213		return Token{ .id = .prereq, .bytes = bytes };
214		},
215		'\r' => {
216		self.state = State{ .prereq_continuation_linefeed = prereq.* };
217		break; // advance
218		},
219		else => {
220		// not continuation
221		try prereq.appendSlice(self.bytes[self.index - 1 .. self.index + 1]);
222		self.state = State{ .prereq = prereq.* };
223		break; // advance
224		},
225		},
226		.prereq_continuation_linefeed => \|prereq\| switch (char) {
227		'\n' => {
228		const bytes = prereq.span();
229		self.index += 1;
230		self.state = State{ .rhs = {} };
231		return Token{ .id = .prereq, .bytes = bytes };
232		},
233		else => {
234		return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
235		},
236		},
237		}
238		}
239		self.index += 1;
240		}
241
242		// eof, handle maybe incomplete token
243		if (self.index == 0) return null;
244		const idx = self.index - 1;
245		switch (self.state) {
246		.lhs,
247		.rhs,
248		.rhs_continuation,
249		.rhs_continuation_linefeed,
250		=> {},
251		.target => \|target\| {
252		return self.errorPosition(idx, target.span(), "incomplete target", .{});
253		},
254		.target_reverse_solidus,
255		.target_dollar_sign,
256		=> {
257		const index = self.index - 1;
258		return self.errorIllegalChar(idx, self.bytes[idx], "incomplete escape", .{});
259		},
260		.target_colon => \|target\| {
261		const bytes = target.span();
262		if (bytes.len != 0) {
263		self.index += 1;
264		self.state = State{ .rhs = {} };
265		return Token{ .id = .target, .bytes = bytes };
266		}
267		// silently ignore null target
268		self.state = State{ .lhs = {} };
269		},
270		.target_colon_reverse_solidus => \|target\| {
271		const bytes = target.span();
272		if (bytes.len != 0) {
273		self.index += 1;
274		self.state = State{ .rhs = {} };
275		return Token{ .id = .target, .bytes = bytes };
276		}
277		// silently ignore null target
278		self.state = State{ .lhs = {} };
279		},
280		.prereq_quote => \|prereq\| {
281		return self.errorPosition(idx, prereq.span(), "incomplete quoted prerequisite", .{});
282		},
283		.prereq => \|prereq\| {
284		const bytes = prereq.span();
285		self.state = State{ .lhs = {} };
286		return Token{ .id = .prereq, .bytes = bytes };
287		},
288		.prereq_continuation => \|prereq\| {
289		const bytes = prereq.span();
290		self.state = State{ .lhs = {} };
291		return Token{ .id = .prereq, .bytes = bytes };
292		},
293		.prereq_continuation_linefeed => \|prereq\| {
294		const bytes = prereq.span();
295		self.state = State{ .lhs = {} };
296		return Token{ .id = .prereq, .bytes = bytes };
297		},
298		}
299		return null;
300		}
301
302		fn errorf(self: *Tokenizer, comptime fmt: []const u8, args: anytype) Error {
303		self.error_text = try std.fmt.allocPrintZ(&self.arena.allocator, fmt, args);
304		return Error.InvalidInput;
305		}
306
307		fn errorPosition(self: *Tokenizer, position: usize, bytes: []const u8, comptime fmt: []const u8, args: anytype) Error {
308		var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0);
309		try buffer.outStream().print(fmt, args);
310		try buffer.appendSlice(" '");
311		var out = makeOutput(std.ArrayListSentineled(u8, 0).appendSlice, &buffer);
312		try printCharValues(&out, bytes);
313		try buffer.appendSlice("'");
314		try buffer.outStream().print(" at position {}", .{position - (bytes.len - 1)});
315		self.error_text = buffer.span();
316		return Error.InvalidInput;
317		}
318
319		fn errorIllegalChar(self: *Tokenizer, position: usize, char: u8, comptime fmt: []const u8, args: anytype) Error {
320		var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0);
321		try buffer.appendSlice("illegal char ");
322		try printUnderstandableChar(&buffer, char);
323		try buffer.outStream().print(" at position {}", .{position});
324		if (fmt.len != 0) try buffer.outStream().print(": " ++ fmt, args);
325		self.error_text = buffer.span();
326		return Error.InvalidInput;
327		}
328
329		const Error = error{
330		OutOfMemory,
331		InvalidInput,
332		};
333
334		const State = union(enum) {
335		lhs: void,
336		target: std.ArrayListSentineled(u8, 0),
337		target_reverse_solidus: std.ArrayListSentineled(u8, 0),
338		target_dollar_sign: std.ArrayListSentineled(u8, 0),
339		target_colon: std.ArrayListSentineled(u8, 0),
340		target_colon_reverse_solidus: std.ArrayListSentineled(u8, 0),
341		rhs: void,
342		rhs_continuation: void,
343		rhs_continuation_linefeed: void,
344		prereq_quote: std.ArrayListSentineled(u8, 0),
345		prereq: std.ArrayListSentineled(u8, 0),
346		prereq_continuation: std.ArrayListSentineled(u8, 0),
347		prereq_continuation_linefeed: std.ArrayListSentineled(u8, 0),
348		};
349
350		const Token = struct {
351		id: ID,
352		bytes: []const u8,
353
354		const ID = enum {
355		target,
356		prereq,
357		};
358		};
359		};
360
361		test "empty file" {
362		try depTokenizer("", "");
363		}
364
365		test "empty whitespace" {
366		try depTokenizer("\n", "");
367		try depTokenizer("\r", "");
368		try depTokenizer("\r\n", "");
369		try depTokenizer(" ", "");
370		}
371
372		test "empty colon" {
373		try depTokenizer(":", "");
374		try depTokenizer("\n:", "");
375		try depTokenizer("\r:", "");
376		try depTokenizer("\r\n:", "");
377		try depTokenizer(" :", "");
378		}
379
380		test "empty target" {
381		try depTokenizer("foo.o:", "target = {foo.o}");
382		try depTokenizer(
383		\\foo.o:
384		\\bar.o:
385		\\abcd.o:
386		,
387		\\target = {foo.o}
388		\\target = {bar.o}
389		\\target = {abcd.o}
390		);
391		}
392
393		test "whitespace empty target" {
394		try depTokenizer("\nfoo.o:", "target = {foo.o}");
395		try depTokenizer("\rfoo.o:", "target = {foo.o}");
396		try depTokenizer("\r\nfoo.o:", "target = {foo.o}");
397		try depTokenizer(" foo.o:", "target = {foo.o}");
398		}
399
400		test "escape empty target" {
401		try depTokenizer("\\ foo.o:", "target = { foo.o}");
402		try depTokenizer("\\#foo.o:", "target = {#foo.o}");
403		try depTokenizer("\\\\foo.o:", "target = {\\foo.o}");
404		try depTokenizer("$$foo.o:", "target = {$foo.o}");
405		}
406
407		test "empty target linefeeds" {
408		try depTokenizer("\n", "");
409		try depTokenizer("\r\n", "");
410
411		const expect = "target = {foo.o}";
412		try depTokenizer(
413		\\foo.o:
414		, expect);
415		try depTokenizer(
416		\\foo.o:
417		\\
418		, expect);
419		try depTokenizer(
420		\\foo.o:
421		, expect);
422		try depTokenizer(
423		\\foo.o:
424		\\
425		, expect);
426		}
427
428		test "empty target linefeeds + continuations" {
429		const expect = "target = {foo.o}";
430		try depTokenizer(
431		\\foo.o:\
432		, expect);
433		try depTokenizer(
434		\\foo.o:\
435		\\
436		, expect);
437		try depTokenizer(
438		\\foo.o:\
439		, expect);
440		try depTokenizer(
441		\\foo.o:\
442		\\
443		, expect);
444		}
445
446		test "empty target linefeeds + hspace + continuations" {
447		const expect = "target = {foo.o}";
448		try depTokenizer(
449		\\foo.o: \
450		, expect);
451		try depTokenizer(
452		\\foo.o: \
453		\\
454		, expect);
455		try depTokenizer(
456		\\foo.o: \
457		, expect);
458		try depTokenizer(
459		\\foo.o: \
460		\\
461		, expect);
462		}
463
464		test "prereq" {
465		const expect =
466		\\target = {foo.o}
467		\\prereq = {foo.c}
468		;
469		try depTokenizer("foo.o: foo.c", expect);
470		try depTokenizer(
471		\\foo.o: \
472		\\foo.c
473		, expect);
474		try depTokenizer(
475		\\foo.o: \
476		\\ foo.c
477		, expect);
478		try depTokenizer(
479		\\foo.o: \
480		\\ foo.c
481		, expect);
482		}
483
484		test "prereq continuation" {
485		const expect =
486		\\target = {foo.o}
487		\\prereq = {foo.h}
488		\\prereq = {bar.h}
489		;
490		try depTokenizer(
491		\\foo.o: foo.h\
492		\\bar.h
493		, expect);
494		try depTokenizer(
495		\\foo.o: foo.h\
496		\\bar.h
497		, expect);
498		}
499
500		test "multiple prereqs" {
501		const expect =
502		\\target = {foo.o}
503		\\prereq = {foo.c}
504		\\prereq = {foo.h}
505		\\prereq = {bar.h}
506		;
507		try depTokenizer("foo.o: foo.c foo.h bar.h", expect);
508		try depTokenizer(
509		\\foo.o: \
510		\\foo.c foo.h bar.h
511		, expect);
512		try depTokenizer(
513		\\foo.o: foo.c foo.h bar.h\
514		, expect);
515		try depTokenizer(
516		\\foo.o: foo.c foo.h bar.h\
517		\\
518		, expect);
519		try depTokenizer(
520		\\foo.o: \
521		\\foo.c \
522		\\ foo.h\
523		\\bar.h
524		\\
525		, expect);
526		try depTokenizer(
527		\\foo.o: \
528		\\foo.c \
529		\\ foo.h\
530		\\bar.h\
531		\\
532		, expect);
533		try depTokenizer(
534		\\foo.o: \
535		\\foo.c \
536		\\ foo.h\
537		\\bar.h\
538		, expect);
539		}
540
541		test "multiple targets and prereqs" {
542		try depTokenizer(
543		\\foo.o: foo.c
544		\\bar.o: bar.c a.h b.h c.h
545		\\abc.o: abc.c \
546		\\ one.h two.h \
547		\\ three.h four.h
548		,
549		\\target = {foo.o}
550		\\prereq = {foo.c}
551		\\target = {bar.o}
552		\\prereq = {bar.c}
553		\\prereq = {a.h}
554		\\prereq = {b.h}
555		\\prereq = {c.h}
556		\\target = {abc.o}
557		\\prereq = {abc.c}
558		\\prereq = {one.h}
559		\\prereq = {two.h}
560		\\prereq = {three.h}
561		\\prereq = {four.h}
562		);
563		try depTokenizer(
564		\\ascii.o: ascii.c
565		\\base64.o: base64.c stdio.h
566		\\elf.o: elf.c a.h b.h c.h
567		\\macho.o: \
568		\\ macho.c\
569		\\ a.h b.h c.h
570		,
571		\\target = {ascii.o}
572		\\prereq = {ascii.c}
573		\\target = {base64.o}
574		\\prereq = {base64.c}
575		\\prereq = {stdio.h}
576		\\target = {elf.o}
577		\\prereq = {elf.c}
578		\\prereq = {a.h}
579		\\prereq = {b.h}
580		\\prereq = {c.h}
581		\\target = {macho.o}
582		\\prereq = {macho.c}
583		\\prereq = {a.h}
584		\\prereq = {b.h}
585		\\prereq = {c.h}
586		);
587		try depTokenizer(
588		\\a$$scii.o: ascii.c
589		\\\\base64.o: "\base64.c" "s t#dio.h"
590		\\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$"
591		\\macho.o: \
592		\\ "macho!.c" \
593		\\ a.h b.h c.h
594		,
595		\\target = {a$scii.o}
596		\\prereq = {ascii.c}
597		\\target = {\base64.o}
598		\\prereq = {\base64.c}
599		\\prereq = {s t#dio.h}
600		\\target = {e\lf.o}
601		\\prereq = {e\lf.c}
602		\\prereq = {a.h$$}
603		\\prereq = {$$b.h c.h$$}
604		\\target = {macho.o}
605		\\prereq = {macho!.c}
606		\\prereq = {a.h}
607		\\prereq = {b.h}
608		\\prereq = {c.h}
609		);
610		}
611
612		test "windows quoted prereqs" {
613		try depTokenizer(
614		\\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c"
615		\\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \
616		\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \
617		\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h"
618		,
619		\\target = {c:\foo.o}
620		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c}
621		\\target = {c:\foo2.o}
622		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c}
623		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h}
624		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h}
625		);
626		}
627
628		test "windows mixed prereqs" {
629		try depTokenizer(
630		\\cimport.o: \
631		\\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \
632		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \
633		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \
634		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \
635		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \
636		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \
637		\\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \
638		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \
639		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \
640		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \
641		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \
642		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \
643		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \
644		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \
645		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \
646		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h"
647		,
648		\\target = {cimport.o}
649		\\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h}
650		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h}
651		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h}
652		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h}
653		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h}
654		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h}
655		\\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h}
656		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h}
657		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h}
658		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h}
659		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h}
660		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h}
661		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h}
662		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h}
663		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h}
664		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h}
665		);
666		}
667
668		test "funky targets" {
669		try depTokenizer(
670		\\C:\Users\anon\foo.o:
671		\\C:\Users\anon\foo\ .o:
672		\\C:\Users\anon\foo\#.o:
673		\\C:\Users\anon\foo$$.o:
674		\\C:\Users\anon\\\ foo.o:
675		\\C:\Users\anon\\#foo.o:
676		\\C:\Users\anon\$$foo.o:
677		\\C:\Users\anon\\\ \ \ \ \ foo.o:
678		,
679		\\target = {C:\Users\anon\foo.o}
680		\\target = {C:\Users\anon\foo .o}
681		\\target = {C:\Users\anon\foo#.o}
682		\\target = {C:\Users\anon\foo$.o}
683		\\target = {C:\Users\anon\ foo.o}
684		\\target = {C:\Users\anon\#foo.o}
685		\\target = {C:\Users\anon\$foo.o}
686		\\target = {C:\Users\anon\ foo.o}
687		);
688		}
689
690		test "error incomplete escape - reverse_solidus" {
691		try depTokenizer("\\",
692		\\ERROR: illegal char '\' at position 0: incomplete escape
693		);
694		try depTokenizer("\t\\",
695		\\ERROR: illegal char '\' at position 1: incomplete escape
696		);
697		try depTokenizer("\n\\",
698		\\ERROR: illegal char '\' at position 1: incomplete escape
699		);
700		try depTokenizer("\r\\",
701		\\ERROR: illegal char '\' at position 1: incomplete escape
702		);
703		try depTokenizer("\r\n\\",
704		\\ERROR: illegal char '\' at position 2: incomplete escape
705		);
706		try depTokenizer(" \\",
707		\\ERROR: illegal char '\' at position 1: incomplete escape
708		);
709		}
710
711		test "error incomplete escape - dollar_sign" {
712		try depTokenizer("$",
713		\\ERROR: illegal char '$' at position 0: incomplete escape
714		);
715		try depTokenizer("\t$",
716		\\ERROR: illegal char '$' at position 1: incomplete escape
717		);
718		try depTokenizer("\n$",
719		\\ERROR: illegal char '$' at position 1: incomplete escape
720		);
721		try depTokenizer("\r$",
722		\\ERROR: illegal char '$' at position 1: incomplete escape
723		);
724		try depTokenizer("\r\n$",
725		\\ERROR: illegal char '$' at position 2: incomplete escape
726		);
727		try depTokenizer(" $",
728		\\ERROR: illegal char '$' at position 1: incomplete escape
729		);
730		}
731
732		test "error incomplete target" {
733		try depTokenizer("foo.o",
734		\\ERROR: incomplete target 'foo.o' at position 0
735		);
736		try depTokenizer("\tfoo.o",
737		\\ERROR: incomplete target 'foo.o' at position 1
738		);
739		try depTokenizer("\nfoo.o",
740		\\ERROR: incomplete target 'foo.o' at position 1
741		);
742		try depTokenizer("\rfoo.o",
743		\\ERROR: incomplete target 'foo.o' at position 1
744		);
745		try depTokenizer("\r\nfoo.o",
746		\\ERROR: incomplete target 'foo.o' at position 2
747		);
748		try depTokenizer(" foo.o",
749		\\ERROR: incomplete target 'foo.o' at position 1
750		);
751
752		try depTokenizer("\\ foo.o",
753		\\ERROR: incomplete target ' foo.o' at position 1
754		);
755		try depTokenizer("\\#foo.o",
756		\\ERROR: incomplete target '#foo.o' at position 1
757		);
758		try depTokenizer("\\\\foo.o",
759		\\ERROR: incomplete target '\foo.o' at position 1
760		);
761		try depTokenizer("$$foo.o",
762		\\ERROR: incomplete target '$foo.o' at position 1
763		);
764		}
765
766		test "error illegal char at position - bad target escape" {
767		try depTokenizer("\\\t",
768		\\ERROR: illegal char \x09 at position 1: bad target escape
769		);
770		try depTokenizer("\\\n",
771		\\ERROR: illegal char \x0A at position 1: bad target escape
772		);
773		try depTokenizer("\\\r",
774		\\ERROR: illegal char \x0D at position 1: bad target escape
775		);
776		try depTokenizer("\\\r\n",
777		\\ERROR: illegal char \x0D at position 1: bad target escape
778		);
779		}
780
781		test "error illegal char at position - execting dollar_sign" {
782		try depTokenizer("$\t",
783		\\ERROR: illegal char \x09 at position 1: expecting '$'
784		);
785		try depTokenizer("$\n",
786		\\ERROR: illegal char \x0A at position 1: expecting '$'
787		);
788		try depTokenizer("$\r",
789		\\ERROR: illegal char \x0D at position 1: expecting '$'
790		);
791		try depTokenizer("$\r\n",
792		\\ERROR: illegal char \x0D at position 1: expecting '$'
793		);
794		}
795
796		test "error illegal char at position - invalid target" {
797		try depTokenizer("foo\t.o",
798		\\ERROR: illegal char \x09 at position 3: invalid target
799		);
800		try depTokenizer("foo\n.o",
801		\\ERROR: illegal char \x0A at position 3: invalid target
802		);
803		try depTokenizer("foo\r.o",
804		\\ERROR: illegal char \x0D at position 3: invalid target
805		);
806		try depTokenizer("foo\r\n.o",
807		\\ERROR: illegal char \x0D at position 3: invalid target
808		);
809		}
810
811		test "error target - continuation expecting end-of-line" {
812		try depTokenizer("foo.o: \\\t",
813		\\target = {foo.o}
814		\\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line
815		);
816		try depTokenizer("foo.o: \\ ",
817		\\target = {foo.o}
818		\\ERROR: illegal char \x20 at position 8: continuation expecting end-of-line
819		);
820		try depTokenizer("foo.o: \\x",
821		\\target = {foo.o}
822		\\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
823		);
824		try depTokenizer("foo.o: \\\x0dx",
825		\\target = {foo.o}
826		\\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
827		);
828		}
829
830		test "error prereq - continuation expecting end-of-line" {
831		try depTokenizer("foo.o: foo.h\\\x0dx",
832		\\target = {foo.o}
833		\\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
834		);
835		}
836
837		// - tokenize input, emit textual representation, and compare to expect
838		fn depTokenizer(input: []const u8, expect: []const u8) !void {
839		var arena_allocator = std.heap.ArenaAllocator.init(std.heap.page_allocator);
840		const arena = &arena_allocator.allocator;
841		defer arena_allocator.deinit();
842
843		var it = Tokenizer.init(arena, input);
844		var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);
845		var i: usize = 0;
846		while (true) {
847		const r = it.next() catch \|err\| {
848		switch (err) {
849		Tokenizer.Error.InvalidInput => {
850		if (i != 0) try buffer.appendSlice("\n");
851		try buffer.appendSlice("ERROR: ");
852		try buffer.appendSlice(it.error_text);
853		},
854		else => return err,
855		}
856		break;
857		};
858		const token = r orelse break;
859		if (i != 0) try buffer.appendSlice("\n");
860		try buffer.appendSlice(@tagName(token.id));
861		try buffer.appendSlice(" = {");
862		for (token.bytes) \|b\| {
863		try buffer.append(printable_char_tab[b]);
864		}
865		try buffer.appendSlice("}");
866		i += 1;
867		}
868		const got: []const u8 = buffer.span();
869
870		if (std.mem.eql(u8, expect, got)) {
871		testing.expect(true);
872		return;
873		}
874
875		var out = makeOutput(std.fs.File.write, try std.io.getStdErr());
876
877		try out.write("\n");
878		try printSection(&out, "<<<< input", input);
879		try printSection(&out, "==== expect", expect);
880		try printSection(&out, ">>>> got", got);
881		try printRuler(&out);
882
883		testing.expect(false);
884		}
885
886		fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
887		try printLabel(out, label, bytes);
888		try hexDump(out, bytes);
889		try printRuler(out);
890		try out.write(bytes);
891		try out.write("\n");
892		}
893
894		fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
895		var buf: [80]u8 = undefined;
896		var text = try std.fmt.bufPrint(buf[0..], "{} {} bytes ", .{ label, bytes.len });
897		try out.write(text);
898		var i: usize = text.len;
899		const end = 79;
900		while (i < 79) : (i += 1) {
901		try out.write([_]u8{label[0]});
902		}
903		try out.write("\n");
904		}
905
906		fn printRuler(out: anytype) !void {
907		var i: usize = 0;
908		const end = 79;
909		while (i < 79) : (i += 1) {
910		try out.write("-");
911		}
912		try out.write("\n");
913		}
914
915		fn hexDump(out: anytype, bytes: []const u8) !void {
916		const n16 = bytes.len >> 4;
917		var line: usize = 0;
918		var offset: usize = 0;
919		while (line < n16) : (line += 1) {
920		try hexDump16(out, offset, bytes[offset .. offset + 16]);
921		offset += 16;
922		}
923
924		const n = bytes.len & 0x0f;
925		if (n > 0) {
926		try printDecValue(out, offset, 8);
927		try out.write(":");
928		try out.write(" ");
929		var end1 = std.math.min(offset + n, offset + 8);
930		for (bytes[offset..end1]) \|b\| {
931		try out.write(" ");
932		try printHexValue(out, b, 2);
933		}
934		var end2 = offset + n;
935		if (end2 > end1) {
936		try out.write(" ");
937		for (bytes[end1..end2]) \|b\| {
938		try out.write(" ");
939		try printHexValue(out, b, 2);
940		}
941		}
942		const short = 16 - n;
943		var i: usize = 0;
944		while (i < short) : (i += 1) {
945		try out.write(" ");
946		}
947		if (end2 > end1) {
948		try out.write(" \|");
949		} else {
950		try out.write(" \|");
951		}
952		try printCharValues(out, bytes[offset..end2]);
953		try out.write("\|\n");
954		offset += n;
955		}
956
957		try printDecValue(out, offset, 8);
958		try out.write(":");
959		try out.write("\n");
960		}
961
962		fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void {
963		try printDecValue(out, offset, 8);
964		try out.write(":");
965		try out.write(" ");
966		for (bytes[0..8]) \|b\| {
967		try out.write(" ");
968		try printHexValue(out, b, 2);
969		}
970		try out.write(" ");
971		for (bytes[8..16]) \|b\| {
972		try out.write(" ");
973		try printHexValue(out, b, 2);
974		}
975		try out.write(" \|");
976		try printCharValues(out, bytes);
977		try out.write("\|\n");
978		}
979
980		fn printDecValue(out: anytype, value: u64, width: u8) !void {
981		var buffer: [20]u8 = undefined;
982		const len = std.fmt.formatIntBuf(buffer[0..], value, 10, false, width);
983		try out.write(buffer[0..len]);
984		}
985
986		fn printHexValue(out: anytype, value: u64, width: u8) !void {
987		var buffer: [16]u8 = undefined;
988		const len = std.fmt.formatIntBuf(buffer[0..], value, 16, false, width);
989		try out.write(buffer[0..len]);
990		}
991
992		fn printCharValues(out: anytype, bytes: []const u8) !void {
993		for (bytes) \|b\| {
994		try out.write(&[_]u8{printable_char_tab[b]});
995		}
996		}
997
998		fn printUnderstandableChar(buffer: *std.ArrayListSentineled(u8, 0), char: u8) !void {
999		if (!std.ascii.isPrint(char) or char == ' ') {
1000		try buffer.outStream().print("\\x{X:2}", .{char});
1001		} else {
1002		try buffer.appendSlice("'");
1003		try buffer.append(printable_char_tab[char]);
1004		try buffer.appendSlice("'");
1005		}
1006		}
1007
1008		// zig fmt: off
1009		const printable_char_tab: []const u8 =
1010		"................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
1011		"@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{\|}~." ++
1012		"................................................................" ++
1013		"................................................................";
1014		// zig fmt: on
1015		comptime {
1016		std.debug.assert(printable_char_tab.len == 256);
1017		}
1018
1019		// Make an output var that wraps a context and output function.
1020		// output: must be a function that takes a `self` idiom parameter
1021		// and a bytes parameter
1022		// context: must be that self
1023		fn makeOutput(comptime output: anytype, context: anytype) Output(output, @TypeOf(context)) {
1024		return Output(output, @TypeOf(context)){
1025		.context = context,
1026		};
1027		}
1028
1029		fn Output(comptime output_func: anytype, comptime Context: type) type {
1030		return struct {
1031		context: Context,
1032
1033		pub const output = output_func;
1034
1035		fn write(self: @This(), bytes: []const u8) !void {
1036		try output_func(self.context, bytes);
1037		}
1038		};
1039		}