Commit d8d379faf1f656743d118e5e5cfa3dba1e537d65 | zig

author	Andrew Kelley <andrew@ziglang.org> 2017-12-11 16:18:06-05:00
committer	Andrew Kelley <andrew@ziglang.org> 2017-12-11 16:18:06-05:00
log	d8d379faf1f656743d118e5e5cfa3dba1e537d65
tree	13a399cacf74453e0758299787eceeb265968088
parent	a3a590a32ab8165955583b6ea76f89f19bb21740

self-hosted: refactor into multiple files

add return expression
add number literal

4 files changed, 1809 insertions(+), 1673 deletions(-)

src-self-hosted

src-self-hosted/ast.zig created+245

...	...	@@ -0,0 +1,245 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3	const ArrayList = std.ArrayList;
	4	const Token = @import("tokenizer.zig").Token;
	5	const mem = std.mem;
	6
	7	pub const Node = struct {
	8	id: Id,
	9
	10	pub const Id = enum {
	11	Root,
	12	VarDecl,
	13	Identifier,
	14	FnProto,
	15	ParamDecl,
	16	AddrOfExpr,
	17	Block,
	18	Return,
	19	IntegerLiteral,
	20	FloatLiteral,
	21	};
	22
	23	pub fn iterate(base: &Node, index: usize) -> ?&Node {
	24	return switch (base.id) {
	25	Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index),
	26	Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index),
	27	Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index),
	28	Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
	29	Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
	30	Id.AddrOfExpr => @fieldParentPtr(NodeAddrOfExpr, "base", base).iterate(index),
	31	Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index),
	32	Id.Return => @fieldParentPtr(NodeReturn, "base", base).iterate(index),
	33	Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
	34	Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
	35	};
	36	}
	37
	38	pub fn destroy(base: &Node, allocator: &mem.Allocator) {
	39	return switch (base.id) {
	40	Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
	41	Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
	42	Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
	43	Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
	44	Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
	45	Id.AddrOfExpr => allocator.destroy(@fieldParentPtr(NodeAddrOfExpr, "base", base)),
	46	Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
	47	Id.Return => allocator.destroy(@fieldParentPtr(NodeReturn, "base", base)),
	48	Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
	49	Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
	50	};
	51	}
	52	};
	53
	54	pub const NodeRoot = struct {
	55	base: Node,
	56	decls: ArrayList(&Node),
	57
	58	pub fn iterate(self: &NodeRoot, index: usize) -> ?&Node {
	59	if (index < self.decls.len) {
	60	return self.decls.items[self.decls.len - index - 1];
	61	}
	62	return null;
	63	}
	64	};
	65
	66	pub const NodeVarDecl = struct {
	67	base: Node,
	68	visib_token: ?Token,
	69	name_token: Token,
	70	eq_token: Token,
	71	mut_token: Token,
	72	comptime_token: ?Token,
	73	extern_token: ?Token,
	74	lib_name: ?&Node,
	75	type_node: ?&Node,
	76	align_node: ?&Node,
	77	init_node: ?&Node,
	78
	79	pub fn iterate(self: &NodeVarDecl, index: usize) -> ?&Node {
	80	var i = index;
	81
	82	if (self.type_node) \|type_node\| {
	83	if (i < 1) return type_node;
	84	i -= 1;
	85	}
	86
	87	if (self.align_node) \|align_node\| {
	88	if (i < 1) return align_node;
	89	i -= 1;
	90	}
	91
	92	if (self.init_node) \|init_node\| {
	93	if (i < 1) return init_node;
	94	i -= 1;
	95	}
	96
	97	return null;
	98	}
	99	};
	100
	101	pub const NodeIdentifier = struct {
	102	base: Node,
	103	name_token: Token,
	104
	105	pub fn iterate(self: &NodeIdentifier, index: usize) -> ?&Node {
	106	return null;
	107	}
	108	};
	109
	110	pub const NodeFnProto = struct {
	111	base: Node,
	112	visib_token: ?Token,
	113	fn_token: Token,
	114	name_token: ?Token,
	115	params: ArrayList(&Node),
	116	return_type: ?&Node,
	117	var_args_token: ?Token,
	118	extern_token: ?Token,
	119	inline_token: ?Token,
	120	cc_token: ?Token,
	121	body_node: ?&Node,
	122	lib_name: ?&Node, // populated if this is an extern declaration
	123	align_expr: ?&Node, // populated if align(A) is present
	124
	125	pub fn iterate(self: &NodeFnProto, index: usize) -> ?&Node {
	126	var i = index;
	127
	128	if (self.body_node) \|body_node\| {
	129	if (i < 1) return body_node;
	130	i -= 1;
	131	}
	132
	133	if (self.return_type) \|return_type\| {
	134	if (i < 1) return return_type;
	135	i -= 1;
	136	}
	137
	138	if (self.align_expr) \|align_expr\| {
	139	if (i < 1) return align_expr;
	140	i -= 1;
	141	}
	142
	143	if (i < self.params.len) return self.params.items[self.params.len - i - 1];
	144	i -= self.params.len;
	145
	146	if (self.lib_name) \|lib_name\| {
	147	if (i < 1) return lib_name;
	148	i -= 1;
	149	}
	150
	151	return null;
	152	}
	153	};
	154
	155	pub const NodeParamDecl = struct {
	156	base: Node,
	157	comptime_token: ?Token,
	158	noalias_token: ?Token,
	159	name_token: ?Token,
	160	type_node: &Node,
	161	var_args_token: ?Token,
	162
	163	pub fn iterate(self: &NodeParamDecl, index: usize) -> ?&Node {
	164	var i = index;
	165
	166	if (i < 1) return self.type_node;
	167	i -= 1;
	168
	169	return null;
	170	}
	171	};
	172
	173	pub const NodeAddrOfExpr = struct {
	174	base: Node,
	175	op_token: Token,
	176	align_expr: ?&Node,
	177	bit_offset_start_token: ?Token,
	178	bit_offset_end_token: ?Token,
	179	const_token: ?Token,
	180	volatile_token: ?Token,
	181	op_expr: &Node,
	182
	183	pub fn iterate(self: &NodeAddrOfExpr, index: usize) -> ?&Node {
	184	var i = index;
	185
	186	if (self.align_expr) \|align_expr\| {
	187	if (i < 1) return align_expr;
	188	i -= 1;
	189	}
	190
	191	if (i < 1) return self.op_expr;
	192	i -= 1;
	193
	194	return null;
	195	}
	196	};
	197
	198	pub const NodeBlock = struct {
	199	base: Node,
	200	begin_token: Token,
	201	end_token: Token,
	202	statements: ArrayList(&Node),
	203
	204	pub fn iterate(self: &NodeBlock, index: usize) -> ?&Node {
	205	var i = index;
	206
	207	if (i < self.statements.len) return self.statements.items[i];
	208	i -= self.statements.len;
	209
	210	return null;
	211	}
	212	};
	213
	214	pub const NodeReturn = struct {
	215	base: Node,
	216	return_token: Token,
	217	expr: &Node,
	218
	219	pub fn iterate(self: &NodeReturn, index: usize) -> ?&Node {
	220	var i = index;
	221
	222	if (i < 1) return self.expr;
	223	i -= 1;
	224
	225	return null;
	226	}
	227	};
	228
	229	pub const NodeIntegerLiteral = struct {
	230	base: Node,
	231	token: Token,
	232
	233	pub fn iterate(self: &NodeIntegerLiteral, index: usize) -> ?&Node {
	234	return null;
	235	}
	236	};
	237
	238	pub const NodeFloatLiteral = struct {
	239	base: Node,
	240	token: Token,
	241
	242	pub fn iterate(self: &NodeFloatLiteral, index: usize) -> ?&Node {
	243	return null;
	244	}
	245	};

src-self-hosted/main.zig+11-1673

...	...	@@ -1,1682 +1,13 @@
1	1	const std = @import("std");
	2	const mem = std.mem;
2	3	const builtin = @import("builtin");
3	4	const io = std.io;
4	5	const os = std.os;
5	6	const heap = std.heap;
6	7	const warn = std.debug.warn;
7		const assert = std.debug.assert;
8		const mem = std.mem;
9		const ArrayList = std.ArrayList;
10		const AlignedArrayList = std.AlignedArrayList;
11		const math = std.math;
12
13
14		const Token = struct {
15		id: Id,
16		start: usize,
17		end: usize,
18
19		const KeywordId = struct {
20		bytes: []const u8,
21		id: Id,
22		};
23
24		const keywords = []KeywordId {
25		KeywordId{.bytes="align", .id = Id.Keyword_align},
26		KeywordId{.bytes="and", .id = Id.Keyword_and},
27		KeywordId{.bytes="asm", .id = Id.Keyword_asm},
28		KeywordId{.bytes="break", .id = Id.Keyword_break},
29		KeywordId{.bytes="coldcc", .id = Id.Keyword_coldcc},
30		KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
31		KeywordId{.bytes="const", .id = Id.Keyword_const},
32		KeywordId{.bytes="continue", .id = Id.Keyword_continue},
33		KeywordId{.bytes="defer", .id = Id.Keyword_defer},
34		KeywordId{.bytes="else", .id = Id.Keyword_else},
35		KeywordId{.bytes="enum", .id = Id.Keyword_enum},
36		KeywordId{.bytes="error", .id = Id.Keyword_error},
37		KeywordId{.bytes="export", .id = Id.Keyword_export},
38		KeywordId{.bytes="extern", .id = Id.Keyword_extern},
39		KeywordId{.bytes="false", .id = Id.Keyword_false},
40		KeywordId{.bytes="fn", .id = Id.Keyword_fn},
41		KeywordId{.bytes="for", .id = Id.Keyword_for},
42		KeywordId{.bytes="goto", .id = Id.Keyword_goto},
43		KeywordId{.bytes="if", .id = Id.Keyword_if},
44		KeywordId{.bytes="inline", .id = Id.Keyword_inline},
45		KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
46		KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
47		KeywordId{.bytes="null", .id = Id.Keyword_null},
48		KeywordId{.bytes="or", .id = Id.Keyword_or},
49		KeywordId{.bytes="packed", .id = Id.Keyword_packed},
50		KeywordId{.bytes="pub", .id = Id.Keyword_pub},
51		KeywordId{.bytes="return", .id = Id.Keyword_return},
52		KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
53		KeywordId{.bytes="struct", .id = Id.Keyword_struct},
54		KeywordId{.bytes="switch", .id = Id.Keyword_switch},
55		KeywordId{.bytes="test", .id = Id.Keyword_test},
56		KeywordId{.bytes="this", .id = Id.Keyword_this},
57		KeywordId{.bytes="true", .id = Id.Keyword_true},
58		KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
59		KeywordId{.bytes="union", .id = Id.Keyword_union},
60		KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
61		KeywordId{.bytes="use", .id = Id.Keyword_use},
62		KeywordId{.bytes="var", .id = Id.Keyword_var},
63		KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
64		KeywordId{.bytes="while", .id = Id.Keyword_while},
65		};
66
67		fn getKeyword(bytes: []const u8) -> ?Id {
68		for (keywords) \|kw\| {
69		if (mem.eql(u8, kw.bytes, bytes)) {
70		return kw.id;
71		}
72		}
73		return null;
74		}
75
76		const StrLitKind = enum {Normal, C};
77
78		const Id = union(enum) {
79		Invalid,
80		Identifier,
81		StringLiteral: StrLitKind,
82		Eof,
83		Builtin,
84		Equal,
85		LParen,
86		RParen,
87		Semicolon,
88		Percent,
89		LBrace,
90		RBrace,
91		Period,
92		Ellipsis2,
93		Ellipsis3,
94		Minus,
95		Arrow,
96		Colon,
97		Slash,
98		Comma,
99		Ampersand,
100		AmpersandEqual,
101		NumberLiteral,
102		Keyword_align,
103		Keyword_and,
104		Keyword_asm,
105		Keyword_break,
106		Keyword_coldcc,
107		Keyword_comptime,
108		Keyword_const,
109		Keyword_continue,
110		Keyword_defer,
111		Keyword_else,
112		Keyword_enum,
113		Keyword_error,
114		Keyword_export,
115		Keyword_extern,
116		Keyword_false,
117		Keyword_fn,
118		Keyword_for,
119		Keyword_goto,
120		Keyword_if,
121		Keyword_inline,
122		Keyword_nakedcc,
123		Keyword_noalias,
124		Keyword_null,
125		Keyword_or,
126		Keyword_packed,
127		Keyword_pub,
128		Keyword_return,
129		Keyword_stdcallcc,
130		Keyword_struct,
131		Keyword_switch,
132		Keyword_test,
133		Keyword_this,
134		Keyword_true,
135		Keyword_undefined,
136		Keyword_union,
137		Keyword_unreachable,
138		Keyword_use,
139		Keyword_var,
140		Keyword_volatile,
141		Keyword_while,
142		};
143		};
144
145		const Tokenizer = struct {
146		buffer: []const u8,
147		index: usize,
148
149		pub const Location = struct {
150		line: usize,
151		column: usize,
152		line_start: usize,
153		line_end: usize,
154		};
155
156		pub fn getTokenLocation(self: &Tokenizer, token: &const Token) -> Location {
157		var loc = Location {
158		.line = 0,
159		.column = 0,
160		.line_start = 0,
161		.line_end = 0,
162		};
163		for (self.buffer) \|c, i\| {
164		if (i == token.start) {
165		loc.line_end = i;
166		while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
167		return loc;
168		}
169		if (c == '\n') {
170		loc.line += 1;
171		loc.column = 0;
172		loc.line_start = i + 1;
173		} else {
174		loc.column += 1;
175		}
176		}
177		return loc;
178		}
179
180		pub fn dump(self: &Tokenizer, token: &const Token) {
181		warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
182		}
183
184		pub fn init(buffer: []const u8) -> Tokenizer {
185		return Tokenizer {
186		.buffer = buffer,
187		.index = 0,
188		};
189		}
190
191		const State = enum {
192		Start,
193		Identifier,
194		Builtin,
195		C,
196		StringLiteral,
197		StringLiteralBackslash,
198		Minus,
199		Slash,
200		LineComment,
201		Zero,
202		NumberLiteral,
203		NumberDot,
204		FloatFraction,
205		FloatExponentUnsigned,
206		FloatExponentNumber,
207		Ampersand,
208		Period,
209		Period2,
210		};
211
212		pub fn next(self: &Tokenizer) -> Token {
213		var state = State.Start;
214		var result = Token {
215		.id = Token.Id.Eof,
216		.start = self.index,
217		.end = undefined,
218		};
219		while (self.index < self.buffer.len) : (self.index += 1) {
220		const c = self.buffer[self.index];
221		switch (state) {
222		State.Start => switch (c) {
223		' ', '\n' => {
224		result.start = self.index + 1;
225		},
226		'c' => {
227		state = State.C;
228		result.id = Token.Id.Identifier;
229		},
230		'"' => {
231		state = State.StringLiteral;
232		result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
233		},
234		'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
235		state = State.Identifier;
236		result.id = Token.Id.Identifier;
237		},
238		'@' => {
239		state = State.Builtin;
240		result.id = Token.Id.Builtin;
241		},
242		'=' => {
243		result.id = Token.Id.Equal;
244		self.index += 1;
245		break;
246		},
247		'(' => {
248		result.id = Token.Id.LParen;
249		self.index += 1;
250		break;
251		},
252		')' => {
253		result.id = Token.Id.RParen;
254		self.index += 1;
255		break;
256		},
257		';' => {
258		result.id = Token.Id.Semicolon;
259		self.index += 1;
260		break;
261		},
262		',' => {
263		result.id = Token.Id.Comma;
264		self.index += 1;
265		break;
266		},
267		':' => {
268		result.id = Token.Id.Colon;
269		self.index += 1;
270		break;
271		},
272		'%' => {
273		result.id = Token.Id.Percent;
274		self.index += 1;
275		break;
276		},
277		'{' => {
278		result.id = Token.Id.LBrace;
279		self.index += 1;
280		break;
281		},
282		'}' => {
283		result.id = Token.Id.RBrace;
284		self.index += 1;
285		break;
286		},
287		'.' => {
288		state = State.Period;
289		},
290		'-' => {
291		state = State.Minus;
292		},
293		'/' => {
294		state = State.Slash;
295		},
296		'&' => {
297		state = State.Ampersand;
298		},
299		'0' => {
300		state = State.Zero;
301		result.id = Token.Id.NumberLiteral;
302		},
303		'1'...'9' => {
304		state = State.NumberLiteral;
305		result.id = Token.Id.NumberLiteral;
306		},
307		else => {
308		result.id = Token.Id.Invalid;
309		self.index += 1;
310		break;
311		},
312		},
313		State.Ampersand => switch (c) {
314		'=' => {
315		result.id = Token.Id.AmpersandEqual;
316		self.index += 1;
317		break;
318		},
319		else => {
320		result.id = Token.Id.Ampersand;
321		break;
322		},
323		},
324		State.Identifier => switch (c) {
325		'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
326		else => {
327		if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
328		result.id = id;
329		}
330		break;
331		},
332		},
333		State.Builtin => switch (c) {
334		'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
335		else => break,
336		},
337		State.C => switch (c) {
338		'\\' => @panic("TODO"),
339		'"' => {
340		state = State.StringLiteral;
341		result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
342		},
343		'a'...'z', 'A'...'Z', '_', '0'...'9' => {
344		state = State.Identifier;
345		},
346		else => break,
347		},
348		State.StringLiteral => switch (c) {
349		'\\' => {
350		state = State.StringLiteralBackslash;
351		},
352		'"' => {
353		self.index += 1;
354		break;
355		},
356		'\n' => break, // Look for this error later.
357		else => {},
358		},
359
360		State.StringLiteralBackslash => switch (c) {
361		'\n' => break, // Look for this error later.
362		else => {
363		state = State.StringLiteral;
364		},
365		},
366
367		State.Minus => switch (c) {
368		'>' => {
369		result.id = Token.Id.Arrow;
370		self.index += 1;
371		break;
372		},
373		else => {
374		result.id = Token.Id.Minus;
375		break;
376		},
377		},
378
379		State.Period => switch (c) {
380		'.' => {
381		state = State.Period2;
382		},
383		else => {
384		result.id = Token.Id.Period;
385		break;
386		},
387		},
388
389		State.Period2 => switch (c) {
390		'.' => {
391		result.id = Token.Id.Ellipsis3;
392		self.index += 1;
393		break;
394		},
395		else => {
396		result.id = Token.Id.Ellipsis2;
397		break;
398		},
399		},
400
401		State.Slash => switch (c) {
402		'/' => {
403		result.id = undefined;
404		state = State.LineComment;
405		},
406		else => {
407		result.id = Token.Id.Slash;
408		break;
409		},
410		},
411		State.LineComment => switch (c) {
412		'\n' => {
413		state = State.Start;
414		result = Token {
415		.id = Token.Id.Eof,
416		.start = self.index + 1,
417		.end = undefined,
418		};
419		},
420		else => {},
421		},
422		State.Zero => switch (c) {
423		'b', 'o', 'x' => {
424		state = State.NumberLiteral;
425		},
426		else => {
427		// reinterpret as a normal number
428		self.index -= 1;
429		state = State.NumberLiteral;
430		},
431		},
432		State.NumberLiteral => switch (c) {
433		'.' => {
434		state = State.NumberDot;
435		},
436		'p', 'P', 'e', 'E' => {
437		state = State.FloatExponentUnsigned;
438		},
439		'0'...'9', 'a'...'f', 'A'...'F' => {},
440		else => break,
441		},
442		State.NumberDot => switch (c) {
443		'.' => {
444		self.index -= 1;
445		state = State.Start;
446		break;
447		},
448		else => {
449		self.index -= 1;
450		state = State.FloatFraction;
451		},
452		},
453		State.FloatFraction => switch (c) {
454		'p', 'P', 'e', 'E' => {
455		state = State.FloatExponentUnsigned;
456		},
457		'0'...'9', 'a'...'f', 'A'...'F' => {},
458		else => break,
459		},
460		State.FloatExponentUnsigned => switch (c) {
461		'+', '-' => {
462		state = State.FloatExponentNumber;
463		},
464		else => {
465		// reinterpret as a normal exponent number
466		self.index -= 1;
467		state = State.FloatExponentNumber;
468		}
469		},
470		State.FloatExponentNumber => switch (c) {
471		'0'...'9', 'a'...'f', 'A'...'F' => {},
472		else => break,
473		},
474		}
475		}
476		result.end = self.index;
477		// TODO check state when returning EOF
478		return result;
479		}
480
481		pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) -> []const u8 {
482		return self.buffer[token.start..token.end];
483		}
484		};
485
486		const Comptime = enum { No, Yes };
487		const NoAlias = enum { No, Yes };
488		const Extern = enum { No, Yes };
489		const VarArgs = enum { No, Yes };
490		const Mutability = enum { Const, Var };
491		const Volatile = enum { No, Yes };
492
493		const Inline = enum {
494		Auto,
495		Always,
496		Never,
497		};
498
499		const Visibility = enum {
500		Private,
501		Pub,
502		Export,
503		};
504
505		const CallingConvention = enum {
506		Auto,
507		C,
508		Cold,
509		Naked,
510		Stdcall,
511		};
512
513		const AstNode = struct {
514		id: Id,
515
516		const Id = enum {
517		Root,
518		VarDecl,
519		Identifier,
520		FnProto,
521		ParamDecl,
522		AddrOfExpr,
523		Block,
524		};
525
526		fn iterate(base: &AstNode, index: usize) -> ?&AstNode {
527		return switch (base.id) {
528		Id.Root => @fieldParentPtr(AstNodeRoot, "base", base).iterate(index),
529		Id.VarDecl => @fieldParentPtr(AstNodeVarDecl, "base", base).iterate(index),
530		Id.Identifier => @fieldParentPtr(AstNodeIdentifier, "base", base).iterate(index),
531		Id.FnProto => @fieldParentPtr(AstNodeFnProto, "base", base).iterate(index),
532		Id.ParamDecl => @fieldParentPtr(AstNodeParamDecl, "base", base).iterate(index),
533		Id.AddrOfExpr => @fieldParentPtr(AstNodeAddrOfExpr, "base", base).iterate(index),
534		Id.Block => @fieldParentPtr(AstNodeBlock, "base", base).iterate(index),
535		};
536		}
537
538		fn destroy(base: &AstNode, allocator: &mem.Allocator) {
539		return switch (base.id) {
540		Id.Root => allocator.destroy(@fieldParentPtr(AstNodeRoot, "base", base)),
541		Id.VarDecl => allocator.destroy(@fieldParentPtr(AstNodeVarDecl, "base", base)),
542		Id.Identifier => allocator.destroy(@fieldParentPtr(AstNodeIdentifier, "base", base)),
543		Id.FnProto => allocator.destroy(@fieldParentPtr(AstNodeFnProto, "base", base)),
544		Id.ParamDecl => allocator.destroy(@fieldParentPtr(AstNodeParamDecl, "base", base)),
545		Id.AddrOfExpr => allocator.destroy(@fieldParentPtr(AstNodeAddrOfExpr, "base", base)),
546		Id.Block => allocator.destroy(@fieldParentPtr(AstNodeBlock, "base", base)),
547		};
548		}
549		};
550
551		const AstNodeRoot = struct {
552		base: AstNode,
553		decls: ArrayList(&AstNode),
554
555		fn iterate(self: &AstNodeRoot, index: usize) -> ?&AstNode {
556		if (index < self.decls.len) {
557		return self.decls.items[self.decls.len - index - 1];
558		}
559		return null;
560		}
561		};
562
563		const AstNodeVarDecl = struct {
564		base: AstNode,
565		visib_token: ?Token,
566		name_token: Token,
567		eq_token: Token,
568		mut_token: Token,
569		comptime_token: ?Token,
570		extern_token: ?Token,
571		lib_name: ?&AstNode,
572		type_node: ?&AstNode,
573		align_node: ?&AstNode,
574		init_node: ?&AstNode,
575
576		fn iterate(self: &AstNodeVarDecl, index: usize) -> ?&AstNode {
577		var i = index;
578
579		if (self.type_node) \|type_node\| {
580		if (i < 1) return type_node;
581		i -= 1;
582		}
583
584		if (self.align_node) \|align_node\| {
585		if (i < 1) return align_node;
586		i -= 1;
587		}
588
589		if (self.init_node) \|init_node\| {
590		if (i < 1) return init_node;
591		i -= 1;
592		}
593
594		return null;
595		}
596		};
597
598		const AstNodeIdentifier = struct {
599		base: AstNode,
600		name_token: Token,
601
602		fn iterate(self: &AstNodeIdentifier, index: usize) -> ?&AstNode {
603		return null;
604		}
605		};
606
607		const AstNodeFnProto = struct {
608		base: AstNode,
609		visib_token: ?Token,
610		fn_token: Token,
611		name_token: ?Token,
612		params: ArrayList(&AstNode),
613		return_type: ?&AstNode,
614		var_args_token: ?Token,
615		extern_token: ?Token,
616		inline_token: ?Token,
617		cc_token: ?Token,
618		body_node: ?&AstNode,
619		lib_name: ?&AstNode, // populated if this is an extern declaration
620		align_expr: ?&AstNode, // populated if align(A) is present
621
622		fn iterate(self: &AstNodeFnProto, index: usize) -> ?&AstNode {
623		var i = index;
624
625		if (self.body_node) \|body_node\| {
626		if (i < 1) return body_node;
627		i -= 1;
628		}
629
630		if (self.return_type) \|return_type\| {
631		if (i < 1) return return_type;
632		i -= 1;
633		}
634
635		if (self.align_expr) \|align_expr\| {
636		if (i < 1) return align_expr;
637		i -= 1;
638		}
639
640		if (i < self.params.len) return self.params.items[self.params.len - i - 1];
641		i -= self.params.len;
642
643		if (self.lib_name) \|lib_name\| {
644		if (i < 1) return lib_name;
645		i -= 1;
646		}
647
648		return null;
649		}
650		};
651
652		const AstNodeParamDecl = struct {
653		base: AstNode,
654		comptime_token: ?Token,
655		noalias_token: ?Token,
656		name_token: ?Token,
657		type_node: &AstNode,
658		var_args_token: ?Token,
659
660		fn iterate(self: &AstNodeParamDecl, index: usize) -> ?&AstNode {
661		var i = index;
662
663		if (i < 1) return self.type_node;
664		i -= 1;
665
666		return null;
667		}
668		};
669
670		const AstNodeAddrOfExpr = struct {
671		base: AstNode,
672		op_token: Token,
673		align_expr: ?&AstNode,
674		bit_offset_start_token: ?Token,
675		bit_offset_end_token: ?Token,
676		const_token: ?Token,
677		volatile_token: ?Token,
678		op_expr: &AstNode,
679
680		fn iterate(self: &AstNodeAddrOfExpr, index: usize) -> ?&AstNode {
681		var i = index;
682
683		if (self.align_expr) \|align_expr\| {
684		if (i < 1) return align_expr;
685		i -= 1;
686		}
687
688		if (i < 1) return self.op_expr;
689		i -= 1;
690
691		return null;
692		}
693		};
694
695		const AstNodeBlock = struct {
696		base: AstNode,
697		begin_token: Token,
698		end_token: Token,
699		statements: ArrayList(&AstNode),
700
701		fn iterate(self: &AstNodeBlock, index: usize) -> ?&AstNode {
702		var i = index;
703
704		if (i < self.statements.len) return self.statements.items[i];
705		i -= self.statements.len;
706
707		return null;
708		}
709		};
710
711		error ParseError;
712
713		const Parser = struct {
714		allocator: &mem.Allocator,
715		tokenizer: &Tokenizer,
716		put_back_tokens: [2]Token,
717		put_back_count: usize,
718		source_file_name: []const u8,
719
720		// This memory contents are used only during a function call. It's used to repurpose memory;
721		// specifically so that freeAst can be guaranteed to succeed.
722		const utility_bytes_align = @alignOf( union { a: RenderAstFrame, b: State, c: RenderState } );
723		utility_bytes: []align(utility_bytes_align) u8,
724
725		fn initUtilityArrayList(self: &Parser, comptime T: type) -> ArrayList(T) {
726		const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
727		self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
728		const typed_slice = ([]T)(self.utility_bytes);
729		return ArrayList(T).fromOwnedSlice(self.allocator, typed_slice);
730		}
731
732		fn deinitUtilityArrayList(self: &Parser, list: var) {
733		self.utility_bytes = ([]align(utility_bytes_align) u8)(list.toOwnedSlice());
734		}
735
736		pub fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) -> Parser {
737		return Parser {
738		.allocator = allocator,
739		.tokenizer = tokenizer,
740		.put_back_tokens = undefined,
741		.put_back_count = 0,
742		.source_file_name = source_file_name,
743		.utility_bytes = []align(utility_bytes_align) u8{},
744		};
745		}
746
747		pub fn deinit(self: &Parser) {
748		self.allocator.free(self.utility_bytes);
749		}
750
751		const TopLevelDeclCtx = struct {
752		visib_token: ?Token,
753		extern_token: ?Token,
754		};
755
756		const DestPtr = union(enum) {
757		Field: &&AstNode,
758		NullableField: &?&AstNode,
759		List: &ArrayList(&AstNode),
760
761		pub fn store(self: &const DestPtr, value: &AstNode) -> %void {
762		switch (*self) {
763		DestPtr.Field => \|ptr\| *ptr = value,
764		DestPtr.NullableField => \|ptr\| *ptr = value,
765		DestPtr.List => \|list\| %return list.append(value),
766		}
767		}
768		};
769
770		const State = union(enum) {
771		TopLevel,
772		TopLevelExtern: ?Token,
773		TopLevelDecl: TopLevelDeclCtx,
774		Expression: DestPtr,
775		GroupedExpression: DestPtr,
776		UnwrapExpression: DestPtr,
777		BoolOrExpression: DestPtr,
778		BoolAndExpression: DestPtr,
779		ComparisonExpression: DestPtr,
780		BinaryOrExpression: DestPtr,
781		BinaryXorExpression: DestPtr,
782		BinaryAndExpression: DestPtr,
783		BitShiftExpression: DestPtr,
784		AdditionExpression: DestPtr,
785		MultiplyExpression: DestPtr,
786		BraceSuffixExpression: DestPtr,
787		PrefixOpExpression: DestPtr,
788		SuffixOpExpression: DestPtr,
789		PrimaryExpression: DestPtr,
790		TypeExpr: DestPtr,
791		VarDecl: &AstNodeVarDecl,
792		VarDeclAlign: &AstNodeVarDecl,
793		VarDeclEq: &AstNodeVarDecl,
794		ExpectToken: @TagType(Token.Id),
795		FnProto: &AstNodeFnProto,
796		FnProtoAlign: &AstNodeFnProto,
797		ParamDecl: &AstNodeFnProto,
798		ParamDeclComma,
799		FnDef: &AstNodeFnProto,
800		Block: &AstNodeBlock,
801		Statement: &AstNodeBlock,
802		};
803
804		pub fn freeAst(self: &Parser, root_node: &AstNodeRoot) {
805		// utility_bytes is big enough to do this iteration since we were able to do
806		// the parsing in the first place
807		comptime assert(@sizeOf(State) >= @sizeOf(&AstNode));
808
809		var stack = self.initUtilityArrayList(&AstNode);
810		defer self.deinitUtilityArrayList(stack);
811
812		stack.append(&root_node.base) %% unreachable;
813		while (stack.popOrNull()) \|node\| {
814		var i: usize = 0;
815		while (node.iterate(i)) \|child\| : (i += 1) {
816		if (child.iterate(0) != null) {
817		stack.append(child) %% unreachable;
818		} else {
819		child.destroy(self.allocator);
820		}
821		}
822		node.destroy(self.allocator);
823		}
824		}
825
826		pub fn parse(self: &Parser) -> %&AstNodeRoot {
827		var stack = self.initUtilityArrayList(State);
828		defer self.deinitUtilityArrayList(stack);
829
830		const root_node = %return self.createRoot();
831		%defer self.allocator.destroy(root_node);
832		%return stack.append(State.TopLevel);
833		%defer self.freeAst(root_node);
834
835		while (true) {
836		//{
837		// const token = self.getNextToken();
838		// warn("{} ", @tagName(token.id));
839		// self.putBackToken(token);
840		// var i: usize = stack.len;
841		// while (i != 0) {
842		// i -= 1;
843		// warn("{} ", @tagName(stack.items[i]));
844		// }
845		// warn("\n");
846		//}
847
848		// This gives us 1 free append that can't fail
849		const state = stack.pop();
850
851		switch (state) {
852		State.TopLevel => {
853		const token = self.getNextToken();
854		switch (token.id) {
855		Token.Id.Keyword_pub, Token.Id.Keyword_export => {
856		stack.append(State { .TopLevelExtern = token }) %% unreachable;
857		continue;
858		},
859		Token.Id.Eof => return root_node,
860		else => {
861		self.putBackToken(token);
862		// TODO shouldn't need this cast
863		stack.append(State { .TopLevelExtern = null }) %% unreachable;
864		continue;
865		},
866		}
867		},
868		State.TopLevelExtern => \|visib_token\| {
869		const token = self.getNextToken();
870		if (token.id == Token.Id.Keyword_extern) {
871		stack.append(State {
872		.TopLevelDecl = TopLevelDeclCtx {
873		.visib_token = visib_token,
874		.extern_token = token,
875		},
876		}) %% unreachable;
877		continue;
878		}
879		self.putBackToken(token);
880		stack.append(State {
881		.TopLevelDecl = TopLevelDeclCtx {
882		.visib_token = visib_token,
883		.extern_token = null,
884		},
885		}) %% unreachable;
886		continue;
887		},
888		State.TopLevelDecl => \|ctx\| {
889		const token = self.getNextToken();
890		switch (token.id) {
891		Token.Id.Keyword_var, Token.Id.Keyword_const => {
892		stack.append(State.TopLevel) %% unreachable;
893		// TODO shouldn't need these casts
894		const var_decl_node = %return self.createAttachVarDecl(&root_node.decls, ctx.visib_token,
895		token, (?Token)(null), ctx.extern_token);
896		%return stack.append(State { .VarDecl = var_decl_node });
897		continue;
898		},
899		Token.Id.Keyword_fn => {
900		stack.append(State.TopLevel) %% unreachable;
901		// TODO shouldn't need these casts
902		const fn_proto = %return self.createAttachFnProto(&root_node.decls, token,
903		ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null));
904		%return stack.append(State { .FnDef = fn_proto });
905		%return stack.append(State { .FnProto = fn_proto });
906		continue;
907		},
908		Token.Id.StringLiteral => {
909		@panic("TODO extern with string literal");
910		},
911		Token.Id.Keyword_coldcc, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
912		stack.append(State.TopLevel) %% unreachable;
913		const fn_token = %return self.eatToken(Token.Id.Keyword_fn);
914		// TODO shouldn't need this cast
915		const fn_proto = %return self.createAttachFnProto(&root_node.decls, fn_token,
916		ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null));
917		%return stack.append(State { .FnDef = fn_proto });
918		%return stack.append(State { .FnProto = fn_proto });
919		continue;
920		},
921		else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)),
922		}
923		},
924		State.VarDecl => \|var_decl\| {
925		var_decl.name_token = %return self.eatToken(Token.Id.Identifier);
926		stack.append(State { .VarDeclAlign = var_decl }) %% unreachable;
927
928		const next_token = self.getNextToken();
929		if (next_token.id == Token.Id.Colon) {
930		%return stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} });
931		continue;
932		}
933
934		self.putBackToken(next_token);
935		continue;
936		},
937		State.VarDeclAlign => \|var_decl\| {
938		stack.append(State { .VarDeclEq = var_decl }) %% unreachable;
939
940		const next_token = self.getNextToken();
941		if (next_token.id == Token.Id.Keyword_align) {
942		%return stack.append(State {
943		.GroupedExpression = DestPtr {
944		.NullableField = &var_decl.align_node
945		}
946		});
947		continue;
948		}
949
950		self.putBackToken(next_token);
951		continue;
952		},
953		State.VarDeclEq => \|var_decl\| {
954		const token = self.getNextToken();
955		if (token.id == Token.Id.Equal) {
956		var_decl.eq_token = token;
957		stack.append(State { .ExpectToken = Token.Id.Semicolon }) %% unreachable;
958		%return stack.append(State {
959		.Expression = DestPtr {.NullableField = &var_decl.init_node},
960		});
961		continue;
962		}
963		if (token.id == Token.Id.Semicolon) {
964		continue;
965		}
966		return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
967		},
968		State.ExpectToken => \|token_id\| {
969		_ = %return self.eatToken(token_id);
970		continue;
971		},
972		State.Expression => \|dest_ptr\| {
973		stack.append(State {.UnwrapExpression = dest_ptr}) %% unreachable;
974		continue;
975		},
976
977		State.UnwrapExpression => \|dest_ptr\| {
978		stack.append(State {.BoolOrExpression = dest_ptr}) %% unreachable;
979		continue;
980		},
981
982		State.BoolOrExpression => \|dest_ptr\| {
983		stack.append(State {.BoolAndExpression = dest_ptr}) %% unreachable;
984		continue;
985		},
986
987		State.BoolAndExpression => \|dest_ptr\| {
988		stack.append(State {.ComparisonExpression = dest_ptr}) %% unreachable;
989		continue;
990		},
991
992		State.ComparisonExpression => \|dest_ptr\| {
993		stack.append(State {.BinaryOrExpression = dest_ptr}) %% unreachable;
994		continue;
995		},
996
997		State.BinaryOrExpression => \|dest_ptr\| {
998		stack.append(State {.BinaryXorExpression = dest_ptr}) %% unreachable;
999		continue;
1000		},
1001
1002		State.BinaryXorExpression => \|dest_ptr\| {
1003		stack.append(State {.BinaryAndExpression = dest_ptr}) %% unreachable;
1004		continue;
1005		},
1006
1007		State.BinaryAndExpression => \|dest_ptr\| {
1008		stack.append(State {.BitShiftExpression = dest_ptr}) %% unreachable;
1009		continue;
1010		},
1011
1012		State.BitShiftExpression => \|dest_ptr\| {
1013		stack.append(State {.AdditionExpression = dest_ptr}) %% unreachable;
1014		continue;
1015		},
1016
1017		State.AdditionExpression => \|dest_ptr\| {
1018		stack.append(State {.MultiplyExpression = dest_ptr}) %% unreachable;
1019		continue;
1020		},
1021
1022		State.MultiplyExpression => \|dest_ptr\| {
1023		stack.append(State {.BraceSuffixExpression = dest_ptr}) %% unreachable;
1024		continue;
1025		},
1026
1027		State.BraceSuffixExpression => \|dest_ptr\| {
1028		stack.append(State {.PrefixOpExpression = dest_ptr}) %% unreachable;
1029		continue;
1030		},
1031
1032		State.PrefixOpExpression => \|dest_ptr\| {
1033		const first_token = self.getNextToken();
1034		if (first_token.id == Token.Id.Ampersand) {
1035		const addr_of_expr = %return self.createAttachAddrOfExpr(dest_ptr, first_token);
1036		var token = self.getNextToken();
1037		if (token.id == Token.Id.Keyword_align) {
1038		@panic("TODO align");
1039		}
1040		if (token.id == Token.Id.Keyword_const) {
1041		addr_of_expr.const_token = token;
1042		token = self.getNextToken();
1043		}
1044		if (token.id == Token.Id.Keyword_volatile) {
1045		addr_of_expr.volatile_token = token;
1046		token = self.getNextToken();
1047		}
1048		self.putBackToken(token);
1049		stack.append(State {
1050		.PrefixOpExpression = DestPtr { .Field = &addr_of_expr.op_expr},
1051		}) %% unreachable;
1052		continue;
1053		}
1054
1055		self.putBackToken(first_token);
1056		stack.append(State { .SuffixOpExpression = dest_ptr }) %% unreachable;
1057		continue;
1058		},
1059
1060		State.SuffixOpExpression => \|dest_ptr\| {
1061		stack.append(State { .PrimaryExpression = dest_ptr }) %% unreachable;
1062		continue;
1063		},
1064
1065		State.PrimaryExpression => \|dest_ptr\| {
1066		const token = self.getNextToken();
1067		switch (token.id) {
1068		Token.Id.Identifier => {
1069		const identifier = %return self.createIdentifier(token);
1070		%return dest_ptr.store(&identifier.base);
1071		continue;
1072		},
1073		else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
1074		}
1075		},
1076
1077		State.TypeExpr => \|dest_ptr\| {
1078		const token = self.getNextToken();
1079		if (token.id == Token.Id.Keyword_var) {
1080		@panic("TODO param with type var");
1081		}
1082		self.putBackToken(token);
1083
1084		stack.append(State { .PrefixOpExpression = dest_ptr }) %% unreachable;
1085		continue;
1086		},
1087
1088		State.FnProto => \|fn_proto\| {
1089		stack.append(State { .FnProtoAlign = fn_proto }) %% unreachable;
1090		%return stack.append(State { .ParamDecl = fn_proto });
1091		%return stack.append(State { .ExpectToken = Token.Id.LParen });
1092
1093		const next_token = self.getNextToken();
1094		if (next_token.id == Token.Id.Identifier) {
1095		fn_proto.name_token = next_token;
1096		continue;
1097		}
1098		self.putBackToken(next_token);
1099		continue;
1100		},
1101
1102		State.FnProtoAlign => \|fn_proto\| {
1103		const token = self.getNextToken();
1104		if (token.id == Token.Id.Keyword_align) {
1105		@panic("TODO fn proto align");
1106		}
1107		if (token.id == Token.Id.Arrow) {
1108		stack.append(State {
1109		.TypeExpr = DestPtr {.NullableField = &fn_proto.return_type},
1110		}) %% unreachable;
1111		continue;
1112		} else {
1113		self.putBackToken(token);
1114		continue;
1115		}
1116		},
1117
1118		State.ParamDecl => \|fn_proto\| {
1119		var token = self.getNextToken();
1120		if (token.id == Token.Id.RParen) {
1121		continue;
1122		}
1123		const param_decl = %return self.createAttachParamDecl(&fn_proto.params);
1124		if (token.id == Token.Id.Keyword_comptime) {
1125		param_decl.comptime_token = token;
1126		token = self.getNextToken();
1127		} else if (token.id == Token.Id.Keyword_noalias) {
1128		param_decl.noalias_token = token;
1129		token = self.getNextToken();
1130		};
1131		if (token.id == Token.Id.Identifier) {
1132		const next_token = self.getNextToken();
1133		if (next_token.id == Token.Id.Colon) {
1134		param_decl.name_token = token;
1135		token = self.getNextToken();
1136		} else {
1137		self.putBackToken(next_token);
1138		}
1139		}
1140		if (token.id == Token.Id.Ellipsis3) {
1141		param_decl.var_args_token = token;
1142		stack.append(State { .ExpectToken = Token.Id.RParen }) %% unreachable;
1143		continue;
1144		} else {
1145		self.putBackToken(token);
1146		}
1147
1148		stack.append(State { .ParamDecl = fn_proto }) %% unreachable;
1149		%return stack.append(State.ParamDeclComma);
1150		%return stack.append(State {
1151		.TypeExpr = DestPtr {.Field = &param_decl.type_node}
1152		});
1153		continue;
1154		},
1155
1156		State.ParamDeclComma => {
1157		const token = self.getNextToken();
1158		switch (token.id) {
1159		Token.Id.RParen => {
1160		_ = stack.pop(); // pop off the ParamDecl
1161		continue;
1162		},
1163		Token.Id.Comma => continue,
1164		else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
1165		}
1166		},
1167
1168		State.FnDef => \|fn_proto\| {
1169		const token = self.getNextToken();
1170		switch(token.id) {
1171		Token.Id.LBrace => {
1172		const block = %return self.createBlock(token);
1173		fn_proto.body_node = &block.base;
1174		stack.append(State { .Block = block }) %% unreachable;
1175		continue;
1176		},
1177		Token.Id.Semicolon => continue,
1178		else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)),
1179		}
1180		},
1181
1182		State.Block => \|block\| {
1183		const token = self.getNextToken();
1184		switch (token.id) {
1185		Token.Id.RBrace => {
1186		block.end_token = token;
1187		continue;
1188		},
1189		else => {
1190		self.putBackToken(token);
1191		stack.append(State { .Block = block }) %% unreachable;
1192		%return stack.append(State { .Statement = block });
1193		continue;
1194		},
1195		}
1196		},
1197
1198		State.Statement => \|block\| {
1199		{
1200		// Look for comptime var, comptime const
1201		const comptime_token = self.getNextToken();
1202		if (comptime_token.id == Token.Id.Keyword_comptime) {
1203		const mut_token = self.getNextToken();
1204		if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
1205		// TODO shouldn't need these casts
1206		const var_decl = %return self.createAttachVarDecl(&block.statements, (?Token)(null),
1207		mut_token, (?Token)(comptime_token), (?Token)(null));
1208		%return stack.append(State { .VarDecl = var_decl });
1209		continue;
1210		}
1211		self.putBackToken(mut_token);
1212		}
1213		self.putBackToken(comptime_token);
1214		}
1215		{
1216		// Look for const, var
1217		const mut_token = self.getNextToken();
1218		if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
1219		// TODO shouldn't need these casts
1220		const var_decl = %return self.createAttachVarDecl(&block.statements, (?Token)(null),
1221		mut_token, (?Token)(null), (?Token)(null));
1222		%return stack.append(State { .VarDecl = var_decl });
1223		continue;
1224		}
1225		self.putBackToken(mut_token);
1226		}
1227
1228		stack.append(State { .ExpectToken = Token.Id.Semicolon }) %% unreachable;
1229		%return stack.append(State { .Expression = DestPtr{.List = &block.statements} });
1230		continue;
1231		},
1232
1233		State.GroupedExpression => @panic("TODO"),
1234		}
1235		unreachable;
1236		}
1237		}
1238
1239		fn createRoot(self: &Parser) -> %&AstNodeRoot {
1240		const node = %return self.allocator.create(AstNodeRoot);
1241		%defer self.allocator.destroy(node);
1242
1243		*node = AstNodeRoot {
1244		.base = AstNode {.id = AstNode.Id.Root},
1245		.decls = ArrayList(&AstNode).init(self.allocator),
1246		};
1247		return node;
1248		}
1249
1250		fn createVarDecl(self: &Parser, visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token,
1251		extern_token: &const ?Token) -> %&AstNodeVarDecl
1252		{
1253		const node = %return self.allocator.create(AstNodeVarDecl);
1254		%defer self.allocator.destroy(node);
1255
1256		*node = AstNodeVarDecl {
1257		.base = AstNode {.id = AstNode.Id.VarDecl},
1258		.visib_token = *visib_token,
1259		.mut_token = *mut_token,
1260		.comptime_token = *comptime_token,
1261		.extern_token = *extern_token,
1262		.type_node = null,
1263		.align_node = null,
1264		.init_node = null,
1265		.lib_name = null,
1266		// initialized later
1267		.name_token = undefined,
1268		.eq_token = undefined,
1269		};
1270		return node;
1271		}
1272
1273		fn createIdentifier(self: &Parser, name_token: &const Token) -> %&AstNodeIdentifier {
1274		const node = %return self.allocator.create(AstNodeIdentifier);
1275		%defer self.allocator.destroy(node);
1276
1277		*node = AstNodeIdentifier {
1278		.base = AstNode {.id = AstNode.Id.Identifier},
1279		.name_token = *name_token,
1280		};
1281		return node;
1282		}
1283
1284		fn createFnProto(self: &Parser, fn_token: &const Token, extern_token: &const ?Token,
1285		cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) -> %&AstNodeFnProto
1286		{
1287		const node = %return self.allocator.create(AstNodeFnProto);
1288		%defer self.allocator.destroy(node);
1289
1290		*node = AstNodeFnProto {
1291		.base = AstNode {.id = AstNode.Id.FnProto},
1292		.visib_token = *visib_token,
1293		.name_token = null,
1294		.fn_token = *fn_token,
1295		.params = ArrayList(&AstNode).init(self.allocator),
1296		.return_type = null,
1297		.var_args_token = null,
1298		.extern_token = *extern_token,
1299		.inline_token = *inline_token,
1300		.cc_token = *cc_token,
1301		.body_node = null,
1302		.lib_name = null,
1303		.align_expr = null,
1304		};
1305		return node;
1306		}
1307
1308		fn createParamDecl(self: &Parser) -> %&AstNodeParamDecl {
1309		const node = %return self.allocator.create(AstNodeParamDecl);
1310		%defer self.allocator.destroy(node);
1311
1312		*node = AstNodeParamDecl {
1313		.base = AstNode {.id = AstNode.Id.ParamDecl},
1314		.comptime_token = null,
1315		.noalias_token = null,
1316		.name_token = null,
1317		.type_node = undefined,
1318		.var_args_token = null,
1319		};
1320		return node;
1321		}
1322
1323		fn createAddrOfExpr(self: &Parser, op_token: &const Token) -> %&AstNodeAddrOfExpr {
1324		const node = %return self.allocator.create(AstNodeAddrOfExpr);
1325		%defer self.allocator.destroy(node);
1326
1327		*node = AstNodeAddrOfExpr {
1328		.base = AstNode {.id = AstNode.Id.AddrOfExpr},
1329		.align_expr = null,
1330		.op_token = *op_token,
1331		.bit_offset_start_token = null,
1332		.bit_offset_end_token = null,
1333		.const_token = null,
1334		.volatile_token = null,
1335		.op_expr = undefined,
1336		};
1337		return node;
1338		}
1339
1340		fn createBlock(self: &Parser, begin_token: &const Token) -> %&AstNodeBlock {
1341		const node = %return self.allocator.create(AstNodeBlock);
1342		%defer self.allocator.destroy(node);
1343
1344		*node = AstNodeBlock {
1345		.base = AstNode {.id = AstNode.Id.Block},
1346		.begin_token = *begin_token,
1347		.end_token = undefined,
1348		.statements = ArrayList(&AstNode).init(self.allocator),
1349		};
1350		return node;
1351		}
1352
1353		fn createAttachAddrOfExpr(self: &Parser, dest_ptr: &const DestPtr, op_token: &const Token) -> %&AstNodeAddrOfExpr {
1354		const node = %return self.createAddrOfExpr(op_token);
1355		%defer self.allocator.destroy(node);
1356		%return dest_ptr.store(&node.base);
1357		return node;
1358		}
1359
1360		fn createAttachParamDecl(self: &Parser, list: &ArrayList(&AstNode)) -> %&AstNodeParamDecl {
1361		const node = %return self.createParamDecl();
1362		%defer self.allocator.destroy(node);
1363		%return list.append(&node.base);
1364		return node;
1365		}
1366
1367		fn createAttachFnProto(self: &Parser, list: &ArrayList(&AstNode), fn_token: &const Token,
1368		extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token,
1369		inline_token: &const ?Token) -> %&AstNodeFnProto
1370		{
1371		const node = %return self.createFnProto(fn_token, extern_token, cc_token, visib_token, inline_token);
1372		%defer self.allocator.destroy(node);
1373		%return list.append(&node.base);
1374		return node;
1375		}
1376
1377		fn createAttachVarDecl(self: &Parser, list: &ArrayList(&AstNode), visib_token: &const ?Token,
1378		mut_token: &const Token, comptime_token: &const ?Token, extern_token: &const ?Token) -> %&AstNodeVarDecl
1379		{
1380		const node = %return self.createVarDecl(visib_token, mut_token, comptime_token, extern_token);
1381		%defer self.allocator.destroy(node);
1382		%return list.append(&node.base);
1383		return node;
1384		}
1385
1386		fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) -> error {
1387		const loc = self.tokenizer.getTokenLocation(token);
1388		warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
1389		warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
1390		{
1391		var i: usize = 0;
1392		while (i < loc.column) : (i += 1) {
1393		warn(" ");
1394		}
1395		}
1396		{
1397		const caret_count = token.end - token.start;
1398		var i: usize = 0;
1399		while (i < caret_count) : (i += 1) {
1400		warn("~");
1401		}
1402		}
1403		warn("\n");
1404		return error.ParseError;
1405		}
1406
1407		fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) -> %void {
1408		if (token.id != id) {
1409		return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
1410		}
1411		}
1412
1413		fn eatToken(self: &Parser, id: @TagType(Token.Id)) -> %Token {
1414		const token = self.getNextToken();
1415		%return self.expectToken(token, id);
1416		return token;
1417		}
1418
1419		fn putBackToken(self: &Parser, token: &const Token) {
1420		self.put_back_tokens[self.put_back_count] = *token;
1421		self.put_back_count += 1;
1422		}
1423
1424		fn getNextToken(self: &Parser) -> Token {
1425		return if (self.put_back_count != 0) {
1426		const put_back_index = self.put_back_count - 1;
1427		const put_back_token = self.put_back_tokens[put_back_index];
1428		self.put_back_count = put_back_index;
1429		put_back_token
1430		} else {
1431		self.tokenizer.next()
1432		};
1433		}
1434
1435		const RenderAstFrame = struct {
1436		node: &AstNode,
1437		indent: usize,
1438		};
1439
1440		pub fn renderAst(self: &Parser, stream: &std.io.OutStream, root_node: &AstNodeRoot) -> %void {
1441		var stack = self.initUtilityArrayList(RenderAstFrame);
1442		defer self.deinitUtilityArrayList(stack);
1443
1444		%return stack.append(RenderAstFrame {
1445		.node = &root_node.base,
1446		.indent = 0,
1447		});
1448
1449		while (stack.popOrNull()) \|frame\| {
1450		{
1451		var i: usize = 0;
1452		while (i < frame.indent) : (i += 1) {
1453		%return stream.print(" ");
1454		}
1455		}
1456		%return stream.print("{}\n", @tagName(frame.node.id));
1457		var child_i: usize = 0;
1458		while (frame.node.iterate(child_i)) \|child\| : (child_i += 1) {
1459		%return stack.append(RenderAstFrame {
1460		.node = child,
1461		.indent = frame.indent + 2,
1462		});
1463		}
1464		}
1465		}
1466
1467
1468		pub const RenderState = union(enum) {
1469		TopLevelDecl: &AstNode,
1470		FnProtoRParen: &AstNodeFnProto,
1471		ParamDecl: &AstNode,
1472		Text: []const u8,
1473		Expression: &AstNode,
1474		AddrOfExprBit: &AstNodeAddrOfExpr,
1475		VarDecl: &AstNodeVarDecl,
1476		VarDeclAlign: &AstNodeVarDecl,
1477		Statement: &AstNode,
1478		PrintIndent,
1479		Indent: usize,
1480		};
1481
1482		pub fn renderSource(self: &Parser, stream: &std.io.OutStream, root_node: &AstNodeRoot) -> %void {
1483		var stack = self.initUtilityArrayList(RenderState);
1484		defer self.deinitUtilityArrayList(stack);
1485
1486		{
1487		var i = root_node.decls.len;
1488		while (i != 0) {
1489		i -= 1;
1490		const decl = root_node.decls.items[i];
1491		%return stack.append(RenderState {.TopLevelDecl = decl});
1492		}
1493		}
1494
1495		const indent_delta = 4;
1496		var indent: usize = 0;
1497		while (stack.popOrNull()) \|state\| {
1498		switch (state) {
1499		RenderState.TopLevelDecl => \|decl\| {
1500		switch (decl.id) {
1501		AstNode.Id.FnProto => {
1502		const fn_proto = @fieldParentPtr(AstNodeFnProto, "base", decl);
1503		if (fn_proto.visib_token) \|visib_token\| {
1504		switch (visib_token.id) {
1505		Token.Id.Keyword_pub => %return stream.print("pub "),
1506		Token.Id.Keyword_export => %return stream.print("export "),
1507		else => unreachable,
1508		};
1509		}
1510		if (fn_proto.extern_token) \|extern_token\| {
1511		%return stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
1512		}
1513		%return stream.print("fn");
1514
1515		if (fn_proto.name_token) \|name_token\| {
1516		%return stream.print(" {}", self.tokenizer.getTokenSlice(name_token));
1517		}
1518
1519		%return stream.print("(");
1520
1521		%return stack.append(RenderState { .Text = "\n" });
1522		if (fn_proto.body_node == null) {
1523		%return stack.append(RenderState { .Text = ";" });
1524		}
1525
1526		%return stack.append(RenderState { .FnProtoRParen = fn_proto});
1527		var i = fn_proto.params.len;
1528		while (i != 0) {
1529		i -= 1;
1530		const param_decl_node = fn_proto.params.items[i];
1531		%return stack.append(RenderState { .ParamDecl = param_decl_node});
1532		if (i != 0) {
1533		%return stack.append(RenderState { .Text = ", " });
1534		}
1535		}
1536		},
1537		AstNode.Id.VarDecl => {
1538		const var_decl = @fieldParentPtr(AstNodeVarDecl, "base", decl);
1539		%return stack.append(RenderState { .Text = "\n"});
1540		%return stack.append(RenderState { .VarDecl = var_decl});
1541
1542		},
1543		else => unreachable,
1544		}
1545		},
1546
1547		RenderState.VarDecl => \|var_decl\| {
1548		if (var_decl.visib_token) \|visib_token\| {
1549		%return stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
1550		}
1551		if (var_decl.extern_token) \|extern_token\| {
1552		%return stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
1553		if (var_decl.lib_name != null) {
1554		@panic("TODO");
1555		}
1556		}
1557		if (var_decl.comptime_token) \|comptime_token\| {
1558		%return stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
1559		}
1560		%return stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token));
1561		%return stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token));
1562
1563		%return stack.append(RenderState { .VarDeclAlign = var_decl });
1564		if (var_decl.type_node) \|type_node\| {
1565		%return stream.print(": ");
1566		%return stack.append(RenderState { .Expression = type_node });
1567		}
1568		},
1569
1570		RenderState.VarDeclAlign => \|var_decl\| {
1571		if (var_decl.align_node != null) {
1572		@panic("TODO");
1573		}
1574		%return stack.append(RenderState { .Text = ";" });
1575		if (var_decl.init_node) \|init_node\| {
1576		%return stream.print(" = ");
1577		%return stack.append(RenderState { .Expression = init_node });
1578		}
1579		},
1580
1581		RenderState.ParamDecl => \|base\| {
1582		const param_decl = @fieldParentPtr(AstNodeParamDecl, "base", base);
1583		if (param_decl.comptime_token) \|comptime_token\| {
1584		%return stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
1585		}
1586		if (param_decl.noalias_token) \|noalias_token\| {
1587		%return stream.print("{} ", self.tokenizer.getTokenSlice(noalias_token));
1588		}
1589		if (param_decl.name_token) \|name_token\| {
1590		%return stream.print("{}: ", self.tokenizer.getTokenSlice(name_token));
1591		}
1592		if (param_decl.var_args_token) \|var_args_token\| {
1593		%return stream.print("{}", self.tokenizer.getTokenSlice(var_args_token));
1594		} else {
1595		%return stack.append(RenderState { .Expression = param_decl.type_node});
1596		}
1597		},
1598		RenderState.Text => \|bytes\| {
1599		%return stream.write(bytes);
1600		},
1601		RenderState.Expression => \|base\| switch (base.id) {
1602		AstNode.Id.Identifier => {
1603		const identifier = @fieldParentPtr(AstNodeIdentifier, "base", base);
1604		%return stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token));
1605		},
1606		AstNode.Id.AddrOfExpr => {
1607		const addr_of_expr = @fieldParentPtr(AstNodeAddrOfExpr, "base", base);
1608		%return stream.print("{}", self.tokenizer.getTokenSlice(addr_of_expr.op_token));
1609		%return stack.append(RenderState { .AddrOfExprBit = addr_of_expr});
1610
1611		if (addr_of_expr.align_expr) \|align_expr\| {
1612		%return stream.print("align(");
1613		%return stack.append(RenderState { .Text = ")"});
1614		%return stack.append(RenderState { .Expression = align_expr});
1615		}
1616		},
1617		AstNode.Id.Block => {
1618		const block = @fieldParentPtr(AstNodeBlock, "base", base);
1619		%return stream.write("{");
1620		%return stack.append(RenderState { .Text = "}"});
1621		%return stack.append(RenderState.PrintIndent);
1622		%return stack.append(RenderState { .Indent = indent});
1623		%return stack.append(RenderState { .Text = "\n"});
1624		var i = block.statements.len;
1625		while (i != 0) {
1626		i -= 1;
1627		const statement_node = block.statements.items[i];
1628		%return stack.append(RenderState { .Statement = statement_node});
1629		%return stack.append(RenderState.PrintIndent);
1630		%return stack.append(RenderState { .Indent = indent + indent_delta});
1631		%return stack.append(RenderState { .Text = "\n" });
1632		}
1633		},
1634		else => unreachable,
1635		},
1636		RenderState.AddrOfExprBit => \|addr_of_expr\| {
1637		if (addr_of_expr.bit_offset_start_token) \|bit_offset_start_token\| {
1638		%return stream.print("{} ", self.tokenizer.getTokenSlice(bit_offset_start_token));
1639		}
1640		if (addr_of_expr.bit_offset_end_token) \|bit_offset_end_token\| {
1641		%return stream.print("{} ", self.tokenizer.getTokenSlice(bit_offset_end_token));
1642		}
1643		if (addr_of_expr.const_token) \|const_token\| {
1644		%return stream.print("{} ", self.tokenizer.getTokenSlice(const_token));
1645		}
1646		if (addr_of_expr.volatile_token) \|volatile_token\| {
1647		%return stream.print("{} ", self.tokenizer.getTokenSlice(volatile_token));
1648		}
1649		%return stack.append(RenderState { .Expression = addr_of_expr.op_expr});
1650		},
1651		RenderState.FnProtoRParen => \|fn_proto\| {
1652		%return stream.print(")");
1653		if (fn_proto.align_expr != null) {
1654		@panic("TODO");
1655		}
1656		if (fn_proto.return_type) \|return_type\| {
1657		%return stream.print(" -> ");
1658		if (fn_proto.body_node) \|body_node\| {
1659		%return stack.append(RenderState { .Expression = body_node});
1660		%return stack.append(RenderState { .Text = " "});
1661		}
1662		%return stack.append(RenderState { .Expression = return_type});
1663		}
1664		},
1665		RenderState.Statement => \|base\| {
1666		switch (base.id) {
1667		AstNode.Id.VarDecl => {
1668		const var_decl = @fieldParentPtr(AstNodeVarDecl, "base", base);
1669		%return stack.append(RenderState { .VarDecl = var_decl});
1670		},
1671		else => unreachable,
1672		}
1673		},
1674		RenderState.Indent => \|new_indent\| indent = new_indent,
1675		RenderState.PrintIndent => %return stream.writeByteNTimes(' ', indent),
1676		}
1677		}
1678		}
1679		};
	8	const Tokenizer = @import("tokenizer.zig").Tokenizer;
	9	const Token = @import("tokenizer.zig").Token;
	10	const Parser = @import("parser.zig").Parser;
1680	11
1681	12	pub fn main() -> %void {
1682	13	main2() %% \|err\| {
...	...	@@ -1795,4 +126,11 @@ test "zig fmt" {
1795	126	\\}
1796	127	\\
1797	128	);
	129
	130	testCanonical(
	131	\\fn foo(argc: c_int, argv: &&u8) -> c_int {
	132	\\ return 0;
	133	\\}
	134	\\
	135	);
1798	136	}

src-self-hosted/parser.zig created+1074

...	...	@@ -0,0 +1,1074 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3	const ArrayList = std.ArrayList;
	4	const mem = std.mem;
	5	const ast = @import("ast.zig");
	6	const Tokenizer = @import("tokenizer.zig").Tokenizer;
	7	const Token = @import("tokenizer.zig").Token;
	8
	9	// TODO when we make parse errors into error types instead of printing directly,
	10	// get rid of this
	11	const warn = std.debug.warn;
	12
	13	error ParseError;
	14
	15	pub const Parser = struct {
	16	allocator: &mem.Allocator,
	17	tokenizer: &Tokenizer,
	18	put_back_tokens: [2]Token,
	19	put_back_count: usize,
	20	source_file_name: []const u8,
	21
	22	// This memory contents are used only during a function call. It's used to repurpose memory;
	23	// specifically so that freeAst can be guaranteed to succeed.
	24	const utility_bytes_align = @alignOf( union { a: RenderAstFrame, b: State, c: RenderState } );
	25	utility_bytes: []align(utility_bytes_align) u8,
	26
	27	pub fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) -> Parser {
	28	return Parser {
	29	.allocator = allocator,
	30	.tokenizer = tokenizer,
	31	.put_back_tokens = undefined,
	32	.put_back_count = 0,
	33	.source_file_name = source_file_name,
	34	.utility_bytes = []align(utility_bytes_align) u8{},
	35	};
	36	}
	37
	38	pub fn deinit(self: &Parser) {
	39	self.allocator.free(self.utility_bytes);
	40	}
	41
	42	const TopLevelDeclCtx = struct {
	43	visib_token: ?Token,
	44	extern_token: ?Token,
	45	};
	46
	47	const DestPtr = union(enum) {
	48	Field: &&ast.Node,
	49	NullableField: &?&ast.Node,
	50	List: &ArrayList(&ast.Node),
	51
	52	pub fn store(self: &const DestPtr, value: &ast.Node) -> %void {
	53	switch (*self) {
	54	DestPtr.Field => \|ptr\| *ptr = value,
	55	DestPtr.NullableField => \|ptr\| *ptr = value,
	56	DestPtr.List => \|list\| %return list.append(value),
	57	}
	58	}
	59	};
	60
	61	const State = union(enum) {
	62	TopLevel,
	63	TopLevelExtern: ?Token,
	64	TopLevelDecl: TopLevelDeclCtx,
	65	Expression: DestPtr,
	66	GroupedExpression: DestPtr,
	67	UnwrapExpression: DestPtr,
	68	BoolOrExpression: DestPtr,
	69	BoolAndExpression: DestPtr,
	70	ComparisonExpression: DestPtr,
	71	BinaryOrExpression: DestPtr,
	72	BinaryXorExpression: DestPtr,
	73	BinaryAndExpression: DestPtr,
	74	BitShiftExpression: DestPtr,
	75	AdditionExpression: DestPtr,
	76	MultiplyExpression: DestPtr,
	77	BraceSuffixExpression: DestPtr,
	78	PrefixOpExpression: DestPtr,
	79	SuffixOpExpression: DestPtr,
	80	PrimaryExpression: DestPtr,
	81	TypeExpr: DestPtr,
	82	VarDecl: &ast.NodeVarDecl,
	83	VarDeclAlign: &ast.NodeVarDecl,
	84	VarDeclEq: &ast.NodeVarDecl,
	85	ExpectToken: @TagType(Token.Id),
	86	FnProto: &ast.NodeFnProto,
	87	FnProtoAlign: &ast.NodeFnProto,
	88	ParamDecl: &ast.NodeFnProto,
	89	ParamDeclComma,
	90	FnDef: &ast.NodeFnProto,
	91	Block: &ast.NodeBlock,
	92	Statement: &ast.NodeBlock,
	93	};
	94
	95	pub fn freeAst(self: &Parser, root_node: &ast.NodeRoot) {
	96	// utility_bytes is big enough to do this iteration since we were able to do
	97	// the parsing in the first place
	98	comptime assert(@sizeOf(State) >= @sizeOf(&ast.Node));
	99
	100	var stack = self.initUtilityArrayList(&ast.Node);
	101	defer self.deinitUtilityArrayList(stack);
	102
	103	stack.append(&root_node.base) %% unreachable;
	104	while (stack.popOrNull()) \|node\| {
	105	var i: usize = 0;
	106	while (node.iterate(i)) \|child\| : (i += 1) {
	107	if (child.iterate(0) != null) {
	108	stack.append(child) %% unreachable;
	109	} else {
	110	child.destroy(self.allocator);
	111	}
	112	}
	113	node.destroy(self.allocator);
	114	}
	115	}
	116
	117	pub fn parse(self: &Parser) -> %&ast.NodeRoot {
	118	var stack = self.initUtilityArrayList(State);
	119	defer self.deinitUtilityArrayList(stack);
	120
	121	const root_node = %return self.createRoot();
	122	%defer self.allocator.destroy(root_node);
	123	%return stack.append(State.TopLevel);
	124	%defer self.freeAst(root_node);
	125
	126	while (true) {
	127	//{
	128	// const token = self.getNextToken();
	129	// warn("{} ", @tagName(token.id));
	130	// self.putBackToken(token);
	131	// var i: usize = stack.len;
	132	// while (i != 0) {
	133	// i -= 1;
	134	// warn("{} ", @tagName(stack.items[i]));
	135	// }
	136	// warn("\n");
	137	//}
	138
	139	// This gives us 1 free append that can't fail
	140	const state = stack.pop();
	141
	142	switch (state) {
	143	State.TopLevel => {
	144	const token = self.getNextToken();
	145	switch (token.id) {
	146	Token.Id.Keyword_pub, Token.Id.Keyword_export => {
	147	stack.append(State { .TopLevelExtern = token }) %% unreachable;
	148	continue;
	149	},
	150	Token.Id.Eof => return root_node,
	151	else => {
	152	self.putBackToken(token);
	153	// TODO shouldn't need this cast
	154	stack.append(State { .TopLevelExtern = null }) %% unreachable;
	155	continue;
	156	},
	157	}
	158	},
	159	State.TopLevelExtern => \|visib_token\| {
	160	const token = self.getNextToken();
	161	if (token.id == Token.Id.Keyword_extern) {
	162	stack.append(State {
	163	.TopLevelDecl = TopLevelDeclCtx {
	164	.visib_token = visib_token,
	165	.extern_token = token,
	166	},
	167	}) %% unreachable;
	168	continue;
	169	}
	170	self.putBackToken(token);
	171	stack.append(State {
	172	.TopLevelDecl = TopLevelDeclCtx {
	173	.visib_token = visib_token,
	174	.extern_token = null,
	175	},
	176	}) %% unreachable;
	177	continue;
	178	},
	179	State.TopLevelDecl => \|ctx\| {
	180	const token = self.getNextToken();
	181	switch (token.id) {
	182	Token.Id.Keyword_var, Token.Id.Keyword_const => {
	183	stack.append(State.TopLevel) %% unreachable;
	184	// TODO shouldn't need these casts
	185	const var_decl_node = %return self.createAttachVarDecl(&root_node.decls, ctx.visib_token,
	186	token, (?Token)(null), ctx.extern_token);
	187	%return stack.append(State { .VarDecl = var_decl_node });
	188	continue;
	189	},
	190	Token.Id.Keyword_fn => {
	191	stack.append(State.TopLevel) %% unreachable;
	192	// TODO shouldn't need these casts
	193	const fn_proto = %return self.createAttachFnProto(&root_node.decls, token,
	194	ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null));
	195	%return stack.append(State { .FnDef = fn_proto });
	196	%return stack.append(State { .FnProto = fn_proto });
	197	continue;
	198	},
	199	Token.Id.StringLiteral => {
	200	@panic("TODO extern with string literal");
	201	},
	202	Token.Id.Keyword_coldcc, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
	203	stack.append(State.TopLevel) %% unreachable;
	204	const fn_token = %return self.eatToken(Token.Id.Keyword_fn);
	205	// TODO shouldn't need this cast
	206	const fn_proto = %return self.createAttachFnProto(&root_node.decls, fn_token,
	207	ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null));
	208	%return stack.append(State { .FnDef = fn_proto });
	209	%return stack.append(State { .FnProto = fn_proto });
	210	continue;
	211	},
	212	else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)),
	213	}
	214	},
	215	State.VarDecl => \|var_decl\| {
	216	var_decl.name_token = %return self.eatToken(Token.Id.Identifier);
	217	stack.append(State { .VarDeclAlign = var_decl }) %% unreachable;
	218
	219	const next_token = self.getNextToken();
	220	if (next_token.id == Token.Id.Colon) {
	221	%return stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} });
	222	continue;
	223	}
	224
	225	self.putBackToken(next_token);
	226	continue;
	227	},
	228	State.VarDeclAlign => \|var_decl\| {
	229	stack.append(State { .VarDeclEq = var_decl }) %% unreachable;
	230
	231	const next_token = self.getNextToken();
	232	if (next_token.id == Token.Id.Keyword_align) {
	233	%return stack.append(State {
	234	.GroupedExpression = DestPtr {
	235	.NullableField = &var_decl.align_node
	236	}
	237	});
	238	continue;
	239	}
	240
	241	self.putBackToken(next_token);
	242	continue;
	243	},
	244	State.VarDeclEq => \|var_decl\| {
	245	const token = self.getNextToken();
	246	if (token.id == Token.Id.Equal) {
	247	var_decl.eq_token = token;
	248	stack.append(State { .ExpectToken = Token.Id.Semicolon }) %% unreachable;
	249	%return stack.append(State {
	250	.Expression = DestPtr {.NullableField = &var_decl.init_node},
	251	});
	252	continue;
	253	}
	254	if (token.id == Token.Id.Semicolon) {
	255	continue;
	256	}
	257	return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
	258	},
	259	State.ExpectToken => \|token_id\| {
	260	_ = %return self.eatToken(token_id);
	261	continue;
	262	},
	263	State.Expression => \|dest_ptr\| {
	264	const token = self.getNextToken();
	265	if (token.id == Token.Id.Keyword_return) {
	266	const return_node = %return self.createAttachReturn(dest_ptr, token);
	267	stack.append(State {.UnwrapExpression = DestPtr {.Field = &return_node.expr} }) %% unreachable;
	268	continue;
	269	}
	270	self.putBackToken(token);
	271	stack.append(State {.UnwrapExpression = dest_ptr}) %% unreachable;
	272	continue;
	273	},
	274
	275	State.UnwrapExpression => \|dest_ptr\| {
	276	stack.append(State {.BoolOrExpression = dest_ptr}) %% unreachable;
	277	continue;
	278	},
	279
	280	State.BoolOrExpression => \|dest_ptr\| {
	281	stack.append(State {.BoolAndExpression = dest_ptr}) %% unreachable;
	282	continue;
	283	},
	284
	285	State.BoolAndExpression => \|dest_ptr\| {
	286	stack.append(State {.ComparisonExpression = dest_ptr}) %% unreachable;
	287	continue;
	288	},
	289
	290	State.ComparisonExpression => \|dest_ptr\| {
	291	stack.append(State {.BinaryOrExpression = dest_ptr}) %% unreachable;
	292	continue;
	293	},
	294
	295	State.BinaryOrExpression => \|dest_ptr\| {
	296	stack.append(State {.BinaryXorExpression = dest_ptr}) %% unreachable;
	297	continue;
	298	},
	299
	300	State.BinaryXorExpression => \|dest_ptr\| {
	301	stack.append(State {.BinaryAndExpression = dest_ptr}) %% unreachable;
	302	continue;
	303	},
	304
	305	State.BinaryAndExpression => \|dest_ptr\| {
	306	stack.append(State {.BitShiftExpression = dest_ptr}) %% unreachable;
	307	continue;
	308	},
	309
	310	State.BitShiftExpression => \|dest_ptr\| {
	311	stack.append(State {.AdditionExpression = dest_ptr}) %% unreachable;
	312	continue;
	313	},
	314
	315	State.AdditionExpression => \|dest_ptr\| {
	316	stack.append(State {.MultiplyExpression = dest_ptr}) %% unreachable;
	317	continue;
	318	},
	319
	320	State.MultiplyExpression => \|dest_ptr\| {
	321	stack.append(State {.BraceSuffixExpression = dest_ptr}) %% unreachable;
	322	continue;
	323	},
	324
	325	State.BraceSuffixExpression => \|dest_ptr\| {
	326	stack.append(State {.PrefixOpExpression = dest_ptr}) %% unreachable;
	327	continue;
	328	},
	329
	330	State.PrefixOpExpression => \|dest_ptr\| {
	331	const first_token = self.getNextToken();
	332	if (first_token.id == Token.Id.Ampersand) {
	333	const addr_of_expr = %return self.createAttachAddrOfExpr(dest_ptr, first_token);
	334	var token = self.getNextToken();
	335	if (token.id == Token.Id.Keyword_align) {
	336	@panic("TODO align");
	337	}
	338	if (token.id == Token.Id.Keyword_const) {
	339	addr_of_expr.const_token = token;
	340	token = self.getNextToken();
	341	}
	342	if (token.id == Token.Id.Keyword_volatile) {
	343	addr_of_expr.volatile_token = token;
	344	token = self.getNextToken();
	345	}
	346	self.putBackToken(token);
	347	stack.append(State {
	348	.PrefixOpExpression = DestPtr { .Field = &addr_of_expr.op_expr},
	349	}) %% unreachable;
	350	continue;
	351	}
	352
	353	self.putBackToken(first_token);
	354	stack.append(State { .SuffixOpExpression = dest_ptr }) %% unreachable;
	355	continue;
	356	},
	357
	358	State.SuffixOpExpression => \|dest_ptr\| {
	359	stack.append(State { .PrimaryExpression = dest_ptr }) %% unreachable;
	360	continue;
	361	},
	362
	363	State.PrimaryExpression => \|dest_ptr\| {
	364	const token = self.getNextToken();
	365	switch (token.id) {
	366	Token.Id.Identifier => {
	367	_ = %return self.createAttachIdentifier(dest_ptr, token);
	368	continue;
	369	},
	370	Token.Id.IntegerLiteral => {
	371	_ = %return self.createAttachIntegerLiteral(dest_ptr, token);
	372	continue;
	373	},
	374	Token.Id.FloatLiteral => {
	375	_ = %return self.createAttachFloatLiteral(dest_ptr, token);
	376	continue;
	377	},
	378	else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
	379	}
	380	},
	381
	382	State.TypeExpr => \|dest_ptr\| {
	383	const token = self.getNextToken();
	384	if (token.id == Token.Id.Keyword_var) {
	385	@panic("TODO param with type var");
	386	}
	387	self.putBackToken(token);
	388
	389	stack.append(State { .PrefixOpExpression = dest_ptr }) %% unreachable;
	390	continue;
	391	},
	392
	393	State.FnProto => \|fn_proto\| {
	394	stack.append(State { .FnProtoAlign = fn_proto }) %% unreachable;
	395	%return stack.append(State { .ParamDecl = fn_proto });
	396	%return stack.append(State { .ExpectToken = Token.Id.LParen });
	397
	398	const next_token = self.getNextToken();
	399	if (next_token.id == Token.Id.Identifier) {
	400	fn_proto.name_token = next_token;
	401	continue;
	402	}
	403	self.putBackToken(next_token);
	404	continue;
	405	},
	406
	407	State.FnProtoAlign => \|fn_proto\| {
	408	const token = self.getNextToken();
	409	if (token.id == Token.Id.Keyword_align) {
	410	@panic("TODO fn proto align");
	411	}
	412	if (token.id == Token.Id.Arrow) {
	413	stack.append(State {
	414	.TypeExpr = DestPtr {.NullableField = &fn_proto.return_type},
	415	}) %% unreachable;
	416	continue;
	417	} else {
	418	self.putBackToken(token);
	419	continue;
	420	}
	421	},
	422
	423	State.ParamDecl => \|fn_proto\| {
	424	var token = self.getNextToken();
	425	if (token.id == Token.Id.RParen) {
	426	continue;
	427	}
	428	const param_decl = %return self.createAttachParamDecl(&fn_proto.params);
	429	if (token.id == Token.Id.Keyword_comptime) {
	430	param_decl.comptime_token = token;
	431	token = self.getNextToken();
	432	} else if (token.id == Token.Id.Keyword_noalias) {
	433	param_decl.noalias_token = token;
	434	token = self.getNextToken();
	435	};
	436	if (token.id == Token.Id.Identifier) {
	437	const next_token = self.getNextToken();
	438	if (next_token.id == Token.Id.Colon) {
	439	param_decl.name_token = token;
	440	token = self.getNextToken();
	441	} else {
	442	self.putBackToken(next_token);
	443	}
	444	}
	445	if (token.id == Token.Id.Ellipsis3) {
	446	param_decl.var_args_token = token;
	447	stack.append(State { .ExpectToken = Token.Id.RParen }) %% unreachable;
	448	continue;
	449	} else {
	450	self.putBackToken(token);
	451	}
	452
	453	stack.append(State { .ParamDecl = fn_proto }) %% unreachable;
	454	%return stack.append(State.ParamDeclComma);
	455	%return stack.append(State {
	456	.TypeExpr = DestPtr {.Field = &param_decl.type_node}
	457	});
	458	continue;
	459	},
	460
	461	State.ParamDeclComma => {
	462	const token = self.getNextToken();
	463	switch (token.id) {
	464	Token.Id.RParen => {
	465	_ = stack.pop(); // pop off the ParamDecl
	466	continue;
	467	},
	468	Token.Id.Comma => continue,
	469	else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
	470	}
	471	},
	472
	473	State.FnDef => \|fn_proto\| {
	474	const token = self.getNextToken();
	475	switch(token.id) {
	476	Token.Id.LBrace => {
	477	const block = %return self.createBlock(token);
	478	fn_proto.body_node = &block.base;
	479	stack.append(State { .Block = block }) %% unreachable;
	480	continue;
	481	},
	482	Token.Id.Semicolon => continue,
	483	else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)),
	484	}
	485	},
	486
	487	State.Block => \|block\| {
	488	const token = self.getNextToken();
	489	switch (token.id) {
	490	Token.Id.RBrace => {
	491	block.end_token = token;
	492	continue;
	493	},
	494	else => {
	495	self.putBackToken(token);
	496	stack.append(State { .Block = block }) %% unreachable;
	497	%return stack.append(State { .Statement = block });
	498	continue;
	499	},
	500	}
	501	},
	502
	503	State.Statement => \|block\| {
	504	{
	505	// Look for comptime var, comptime const
	506	const comptime_token = self.getNextToken();
	507	if (comptime_token.id == Token.Id.Keyword_comptime) {
	508	const mut_token = self.getNextToken();
	509	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
	510	// TODO shouldn't need these casts
	511	const var_decl = %return self.createAttachVarDecl(&block.statements, (?Token)(null),
	512	mut_token, (?Token)(comptime_token), (?Token)(null));
	513	%return stack.append(State { .VarDecl = var_decl });
	514	continue;
	515	}
	516	self.putBackToken(mut_token);
	517	}
	518	self.putBackToken(comptime_token);
	519	}
	520	{
	521	// Look for const, var
	522	const mut_token = self.getNextToken();
	523	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
	524	// TODO shouldn't need these casts
	525	const var_decl = %return self.createAttachVarDecl(&block.statements, (?Token)(null),
	526	mut_token, (?Token)(null), (?Token)(null));
	527	%return stack.append(State { .VarDecl = var_decl });
	528	continue;
	529	}
	530	self.putBackToken(mut_token);
	531	}
	532
	533	stack.append(State { .ExpectToken = Token.Id.Semicolon }) %% unreachable;
	534	%return stack.append(State { .Expression = DestPtr{.List = &block.statements} });
	535	continue;
	536	},
	537
	538	State.GroupedExpression => @panic("TODO"),
	539	}
	540	unreachable;
	541	}
	542	}
	543
	544	fn createRoot(self: &Parser) -> %&ast.NodeRoot {
	545	const node = %return self.allocator.create(ast.NodeRoot);
	546	%defer self.allocator.destroy(node);
	547
	548	*node = ast.NodeRoot {
	549	.base = ast.Node {.id = ast.Node.Id.Root},
	550	.decls = ArrayList(&ast.Node).init(self.allocator),
	551	};
	552	return node;
	553	}
	554
	555	fn createVarDecl(self: &Parser, visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token,
	556	extern_token: &const ?Token) -> %&ast.NodeVarDecl
	557	{
	558	const node = %return self.allocator.create(ast.NodeVarDecl);
	559	%defer self.allocator.destroy(node);
	560
	561	*node = ast.NodeVarDecl {
	562	.base = ast.Node {.id = ast.Node.Id.VarDecl},
	563	.visib_token = *visib_token,
	564	.mut_token = *mut_token,
	565	.comptime_token = *comptime_token,
	566	.extern_token = *extern_token,
	567	.type_node = null,
	568	.align_node = null,
	569	.init_node = null,
	570	.lib_name = null,
	571	// initialized later
	572	.name_token = undefined,
	573	.eq_token = undefined,
	574	};
	575	return node;
	576	}
	577
	578	fn createFnProto(self: &Parser, fn_token: &const Token, extern_token: &const ?Token,
	579	cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) -> %&ast.NodeFnProto
	580	{
	581	const node = %return self.allocator.create(ast.NodeFnProto);
	582	%defer self.allocator.destroy(node);
	583
	584	*node = ast.NodeFnProto {
	585	.base = ast.Node {.id = ast.Node.Id.FnProto},
	586	.visib_token = *visib_token,
	587	.name_token = null,
	588	.fn_token = *fn_token,
	589	.params = ArrayList(&ast.Node).init(self.allocator),
	590	.return_type = null,
	591	.var_args_token = null,
	592	.extern_token = *extern_token,
	593	.inline_token = *inline_token,
	594	.cc_token = *cc_token,
	595	.body_node = null,
	596	.lib_name = null,
	597	.align_expr = null,
	598	};
	599	return node;
	600	}
	601
	602	fn createParamDecl(self: &Parser) -> %&ast.NodeParamDecl {
	603	const node = %return self.allocator.create(ast.NodeParamDecl);
	604	%defer self.allocator.destroy(node);
	605
	606	*node = ast.NodeParamDecl {
	607	.base = ast.Node {.id = ast.Node.Id.ParamDecl},
	608	.comptime_token = null,
	609	.noalias_token = null,
	610	.name_token = null,
	611	.type_node = undefined,
	612	.var_args_token = null,
	613	};
	614	return node;
	615	}
	616
	617	fn createAddrOfExpr(self: &Parser, op_token: &const Token) -> %&ast.NodeAddrOfExpr {
	618	const node = %return self.allocator.create(ast.NodeAddrOfExpr);
	619	%defer self.allocator.destroy(node);
	620
	621	*node = ast.NodeAddrOfExpr {
	622	.base = ast.Node {.id = ast.Node.Id.AddrOfExpr},
	623	.align_expr = null,
	624	.op_token = *op_token,
	625	.bit_offset_start_token = null,
	626	.bit_offset_end_token = null,
	627	.const_token = null,
	628	.volatile_token = null,
	629	.op_expr = undefined,
	630	};
	631	return node;
	632	}
	633
	634	fn createBlock(self: &Parser, begin_token: &const Token) -> %&ast.NodeBlock {
	635	const node = %return self.allocator.create(ast.NodeBlock);
	636	%defer self.allocator.destroy(node);
	637
	638	*node = ast.NodeBlock {
	639	.base = ast.Node {.id = ast.Node.Id.Block},
	640	.begin_token = *begin_token,
	641	.end_token = undefined,
	642	.statements = ArrayList(&ast.Node).init(self.allocator),
	643	};
	644	return node;
	645	}
	646
	647	fn createReturn(self: &Parser, return_token: &const Token) -> %&ast.NodeReturn {
	648	const node = %return self.allocator.create(ast.NodeReturn);
	649	%defer self.allocator.destroy(node);
	650
	651	*node = ast.NodeReturn {
	652	.base = ast.Node {.id = ast.Node.Id.Return},
	653	.return_token = *return_token,
	654	.expr = undefined,
	655	};
	656	return node;
	657	}
	658
	659	fn createIdentifier(self: &Parser, name_token: &const Token) -> %&ast.NodeIdentifier {
	660	const node = %return self.allocator.create(ast.NodeIdentifier);
	661	%defer self.allocator.destroy(node);
	662
	663	*node = ast.NodeIdentifier {
	664	.base = ast.Node {.id = ast.Node.Id.Identifier},
	665	.name_token = *name_token,
	666	};
	667	return node;
	668	}
	669
	670	fn createIntegerLiteral(self: &Parser, token: &const Token) -> %&ast.NodeIntegerLiteral {
	671	const node = %return self.allocator.create(ast.NodeIntegerLiteral);
	672	%defer self.allocator.destroy(node);
	673
	674	*node = ast.NodeIntegerLiteral {
	675	.base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
	676	.token = *token,
	677	};
	678	return node;
	679	}
	680
	681	fn createFloatLiteral(self: &Parser, token: &const Token) -> %&ast.NodeFloatLiteral {
	682	const node = %return self.allocator.create(ast.NodeFloatLiteral);
	683	%defer self.allocator.destroy(node);
	684
	685	*node = ast.NodeFloatLiteral {
	686	.base = ast.Node {.id = ast.Node.Id.FloatLiteral},
	687	.token = *token,
	688	};
	689	return node;
	690	}
	691
	692	fn createAttachFloatLiteral(self: &Parser, dest_ptr: &const DestPtr, token: &const Token) -> %&ast.NodeFloatLiteral {
	693	const node = %return self.createFloatLiteral(token);
	694	%defer self.allocator.destroy(node);
	695	%return dest_ptr.store(&node.base);
	696	return node;
	697	}
	698
	699	fn createAttachIntegerLiteral(self: &Parser, dest_ptr: &const DestPtr, token: &const Token) -> %&ast.NodeIntegerLiteral {
	700	const node = %return self.createIntegerLiteral(token);
	701	%defer self.allocator.destroy(node);
	702	%return dest_ptr.store(&node.base);
	703	return node;
	704	}
	705
	706	fn createAttachIdentifier(self: &Parser, dest_ptr: &const DestPtr, name_token: &const Token) -> %&ast.NodeIdentifier {
	707	const node = %return self.createIdentifier(name_token);
	708	%defer self.allocator.destroy(node);
	709	%return dest_ptr.store(&node.base);
	710	return node;
	711	}
	712
	713	fn createAttachReturn(self: &Parser, dest_ptr: &const DestPtr, return_token: &const Token) -> %&ast.NodeReturn {
	714	const node = %return self.createReturn(return_token);
	715	%defer self.allocator.destroy(node);
	716	%return dest_ptr.store(&node.base);
	717	return node;
	718	}
	719
	720	fn createAttachAddrOfExpr(self: &Parser, dest_ptr: &const DestPtr, op_token: &const Token) -> %&ast.NodeAddrOfExpr {
	721	const node = %return self.createAddrOfExpr(op_token);
	722	%defer self.allocator.destroy(node);
	723	%return dest_ptr.store(&node.base);
	724	return node;
	725	}
	726
	727	fn createAttachParamDecl(self: &Parser, list: &ArrayList(&ast.Node)) -> %&ast.NodeParamDecl {
	728	const node = %return self.createParamDecl();
	729	%defer self.allocator.destroy(node);
	730	%return list.append(&node.base);
	731	return node;
	732	}
	733
	734	fn createAttachFnProto(self: &Parser, list: &ArrayList(&ast.Node), fn_token: &const Token,
	735	extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token,
	736	inline_token: &const ?Token) -> %&ast.NodeFnProto
	737	{
	738	const node = %return self.createFnProto(fn_token, extern_token, cc_token, visib_token, inline_token);
	739	%defer self.allocator.destroy(node);
	740	%return list.append(&node.base);
	741	return node;
	742	}
	743
	744	fn createAttachVarDecl(self: &Parser, list: &ArrayList(&ast.Node), visib_token: &const ?Token,
	745	mut_token: &const Token, comptime_token: &const ?Token, extern_token: &const ?Token) -> %&ast.NodeVarDecl
	746	{
	747	const node = %return self.createVarDecl(visib_token, mut_token, comptime_token, extern_token);
	748	%defer self.allocator.destroy(node);
	749	%return list.append(&node.base);
	750	return node;
	751	}
	752
	753	fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) -> error {
	754	const loc = self.tokenizer.getTokenLocation(token);
	755	warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
	756	warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
	757	{
	758	var i: usize = 0;
	759	while (i < loc.column) : (i += 1) {
	760	warn(" ");
	761	}
	762	}
	763	{
	764	const caret_count = token.end - token.start;
	765	var i: usize = 0;
	766	while (i < caret_count) : (i += 1) {
	767	warn("~");
	768	}
	769	}
	770	warn("\n");
	771	return error.ParseError;
	772	}
	773
	774	fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) -> %void {
	775	if (token.id != id) {
	776	return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
	777	}
	778	}
	779
	780	fn eatToken(self: &Parser, id: @TagType(Token.Id)) -> %Token {
	781	const token = self.getNextToken();
	782	%return self.expectToken(token, id);
	783	return token;
	784	}
	785
	786	fn putBackToken(self: &Parser, token: &const Token) {
	787	self.put_back_tokens[self.put_back_count] = *token;
	788	self.put_back_count += 1;
	789	}
	790
	791	fn getNextToken(self: &Parser) -> Token {
	792	return if (self.put_back_count != 0) {
	793	const put_back_index = self.put_back_count - 1;
	794	const put_back_token = self.put_back_tokens[put_back_index];
	795	self.put_back_count = put_back_index;
	796	put_back_token
	797	} else {
	798	self.tokenizer.next()
	799	};
	800	}
	801
	802	const RenderAstFrame = struct {
	803	node: &ast.Node,
	804	indent: usize,
	805	};
	806
	807	pub fn renderAst(self: &Parser, stream: &std.io.OutStream, root_node: &ast.NodeRoot) -> %void {
	808	var stack = self.initUtilityArrayList(RenderAstFrame);
	809	defer self.deinitUtilityArrayList(stack);
	810
	811	%return stack.append(RenderAstFrame {
	812	.node = &root_node.base,
	813	.indent = 0,
	814	});
	815
	816	while (stack.popOrNull()) \|frame\| {
	817	{
	818	var i: usize = 0;
	819	while (i < frame.indent) : (i += 1) {
	820	%return stream.print(" ");
	821	}
	822	}
	823	%return stream.print("{}\n", @tagName(frame.node.id));
	824	var child_i: usize = 0;
	825	while (frame.node.iterate(child_i)) \|child\| : (child_i += 1) {
	826	%return stack.append(RenderAstFrame {
	827	.node = child,
	828	.indent = frame.indent + 2,
	829	});
	830	}
	831	}
	832	}
	833
	834	const RenderState = union(enum) {
	835	TopLevelDecl: &ast.Node,
	836	FnProtoRParen: &ast.NodeFnProto,
	837	ParamDecl: &ast.Node,
	838	Text: []const u8,
	839	Expression: &ast.Node,
	840	AddrOfExprBit: &ast.NodeAddrOfExpr,
	841	VarDecl: &ast.NodeVarDecl,
	842	VarDeclAlign: &ast.NodeVarDecl,
	843	Statement: &ast.Node,
	844	PrintIndent,
	845	Indent: usize,
	846	};
	847
	848	pub fn renderSource(self: &Parser, stream: &std.io.OutStream, root_node: &ast.NodeRoot) -> %void {
	849	var stack = self.initUtilityArrayList(RenderState);
	850	defer self.deinitUtilityArrayList(stack);
	851
	852	{
	853	var i = root_node.decls.len;
	854	while (i != 0) {
	855	i -= 1;
	856	const decl = root_node.decls.items[i];
	857	%return stack.append(RenderState {.TopLevelDecl = decl});
	858	}
	859	}
	860
	861	const indent_delta = 4;
	862	var indent: usize = 0;
	863	while (stack.popOrNull()) \|state\| {
	864	switch (state) {
	865	RenderState.TopLevelDecl => \|decl\| {
	866	switch (decl.id) {
	867	ast.Node.Id.FnProto => {
	868	const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl);
	869	if (fn_proto.visib_token) \|visib_token\| {
	870	switch (visib_token.id) {
	871	Token.Id.Keyword_pub => %return stream.print("pub "),
	872	Token.Id.Keyword_export => %return stream.print("export "),
	873	else => unreachable,
	874	};
	875	}
	876	if (fn_proto.extern_token) \|extern_token\| {
	877	%return stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
	878	}
	879	%return stream.print("fn");
	880
	881	if (fn_proto.name_token) \|name_token\| {
	882	%return stream.print(" {}", self.tokenizer.getTokenSlice(name_token));
	883	}
	884
	885	%return stream.print("(");
	886
	887	%return stack.append(RenderState { .Text = "\n" });
	888	if (fn_proto.body_node == null) {
	889	%return stack.append(RenderState { .Text = ";" });
	890	}
	891
	892	%return stack.append(RenderState { .FnProtoRParen = fn_proto});
	893	var i = fn_proto.params.len;
	894	while (i != 0) {
	895	i -= 1;
	896	const param_decl_node = fn_proto.params.items[i];
	897	%return stack.append(RenderState { .ParamDecl = param_decl_node});
	898	if (i != 0) {
	899	%return stack.append(RenderState { .Text = ", " });
	900	}
	901	}
	902	},
	903	ast.Node.Id.VarDecl => {
	904	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
	905	%return stack.append(RenderState { .Text = "\n"});
	906	%return stack.append(RenderState { .VarDecl = var_decl});
	907
	908	},
	909	else => unreachable,
	910	}
	911	},
	912
	913	RenderState.VarDecl => \|var_decl\| {
	914	if (var_decl.visib_token) \|visib_token\| {
	915	%return stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
	916	}
	917	if (var_decl.extern_token) \|extern_token\| {
	918	%return stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
	919	if (var_decl.lib_name != null) {
	920	@panic("TODO");
	921	}
	922	}
	923	if (var_decl.comptime_token) \|comptime_token\| {
	924	%return stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
	925	}
	926	%return stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token));
	927	%return stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token));
	928
	929	%return stack.append(RenderState { .VarDeclAlign = var_decl });
	930	if (var_decl.type_node) \|type_node\| {
	931	%return stream.print(": ");
	932	%return stack.append(RenderState { .Expression = type_node });
	933	}
	934	},
	935
	936	RenderState.VarDeclAlign => \|var_decl\| {
	937	if (var_decl.align_node != null) {
	938	@panic("TODO");
	939	}
	940	%return stack.append(RenderState { .Text = ";" });
	941	if (var_decl.init_node) \|init_node\| {
	942	%return stream.print(" = ");
	943	%return stack.append(RenderState { .Expression = init_node });
	944	}
	945	},
	946
	947	RenderState.ParamDecl => \|base\| {
	948	const param_decl = @fieldParentPtr(ast.NodeParamDecl, "base", base);
	949	if (param_decl.comptime_token) \|comptime_token\| {
	950	%return stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
	951	}
	952	if (param_decl.noalias_token) \|noalias_token\| {
	953	%return stream.print("{} ", self.tokenizer.getTokenSlice(noalias_token));
	954	}
	955	if (param_decl.name_token) \|name_token\| {
	956	%return stream.print("{}: ", self.tokenizer.getTokenSlice(name_token));
	957	}
	958	if (param_decl.var_args_token) \|var_args_token\| {
	959	%return stream.print("{}", self.tokenizer.getTokenSlice(var_args_token));
	960	} else {
	961	%return stack.append(RenderState { .Expression = param_decl.type_node});
	962	}
	963	},
	964	RenderState.Text => \|bytes\| {
	965	%return stream.write(bytes);
	966	},
	967	RenderState.Expression => \|base\| switch (base.id) {
	968	ast.Node.Id.Identifier => {
	969	const identifier = @fieldParentPtr(ast.NodeIdentifier, "base", base);
	970	%return stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token));
	971	},
	972	ast.Node.Id.AddrOfExpr => {
	973	const addr_of_expr = @fieldParentPtr(ast.NodeAddrOfExpr, "base", base);
	974	%return stream.print("{}", self.tokenizer.getTokenSlice(addr_of_expr.op_token));
	975	%return stack.append(RenderState { .AddrOfExprBit = addr_of_expr});
	976
	977	if (addr_of_expr.align_expr) \|align_expr\| {
	978	%return stream.print("align(");
	979	%return stack.append(RenderState { .Text = ")"});
	980	%return stack.append(RenderState { .Expression = align_expr});
	981	}
	982	},
	983	ast.Node.Id.Block => {
	984	const block = @fieldParentPtr(ast.NodeBlock, "base", base);
	985	%return stream.write("{");
	986	%return stack.append(RenderState { .Text = "}"});
	987	%return stack.append(RenderState.PrintIndent);
	988	%return stack.append(RenderState { .Indent = indent});
	989	%return stack.append(RenderState { .Text = "\n"});
	990	var i = block.statements.len;
	991	while (i != 0) {
	992	i -= 1;
	993	const statement_node = block.statements.items[i];
	994	%return stack.append(RenderState { .Statement = statement_node});
	995	%return stack.append(RenderState.PrintIndent);
	996	%return stack.append(RenderState { .Indent = indent + indent_delta});
	997	%return stack.append(RenderState { .Text = "\n" });
	998	}
	999	},
	1000	ast.Node.Id.Return => {
	1001	const return_node = @fieldParentPtr(ast.NodeReturn, "base", base);
	1002	%return stream.write("return ");
	1003	%return stack.append(RenderState { .Expression = return_node.expr });
	1004	},
	1005	ast.Node.Id.IntegerLiteral => {
	1006	const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);
	1007	%return stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));
	1008	},
	1009	ast.Node.Id.FloatLiteral => {
	1010	const float_literal = @fieldParentPtr(ast.NodeFloatLiteral, "base", base);
	1011	%return stream.print("{}", self.tokenizer.getTokenSlice(float_literal.token));
	1012	},
	1013	else => unreachable,
	1014	},
	1015	RenderState.AddrOfExprBit => \|addr_of_expr\| {
	1016	if (addr_of_expr.bit_offset_start_token) \|bit_offset_start_token\| {
	1017	%return stream.print("{} ", self.tokenizer.getTokenSlice(bit_offset_start_token));
	1018	}
	1019	if (addr_of_expr.bit_offset_end_token) \|bit_offset_end_token\| {
	1020	%return stream.print("{} ", self.tokenizer.getTokenSlice(bit_offset_end_token));
	1021	}
	1022	if (addr_of_expr.const_token) \|const_token\| {
	1023	%return stream.print("{} ", self.tokenizer.getTokenSlice(const_token));
	1024	}
	1025	if (addr_of_expr.volatile_token) \|volatile_token\| {
	1026	%return stream.print("{} ", self.tokenizer.getTokenSlice(volatile_token));
	1027	}
	1028	%return stack.append(RenderState { .Expression = addr_of_expr.op_expr});
	1029	},
	1030	RenderState.FnProtoRParen => \|fn_proto\| {
	1031	%return stream.print(")");
	1032	if (fn_proto.align_expr != null) {
	1033	@panic("TODO");
	1034	}
	1035	if (fn_proto.return_type) \|return_type\| {
	1036	%return stream.print(" -> ");
	1037	if (fn_proto.body_node) \|body_node\| {
	1038	%return stack.append(RenderState { .Expression = body_node});
	1039	%return stack.append(RenderState { .Text = " "});
	1040	}
	1041	%return stack.append(RenderState { .Expression = return_type});
	1042	}
	1043	},
	1044	RenderState.Statement => \|base\| {
	1045	switch (base.id) {
	1046	ast.Node.Id.VarDecl => {
	1047	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
	1048	%return stack.append(RenderState { .VarDecl = var_decl});
	1049	},
	1050	else => {
	1051	%return stack.append(RenderState { .Text = ";"});
	1052	%return stack.append(RenderState { .Expression = base});
	1053	},
	1054	}
	1055	},
	1056	RenderState.Indent => \|new_indent\| indent = new_indent,
	1057	RenderState.PrintIndent => %return stream.writeByteNTimes(' ', indent),
	1058	}
	1059	}
	1060	}
	1061
	1062	fn initUtilityArrayList(self: &Parser, comptime T: type) -> ArrayList(T) {
	1063	const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
	1064	self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
	1065	const typed_slice = ([]T)(self.utility_bytes);
	1066	return ArrayList(T).fromOwnedSlice(self.allocator, typed_slice);
	1067	}
	1068
	1069	fn deinitUtilityArrayList(self: &Parser, list: var) {
	1070	self.utility_bytes = ([]align(utility_bytes_align) u8)(list.toOwnedSlice());
	1071	}
	1072
	1073	};
	1074

src-self-hosted/tokenizer.zig created+479

...	...	@@ -0,0 +1,479 @@
	1	const std = @import("std");
	2	const mem = std.mem;
	3
	4	pub const Token = struct {
	5	id: Id,
	6	start: usize,
	7	end: usize,
	8
	9	const KeywordId = struct {
	10	bytes: []const u8,
	11	id: Id,
	12	};
	13
	14	const keywords = []KeywordId {
	15	KeywordId{.bytes="align", .id = Id.Keyword_align},
	16	KeywordId{.bytes="and", .id = Id.Keyword_and},
	17	KeywordId{.bytes="asm", .id = Id.Keyword_asm},
	18	KeywordId{.bytes="break", .id = Id.Keyword_break},
	19	KeywordId{.bytes="coldcc", .id = Id.Keyword_coldcc},
	20	KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
	21	KeywordId{.bytes="const", .id = Id.Keyword_const},
	22	KeywordId{.bytes="continue", .id = Id.Keyword_continue},
	23	KeywordId{.bytes="defer", .id = Id.Keyword_defer},
	24	KeywordId{.bytes="else", .id = Id.Keyword_else},
	25	KeywordId{.bytes="enum", .id = Id.Keyword_enum},
	26	KeywordId{.bytes="error", .id = Id.Keyword_error},
	27	KeywordId{.bytes="export", .id = Id.Keyword_export},
	28	KeywordId{.bytes="extern", .id = Id.Keyword_extern},
	29	KeywordId{.bytes="false", .id = Id.Keyword_false},
	30	KeywordId{.bytes="fn", .id = Id.Keyword_fn},
	31	KeywordId{.bytes="for", .id = Id.Keyword_for},
	32	KeywordId{.bytes="goto", .id = Id.Keyword_goto},
	33	KeywordId{.bytes="if", .id = Id.Keyword_if},
	34	KeywordId{.bytes="inline", .id = Id.Keyword_inline},
	35	KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
	36	KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
	37	KeywordId{.bytes="null", .id = Id.Keyword_null},
	38	KeywordId{.bytes="or", .id = Id.Keyword_or},
	39	KeywordId{.bytes="packed", .id = Id.Keyword_packed},
	40	KeywordId{.bytes="pub", .id = Id.Keyword_pub},
	41	KeywordId{.bytes="return", .id = Id.Keyword_return},
	42	KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
	43	KeywordId{.bytes="struct", .id = Id.Keyword_struct},
	44	KeywordId{.bytes="switch", .id = Id.Keyword_switch},
	45	KeywordId{.bytes="test", .id = Id.Keyword_test},
	46	KeywordId{.bytes="this", .id = Id.Keyword_this},
	47	KeywordId{.bytes="true", .id = Id.Keyword_true},
	48	KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
	49	KeywordId{.bytes="union", .id = Id.Keyword_union},
	50	KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
	51	KeywordId{.bytes="use", .id = Id.Keyword_use},
	52	KeywordId{.bytes="var", .id = Id.Keyword_var},
	53	KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
	54	KeywordId{.bytes="while", .id = Id.Keyword_while},
	55	};
	56
	57	fn getKeyword(bytes: []const u8) -> ?Id {
	58	for (keywords) \|kw\| {
	59	if (mem.eql(u8, kw.bytes, bytes)) {
	60	return kw.id;
	61	}
	62	}
	63	return null;
	64	}
	65
	66	const StrLitKind = enum {Normal, C};
	67
	68	pub const Id = union(enum) {
	69	Invalid,
	70	Identifier,
	71	StringLiteral: StrLitKind,
	72	Eof,
	73	Builtin,
	74	Equal,
	75	LParen,
	76	RParen,
	77	Semicolon,
	78	Percent,
	79	LBrace,
	80	RBrace,
	81	Period,
	82	Ellipsis2,
	83	Ellipsis3,
	84	Minus,
	85	Arrow,
	86	Colon,
	87	Slash,
	88	Comma,
	89	Ampersand,
	90	AmpersandEqual,
	91	IntegerLiteral,
	92	FloatLiteral,
	93	Keyword_align,
	94	Keyword_and,
	95	Keyword_asm,
	96	Keyword_break,
	97	Keyword_coldcc,
	98	Keyword_comptime,
	99	Keyword_const,
	100	Keyword_continue,
	101	Keyword_defer,
	102	Keyword_else,
	103	Keyword_enum,
	104	Keyword_error,
	105	Keyword_export,
	106	Keyword_extern,
	107	Keyword_false,
	108	Keyword_fn,
	109	Keyword_for,
	110	Keyword_goto,
	111	Keyword_if,
	112	Keyword_inline,
	113	Keyword_nakedcc,
	114	Keyword_noalias,
	115	Keyword_null,
	116	Keyword_or,
	117	Keyword_packed,
	118	Keyword_pub,
	119	Keyword_return,
	120	Keyword_stdcallcc,
	121	Keyword_struct,
	122	Keyword_switch,
	123	Keyword_test,
	124	Keyword_this,
	125	Keyword_true,
	126	Keyword_undefined,
	127	Keyword_union,
	128	Keyword_unreachable,
	129	Keyword_use,
	130	Keyword_var,
	131	Keyword_volatile,
	132	Keyword_while,
	133	};
	134	};
	135
	136	pub const Tokenizer = struct {
	137	buffer: []const u8,
	138	index: usize,
	139
	140	pub const Location = struct {
	141	line: usize,
	142	column: usize,
	143	line_start: usize,
	144	line_end: usize,
	145	};
	146
	147	pub fn getTokenLocation(self: &Tokenizer, token: &const Token) -> Location {
	148	var loc = Location {
	149	.line = 0,
	150	.column = 0,
	151	.line_start = 0,
	152	.line_end = 0,
	153	};
	154	for (self.buffer) \|c, i\| {
	155	if (i == token.start) {
	156	loc.line_end = i;
	157	while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
	158	return loc;
	159	}
	160	if (c == '\n') {
	161	loc.line += 1;
	162	loc.column = 0;
	163	loc.line_start = i + 1;
	164	} else {
	165	loc.column += 1;
	166	}
	167	}
	168	return loc;
	169	}
	170
	171	/// For debugging purposes
	172	pub fn dump(self: &Tokenizer, token: &const Token) {
	173	std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
	174	}
	175
	176	pub fn init(buffer: []const u8) -> Tokenizer {
	177	return Tokenizer {
	178	.buffer = buffer,
	179	.index = 0,
	180	};
	181	}
	182
	183	const State = enum {
	184	Start,
	185	Identifier,
	186	Builtin,
	187	C,
	188	StringLiteral,
	189	StringLiteralBackslash,
	190	Minus,
	191	Slash,
	192	LineComment,
	193	Zero,
	194	IntegerLiteral,
	195	NumberDot,
	196	FloatFraction,
	197	FloatExponentUnsigned,
	198	FloatExponentNumber,
	199	Ampersand,
	200	Period,
	201	Period2,
	202	};
	203
	204	pub fn next(self: &Tokenizer) -> Token {
	205	var state = State.Start;
	206	var result = Token {
	207	.id = Token.Id.Eof,
	208	.start = self.index,
	209	.end = undefined,
	210	};
	211	while (self.index < self.buffer.len) : (self.index += 1) {
	212	const c = self.buffer[self.index];
	213	switch (state) {
	214	State.Start => switch (c) {
	215	' ', '\n' => {
	216	result.start = self.index + 1;
	217	},
	218	'c' => {
	219	state = State.C;
	220	result.id = Token.Id.Identifier;
	221	},
	222	'"' => {
	223	state = State.StringLiteral;
	224	result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
	225	},
	226	'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
	227	state = State.Identifier;
	228	result.id = Token.Id.Identifier;
	229	},
	230	'@' => {
	231	state = State.Builtin;
	232	result.id = Token.Id.Builtin;
	233	},
	234	'=' => {
	235	result.id = Token.Id.Equal;
	236	self.index += 1;
	237	break;
	238	},
	239	'(' => {
	240	result.id = Token.Id.LParen;
	241	self.index += 1;
	242	break;
	243	},
	244	')' => {
	245	result.id = Token.Id.RParen;
	246	self.index += 1;
	247	break;
	248	},
	249	';' => {
	250	result.id = Token.Id.Semicolon;
	251	self.index += 1;
	252	break;
	253	},
	254	',' => {
	255	result.id = Token.Id.Comma;
	256	self.index += 1;
	257	break;
	258	},
	259	':' => {
	260	result.id = Token.Id.Colon;
	261	self.index += 1;
	262	break;
	263	},
	264	'%' => {
	265	result.id = Token.Id.Percent;
	266	self.index += 1;
	267	break;
	268	},
	269	'{' => {
	270	result.id = Token.Id.LBrace;
	271	self.index += 1;
	272	break;
	273	},
	274	'}' => {
	275	result.id = Token.Id.RBrace;
	276	self.index += 1;
	277	break;
	278	},
	279	'.' => {
	280	state = State.Period;
	281	},
	282	'-' => {
	283	state = State.Minus;
	284	},
	285	'/' => {
	286	state = State.Slash;
	287	},
	288	'&' => {
	289	state = State.Ampersand;
	290	},
	291	'0' => {
	292	state = State.Zero;
	293	result.id = Token.Id.IntegerLiteral;
	294	},
	295	'1'...'9' => {
	296	state = State.IntegerLiteral;
	297	result.id = Token.Id.IntegerLiteral;
	298	},
	299	else => {
	300	result.id = Token.Id.Invalid;
	301	self.index += 1;
	302	break;
	303	},
	304	},
	305	State.Ampersand => switch (c) {
	306	'=' => {
	307	result.id = Token.Id.AmpersandEqual;
	308	self.index += 1;
	309	break;
	310	},
	311	else => {
	312	result.id = Token.Id.Ampersand;
	313	break;
	314	},
	315	},
	316	State.Identifier => switch (c) {
	317	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	318	else => {
	319	if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
	320	result.id = id;
	321	}
	322	break;
	323	},
	324	},
	325	State.Builtin => switch (c) {
	326	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	327	else => break,
	328	},
	329	State.C => switch (c) {
	330	'\\' => @panic("TODO"),
	331	'"' => {
	332	state = State.StringLiteral;
	333	result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
	334	},
	335	'a'...'z', 'A'...'Z', '_', '0'...'9' => {
	336	state = State.Identifier;
	337	},
	338	else => break,
	339	},
	340	State.StringLiteral => switch (c) {
	341	'\\' => {
	342	state = State.StringLiteralBackslash;
	343	},
	344	'"' => {
	345	self.index += 1;
	346	break;
	347	},
	348	'\n' => break, // Look for this error later.
	349	else => {},
	350	},
	351
	352	State.StringLiteralBackslash => switch (c) {
	353	'\n' => break, // Look for this error later.
	354	else => {
	355	state = State.StringLiteral;
	356	},
	357	},
	358
	359	State.Minus => switch (c) {
	360	'>' => {
	361	result.id = Token.Id.Arrow;
	362	self.index += 1;
	363	break;
	364	},
	365	else => {
	366	result.id = Token.Id.Minus;
	367	break;
	368	},
	369	},
	370
	371	State.Period => switch (c) {
	372	'.' => {
	373	state = State.Period2;
	374	},
	375	else => {
	376	result.id = Token.Id.Period;
	377	break;
	378	},
	379	},
	380
	381	State.Period2 => switch (c) {
	382	'.' => {
	383	result.id = Token.Id.Ellipsis3;
	384	self.index += 1;
	385	break;
	386	},
	387	else => {
	388	result.id = Token.Id.Ellipsis2;
	389	break;
	390	},
	391	},
	392
	393	State.Slash => switch (c) {
	394	'/' => {
	395	result.id = undefined;
	396	state = State.LineComment;
	397	},
	398	else => {
	399	result.id = Token.Id.Slash;
	400	break;
	401	},
	402	},
	403	State.LineComment => switch (c) {
	404	'\n' => {
	405	state = State.Start;
	406	result = Token {
	407	.id = Token.Id.Eof,
	408	.start = self.index + 1,
	409	.end = undefined,
	410	};
	411	},
	412	else => {},
	413	},
	414	State.Zero => switch (c) {
	415	'b', 'o', 'x' => {
	416	state = State.IntegerLiteral;
	417	},
	418	else => {
	419	// reinterpret as a normal number
	420	self.index -= 1;
	421	state = State.IntegerLiteral;
	422	},
	423	},
	424	State.IntegerLiteral => switch (c) {
	425	'.' => {
	426	state = State.NumberDot;
	427	},
	428	'p', 'P', 'e', 'E' => {
	429	state = State.FloatExponentUnsigned;
	430	},
	431	'0'...'9', 'a'...'f', 'A'...'F' => {},
	432	else => break,
	433	},
	434	State.NumberDot => switch (c) {
	435	'.' => {
	436	self.index -= 1;
	437	state = State.Start;
	438	break;
	439	},
	440	else => {
	441	self.index -= 1;
	442	result.id = Token.Id.FloatLiteral;
	443	state = State.FloatFraction;
	444	},
	445	},
	446	State.FloatFraction => switch (c) {
	447	'p', 'P', 'e', 'E' => {
	448	state = State.FloatExponentUnsigned;
	449	},
	450	'0'...'9', 'a'...'f', 'A'...'F' => {},
	451	else => break,
	452	},
	453	State.FloatExponentUnsigned => switch (c) {
	454	'+', '-' => {
	455	state = State.FloatExponentNumber;
	456	},
	457	else => {
	458	// reinterpret as a normal exponent number
	459	self.index -= 1;
	460	state = State.FloatExponentNumber;
	461	}
	462	},
	463	State.FloatExponentNumber => switch (c) {
	464	'0'...'9', 'a'...'f', 'A'...'F' => {},
	465	else => break,
	466	},
	467	}
	468	}
	469	result.end = self.index;
	470	// TODO check state when returning EOF
	471	return result;
	472	}
	473
	474	pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) -> []const u8 {
	475	return self.buffer[token.start..token.end];
	476	}
	477	};
	478
	479