Commit d8d379faf1f656743d118e5e5cfa3dba1e537d65 | zig

author	Andrew Kelley <andrew@ziglang.org> 2017-12-11 16:18:06-05:00
committer	Andrew Kelley <andrew@ziglang.org> 2017-12-11 16:18:06-05:00
log	d8d379faf1f656743d118e5e5cfa3dba1e537d65
tree	13a399cacf74453e0758299787eceeb265968088
parent	a3a590a32ab8165955583b6ea76f89f19bb21740

self-hosted: refactor into multiple files

add return expression
add number literal

4 files changed, 1809 insertions(+), 1673 deletions(-)

src-self-hosted

src-self-hosted/ast.zig created+245

...	@@ -0,0 +1,245 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const ArrayList = std.ArrayList;
4	const Token = @import("tokenizer.zig").Token;
5	const mem = std.mem;
6
7	pub const Node = struct {
8	id: Id,
9
10	pub const Id = enum {
11	Root,
12	VarDecl,
13	Identifier,
14	FnProto,
15	ParamDecl,
16	AddrOfExpr,
17	Block,
18	Return,
19	IntegerLiteral,
20	FloatLiteral,
21	};
22
23	pub fn iterate(base: &Node, index: usize) -> ?&Node {
24	return switch (base.id) {
25	Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index),
26	Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index),
27	Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index),
28	Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
29	Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
30	Id.AddrOfExpr => @fieldParentPtr(NodeAddrOfExpr, "base", base).iterate(index),
31	Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index),
32	Id.Return => @fieldParentPtr(NodeReturn, "base", base).iterate(index),
33	Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
34	Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
35	};
36	}
37
38	pub fn destroy(base: &Node, allocator: &mem.Allocator) {
39	return switch (base.id) {
40	Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
41	Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
42	Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
43	Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
44	Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
45	Id.AddrOfExpr => allocator.destroy(@fieldParentPtr(NodeAddrOfExpr, "base", base)),
46	Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
47	Id.Return => allocator.destroy(@fieldParentPtr(NodeReturn, "base", base)),
48	Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
49	Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
50	};
51	}
52	};
53
54	pub const NodeRoot = struct {
55	base: Node,
56	decls: ArrayList(&Node),
57
58	pub fn iterate(self: &NodeRoot, index: usize) -> ?&Node {
59	if (index < self.decls.len) {
60	return self.decls.items[self.decls.len - index - 1];
61	}
62	return null;
63	}
64	};
65
66	pub const NodeVarDecl = struct {
67	base: Node,
68	visib_token: ?Token,
69	name_token: Token,
70	eq_token: Token,
71	mut_token: Token,
72	comptime_token: ?Token,
73	extern_token: ?Token,
74	lib_name: ?&Node,
75	type_node: ?&Node,
76	align_node: ?&Node,
77	init_node: ?&Node,
78
79	pub fn iterate(self: &NodeVarDecl, index: usize) -> ?&Node {
80	var i = index;
81
82	if (self.type_node) \|type_node\| {
83	if (i < 1) return type_node;
84	i -= 1;
85	}
86
87	if (self.align_node) \|align_node\| {
88	if (i < 1) return align_node;
89	i -= 1;
90	}
91
92	if (self.init_node) \|init_node\| {
93	if (i < 1) return init_node;
94	i -= 1;
95	}
96
97	return null;
98	}
99	};
100
101	pub const NodeIdentifier = struct {
102	base: Node,
103	name_token: Token,
104
105	pub fn iterate(self: &NodeIdentifier, index: usize) -> ?&Node {
106	return null;
107	}
108	};
109
110	pub const NodeFnProto = struct {
111	base: Node,
112	visib_token: ?Token,
113	fn_token: Token,
114	name_token: ?Token,
115	params: ArrayList(&Node),
116	return_type: ?&Node,
117	var_args_token: ?Token,
118	extern_token: ?Token,
119	inline_token: ?Token,
120	cc_token: ?Token,
121	body_node: ?&Node,
122	lib_name: ?&Node, // populated if this is an extern declaration
123	align_expr: ?&Node, // populated if align(A) is present
124
125	pub fn iterate(self: &NodeFnProto, index: usize) -> ?&Node {
126	var i = index;
127
128	if (self.body_node) \|body_node\| {
129	if (i < 1) return body_node;
130	i -= 1;
131	}
132
133	if (self.return_type) \|return_type\| {
134	if (i < 1) return return_type;
135	i -= 1;
136	}
137
138	if (self.align_expr) \|align_expr\| {
139	if (i < 1) return align_expr;
140	i -= 1;
141	}
142
143	if (i < self.params.len) return self.params.items[self.params.len - i - 1];
144	i -= self.params.len;
145
146	if (self.lib_name) \|lib_name\| {
147	if (i < 1) return lib_name;
148	i -= 1;
149	}
150
151	return null;
152	}
153	};
154
155	pub const NodeParamDecl = struct {
156	base: Node,
157	comptime_token: ?Token,
158	noalias_token: ?Token,
159	name_token: ?Token,
160	type_node: &Node,
161	var_args_token: ?Token,
162
163	pub fn iterate(self: &NodeParamDecl, index: usize) -> ?&Node {
164	var i = index;
165
166	if (i < 1) return self.type_node;
167	i -= 1;
168
169	return null;
170	}
171	};
172
173	pub const NodeAddrOfExpr = struct {
174	base: Node,
175	op_token: Token,
176	align_expr: ?&Node,
177	bit_offset_start_token: ?Token,
178	bit_offset_end_token: ?Token,
179	const_token: ?Token,
180	volatile_token: ?Token,
181	op_expr: &Node,
182
183	pub fn iterate(self: &NodeAddrOfExpr, index: usize) -> ?&Node {
184	var i = index;
185
186	if (self.align_expr) \|align_expr\| {
187	if (i < 1) return align_expr;
188	i -= 1;
189	}
190
191	if (i < 1) return self.op_expr;
192	i -= 1;
193
194	return null;
195	}
196	};
197
198	pub const NodeBlock = struct {
199	base: Node,
200	begin_token: Token,
201	end_token: Token,
202	statements: ArrayList(&Node),
203
204	pub fn iterate(self: &NodeBlock, index: usize) -> ?&Node {
205	var i = index;
206
207	if (i < self.statements.len) return self.statements.items[i];
208	i -= self.statements.len;
209
210	return null;
211	}
212	};
213
214	pub const NodeReturn = struct {
215	base: Node,
216	return_token: Token,
217	expr: &Node,
218
219	pub fn iterate(self: &NodeReturn, index: usize) -> ?&Node {
220	var i = index;
221
222	if (i < 1) return self.expr;
223	i -= 1;
224
225	return null;
226	}
227	};
228
229	pub const NodeIntegerLiteral = struct {
230	base: Node,
231	token: Token,
232
233	pub fn iterate(self: &NodeIntegerLiteral, index: usize) -> ?&Node {
234	return null;
235	}
236	};
237
238	pub const NodeFloatLiteral = struct {
239	base: Node,
240	token: Token,
241
242	pub fn iterate(self: &NodeFloatLiteral, index: usize) -> ?&Node {
243	return null;
244	}
245	};

src-self-hosted/main.zig+11-1673

...	@@ -1,1682 +1,13 @@	...	@@ -1,1682 +1,13 @@
1	const std = @import("std");	1	const std = @import("std");
		2	const mem = std.mem;
2	const builtin = @import("builtin");	3	const builtin = @import("builtin");
3	const io = std.io;	4	const io = std.io;
4	const os = std.os;	5	const os = std.os;
5	const heap = std.heap;	6	const heap = std.heap;
6	const warn = std.debug.warn;	7	const warn = std.debug.warn;
7	const assert = std.debug.assert;	8	const Tokenizer = @import("tokenizer.zig").Tokenizer;
8	const mem = std.mem;	9	const Token = @import("tokenizer.zig").Token;
9	const ArrayList = std.ArrayList;	10	const Parser = @import("parser.zig").Parser;
10	const AlignedArrayList = std.AlignedArrayList;
11	const math = std.math;
12
13
14	const Token = struct {
15	id: Id,
16	start: usize,
17	end: usize,
18
19	const KeywordId = struct {
20	bytes: []const u8,
21	id: Id,
22	};
23
24	const keywords = []KeywordId {
25	KeywordId{.bytes="align", .id = Id.Keyword_align},
26	KeywordId{.bytes="and", .id = Id.Keyword_and},
27	KeywordId{.bytes="asm", .id = Id.Keyword_asm},
28	KeywordId{.bytes="break", .id = Id.Keyword_break},
29	KeywordId{.bytes="coldcc", .id = Id.Keyword_coldcc},
30	KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
31	KeywordId{.bytes="const", .id = Id.Keyword_const},
32	KeywordId{.bytes="continue", .id = Id.Keyword_continue},
33	KeywordId{.bytes="defer", .id = Id.Keyword_defer},
34	KeywordId{.bytes="else", .id = Id.Keyword_else},
35	KeywordId{.bytes="enum", .id = Id.Keyword_enum},
36	KeywordId{.bytes="error", .id = Id.Keyword_error},
37	KeywordId{.bytes="export", .id = Id.Keyword_export},
38	KeywordId{.bytes="extern", .id = Id.Keyword_extern},
39	KeywordId{.bytes="false", .id = Id.Keyword_false},
40	KeywordId{.bytes="fn", .id = Id.Keyword_fn},
41	KeywordId{.bytes="for", .id = Id.Keyword_for},
42	KeywordId{.bytes="goto", .id = Id.Keyword_goto},
43	KeywordId{.bytes="if", .id = Id.Keyword_if},
44	KeywordId{.bytes="inline", .id = Id.Keyword_inline},
45	KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
46	KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
47	KeywordId{.bytes="null", .id = Id.Keyword_null},
48	KeywordId{.bytes="or", .id = Id.Keyword_or},
49	KeywordId{.bytes="packed", .id = Id.Keyword_packed},
50	KeywordId{.bytes="pub", .id = Id.Keyword_pub},
51	KeywordId{.bytes="return", .id = Id.Keyword_return},
52	KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
53	KeywordId{.bytes="struct", .id = Id.Keyword_struct},
54	KeywordId{.bytes="switch", .id = Id.Keyword_switch},
55	KeywordId{.bytes="test", .id = Id.Keyword_test},
56	KeywordId{.bytes="this", .id = Id.Keyword_this},
57	KeywordId{.bytes="true", .id = Id.Keyword_true},
58	KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
59	KeywordId{.bytes="union", .id = Id.Keyword_union},
60	KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
61	KeywordId{.bytes="use", .id = Id.Keyword_use},
62	KeywordId{.bytes="var", .id = Id.Keyword_var},
63	KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
64	KeywordId{.bytes="while", .id = Id.Keyword_while},
65	};
66
67	fn getKeyword(bytes: []const u8) -> ?Id {
68	for (keywords) \|kw\| {
69	if (mem.eql(u8, kw.bytes, bytes)) {
70	return kw.id;
71	}
72	}
73	return null;
74	}
75
76	const StrLitKind = enum {Normal, C};
77
78	const Id = union(enum) {
79	Invalid,
80	Identifier,
81	StringLiteral: StrLitKind,
82	Eof,
83	Builtin,
84	Equal,
85	LParen,
86	RParen,
87	Semicolon,
88	Percent,
89	LBrace,
90	RBrace,
91	Period,
92	Ellipsis2,
93	Ellipsis3,
94	Minus,
95	Arrow,
96	Colon,
97	Slash,
98	Comma,
99	Ampersand,
100	AmpersandEqual,
101	NumberLiteral,
102	Keyword_align,
103	Keyword_and,
104	Keyword_asm,
105	Keyword_break,
106	Keyword_coldcc,
107	Keyword_comptime,
108	Keyword_const,
109	Keyword_continue,
110	Keyword_defer,
111	Keyword_else,
112	Keyword_enum,
113	Keyword_error,
114	Keyword_export,
115	Keyword_extern,
116	Keyword_false,
117	Keyword_fn,
118	Keyword_for,
119	Keyword_goto,
120	Keyword_if,
121	Keyword_inline,
122	Keyword_nakedcc,
123	Keyword_noalias,
124	Keyword_null,
125	Keyword_or,
126	Keyword_packed,
127	Keyword_pub,
128	Keyword_return,
129	Keyword_stdcallcc,
130	Keyword_struct,
131	Keyword_switch,
132	Keyword_test,
133	Keyword_this,
134	Keyword_true,
135	Keyword_undefined,
136	Keyword_union,
137	Keyword_unreachable,
138	Keyword_use,
139	Keyword_var,
140	Keyword_volatile,
141	Keyword_while,
142	};
143	};
144
145	const Tokenizer = struct {
146	buffer: []const u8,
147	index: usize,
148
149	pub const Location = struct {
150	line: usize,
151	column: usize,
152	line_start: usize,
153	line_end: usize,
154	};
155
156	pub fn getTokenLocation(self: &Tokenizer, token: &const Token) -> Location {
157	var loc = Location {
158	.line = 0,
159	.column = 0,
160	.line_start = 0,
161	.line_end = 0,
162	};
163	for (self.buffer) \|c, i\| {
164	if (i == token.start) {
165	loc.line_end = i;
166	while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
167	return loc;
168	}
169	if (c == '\n') {
170	loc.line += 1;
171	loc.column = 0;
172	loc.line_start = i + 1;
173	} else {
174	loc.column += 1;
175	}
176	}
177	return loc;
178	}
179
180	pub fn dump(self: &Tokenizer, token: &const Token) {
181	warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
182	}
183
184	pub fn init(buffer: []const u8) -> Tokenizer {
185	return Tokenizer {
186	.buffer = buffer,
187	.index = 0,
188	};
189	}
190
191	const State = enum {
192	Start,
193	Identifier,
194	Builtin,
195	C,
196	StringLiteral,
197	StringLiteralBackslash,
198	Minus,
199	Slash,
200	LineComment,
201	Zero,
202	NumberLiteral,
203	NumberDot,
204	FloatFraction,
205	FloatExponentUnsigned,
206	FloatExponentNumber,
207	Ampersand,
208	Period,
209	Period2,
210	};
211
212	pub fn next(self: &Tokenizer) -> Token {
213	var state = State.Start;
214	var result = Token {
215	.id = Token.Id.Eof,
216	.start = self.index,
217	.end = undefined,
218	};
219	while (self.index < self.buffer.len) : (self.index += 1) {
220	const c = self.buffer[self.index];
221	switch (state) {
222	State.Start => switch (c) {
223	' ', '\n' => {
224	result.start = self.index + 1;
225	},
226	'c' => {
227	state = State.C;
228	result.id = Token.Id.Identifier;
229	},
230	'"' => {
231	state = State.StringLiteral;
232	result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
233	},
234	'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
235	state = State.Identifier;
236	result.id = Token.Id.Identifier;
237	},
238	'@' => {
239	state = State.Builtin;
240	result.id = Token.Id.Builtin;
241	},
242	'=' => {
243	result.id = Token.Id.Equal;
244	self.index += 1;
245	break;
246	},
247	'(' => {
248	result.id = Token.Id.LParen;
249	self.index += 1;
250	break;
251	},
252	')' => {
253	result.id = Token.Id.RParen;
254	self.index += 1;
255	break;
256	},
257	';' => {
258	result.id = Token.Id.Semicolon;
259	self.index += 1;
260	break;
261	},
262	',' => {
263	result.id = Token.Id.Comma;
264	self.index += 1;
265	break;
266	},
267	':' => {
268	result.id = Token.Id.Colon;
269	self.index += 1;
270	break;
271	},
272	'%' => {
273	result.id = Token.Id.Percent;
274	self.index += 1;
275	break;
276	},
277	'{' => {
278	result.id = Token.Id.LBrace;
279	self.index += 1;
280	break;
281	},
282	'}' => {
283	result.id = Token.Id.RBrace;
284	self.index += 1;
285	break;
286	},
287	'.' => {
288	state = State.Period;
289	},
290	'-' => {
291	state = State.Minus;
292	},
293	'/' => {
294	state = State.Slash;
295	},
296	'&' => {
297	state = State.Ampersand;
298	},
299	'0' => {
300	state = State.Zero;
301	result.id = Token.Id.NumberLiteral;
302	},
303	'1'...'9' => {
304	state = State.NumberLiteral;
305	result.id = Token.Id.NumberLiteral;
306	},
307	else => {
308	result.id = Token.Id.Invalid;
309	self.index += 1;
310	break;
311	},
312	},
313	State.Ampersand => switch (c) {
314	'=' => {
315	result.id = Token.Id.AmpersandEqual;
316	self.index += 1;
317	break;
318	},
319	else => {
320	result.id = Token.Id.Ampersand;
321	break;
322	},
323	},
324	State.Identifier => switch (c) {
325	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
326	else => {
327	if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
328	result.id = id;
329	}
330	break;
331	},
332	},
333	State.Builtin => switch (c) {
334	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
335	else => break,
336	},
337	State.C => switch (c) {
338	'\\' => @panic("TODO"),
339	'"' => {
340	state = State.StringLiteral;
341	result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
342	},
343	'a'...'z', 'A'...'Z', '_', '0'...'9' => {
344	state = State.Identifier;
345	},
346	else => break,
347	},
348	State.StringLiteral => switch (c) {
349	'\\' => {
350	state = State.StringLiteralBackslash;
351	},
352	'"' => {
353	self.index += 1;
354	break;
355	},
356	'\n' => break, // Look for this error later.
357	else => {},
358	},
359
360	State.StringLiteralBackslash => switch (c) {
361	'\n' => break, // Look for this error later.
362	else => {
363	state = State.StringLiteral;
364	},
365	},
366
367	State.Minus => switch (c) {
368	'>' => {
369	result.id = Token.Id.Arrow;
370	self.index += 1;
371	break;
372	},
373	else => {
374	result.id = Token.Id.Minus;
375	break;
376	},
377	},
378
379	State.Period => switch (c) {
380	'.' => {
381	state = State.Period2;
382	},
383	else => {
384	result.id = Token.Id.Period;
385	break;
386	},
387	},
388
389	State.Period2 => switch (c) {
390	'.' => {
391	result.id = Token.Id.Ellipsis3;
392	self.index += 1;
393	break;
394	},
395	else => {
396	result.id = Token.Id.Ellipsis2;
397	break;
398	},
399	},
400
401	State.Slash => switch (c) {
402	'/' => {
403	result.id = undefined;
404	state = State.LineComment;
405	},
406	else => {
407	result.id = Token.Id.Slash;
408	break;
409	},
410	},
411	State.LineComment => switch (c) {
412	'\n' => {
413	state = State.Start;
414	result = Token {
415	.id = Token.Id.Eof,
416	.start = self.index + 1,
417	.end = undefined,
418	};
419	},
420	else => {},
421	},
422	State.Zero => switch (c) {
423	'b', 'o', 'x' => {
424	state = State.NumberLiteral;
425	},
426	else => {
427	// reinterpret as a normal number
428	self.index -= 1;
429	state = State.NumberLiteral;
430	},
431	},
432	State.NumberLiteral => switch (c) {
433	'.' => {
434	state = State.NumberDot;
435	},
436	'p', 'P', 'e', 'E' => {
437	state = State.FloatExponentUnsigned;
438	},
439	'0'...'9', 'a'...'f', 'A'...'F' => {},
440	else => break,
441	},
442	State.NumberDot => switch (c) {
443	'.' => {
444	self.index -= 1;
445	state = State.Start;
446	break;
447	},
448	else => {
449	self.index -= 1;
450	state = State.FloatFraction;
451	},
452	},
453	State.FloatFraction => switch (c) {
454	'p', 'P', 'e', 'E' => {
455	state = State.FloatExponentUnsigned;
456	},
457	'0'...'9', 'a'...'f', 'A'...'F' => {},
458	else => break,
459	},
460	State.FloatExponentUnsigned => switch (c) {
461	'+', '-' => {
462	state = State.FloatExponentNumber;
463	},
464	else => {
465	// reinterpret as a normal exponent number
466	self.index -= 1;
467	state = State.FloatExponentNumber;
468	}
469	},
470	State.FloatExponentNumber => switch (c) {
471	'0'...'9', 'a'...'f', 'A'...'F' => {},
472	else => break,
473	},
474	}
475	}
476	result.end = self.index;
477	// TODO check state when returning EOF
478	return result;
479	}
480
481	pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) -> []const u8 {
482	return self.buffer[token.start..token.end];
483	}
484	};
485
486	const Comptime = enum { No, Yes };
487	const NoAlias = enum { No, Yes };
488	const Extern = enum { No, Yes };
489	const VarArgs = enum { No, Yes };
490	const Mutability = enum { Const, Var };
491	const Volatile = enum { No, Yes };
492
493	const Inline = enum {
494	Auto,
495	Always,
496	Never,
497	};
498
499	const Visibility = enum {
500	Private,
501	Pub,
502	Export,
503	};
504
505	const CallingConvention = enum {
506	Auto,
507	C,
508	Cold,
509	Naked,
510	Stdcall,
511	};
512
513	const AstNode = struct {
514	id: Id,
515
516	const Id = enum {
517	Root,
518	VarDecl,
519	Identifier,
520	FnProto,
521	ParamDecl,
522	AddrOfExpr,
523	Block,
524	};
525
526	fn iterate(base: &AstNode, index: usize) -> ?&AstNode {
527	return switch (base.id) {
528	Id.Root => @fieldParentPtr(AstNodeRoot, "base", base).iterate(index),
529	Id.VarDecl => @fieldParentPtr(AstNodeVarDecl, "base", base).iterate(index),
530	Id.Identifier => @fieldParentPtr(AstNodeIdentifier, "base", base).iterate(index),
531	Id.FnProto => @fieldParentPtr(AstNodeFnProto, "base", base).iterate(index),
532	Id.ParamDecl => @fieldParentPtr(AstNodeParamDecl, "base", base).iterate(index),
533	Id.AddrOfExpr => @fieldParentPtr(AstNodeAddrOfExpr, "base", base).iterate(index),
534	Id.Block => @fieldParentPtr(AstNodeBlock, "base", base).iterate(index),
535	};
536	}
537
538	fn destroy(base: &AstNode, allocator: &mem.Allocator) {
539	return switch (base.id) {
540	Id.Root => allocator.destroy(@fieldParentPtr(AstNodeRoot, "base", base)),
541	Id.VarDecl => allocator.destroy(@fieldParentPtr(AstNodeVarDecl, "base", base)),
542	Id.Identifier => allocator.destroy(@fieldParentPtr(AstNodeIdentifier, "base", base)),
543	Id.FnProto => allocator.destroy(@fieldParentPtr(AstNodeFnProto, "base", base)),
544	Id.ParamDecl => allocator.destroy(@fieldParentPtr(AstNodeParamDecl, "base", base)),
545	Id.AddrOfExpr => allocator.destroy(@fieldParentPtr(AstNodeAddrOfExpr, "base", base)),
546	Id.Block => allocator.destroy(@fieldParentPtr(AstNodeBlock, "base", base)),
547	};
548	}
549	};
550
551	const AstNodeRoot = struct {
552	base: AstNode,
553	decls: ArrayList(&AstNode),
554
555	fn iterate(self: &AstNodeRoot, index: usize) -> ?&AstNode {
556	if (index < self.decls.len) {
557	return self.decls.items[self.decls.len - index - 1];
558	}
559	return null;
560	}
561	};
562
563	const AstNodeVarDecl = struct {
564	base: AstNode,
565	visib_token: ?Token,
566	name_token: Token,
567	eq_token: Token,
568	mut_token: Token,
569	comptime_token: ?Token,
570	extern_token: ?Token,
571	lib_name: ?&AstNode,
572	type_node: ?&AstNode,
573	align_node: ?&AstNode,
574	init_node: ?&AstNode,
575
576	fn iterate(self: &AstNodeVarDecl, index: usize) -> ?&AstNode {
577	var i = index;
578
579	if (self.type_node) \|type_node\| {
580	if (i < 1) return type_node;
581	i -= 1;
582	}
583
584	if (self.align_node) \|align_node\| {
585	if (i < 1) return align_node;
586	i -= 1;
587	}
588
589	if (self.init_node) \|init_node\| {
590	if (i < 1) return init_node;
591	i -= 1;
592	}
593
594	return null;
595	}
596	};
597
598	const AstNodeIdentifier = struct {
599	base: AstNode,
600	name_token: Token,
601
602	fn iterate(self: &AstNodeIdentifier, index: usize) -> ?&AstNode {
603	return null;
604	}
605	};
606
607	const AstNodeFnProto = struct {
608	base: AstNode,
609	visib_token: ?Token,
610	fn_token: Token,
611	name_token: ?Token,
612	params: ArrayList(&AstNode),
613	return_type: ?&AstNode,
614	var_args_token: ?Token,
615	extern_token: ?Token,
616	inline_token: ?Token,
617	cc_token: ?Token,
618	body_node: ?&AstNode,
619	lib_name: ?&AstNode, // populated if this is an extern declaration
620	align_expr: ?&AstNode, // populated if align(A) is present
621
622	fn iterate(self: &AstNodeFnProto, index: usize) -> ?&AstNode {
623	var i = index;
624
625	if (self.body_node) \|body_node\| {
626	if (i < 1) return body_node;
627	i -= 1;
628	}
629
630	if (self.return_type) \|return_type\| {
631	if (i < 1) return return_type;
632	i -= 1;
633	}
634
635	if (self.align_expr) \|align_expr\| {
636	if (i < 1) return align_expr;
637	i -= 1;
638	}
639
640	if (i < self.params.len) return self.params.items[self.params.len - i - 1];
641	i -= self.params.len;
642
643	if (self.lib_name) \|lib_name\| {
644	if (i < 1) return lib_name;
645	i -= 1;
646	}
647
648	return null;
649	}
650	};
651
652	const AstNodeParamDecl = struct {
653	base: AstNode,
654	comptime_token: ?Token,
655	noalias_token: ?Token,
656	name_token: ?Token,
657	type_node: &AstNode,
658	var_args_token: ?Token,
659
660	fn iterate(self: &AstNodeParamDecl, index: usize) -> ?&AstNode {
661	var i = index;
662
663	if (i < 1) return self.type_node;
664	i -= 1;
665
666	return null;
667	}
668	};
669
670	const AstNodeAddrOfExpr = struct {
671	base: AstNode,
672	op_token: Token,
673	align_expr: ?&AstNode,
674	bit_offset_start_token: ?Token,
675	bit_offset_end_token: ?Token,
676	const_token: ?Token,
677	volatile_token: ?Token,
678	op_expr: &AstNode,
679
680	fn iterate(self: &AstNodeAddrOfExpr, index: usize) -> ?&AstNode {
681	var i = index;
682
683	if (self.align_expr) \|align_expr\| {
684	if (i < 1) return align_expr;
685	i -= 1;
686	}
687
688	if (i < 1) return self.op_expr;
689	i -= 1;
690
691	return null;
692	}
693	};
694
695	const AstNodeBlock = struct {
696	base: AstNode,
697	begin_token: Token,
698	end_token: Token,
699	statements: ArrayList(&AstNode),
700
701	fn iterate(self: &AstNodeBlock, index: usize) -> ?&AstNode {
702	var i = index;
703
704	if (i < self.statements.len) return self.statements.items[i];
705	i -= self.statements.len;
706
707	return null;
708	}
709	};
710
711	error ParseError;
712
713	const Parser = struct {
714	allocator: &mem.Allocator,
715	tokenizer: &Tokenizer,
716	put_back_tokens: [2]Token,
717	put_back_count: usize,
718	source_file_name: []const u8,
719
720	// This memory contents are used only during a function call. It's used to repurpose memory;
721	// specifically so that freeAst can be guaranteed to succeed.
722	const utility_bytes_align = @alignOf( union { a: RenderAstFrame, b: State, c: RenderState } );
723	utility_bytes: []align(utility_bytes_align) u8,
724
725	fn initUtilityArrayList(self: &Parser, comptime T: type) -> ArrayList(T) {
726	const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
727	self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
728	const typed_slice = ([]T)(self.utility_bytes);
729	return ArrayList(T).fromOwnedSlice(self.allocator, typed_slice);
730	}
731
732	fn deinitUtilityArrayList(self: &Parser, list: var) {
733	self.utility_bytes = ([]align(utility_bytes_align) u8)(list.toOwnedSlice());
734	}
735
736	pub fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) -> Parser {
737	return Parser {
738	.allocator = allocator,
739	.tokenizer = tokenizer,
740	.put_back_tokens = undefined,
741	.put_back_count = 0,
742	.source_file_name = source_file_name,
743	.utility_bytes = []align(utility_bytes_align) u8{},
744	};
745	}
746
747	pub fn deinit(self: &Parser) {
748	self.allocator.free(self.utility_bytes);
749	}
750
751	const TopLevelDeclCtx = struct {
752	visib_token: ?Token,
753	extern_token: ?Token,
754	};
755
756	const DestPtr = union(enum) {
757	Field: &&AstNode,
758	NullableField: &?&AstNode,
759	List: &ArrayList(&AstNode),
760
761	pub fn store(self: &const DestPtr, value: &AstNode) -> %void {
762	switch (*self) {
763	DestPtr.Field => \|ptr\| *ptr = value,
764	DestPtr.NullableField => \|ptr\| *ptr = value,
765	DestPtr.List => \|list\| %return list.append(value),
766	}
767	}
768	};
769
770	const State = union(enum) {
771	TopLevel,
772	TopLevelExtern: ?Token,
773	TopLevelDecl: TopLevelDeclCtx,
774	Expression: DestPtr,
775	GroupedExpression: DestPtr,
776	UnwrapExpression: DestPtr,
777	BoolOrExpression: DestPtr,
778	BoolAndExpression: DestPtr,
779	ComparisonExpression: DestPtr,
780	BinaryOrExpression: DestPtr,
781	BinaryXorExpression: DestPtr,
782	BinaryAndExpression: DestPtr,
783	BitShiftExpression: DestPtr,
784	AdditionExpression: DestPtr,
785	MultiplyExpression: DestPtr,
786	BraceSuffixExpression: DestPtr,
787	PrefixOpExpression: DestPtr,
788	SuffixOpExpression: DestPtr,
789	PrimaryExpression: DestPtr,
790	TypeExpr: DestPtr,
791	VarDecl: &AstNodeVarDecl,
792	VarDeclAlign: &AstNodeVarDecl,
793	VarDeclEq: &AstNodeVarDecl,
794	ExpectToken: @TagType(Token.Id),
795	FnProto: &AstNodeFnProto,
796	FnProtoAlign: &AstNodeFnProto,
797	ParamDecl: &AstNodeFnProto,
798	ParamDeclComma,
799	FnDef: &AstNodeFnProto,
800	Block: &AstNodeBlock,
801	Statement: &AstNodeBlock,
802	};
803
804	pub fn freeAst(self: &Parser, root_node: &AstNodeRoot) {
805	// utility_bytes is big enough to do this iteration since we were able to do
806	// the parsing in the first place
807	comptime assert(@sizeOf(State) >= @sizeOf(&AstNode));
808
809	var stack = self.initUtilityArrayList(&AstNode);
810	defer self.deinitUtilityArrayList(stack);
811
812	stack.append(&root_node.base) %% unreachable;
813	while (stack.popOrNull()) \|node\| {
814	var i: usize = 0;
815	while (node.iterate(i)) \|child\| : (i += 1) {
816	if (child.iterate(0) != null) {
817	stack.append(child) %% unreachable;
818	} else {
819	child.destroy(self.allocator);
820	}
821	}
822	node.destroy(self.allocator);
823	}
824	}
825
826	pub fn parse(self: &Parser) -> %&AstNodeRoot {
827	var stack = self.initUtilityArrayList(State);
828	defer self.deinitUtilityArrayList(stack);
829
830	const root_node = %return self.createRoot();
831	%defer self.allocator.destroy(root_node);
832	%return stack.append(State.TopLevel);
833	%defer self.freeAst(root_node);
834
835	while (true) {
836	//{
837	// const token = self.getNextToken();
838	// warn("{} ", @tagName(token.id));
839	// self.putBackToken(token);
840	// var i: usize = stack.len;
841	// while (i != 0) {
842	// i -= 1;
843	// warn("{} ", @tagName(stack.items[i]));
844	// }
845	// warn("\n");
846	//}
847
848	// This gives us 1 free append that can't fail
849	const state = stack.pop();
850
851	switch (state) {
852	State.TopLevel => {
853	const token = self.getNextToken();
854	switch (token.id) {
855	Token.Id.Keyword_pub, Token.Id.Keyword_export => {
856	stack.append(State { .TopLevelExtern = token }) %% unreachable;
857	continue;
858	},
859	Token.Id.Eof => return root_node,
860	else => {
861	self.putBackToken(token);
862	// TODO shouldn't need this cast
863	stack.append(State { .TopLevelExtern = null }) %% unreachable;
864	continue;
865	},
866	}
867	},
868	State.TopLevelExtern => \|visib_token\| {
869	const token = self.getNextToken();
870	if (token.id == Token.Id.Keyword_extern) {
871	stack.append(State {
872	.TopLevelDecl = TopLevelDeclCtx {
873	.visib_token = visib_token,
874	.extern_token = token,
875	},
876	}) %% unreachable;
877	continue;
878	}
879	self.putBackToken(token);
880	stack.append(State {
881	.TopLevelDecl = TopLevelDeclCtx {
882	.visib_token = visib_token,
883	.extern_token = null,
884	},
885	}) %% unreachable;
886	continue;
887	},
888	State.TopLevelDecl => \|ctx\| {
889	const token = self.getNextToken();
890	switch (token.id) {
891	Token.Id.Keyword_var, Token.Id.Keyword_const => {
892	stack.append(State.TopLevel) %% unreachable;
893	// TODO shouldn't need these casts
894	const var_decl_node = %return self.createAttachVarDecl(&root_node.decls, ctx.visib_token,
895	token, (?Token)(null), ctx.extern_token);
896	%return stack.append(State { .VarDecl = var_decl_node });
897	continue;
898	},
899	Token.Id.Keyword_fn => {
900	stack.append(State.TopLevel) %% unreachable;
901	// TODO shouldn't need these casts
902	const fn_proto = %return self.createAttachFnProto(&root_node.decls, token,
903	ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null));
904	%return stack.append(State { .FnDef = fn_proto });
905	%return stack.append(State { .FnProto = fn_proto });
906	continue;
907	},
908	Token.Id.StringLiteral => {
909	@panic("TODO extern with string literal");
910	},
911	Token.Id.Keyword_coldcc, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
912	stack.append(State.TopLevel) %% unreachable;
913	const fn_token = %return self.eatToken(Token.Id.Keyword_fn);
914	// TODO shouldn't need this cast
915	const fn_proto = %return self.createAttachFnProto(&root_node.decls, fn_token,
916	ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null));
917	%return stack.append(State { .FnDef = fn_proto });
918	%return stack.append(State { .FnProto = fn_proto });
919	continue;
920	},
921	else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)),
922	}
923	},
924	State.VarDecl => \|var_decl\| {
925	var_decl.name_token = %return self.eatToken(Token.Id.Identifier);
926	stack.append(State { .VarDeclAlign = var_decl }) %% unreachable;
927
928	const next_token = self.getNextToken();
929	if (next_token.id == Token.Id.Colon) {
930	%return stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} });
931	continue;
932	}
933
934	self.putBackToken(next_token);
935	continue;
936	},
937	State.VarDeclAlign => \|var_decl\| {
938	stack.append(State { .VarDeclEq = var_decl }) %% unreachable;
939
940	const next_token = self.getNextToken();
941	if (next_token.id == Token.Id.Keyword_align) {
942	%return stack.append(State {
943	.GroupedExpression = DestPtr {
944	.NullableField = &var_decl.align_node
945	}
946	});
947	continue;
948	}
949
950	self.putBackToken(next_token);
951	continue;
952	},
953	State.VarDeclEq => \|var_decl\| {
954	const token = self.getNextToken();
955	if (token.id == Token.Id.Equal) {
956	var_decl.eq_token = token;
957	stack.append(State { .ExpectToken = Token.Id.Semicolon }) %% unreachable;
958	%return stack.append(State {
959	.Expression = DestPtr {.NullableField = &var_decl.init_node},
960	});
961	continue;
962	}
963	if (token.id == Token.Id.Semicolon) {
964	continue;
965	}
966	return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
967	},
968	State.ExpectToken => \|token_id\| {
969	_ = %return self.eatToken(token_id);
970	continue;
971	},
972	State.Expression => \|dest_ptr\| {
973	stack.append(State {.UnwrapExpression = dest_ptr}) %% unreachable;
974	continue;
975	},
976
977	State.UnwrapExpression => \|dest_ptr\| {
978	stack.append(State {.BoolOrExpression = dest_ptr}) %% unreachable;
979	continue;
980	},
981
982	State.BoolOrExpression => \|dest_ptr\| {
983	stack.append(State {.BoolAndExpression = dest_ptr}) %% unreachable;
984	continue;
985	},
986
987	State.BoolAndExpression => \|dest_ptr\| {
988	stack.append(State {.ComparisonExpression = dest_ptr}) %% unreachable;
989	continue;
990	},
991
992	State.ComparisonExpression => \|dest_ptr\| {
993	stack.append(State {.BinaryOrExpression = dest_ptr}) %% unreachable;
994	continue;
995	},
996
997	State.BinaryOrExpression => \|dest_ptr\| {
998	stack.append(State {.BinaryXorExpression = dest_ptr}) %% unreachable;
999	continue;
1000	},
1001
1002	State.BinaryXorExpression => \|dest_ptr\| {
1003	stack.append(State {.BinaryAndExpression = dest_ptr}) %% unreachable;
1004	continue;
1005	},
1006
1007	State.BinaryAndExpression => \|dest_ptr\| {
1008	stack.append(State {.BitShiftExpression = dest_ptr}) %% unreachable;
1009	continue;
1010	},
1011
1012	State.BitShiftExpression => \|dest_ptr\| {
1013	stack.append(State {.AdditionExpression = dest_ptr}) %% unreachable;
1014	continue;
1015	},
1016
1017	State.AdditionExpression => \|dest_ptr\| {
1018	stack.append(State {.MultiplyExpression = dest_ptr}) %% unreachable;
1019	continue;
1020	},
1021
1022	State.MultiplyExpression => \|dest_ptr\| {
1023	stack.append(State {.BraceSuffixExpression = dest_ptr}) %% unreachable;
1024	continue;
1025	},
1026
1027	State.BraceSuffixExpression => \|dest_ptr\| {
1028	stack.append(State {.PrefixOpExpression = dest_ptr}) %% unreachable;
1029	continue;
1030	},
1031
1032	State.PrefixOpExpression => \|dest_ptr\| {
1033	const first_token = self.getNextToken();
1034	if (first_token.id == Token.Id.Ampersand) {
1035	const addr_of_expr = %return self.createAttachAddrOfExpr(dest_ptr, first_token);
1036	var token = self.getNextToken();
1037	if (token.id == Token.Id.Keyword_align) {
1038	@panic("TODO align");
1039	}
1040	if (token.id == Token.Id.Keyword_const) {
1041	addr_of_expr.const_token = token;
1042	token = self.getNextToken();
1043	}
1044	if (token.id == Token.Id.Keyword_volatile) {
1045	addr_of_expr.volatile_token = token;
1046	token = self.getNextToken();
1047	}
1048	self.putBackToken(token);
1049	stack.append(State {
1050	.PrefixOpExpression = DestPtr { .Field = &addr_of_expr.op_expr},
1051	}) %% unreachable;
1052	continue;
1053	}
1054
1055	self.putBackToken(first_token);
1056	stack.append(State { .SuffixOpExpression = dest_ptr }) %% unreachable;
1057	continue;
1058	},
1059
1060	State.SuffixOpExpression => \|dest_ptr\| {
1061	stack.append(State { .PrimaryExpression = dest_ptr }) %% unreachable;
1062	continue;
1063	},
1064
1065	State.PrimaryExpression => \|dest_ptr\| {
1066	const token = self.getNextToken();
1067	switch (token.id) {
1068	Token.Id.Identifier => {
1069	const identifier = %return self.createIdentifier(token);
1070	%return dest_ptr.store(&identifier.base);
1071	continue;
1072	},
1073	else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
1074	}
1075	},
1076
1077	State.TypeExpr => \|dest_ptr\| {
1078	const token = self.getNextToken();
1079	if (token.id == Token.Id.Keyword_var) {
1080	@panic("TODO param with type var");
1081	}
1082	self.putBackToken(token);
1083
1084	stack.append(State { .PrefixOpExpression = dest_ptr }) %% unreachable;
1085	continue;
1086	},
1087
1088	State.FnProto => \|fn_proto\| {
1089	stack.append(State { .FnProtoAlign = fn_proto }) %% unreachable;
1090	%return stack.append(State { .ParamDecl = fn_proto });
1091	%return stack.append(State { .ExpectToken = Token.Id.LParen });
1092
1093	const next_token = self.getNextToken();
1094	if (next_token.id == Token.Id.Identifier) {
1095	fn_proto.name_token = next_token;
1096	continue;
1097	}
1098	self.putBackToken(next_token);
1099	continue;
1100	},
1101
1102	State.FnProtoAlign => \|fn_proto\| {
1103	const token = self.getNextToken();
1104	if (token.id == Token.Id.Keyword_align) {
1105	@panic("TODO fn proto align");
1106	}
1107	if (token.id == Token.Id.Arrow) {
1108	stack.append(State {
1109	.TypeExpr = DestPtr {.NullableField = &fn_proto.return_type},
1110	}) %% unreachable;
1111	continue;
1112	} else {
1113	self.putBackToken(token);
1114	continue;
1115	}
1116	},
1117
1118	State.ParamDecl => \|fn_proto\| {
1119	var token = self.getNextToken();
1120	if (token.id == Token.Id.RParen) {
1121	continue;
1122	}
1123	const param_decl = %return self.createAttachParamDecl(&fn_proto.params);
1124	if (token.id == Token.Id.Keyword_comptime) {
1125	param_decl.comptime_token = token;
1126	token = self.getNextToken();
1127	} else if (token.id == Token.Id.Keyword_noalias) {
1128	param_decl.noalias_token = token;
1129	token = self.getNextToken();
1130	};
1131	if (token.id == Token.Id.Identifier) {
1132	const next_token = self.getNextToken();
1133	if (next_token.id == Token.Id.Colon) {
1134	param_decl.name_token = token;
1135	token = self.getNextToken();
1136	} else {
1137	self.putBackToken(next_token);
1138	}
1139	}
1140	if (token.id == Token.Id.Ellipsis3) {
1141	param_decl.var_args_token = token;
1142	stack.append(State { .ExpectToken = Token.Id.RParen }) %% unreachable;
1143	continue;
1144	} else {
1145	self.putBackToken(token);
1146	}
1147
1148	stack.append(State { .ParamDecl = fn_proto }) %% unreachable;
1149	%return stack.append(State.ParamDeclComma);
1150	%return stack.append(State {
1151	.TypeExpr = DestPtr {.Field = &param_decl.type_node}
1152	});
1153	continue;
1154	},
1155
1156	State.ParamDeclComma => {
1157	const token = self.getNextToken();
1158	switch (token.id) {
1159	Token.Id.RParen => {
1160	_ = stack.pop(); // pop off the ParamDecl
1161	continue;
1162	},
1163	Token.Id.Comma => continue,
1164	else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
1165	}
1166	},
1167
1168	State.FnDef => \|fn_proto\| {
1169	const token = self.getNextToken();
1170	switch(token.id) {
1171	Token.Id.LBrace => {
1172	const block = %return self.createBlock(token);
1173	fn_proto.body_node = &block.base;
1174	stack.append(State { .Block = block }) %% unreachable;
1175	continue;
1176	},
1177	Token.Id.Semicolon => continue,
1178	else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)),
1179	}
1180	},
1181
1182	State.Block => \|block\| {
1183	const token = self.getNextToken();
1184	switch (token.id) {
1185	Token.Id.RBrace => {
1186	block.end_token = token;
1187	continue;
1188	},
1189	else => {
1190	self.putBackToken(token);
1191	stack.append(State { .Block = block }) %% unreachable;
1192	%return stack.append(State { .Statement = block });
1193	continue;
1194	},
1195	}
1196	},
1197
1198	State.Statement => \|block\| {
1199	{
1200	// Look for comptime var, comptime const
1201	const comptime_token = self.getNextToken();
1202	if (comptime_token.id == Token.Id.Keyword_comptime) {
1203	const mut_token = self.getNextToken();
1204	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
1205	// TODO shouldn't need these casts
1206	const var_decl = %return self.createAttachVarDecl(&block.statements, (?Token)(null),
1207	mut_token, (?Token)(comptime_token), (?Token)(null));
1208	%return stack.append(State { .VarDecl = var_decl });
1209	continue;
1210	}
1211	self.putBackToken(mut_token);
1212	}
1213	self.putBackToken(comptime_token);
1214	}
1215	{
1216	// Look for const, var
1217	const mut_token = self.getNextToken();
1218	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
1219	// TODO shouldn't need these casts
1220	const var_decl = %return self.createAttachVarDecl(&block.statements, (?Token)(null),
1221	mut_token, (?Token)(null), (?Token)(null));
1222	%return stack.append(State { .VarDecl = var_decl });
1223	continue;
1224	}
1225	self.putBackToken(mut_token);
1226	}
1227
1228	stack.append(State { .ExpectToken = Token.Id.Semicolon }) %% unreachable;
1229	%return stack.append(State { .Expression = DestPtr{.List = &block.statements} });
1230	continue;
1231	},
1232
1233	State.GroupedExpression => @panic("TODO"),
1234	}
1235	unreachable;
1236	}
1237	}
1238
1239	fn createRoot(self: &Parser) -> %&AstNodeRoot {
1240	const node = %return self.allocator.create(AstNodeRoot);
1241	%defer self.allocator.destroy(node);
1242
1243	*node = AstNodeRoot {
1244	.base = AstNode {.id = AstNode.Id.Root},
1245	.decls = ArrayList(&AstNode).init(self.allocator),
1246	};
1247	return node;
1248	}
1249
1250	fn createVarDecl(self: &Parser, visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token,
1251	extern_token: &const ?Token) -> %&AstNodeVarDecl
1252	{
1253	const node = %return self.allocator.create(AstNodeVarDecl);
1254	%defer self.allocator.destroy(node);
1255
1256	*node = AstNodeVarDecl {
1257	.base = AstNode {.id = AstNode.Id.VarDecl},
1258	.visib_token = *visib_token,
1259	.mut_token = *mut_token,
1260	.comptime_token = *comptime_token,
1261	.extern_token = *extern_token,
1262	.type_node = null,
1263	.align_node = null,
1264	.init_node = null,
1265	.lib_name = null,
1266	// initialized later
1267	.name_token = undefined,
1268	.eq_token = undefined,
1269	};
1270	return node;
1271	}
1272
1273	fn createIdentifier(self: &Parser, name_token: &const Token) -> %&AstNodeIdentifier {
1274	const node = %return self.allocator.create(AstNodeIdentifier);
1275	%defer self.allocator.destroy(node);
1276
1277	*node = AstNodeIdentifier {
1278	.base = AstNode {.id = AstNode.Id.Identifier},
1279	.name_token = *name_token,
1280	};
1281	return node;
1282	}
1283
1284	fn createFnProto(self: &Parser, fn_token: &const Token, extern_token: &const ?Token,
1285	cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) -> %&AstNodeFnProto
1286	{
1287	const node = %return self.allocator.create(AstNodeFnProto);
1288	%defer self.allocator.destroy(node);
1289
1290	*node = AstNodeFnProto {
1291	.base = AstNode {.id = AstNode.Id.FnProto},
1292	.visib_token = *visib_token,
1293	.name_token = null,
1294	.fn_token = *fn_token,
1295	.params = ArrayList(&AstNode).init(self.allocator),
1296	.return_type = null,
1297	.var_args_token = null,
1298	.extern_token = *extern_token,
1299	.inline_token = *inline_token,
1300	.cc_token = *cc_token,
1301	.body_node = null,
1302	.lib_name = null,
1303	.align_expr = null,
1304	};
1305	return node;
1306	}
1307
1308	fn createParamDecl(self: &Parser) -> %&AstNodeParamDecl {
1309	const node = %return self.allocator.create(AstNodeParamDecl);
1310	%defer self.allocator.destroy(node);
1311
1312	*node = AstNodeParamDecl {
1313	.base = AstNode {.id = AstNode.Id.ParamDecl},
1314	.comptime_token = null,
1315	.noalias_token = null,
1316	.name_token = null,
1317	.type_node = undefined,
1318	.var_args_token = null,
1319	};
1320	return node;
1321	}
1322
1323	fn createAddrOfExpr(self: &Parser, op_token: &const Token) -> %&AstNodeAddrOfExpr {
1324	const node = %return self.allocator.create(AstNodeAddrOfExpr);
1325	%defer self.allocator.destroy(node);
1326
1327	*node = AstNodeAddrOfExpr {
1328	.base = AstNode {.id = AstNode.Id.AddrOfExpr},
1329	.align_expr = null,
1330	.op_token = *op_token,
1331	.bit_offset_start_token = null,
1332	.bit_offset_end_token = null,
1333	.const_token = null,
1334	.volatile_token = null,
1335	.op_expr = undefined,
1336	};
1337	return node;
1338	}
1339
1340	fn createBlock(self: &Parser, begin_token: &const Token) -> %&AstNodeBlock {
1341	const node = %return self.allocator.create(AstNodeBlock);
1342	%defer self.allocator.destroy(node);
1343
1344	*node = AstNodeBlock {
1345	.base = AstNode {.id = AstNode.Id.Block},
1346	.begin_token = *begin_token,
1347	.end_token = undefined,
1348	.statements = ArrayList(&AstNode).init(self.allocator),
1349	};
1350	return node;
1351	}
1352
1353	fn createAttachAddrOfExpr(self: &Parser, dest_ptr: &const DestPtr, op_token: &const Token) -> %&AstNodeAddrOfExpr {
1354	const node = %return self.createAddrOfExpr(op_token);
1355	%defer self.allocator.destroy(node);
1356	%return dest_ptr.store(&node.base);
1357	return node;
1358	}
1359
1360	fn createAttachParamDecl(self: &Parser, list: &ArrayList(&AstNode)) -> %&AstNodeParamDecl {
1361	const node = %return self.createParamDecl();
1362	%defer self.allocator.destroy(node);
1363	%return list.append(&node.base);
1364	return node;
1365	}
1366
1367	fn createAttachFnProto(self: &Parser, list: &ArrayList(&AstNode), fn_token: &const Token,
1368	extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token,
1369	inline_token: &const ?Token) -> %&AstNodeFnProto
1370	{
1371	const node = %return self.createFnProto(fn_token, extern_token, cc_token, visib_token, inline_token);
1372	%defer self.allocator.destroy(node);
1373	%return list.append(&node.base);
1374	return node;
1375	}
1376
1377	fn createAttachVarDecl(self: &Parser, list: &ArrayList(&AstNode), visib_token: &const ?Token,
1378	mut_token: &const Token, comptime_token: &const ?Token, extern_token: &const ?Token) -> %&AstNodeVarDecl
1379	{
1380	const node = %return self.createVarDecl(visib_token, mut_token, comptime_token, extern_token);
1381	%defer self.allocator.destroy(node);
1382	%return list.append(&node.base);
1383	return node;
1384	}
1385
1386	fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) -> error {
1387	const loc = self.tokenizer.getTokenLocation(token);
1388	warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
1389	warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
1390	{
1391	var i: usize = 0;
1392	while (i < loc.column) : (i += 1) {
1393	warn(" ");
1394	}
1395	}
1396	{
1397	const caret_count = token.end - token.start;
1398	var i: usize = 0;
1399	while (i < caret_count) : (i += 1) {
1400	warn("~");
1401	}
1402	}
1403	warn("\n");
1404	return error.ParseError;
1405	}
1406
1407	fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) -> %void {
1408	if (token.id != id) {
1409	return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
1410	}
1411	}
1412
1413	fn eatToken(self: &Parser, id: @TagType(Token.Id)) -> %Token {
1414	const token = self.getNextToken();
1415	%return self.expectToken(token, id);
1416	return token;
1417	}
1418
1419	fn putBackToken(self: &Parser, token: &const Token) {
1420	self.put_back_tokens[self.put_back_count] = *token;
1421	self.put_back_count += 1;
1422	}
1423
1424	fn getNextToken(self: &Parser) -> Token {
1425	return if (self.put_back_count != 0) {
1426	const put_back_index = self.put_back_count - 1;
1427	const put_back_token = self.put_back_tokens[put_back_index];
1428	self.put_back_count = put_back_index;
1429	put_back_token
1430	} else {
1431	self.tokenizer.next()
1432	};
1433	}
1434
1435	const RenderAstFrame = struct {
1436	node: &AstNode,
1437	indent: usize,
1438	};
1439
1440	pub fn renderAst(self: &Parser, stream: &std.io.OutStream, root_node: &AstNodeRoot) -> %void {
1441	var stack = self.initUtilityArrayList(RenderAstFrame);
1442	defer self.deinitUtilityArrayList(stack);
1443
1444	%return stack.append(RenderAstFrame {
1445	.node = &root_node.base,
1446	.indent = 0,
1447	});
1448
1449	while (stack.popOrNull()) \|frame\| {
1450	{
1451	var i: usize = 0;
1452	while (i < frame.indent) : (i += 1) {
1453	%return stream.print(" ");
1454	}
1455	}
1456	%return stream.print("{}\n", @tagName(frame.node.id));
1457	var child_i: usize = 0;
1458	while (frame.node.iterate(child_i)) \|child\| : (child_i += 1) {
1459	%return stack.append(RenderAstFrame {
1460	.node = child,
1461	.indent = frame.indent + 2,
1462	});
1463	}
1464	}
1465	}
1466
1467
1468	pub const RenderState = union(enum) {
1469	TopLevelDecl: &AstNode,
1470	FnProtoRParen: &AstNodeFnProto,
1471	ParamDecl: &AstNode,
1472	Text: []const u8,
1473	Expression: &AstNode,
1474	AddrOfExprBit: &AstNodeAddrOfExpr,
1475	VarDecl: &AstNodeVarDecl,
1476	VarDeclAlign: &AstNodeVarDecl,
1477	Statement: &AstNode,
1478	PrintIndent,
1479	Indent: usize,
1480	};
1481
1482	pub fn renderSource(self: &Parser, stream: &std.io.OutStream, root_node: &AstNodeRoot) -> %void {
1483	var stack = self.initUtilityArrayList(RenderState);
1484	defer self.deinitUtilityArrayList(stack);
1485
1486	{
1487	var i = root_node.decls.len;
1488	while (i != 0) {
1489	i -= 1;
1490	const decl = root_node.decls.items[i];
1491	%return stack.append(RenderState {.TopLevelDecl = decl});
1492	}
1493	}
1494
1495	const indent_delta = 4;
1496	var indent: usize = 0;
1497	while (stack.popOrNull()) \|state\| {
1498	switch (state) {
1499	RenderState.TopLevelDecl => \|decl\| {
1500	switch (decl.id) {
1501	AstNode.Id.FnProto => {
1502	const fn_proto = @fieldParentPtr(AstNodeFnProto, "base", decl);
1503	if (fn_proto.visib_token) \|visib_token\| {
1504	switch (visib_token.id) {
1505	Token.Id.Keyword_pub => %return stream.print("pub "),
1506	Token.Id.Keyword_export => %return stream.print("export "),
1507	else => unreachable,
1508	};
1509	}
1510	if (fn_proto.extern_token) \|extern_token\| {
1511	%return stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
1512	}
1513	%return stream.print("fn");
1514
1515	if (fn_proto.name_token) \|name_token\| {
1516	%return stream.print(" {}", self.tokenizer.getTokenSlice(name_token));
1517	}
1518
1519	%return stream.print("(");
1520
1521	%return stack.append(RenderState { .Text = "\n" });
1522	if (fn_proto.body_node == null) {
1523	%return stack.append(RenderState { .Text = ";" });
1524	}
1525
1526	%return stack.append(RenderState { .FnProtoRParen = fn_proto});
1527	var i = fn_proto.params.len;
1528	while (i != 0) {
1529	i -= 1;
1530	const param_decl_node = fn_proto.params.items[i];
1531	%return stack.append(RenderState { .ParamDecl = param_decl_node});
1532	if (i != 0) {
1533	%return stack.append(RenderState { .Text = ", " });
1534	}
1535	}
1536	},
1537	AstNode.Id.VarDecl => {
1538	const var_decl = @fieldParentPtr(AstNodeVarDecl, "base", decl);
1539	%return stack.append(RenderState { .Text = "\n"});
1540	%return stack.append(RenderState { .VarDecl = var_decl});
1541
1542	},
1543	else => unreachable,
1544	}
1545	},
1546
1547	RenderState.VarDecl => \|var_decl\| {
1548	if (var_decl.visib_token) \|visib_token\| {
1549	%return stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
1550	}
1551	if (var_decl.extern_token) \|extern_token\| {
1552	%return stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
1553	if (var_decl.lib_name != null) {
1554	@panic("TODO");
1555	}
1556	}
1557	if (var_decl.comptime_token) \|comptime_token\| {
1558	%return stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
1559	}
1560	%return stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token));
1561	%return stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token));
1562
1563	%return stack.append(RenderState { .VarDeclAlign = var_decl });
1564	if (var_decl.type_node) \|type_node\| {
1565	%return stream.print(": ");
1566	%return stack.append(RenderState { .Expression = type_node });
1567	}
1568	},
1569
1570	RenderState.VarDeclAlign => \|var_decl\| {
1571	if (var_decl.align_node != null) {
1572	@panic("TODO");
1573	}
1574	%return stack.append(RenderState { .Text = ";" });
1575	if (var_decl.init_node) \|init_node\| {
1576	%return stream.print(" = ");
1577	%return stack.append(RenderState { .Expression = init_node });
1578	}
1579	},
1580
1581	RenderState.ParamDecl => \|base\| {
1582	const param_decl = @fieldParentPtr(AstNodeParamDecl, "base", base);
1583	if (param_decl.comptime_token) \|comptime_token\| {
1584	%return stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
1585	}
1586	if (param_decl.noalias_token) \|noalias_token\| {
1587	%return stream.print("{} ", self.tokenizer.getTokenSlice(noalias_token));
1588	}
1589	if (param_decl.name_token) \|name_token\| {
1590	%return stream.print("{}: ", self.tokenizer.getTokenSlice(name_token));
1591	}
1592	if (param_decl.var_args_token) \|var_args_token\| {
1593	%return stream.print("{}", self.tokenizer.getTokenSlice(var_args_token));
1594	} else {
1595	%return stack.append(RenderState { .Expression = param_decl.type_node});
1596	}
1597	},
1598	RenderState.Text => \|bytes\| {
1599	%return stream.write(bytes);
1600	},
1601	RenderState.Expression => \|base\| switch (base.id) {
1602	AstNode.Id.Identifier => {
1603	const identifier = @fieldParentPtr(AstNodeIdentifier, "base", base);
1604	%return stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token));
1605	},
1606	AstNode.Id.AddrOfExpr => {
1607	const addr_of_expr = @fieldParentPtr(AstNodeAddrOfExpr, "base", base);
1608	%return stream.print("{}", self.tokenizer.getTokenSlice(addr_of_expr.op_token));
1609	%return stack.append(RenderState { .AddrOfExprBit = addr_of_expr});
1610
1611	if (addr_of_expr.align_expr) \|align_expr\| {
1612	%return stream.print("align(");
1613	%return stack.append(RenderState { .Text = ")"});
1614	%return stack.append(RenderState { .Expression = align_expr});
1615	}
1616	},
1617	AstNode.Id.Block => {
1618	const block = @fieldParentPtr(AstNodeBlock, "base", base);
1619	%return stream.write("{");
1620	%return stack.append(RenderState { .Text = "}"});
1621	%return stack.append(RenderState.PrintIndent);
1622	%return stack.append(RenderState { .Indent = indent});
1623	%return stack.append(RenderState { .Text = "\n"});
1624	var i = block.statements.len;
1625	while (i != 0) {
1626	i -= 1;
1627	const statement_node = block.statements.items[i];
1628	%return stack.append(RenderState { .Statement = statement_node});
1629	%return stack.append(RenderState.PrintIndent);
1630	%return stack.append(RenderState { .Indent = indent + indent_delta});
1631	%return stack.append(RenderState { .Text = "\n" });
1632	}
1633	},
1634	else => unreachable,
1635	},
1636	RenderState.AddrOfExprBit => \|addr_of_expr\| {
1637	if (addr_of_expr.bit_offset_start_token) \|bit_offset_start_token\| {
1638	%return stream.print("{} ", self.tokenizer.getTokenSlice(bit_offset_start_token));
1639	}
1640	if (addr_of_expr.bit_offset_end_token) \|bit_offset_end_token\| {
1641	%return stream.print("{} ", self.tokenizer.getTokenSlice(bit_offset_end_token));
1642	}
1643	if (addr_of_expr.const_token) \|const_token\| {
1644	%return stream.print("{} ", self.tokenizer.getTokenSlice(const_token));
1645	}
1646	if (addr_of_expr.volatile_token) \|volatile_token\| {
1647	%return stream.print("{} ", self.tokenizer.getTokenSlice(volatile_token));
1648	}
1649	%return stack.append(RenderState { .Expression = addr_of_expr.op_expr});
1650	},
1651	RenderState.FnProtoRParen => \|fn_proto\| {
1652	%return stream.print(")");
1653	if (fn_proto.align_expr != null) {
1654	@panic("TODO");
1655	}
1656	if (fn_proto.return_type) \|return_type\| {
1657	%return stream.print(" -> ");
1658	if (fn_proto.body_node) \|body_node\| {
1659	%return stack.append(RenderState { .Expression = body_node});
1660	%return stack.append(RenderState { .Text = " "});
1661	}
1662	%return stack.append(RenderState { .Expression = return_type});
1663	}
1664	},
1665	RenderState.Statement => \|base\| {
1666	switch (base.id) {
1667	AstNode.Id.VarDecl => {
1668	const var_decl = @fieldParentPtr(AstNodeVarDecl, "base", base);
1669	%return stack.append(RenderState { .VarDecl = var_decl});
1670	},
1671	else => unreachable,
1672	}
1673	},
1674	RenderState.Indent => \|new_indent\| indent = new_indent,
1675	RenderState.PrintIndent => %return stream.writeByteNTimes(' ', indent),
1676	}
1677	}
1678	}
1679	};
1680		11
1681	pub fn main() -> %void {	12	pub fn main() -> %void {
1682	main2() %% \|err\| {	13	main2() %% \|err\| {
...	@@ -1795,4 +126,11 @@ test "zig fmt" {	...	@@ -1795,4 +126,11 @@ test "zig fmt" {
1795	\\}	126	\\}
1796	\\	127	\\
1797	);	128	);
		129
		130	testCanonical(
		131	\\fn foo(argc: c_int, argv: &&u8) -> c_int {
		132	\\ return 0;
		133	\\}
		134	\\
		135	);
1798	}	136	}

src-self-hosted/parser.zig created+1074

...	@@ -0,0 +1,1074 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const ArrayList = std.ArrayList;
4	const mem = std.mem;
5	const ast = @import("ast.zig");
6	const Tokenizer = @import("tokenizer.zig").Tokenizer;
7	const Token = @import("tokenizer.zig").Token;
8
9	// TODO when we make parse errors into error types instead of printing directly,
10	// get rid of this
11	const warn = std.debug.warn;
12
13	error ParseError;
14
15	pub const Parser = struct {
16	allocator: &mem.Allocator,
17	tokenizer: &Tokenizer,
18	put_back_tokens: [2]Token,
19	put_back_count: usize,
20	source_file_name: []const u8,
21
22	// This memory contents are used only during a function call. It's used to repurpose memory;
23	// specifically so that freeAst can be guaranteed to succeed.
24	const utility_bytes_align = @alignOf( union { a: RenderAstFrame, b: State, c: RenderState } );
25	utility_bytes: []align(utility_bytes_align) u8,
26
27	pub fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) -> Parser {
28	return Parser {
29	.allocator = allocator,
30	.tokenizer = tokenizer,
31	.put_back_tokens = undefined,
32	.put_back_count = 0,
33	.source_file_name = source_file_name,
34	.utility_bytes = []align(utility_bytes_align) u8{},
35	};
36	}
37
38	pub fn deinit(self: &Parser) {
39	self.allocator.free(self.utility_bytes);
40	}
41
42	const TopLevelDeclCtx = struct {
43	visib_token: ?Token,
44	extern_token: ?Token,
45	};
46
47	const DestPtr = union(enum) {
48	Field: &&ast.Node,
49	NullableField: &?&ast.Node,
50	List: &ArrayList(&ast.Node),
51
52	pub fn store(self: &const DestPtr, value: &ast.Node) -> %void {
53	switch (*self) {
54	DestPtr.Field => \|ptr\| *ptr = value,
55	DestPtr.NullableField => \|ptr\| *ptr = value,
56	DestPtr.List => \|list\| %return list.append(value),
57	}
58	}
59	};
60
61	const State = union(enum) {
62	TopLevel,
63	TopLevelExtern: ?Token,
64	TopLevelDecl: TopLevelDeclCtx,
65	Expression: DestPtr,
66	GroupedExpression: DestPtr,
67	UnwrapExpression: DestPtr,
68	BoolOrExpression: DestPtr,
69	BoolAndExpression: DestPtr,
70	ComparisonExpression: DestPtr,
71	BinaryOrExpression: DestPtr,
72	BinaryXorExpression: DestPtr,
73	BinaryAndExpression: DestPtr,
74	BitShiftExpression: DestPtr,
75	AdditionExpression: DestPtr,
76	MultiplyExpression: DestPtr,
77	BraceSuffixExpression: DestPtr,
78	PrefixOpExpression: DestPtr,
79	SuffixOpExpression: DestPtr,
80	PrimaryExpression: DestPtr,
81	TypeExpr: DestPtr,
82	VarDecl: &ast.NodeVarDecl,
83	VarDeclAlign: &ast.NodeVarDecl,
84	VarDeclEq: &ast.NodeVarDecl,
85	ExpectToken: @TagType(Token.Id),
86	FnProto: &ast.NodeFnProto,
87	FnProtoAlign: &ast.NodeFnProto,
88	ParamDecl: &ast.NodeFnProto,
89	ParamDeclComma,
90	FnDef: &ast.NodeFnProto,
91	Block: &ast.NodeBlock,
92	Statement: &ast.NodeBlock,
93	};
94
95	pub fn freeAst(self: &Parser, root_node: &ast.NodeRoot) {
96	// utility_bytes is big enough to do this iteration since we were able to do
97	// the parsing in the first place
98	comptime assert(@sizeOf(State) >= @sizeOf(&ast.Node));
99
100	var stack = self.initUtilityArrayList(&ast.Node);
101	defer self.deinitUtilityArrayList(stack);
102
103	stack.append(&root_node.base) %% unreachable;
104	while (stack.popOrNull()) \|node\| {
105	var i: usize = 0;
106	while (node.iterate(i)) \|child\| : (i += 1) {
107	if (child.iterate(0) != null) {
108	stack.append(child) %% unreachable;
109	} else {
110	child.destroy(self.allocator);
111	}
112	}
113	node.destroy(self.allocator);
114	}
115	}
116
117	pub fn parse(self: &Parser) -> %&ast.NodeRoot {
118	var stack = self.initUtilityArrayList(State);
119	defer self.deinitUtilityArrayList(stack);
120
121	const root_node = %return self.createRoot();
122	%defer self.allocator.destroy(root_node);
123	%return stack.append(State.TopLevel);
124	%defer self.freeAst(root_node);
125
126	while (true) {
127	//{
128	// const token = self.getNextToken();
129	// warn("{} ", @tagName(token.id));
130	// self.putBackToken(token);
131	// var i: usize = stack.len;
132	// while (i != 0) {
133	// i -= 1;
134	// warn("{} ", @tagName(stack.items[i]));
135	// }
136	// warn("\n");
137	//}
138
139	// This gives us 1 free append that can't fail
140	const state = stack.pop();
141
142	switch (state) {
143	State.TopLevel => {
144	const token = self.getNextToken();
145	switch (token.id) {
146	Token.Id.Keyword_pub, Token.Id.Keyword_export => {
147	stack.append(State { .TopLevelExtern = token }) %% unreachable;
148	continue;
149	},
150	Token.Id.Eof => return root_node,
151	else => {
152	self.putBackToken(token);
153	// TODO shouldn't need this cast
154	stack.append(State { .TopLevelExtern = null }) %% unreachable;
155	continue;
156	},
157	}
158	},
159	State.TopLevelExtern => \|visib_token\| {
160	const token = self.getNextToken();
161	if (token.id == Token.Id.Keyword_extern) {
162	stack.append(State {
163	.TopLevelDecl = TopLevelDeclCtx {
164	.visib_token = visib_token,
165	.extern_token = token,
166	},
167	}) %% unreachable;
168	continue;
169	}
170	self.putBackToken(token);
171	stack.append(State {
172	.TopLevelDecl = TopLevelDeclCtx {
173	.visib_token = visib_token,
174	.extern_token = null,
175	},
176	}) %% unreachable;
177	continue;
178	},
179	State.TopLevelDecl => \|ctx\| {
180	const token = self.getNextToken();
181	switch (token.id) {
182	Token.Id.Keyword_var, Token.Id.Keyword_const => {
183	stack.append(State.TopLevel) %% unreachable;
184	// TODO shouldn't need these casts
185	const var_decl_node = %return self.createAttachVarDecl(&root_node.decls, ctx.visib_token,
186	token, (?Token)(null), ctx.extern_token);
187	%return stack.append(State { .VarDecl = var_decl_node });
188	continue;
189	},
190	Token.Id.Keyword_fn => {
191	stack.append(State.TopLevel) %% unreachable;
192	// TODO shouldn't need these casts
193	const fn_proto = %return self.createAttachFnProto(&root_node.decls, token,
194	ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null));
195	%return stack.append(State { .FnDef = fn_proto });
196	%return stack.append(State { .FnProto = fn_proto });
197	continue;
198	},
199	Token.Id.StringLiteral => {
200	@panic("TODO extern with string literal");
201	},
202	Token.Id.Keyword_coldcc, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
203	stack.append(State.TopLevel) %% unreachable;
204	const fn_token = %return self.eatToken(Token.Id.Keyword_fn);
205	// TODO shouldn't need this cast
206	const fn_proto = %return self.createAttachFnProto(&root_node.decls, fn_token,
207	ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null));
208	%return stack.append(State { .FnDef = fn_proto });
209	%return stack.append(State { .FnProto = fn_proto });
210	continue;
211	},
212	else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)),
213	}
214	},
215	State.VarDecl => \|var_decl\| {
216	var_decl.name_token = %return self.eatToken(Token.Id.Identifier);
217	stack.append(State { .VarDeclAlign = var_decl }) %% unreachable;
218
219	const next_token = self.getNextToken();
220	if (next_token.id == Token.Id.Colon) {
221	%return stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} });
222	continue;
223	}
224
225	self.putBackToken(next_token);
226	continue;
227	},
228	State.VarDeclAlign => \|var_decl\| {
229	stack.append(State { .VarDeclEq = var_decl }) %% unreachable;
230
231	const next_token = self.getNextToken();
232	if (next_token.id == Token.Id.Keyword_align) {
233	%return stack.append(State {
234	.GroupedExpression = DestPtr {
235	.NullableField = &var_decl.align_node
236	}
237	});
238	continue;
239	}
240
241	self.putBackToken(next_token);
242	continue;
243	},
244	State.VarDeclEq => \|var_decl\| {
245	const token = self.getNextToken();
246	if (token.id == Token.Id.Equal) {
247	var_decl.eq_token = token;
248	stack.append(State { .ExpectToken = Token.Id.Semicolon }) %% unreachable;
249	%return stack.append(State {
250	.Expression = DestPtr {.NullableField = &var_decl.init_node},
251	});
252	continue;
253	}
254	if (token.id == Token.Id.Semicolon) {
255	continue;
256	}
257	return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
258	},
259	State.ExpectToken => \|token_id\| {
260	_ = %return self.eatToken(token_id);
261	continue;
262	},
263	State.Expression => \|dest_ptr\| {
264	const token = self.getNextToken();
265	if (token.id == Token.Id.Keyword_return) {
266	const return_node = %return self.createAttachReturn(dest_ptr, token);
267	stack.append(State {.UnwrapExpression = DestPtr {.Field = &return_node.expr} }) %% unreachable;
268	continue;
269	}
270	self.putBackToken(token);
271	stack.append(State {.UnwrapExpression = dest_ptr}) %% unreachable;
272	continue;
273	},
274
275	State.UnwrapExpression => \|dest_ptr\| {
276	stack.append(State {.BoolOrExpression = dest_ptr}) %% unreachable;
277	continue;
278	},
279
280	State.BoolOrExpression => \|dest_ptr\| {
281	stack.append(State {.BoolAndExpression = dest_ptr}) %% unreachable;
282	continue;
283	},
284
285	State.BoolAndExpression => \|dest_ptr\| {
286	stack.append(State {.ComparisonExpression = dest_ptr}) %% unreachable;
287	continue;
288	},
289
290	State.ComparisonExpression => \|dest_ptr\| {
291	stack.append(State {.BinaryOrExpression = dest_ptr}) %% unreachable;
292	continue;
293	},
294
295	State.BinaryOrExpression => \|dest_ptr\| {
296	stack.append(State {.BinaryXorExpression = dest_ptr}) %% unreachable;
297	continue;
298	},
299
300	State.BinaryXorExpression => \|dest_ptr\| {
301	stack.append(State {.BinaryAndExpression = dest_ptr}) %% unreachable;
302	continue;
303	},
304
305	State.BinaryAndExpression => \|dest_ptr\| {
306	stack.append(State {.BitShiftExpression = dest_ptr}) %% unreachable;
307	continue;
308	},
309
310	State.BitShiftExpression => \|dest_ptr\| {
311	stack.append(State {.AdditionExpression = dest_ptr}) %% unreachable;
312	continue;
313	},
314
315	State.AdditionExpression => \|dest_ptr\| {
316	stack.append(State {.MultiplyExpression = dest_ptr}) %% unreachable;
317	continue;
318	},
319
320	State.MultiplyExpression => \|dest_ptr\| {
321	stack.append(State {.BraceSuffixExpression = dest_ptr}) %% unreachable;
322	continue;
323	},
324
325	State.BraceSuffixExpression => \|dest_ptr\| {
326	stack.append(State {.PrefixOpExpression = dest_ptr}) %% unreachable;
327	continue;
328	},
329
330	State.PrefixOpExpression => \|dest_ptr\| {
331	const first_token = self.getNextToken();
332	if (first_token.id == Token.Id.Ampersand) {
333	const addr_of_expr = %return self.createAttachAddrOfExpr(dest_ptr, first_token);
334	var token = self.getNextToken();
335	if (token.id == Token.Id.Keyword_align) {
336	@panic("TODO align");
337	}
338	if (token.id == Token.Id.Keyword_const) {
339	addr_of_expr.const_token = token;
340	token = self.getNextToken();
341	}
342	if (token.id == Token.Id.Keyword_volatile) {
343	addr_of_expr.volatile_token = token;
344	token = self.getNextToken();
345	}
346	self.putBackToken(token);
347	stack.append(State {
348	.PrefixOpExpression = DestPtr { .Field = &addr_of_expr.op_expr},
349	}) %% unreachable;
350	continue;
351	}
352
353	self.putBackToken(first_token);
354	stack.append(State { .SuffixOpExpression = dest_ptr }) %% unreachable;
355	continue;
356	},
357
358	State.SuffixOpExpression => \|dest_ptr\| {
359	stack.append(State { .PrimaryExpression = dest_ptr }) %% unreachable;
360	continue;
361	},
362
363	State.PrimaryExpression => \|dest_ptr\| {
364	const token = self.getNextToken();
365	switch (token.id) {
366	Token.Id.Identifier => {
367	_ = %return self.createAttachIdentifier(dest_ptr, token);
368	continue;
369	},
370	Token.Id.IntegerLiteral => {
371	_ = %return self.createAttachIntegerLiteral(dest_ptr, token);
372	continue;
373	},
374	Token.Id.FloatLiteral => {
375	_ = %return self.createAttachFloatLiteral(dest_ptr, token);
376	continue;
377	},
378	else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
379	}
380	},
381
382	State.TypeExpr => \|dest_ptr\| {
383	const token = self.getNextToken();
384	if (token.id == Token.Id.Keyword_var) {
385	@panic("TODO param with type var");
386	}
387	self.putBackToken(token);
388
389	stack.append(State { .PrefixOpExpression = dest_ptr }) %% unreachable;
390	continue;
391	},
392
393	State.FnProto => \|fn_proto\| {
394	stack.append(State { .FnProtoAlign = fn_proto }) %% unreachable;
395	%return stack.append(State { .ParamDecl = fn_proto });
396	%return stack.append(State { .ExpectToken = Token.Id.LParen });
397
398	const next_token = self.getNextToken();
399	if (next_token.id == Token.Id.Identifier) {
400	fn_proto.name_token = next_token;
401	continue;
402	}
403	self.putBackToken(next_token);
404	continue;
405	},
406
407	State.FnProtoAlign => \|fn_proto\| {
408	const token = self.getNextToken();
409	if (token.id == Token.Id.Keyword_align) {
410	@panic("TODO fn proto align");
411	}
412	if (token.id == Token.Id.Arrow) {
413	stack.append(State {
414	.TypeExpr = DestPtr {.NullableField = &fn_proto.return_type},
415	}) %% unreachable;
416	continue;
417	} else {
418	self.putBackToken(token);
419	continue;
420	}
421	},
422
423	State.ParamDecl => \|fn_proto\| {
424	var token = self.getNextToken();
425	if (token.id == Token.Id.RParen) {
426	continue;
427	}
428	const param_decl = %return self.createAttachParamDecl(&fn_proto.params);
429	if (token.id == Token.Id.Keyword_comptime) {
430	param_decl.comptime_token = token;
431	token = self.getNextToken();
432	} else if (token.id == Token.Id.Keyword_noalias) {
433	param_decl.noalias_token = token;
434	token = self.getNextToken();
435	};
436	if (token.id == Token.Id.Identifier) {
437	const next_token = self.getNextToken();
438	if (next_token.id == Token.Id.Colon) {
439	param_decl.name_token = token;
440	token = self.getNextToken();
441	} else {
442	self.putBackToken(next_token);
443	}
444	}
445	if (token.id == Token.Id.Ellipsis3) {
446	param_decl.var_args_token = token;
447	stack.append(State { .ExpectToken = Token.Id.RParen }) %% unreachable;
448	continue;
449	} else {
450	self.putBackToken(token);
451	}
452
453	stack.append(State { .ParamDecl = fn_proto }) %% unreachable;
454	%return stack.append(State.ParamDeclComma);
455	%return stack.append(State {
456	.TypeExpr = DestPtr {.Field = &param_decl.type_node}
457	});
458	continue;
459	},
460
461	State.ParamDeclComma => {
462	const token = self.getNextToken();
463	switch (token.id) {
464	Token.Id.RParen => {
465	_ = stack.pop(); // pop off the ParamDecl
466	continue;
467	},
468	Token.Id.Comma => continue,
469	else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
470	}
471	},
472
473	State.FnDef => \|fn_proto\| {
474	const token = self.getNextToken();
475	switch(token.id) {
476	Token.Id.LBrace => {
477	const block = %return self.createBlock(token);
478	fn_proto.body_node = &block.base;
479	stack.append(State { .Block = block }) %% unreachable;
480	continue;
481	},
482	Token.Id.Semicolon => continue,
483	else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)),
484	}
485	},
486
487	State.Block => \|block\| {
488	const token = self.getNextToken();
489	switch (token.id) {
490	Token.Id.RBrace => {
491	block.end_token = token;
492	continue;
493	},
494	else => {
495	self.putBackToken(token);
496	stack.append(State { .Block = block }) %% unreachable;
497	%return stack.append(State { .Statement = block });
498	continue;
499	},
500	}
501	},
502
503	State.Statement => \|block\| {
504	{
505	// Look for comptime var, comptime const
506	const comptime_token = self.getNextToken();
507	if (comptime_token.id == Token.Id.Keyword_comptime) {
508	const mut_token = self.getNextToken();
509	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
510	// TODO shouldn't need these casts
511	const var_decl = %return self.createAttachVarDecl(&block.statements, (?Token)(null),
512	mut_token, (?Token)(comptime_token), (?Token)(null));
513	%return stack.append(State { .VarDecl = var_decl });
514	continue;
515	}
516	self.putBackToken(mut_token);
517	}
518	self.putBackToken(comptime_token);
519	}
520	{
521	// Look for const, var
522	const mut_token = self.getNextToken();
523	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
524	// TODO shouldn't need these casts
525	const var_decl = %return self.createAttachVarDecl(&block.statements, (?Token)(null),
526	mut_token, (?Token)(null), (?Token)(null));
527	%return stack.append(State { .VarDecl = var_decl });
528	continue;
529	}
530	self.putBackToken(mut_token);
531	}
532
533	stack.append(State { .ExpectToken = Token.Id.Semicolon }) %% unreachable;
534	%return stack.append(State { .Expression = DestPtr{.List = &block.statements} });
535	continue;
536	},
537
538	State.GroupedExpression => @panic("TODO"),
539	}
540	unreachable;
541	}
542	}
543
544	fn createRoot(self: &Parser) -> %&ast.NodeRoot {
545	const node = %return self.allocator.create(ast.NodeRoot);
546	%defer self.allocator.destroy(node);
547
548	*node = ast.NodeRoot {
549	.base = ast.Node {.id = ast.Node.Id.Root},
550	.decls = ArrayList(&ast.Node).init(self.allocator),
551	};
552	return node;
553	}
554
555	fn createVarDecl(self: &Parser, visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token,
556	extern_token: &const ?Token) -> %&ast.NodeVarDecl
557	{
558	const node = %return self.allocator.create(ast.NodeVarDecl);
559	%defer self.allocator.destroy(node);
560
561	*node = ast.NodeVarDecl {
562	.base = ast.Node {.id = ast.Node.Id.VarDecl},
563	.visib_token = *visib_token,
564	.mut_token = *mut_token,
565	.comptime_token = *comptime_token,
566	.extern_token = *extern_token,
567	.type_node = null,
568	.align_node = null,
569	.init_node = null,
570	.lib_name = null,
571	// initialized later
572	.name_token = undefined,
573	.eq_token = undefined,
574	};
575	return node;
576	}
577
578	fn createFnProto(self: &Parser, fn_token: &const Token, extern_token: &const ?Token,
579	cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) -> %&ast.NodeFnProto
580	{
581	const node = %return self.allocator.create(ast.NodeFnProto);
582	%defer self.allocator.destroy(node);
583
584	*node = ast.NodeFnProto {
585	.base = ast.Node {.id = ast.Node.Id.FnProto},
586	.visib_token = *visib_token,
587	.name_token = null,
588	.fn_token = *fn_token,
589	.params = ArrayList(&ast.Node).init(self.allocator),
590	.return_type = null,
591	.var_args_token = null,
592	.extern_token = *extern_token,
593	.inline_token = *inline_token,
594	.cc_token = *cc_token,
595	.body_node = null,
596	.lib_name = null,
597	.align_expr = null,
598	};
599	return node;
600	}
601
602	fn createParamDecl(self: &Parser) -> %&ast.NodeParamDecl {
603	const node = %return self.allocator.create(ast.NodeParamDecl);
604	%defer self.allocator.destroy(node);
605
606	*node = ast.NodeParamDecl {
607	.base = ast.Node {.id = ast.Node.Id.ParamDecl},
608	.comptime_token = null,
609	.noalias_token = null,
610	.name_token = null,
611	.type_node = undefined,
612	.var_args_token = null,
613	};
614	return node;
615	}
616
617	fn createAddrOfExpr(self: &Parser, op_token: &const Token) -> %&ast.NodeAddrOfExpr {
618	const node = %return self.allocator.create(ast.NodeAddrOfExpr);
619	%defer self.allocator.destroy(node);
620
621	*node = ast.NodeAddrOfExpr {
622	.base = ast.Node {.id = ast.Node.Id.AddrOfExpr},
623	.align_expr = null,
624	.op_token = *op_token,
625	.bit_offset_start_token = null,
626	.bit_offset_end_token = null,
627	.const_token = null,
628	.volatile_token = null,
629	.op_expr = undefined,
630	};
631	return node;
632	}
633
634	fn createBlock(self: &Parser, begin_token: &const Token) -> %&ast.NodeBlock {
635	const node = %return self.allocator.create(ast.NodeBlock);
636	%defer self.allocator.destroy(node);
637
638	*node = ast.NodeBlock {
639	.base = ast.Node {.id = ast.Node.Id.Block},
640	.begin_token = *begin_token,
641	.end_token = undefined,
642	.statements = ArrayList(&ast.Node).init(self.allocator),
643	};
644	return node;
645	}
646
647	fn createReturn(self: &Parser, return_token: &const Token) -> %&ast.NodeReturn {
648	const node = %return self.allocator.create(ast.NodeReturn);
649	%defer self.allocator.destroy(node);
650
651	*node = ast.NodeReturn {
652	.base = ast.Node {.id = ast.Node.Id.Return},
653	.return_token = *return_token,
654	.expr = undefined,
655	};
656	return node;
657	}
658
659	fn createIdentifier(self: &Parser, name_token: &const Token) -> %&ast.NodeIdentifier {
660	const node = %return self.allocator.create(ast.NodeIdentifier);
661	%defer self.allocator.destroy(node);
662
663	*node = ast.NodeIdentifier {
664	.base = ast.Node {.id = ast.Node.Id.Identifier},
665	.name_token = *name_token,
666	};
667	return node;
668	}
669
670	fn createIntegerLiteral(self: &Parser, token: &const Token) -> %&ast.NodeIntegerLiteral {
671	const node = %return self.allocator.create(ast.NodeIntegerLiteral);
672	%defer self.allocator.destroy(node);
673
674	*node = ast.NodeIntegerLiteral {
675	.base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
676	.token = *token,
677	};
678	return node;
679	}
680
681	fn createFloatLiteral(self: &Parser, token: &const Token) -> %&ast.NodeFloatLiteral {
682	const node = %return self.allocator.create(ast.NodeFloatLiteral);
683	%defer self.allocator.destroy(node);
684
685	*node = ast.NodeFloatLiteral {
686	.base = ast.Node {.id = ast.Node.Id.FloatLiteral},
687	.token = *token,
688	};
689	return node;
690	}
691
692	fn createAttachFloatLiteral(self: &Parser, dest_ptr: &const DestPtr, token: &const Token) -> %&ast.NodeFloatLiteral {
693	const node = %return self.createFloatLiteral(token);
694	%defer self.allocator.destroy(node);
695	%return dest_ptr.store(&node.base);
696	return node;
697	}
698
699	fn createAttachIntegerLiteral(self: &Parser, dest_ptr: &const DestPtr, token: &const Token) -> %&ast.NodeIntegerLiteral {
700	const node = %return self.createIntegerLiteral(token);
701	%defer self.allocator.destroy(node);
702	%return dest_ptr.store(&node.base);
703	return node;
704	}
705
706	fn createAttachIdentifier(self: &Parser, dest_ptr: &const DestPtr, name_token: &const Token) -> %&ast.NodeIdentifier {
707	const node = %return self.createIdentifier(name_token);
708	%defer self.allocator.destroy(node);
709	%return dest_ptr.store(&node.base);
710	return node;
711	}
712
713	fn createAttachReturn(self: &Parser, dest_ptr: &const DestPtr, return_token: &const Token) -> %&ast.NodeReturn {
714	const node = %return self.createReturn(return_token);
715	%defer self.allocator.destroy(node);
716	%return dest_ptr.store(&node.base);
717	return node;
718	}
719
720	fn createAttachAddrOfExpr(self: &Parser, dest_ptr: &const DestPtr, op_token: &const Token) -> %&ast.NodeAddrOfExpr {
721	const node = %return self.createAddrOfExpr(op_token);
722	%defer self.allocator.destroy(node);
723	%return dest_ptr.store(&node.base);
724	return node;
725	}
726
727	fn createAttachParamDecl(self: &Parser, list: &ArrayList(&ast.Node)) -> %&ast.NodeParamDecl {
728	const node = %return self.createParamDecl();
729	%defer self.allocator.destroy(node);
730	%return list.append(&node.base);
731	return node;
732	}
733
734	fn createAttachFnProto(self: &Parser, list: &ArrayList(&ast.Node), fn_token: &const Token,
735	extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token,
736	inline_token: &const ?Token) -> %&ast.NodeFnProto
737	{
738	const node = %return self.createFnProto(fn_token, extern_token, cc_token, visib_token, inline_token);
739	%defer self.allocator.destroy(node);
740	%return list.append(&node.base);
741	return node;
742	}
743
744	fn createAttachVarDecl(self: &Parser, list: &ArrayList(&ast.Node), visib_token: &const ?Token,
745	mut_token: &const Token, comptime_token: &const ?Token, extern_token: &const ?Token) -> %&ast.NodeVarDecl
746	{
747	const node = %return self.createVarDecl(visib_token, mut_token, comptime_token, extern_token);
748	%defer self.allocator.destroy(node);
749	%return list.append(&node.base);
750	return node;
751	}
752
753	fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) -> error {
754	const loc = self.tokenizer.getTokenLocation(token);
755	warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
756	warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
757	{
758	var i: usize = 0;
759	while (i < loc.column) : (i += 1) {
760	warn(" ");
761	}
762	}
763	{
764	const caret_count = token.end - token.start;
765	var i: usize = 0;
766	while (i < caret_count) : (i += 1) {
767	warn("~");
768	}
769	}
770	warn("\n");
771	return error.ParseError;
772	}
773
774	fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) -> %void {
775	if (token.id != id) {
776	return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
777	}
778	}
779
780	fn eatToken(self: &Parser, id: @TagType(Token.Id)) -> %Token {
781	const token = self.getNextToken();
782	%return self.expectToken(token, id);
783	return token;
784	}
785
786	fn putBackToken(self: &Parser, token: &const Token) {
787	self.put_back_tokens[self.put_back_count] = *token;
788	self.put_back_count += 1;
789	}
790
791	fn getNextToken(self: &Parser) -> Token {
792	return if (self.put_back_count != 0) {
793	const put_back_index = self.put_back_count - 1;
794	const put_back_token = self.put_back_tokens[put_back_index];
795	self.put_back_count = put_back_index;
796	put_back_token
797	} else {
798	self.tokenizer.next()
799	};
800	}
801
802	const RenderAstFrame = struct {
803	node: &ast.Node,
804	indent: usize,
805	};
806
807	pub fn renderAst(self: &Parser, stream: &std.io.OutStream, root_node: &ast.NodeRoot) -> %void {
808	var stack = self.initUtilityArrayList(RenderAstFrame);
809	defer self.deinitUtilityArrayList(stack);
810
811	%return stack.append(RenderAstFrame {
812	.node = &root_node.base,
813	.indent = 0,
814	});
815
816	while (stack.popOrNull()) \|frame\| {
817	{
818	var i: usize = 0;
819	while (i < frame.indent) : (i += 1) {
820	%return stream.print(" ");
821	}
822	}
823	%return stream.print("{}\n", @tagName(frame.node.id));
824	var child_i: usize = 0;
825	while (frame.node.iterate(child_i)) \|child\| : (child_i += 1) {
826	%return stack.append(RenderAstFrame {
827	.node = child,
828	.indent = frame.indent + 2,
829	});
830	}
831	}
832	}
833
834	const RenderState = union(enum) {
835	TopLevelDecl: &ast.Node,
836	FnProtoRParen: &ast.NodeFnProto,
837	ParamDecl: &ast.Node,
838	Text: []const u8,
839	Expression: &ast.Node,
840	AddrOfExprBit: &ast.NodeAddrOfExpr,
841	VarDecl: &ast.NodeVarDecl,
842	VarDeclAlign: &ast.NodeVarDecl,
843	Statement: &ast.Node,
844	PrintIndent,
845	Indent: usize,
846	};
847
848	pub fn renderSource(self: &Parser, stream: &std.io.OutStream, root_node: &ast.NodeRoot) -> %void {
849	var stack = self.initUtilityArrayList(RenderState);
850	defer self.deinitUtilityArrayList(stack);
851
852	{
853	var i = root_node.decls.len;
854	while (i != 0) {
855	i -= 1;
856	const decl = root_node.decls.items[i];
857	%return stack.append(RenderState {.TopLevelDecl = decl});
858	}
859	}
860
861	const indent_delta = 4;
862	var indent: usize = 0;
863	while (stack.popOrNull()) \|state\| {
864	switch (state) {
865	RenderState.TopLevelDecl => \|decl\| {
866	switch (decl.id) {
867	ast.Node.Id.FnProto => {
868	const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl);
869	if (fn_proto.visib_token) \|visib_token\| {
870	switch (visib_token.id) {
871	Token.Id.Keyword_pub => %return stream.print("pub "),
872	Token.Id.Keyword_export => %return stream.print("export "),
873	else => unreachable,
874	};
875	}
876	if (fn_proto.extern_token) \|extern_token\| {
877	%return stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
878	}
879	%return stream.print("fn");
880
881	if (fn_proto.name_token) \|name_token\| {
882	%return stream.print(" {}", self.tokenizer.getTokenSlice(name_token));
883	}
884
885	%return stream.print("(");
886
887	%return stack.append(RenderState { .Text = "\n" });
888	if (fn_proto.body_node == null) {
889	%return stack.append(RenderState { .Text = ";" });
890	}
891
892	%return stack.append(RenderState { .FnProtoRParen = fn_proto});
893	var i = fn_proto.params.len;
894	while (i != 0) {
895	i -= 1;
896	const param_decl_node = fn_proto.params.items[i];
897	%return stack.append(RenderState { .ParamDecl = param_decl_node});
898	if (i != 0) {
899	%return stack.append(RenderState { .Text = ", " });
900	}
901	}
902	},
903	ast.Node.Id.VarDecl => {
904	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
905	%return stack.append(RenderState { .Text = "\n"});
906	%return stack.append(RenderState { .VarDecl = var_decl});
907
908	},
909	else => unreachable,
910	}
911	},
912
913	RenderState.VarDecl => \|var_decl\| {
914	if (var_decl.visib_token) \|visib_token\| {
915	%return stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
916	}
917	if (var_decl.extern_token) \|extern_token\| {
918	%return stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
919	if (var_decl.lib_name != null) {
920	@panic("TODO");
921	}
922	}
923	if (var_decl.comptime_token) \|comptime_token\| {
924	%return stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
925	}
926	%return stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token));
927	%return stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token));
928
929	%return stack.append(RenderState { .VarDeclAlign = var_decl });
930	if (var_decl.type_node) \|type_node\| {
931	%return stream.print(": ");
932	%return stack.append(RenderState { .Expression = type_node });
933	}
934	},
935
936	RenderState.VarDeclAlign => \|var_decl\| {
937	if (var_decl.align_node != null) {
938	@panic("TODO");
939	}
940	%return stack.append(RenderState { .Text = ";" });
941	if (var_decl.init_node) \|init_node\| {
942	%return stream.print(" = ");
943	%return stack.append(RenderState { .Expression = init_node });
944	}
945	},
946
947	RenderState.ParamDecl => \|base\| {
948	const param_decl = @fieldParentPtr(ast.NodeParamDecl, "base", base);
949	if (param_decl.comptime_token) \|comptime_token\| {
950	%return stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
951	}
952	if (param_decl.noalias_token) \|noalias_token\| {
953	%return stream.print("{} ", self.tokenizer.getTokenSlice(noalias_token));
954	}
955	if (param_decl.name_token) \|name_token\| {
956	%return stream.print("{}: ", self.tokenizer.getTokenSlice(name_token));
957	}
958	if (param_decl.var_args_token) \|var_args_token\| {
959	%return stream.print("{}", self.tokenizer.getTokenSlice(var_args_token));
960	} else {
961	%return stack.append(RenderState { .Expression = param_decl.type_node});
962	}
963	},
964	RenderState.Text => \|bytes\| {
965	%return stream.write(bytes);
966	},
967	RenderState.Expression => \|base\| switch (base.id) {
968	ast.Node.Id.Identifier => {
969	const identifier = @fieldParentPtr(ast.NodeIdentifier, "base", base);
970	%return stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token));
971	},
972	ast.Node.Id.AddrOfExpr => {
973	const addr_of_expr = @fieldParentPtr(ast.NodeAddrOfExpr, "base", base);
974	%return stream.print("{}", self.tokenizer.getTokenSlice(addr_of_expr.op_token));
975	%return stack.append(RenderState { .AddrOfExprBit = addr_of_expr});
976
977	if (addr_of_expr.align_expr) \|align_expr\| {
978	%return stream.print("align(");
979	%return stack.append(RenderState { .Text = ")"});
980	%return stack.append(RenderState { .Expression = align_expr});
981	}
982	},
983	ast.Node.Id.Block => {
984	const block = @fieldParentPtr(ast.NodeBlock, "base", base);
985	%return stream.write("{");
986	%return stack.append(RenderState { .Text = "}"});
987	%return stack.append(RenderState.PrintIndent);
988	%return stack.append(RenderState { .Indent = indent});
989	%return stack.append(RenderState { .Text = "\n"});
990	var i = block.statements.len;
991	while (i != 0) {
992	i -= 1;
993	const statement_node = block.statements.items[i];
994	%return stack.append(RenderState { .Statement = statement_node});
995	%return stack.append(RenderState.PrintIndent);
996	%return stack.append(RenderState { .Indent = indent + indent_delta});
997	%return stack.append(RenderState { .Text = "\n" });
998	}
999	},
1000	ast.Node.Id.Return => {
1001	const return_node = @fieldParentPtr(ast.NodeReturn, "base", base);
1002	%return stream.write("return ");
1003	%return stack.append(RenderState { .Expression = return_node.expr });
1004	},
1005	ast.Node.Id.IntegerLiteral => {
1006	const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);
1007	%return stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));
1008	},
1009	ast.Node.Id.FloatLiteral => {
1010	const float_literal = @fieldParentPtr(ast.NodeFloatLiteral, "base", base);
1011	%return stream.print("{}", self.tokenizer.getTokenSlice(float_literal.token));
1012	},
1013	else => unreachable,
1014	},
1015	RenderState.AddrOfExprBit => \|addr_of_expr\| {
1016	if (addr_of_expr.bit_offset_start_token) \|bit_offset_start_token\| {
1017	%return stream.print("{} ", self.tokenizer.getTokenSlice(bit_offset_start_token));
1018	}
1019	if (addr_of_expr.bit_offset_end_token) \|bit_offset_end_token\| {
1020	%return stream.print("{} ", self.tokenizer.getTokenSlice(bit_offset_end_token));
1021	}
1022	if (addr_of_expr.const_token) \|const_token\| {
1023	%return stream.print("{} ", self.tokenizer.getTokenSlice(const_token));
1024	}
1025	if (addr_of_expr.volatile_token) \|volatile_token\| {
1026	%return stream.print("{} ", self.tokenizer.getTokenSlice(volatile_token));
1027	}
1028	%return stack.append(RenderState { .Expression = addr_of_expr.op_expr});
1029	},
1030	RenderState.FnProtoRParen => \|fn_proto\| {
1031	%return stream.print(")");
1032	if (fn_proto.align_expr != null) {
1033	@panic("TODO");
1034	}
1035	if (fn_proto.return_type) \|return_type\| {
1036	%return stream.print(" -> ");
1037	if (fn_proto.body_node) \|body_node\| {
1038	%return stack.append(RenderState { .Expression = body_node});
1039	%return stack.append(RenderState { .Text = " "});
1040	}
1041	%return stack.append(RenderState { .Expression = return_type});
1042	}
1043	},
1044	RenderState.Statement => \|base\| {
1045	switch (base.id) {
1046	ast.Node.Id.VarDecl => {
1047	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
1048	%return stack.append(RenderState { .VarDecl = var_decl});
1049	},
1050	else => {
1051	%return stack.append(RenderState { .Text = ";"});
1052	%return stack.append(RenderState { .Expression = base});
1053	},
1054	}
1055	},
1056	RenderState.Indent => \|new_indent\| indent = new_indent,
1057	RenderState.PrintIndent => %return stream.writeByteNTimes(' ', indent),
1058	}
1059	}
1060	}
1061
1062	fn initUtilityArrayList(self: &Parser, comptime T: type) -> ArrayList(T) {
1063	const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
1064	self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
1065	const typed_slice = ([]T)(self.utility_bytes);
1066	return ArrayList(T).fromOwnedSlice(self.allocator, typed_slice);
1067	}
1068
1069	fn deinitUtilityArrayList(self: &Parser, list: var) {
1070	self.utility_bytes = ([]align(utility_bytes_align) u8)(list.toOwnedSlice());
1071	}
1072
1073	};
1074

src-self-hosted/tokenizer.zig created+479

...	@@ -0,0 +1,479 @@
1	const std = @import("std");
2	const mem = std.mem;
3
4	pub const Token = struct {
5	id: Id,
6	start: usize,
7	end: usize,
8
9	const KeywordId = struct {
10	bytes: []const u8,
11	id: Id,
12	};
13
14	const keywords = []KeywordId {
15	KeywordId{.bytes="align", .id = Id.Keyword_align},
16	KeywordId{.bytes="and", .id = Id.Keyword_and},
17	KeywordId{.bytes="asm", .id = Id.Keyword_asm},
18	KeywordId{.bytes="break", .id = Id.Keyword_break},
19	KeywordId{.bytes="coldcc", .id = Id.Keyword_coldcc},
20	KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
21	KeywordId{.bytes="const", .id = Id.Keyword_const},
22	KeywordId{.bytes="continue", .id = Id.Keyword_continue},
23	KeywordId{.bytes="defer", .id = Id.Keyword_defer},
24	KeywordId{.bytes="else", .id = Id.Keyword_else},
25	KeywordId{.bytes="enum", .id = Id.Keyword_enum},
26	KeywordId{.bytes="error", .id = Id.Keyword_error},
27	KeywordId{.bytes="export", .id = Id.Keyword_export},
28	KeywordId{.bytes="extern", .id = Id.Keyword_extern},
29	KeywordId{.bytes="false", .id = Id.Keyword_false},
30	KeywordId{.bytes="fn", .id = Id.Keyword_fn},
31	KeywordId{.bytes="for", .id = Id.Keyword_for},
32	KeywordId{.bytes="goto", .id = Id.Keyword_goto},
33	KeywordId{.bytes="if", .id = Id.Keyword_if},
34	KeywordId{.bytes="inline", .id = Id.Keyword_inline},
35	KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
36	KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
37	KeywordId{.bytes="null", .id = Id.Keyword_null},
38	KeywordId{.bytes="or", .id = Id.Keyword_or},
39	KeywordId{.bytes="packed", .id = Id.Keyword_packed},
40	KeywordId{.bytes="pub", .id = Id.Keyword_pub},
41	KeywordId{.bytes="return", .id = Id.Keyword_return},
42	KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
43	KeywordId{.bytes="struct", .id = Id.Keyword_struct},
44	KeywordId{.bytes="switch", .id = Id.Keyword_switch},
45	KeywordId{.bytes="test", .id = Id.Keyword_test},
46	KeywordId{.bytes="this", .id = Id.Keyword_this},
47	KeywordId{.bytes="true", .id = Id.Keyword_true},
48	KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
49	KeywordId{.bytes="union", .id = Id.Keyword_union},
50	KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
51	KeywordId{.bytes="use", .id = Id.Keyword_use},
52	KeywordId{.bytes="var", .id = Id.Keyword_var},
53	KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
54	KeywordId{.bytes="while", .id = Id.Keyword_while},
55	};
56
57	fn getKeyword(bytes: []const u8) -> ?Id {
58	for (keywords) \|kw\| {
59	if (mem.eql(u8, kw.bytes, bytes)) {
60	return kw.id;
61	}
62	}
63	return null;
64	}
65
66	const StrLitKind = enum {Normal, C};
67
68	pub const Id = union(enum) {
69	Invalid,
70	Identifier,
71	StringLiteral: StrLitKind,
72	Eof,
73	Builtin,
74	Equal,
75	LParen,
76	RParen,
77	Semicolon,
78	Percent,
79	LBrace,
80	RBrace,
81	Period,
82	Ellipsis2,
83	Ellipsis3,
84	Minus,
85	Arrow,
86	Colon,
87	Slash,
88	Comma,
89	Ampersand,
90	AmpersandEqual,
91	IntegerLiteral,
92	FloatLiteral,
93	Keyword_align,
94	Keyword_and,
95	Keyword_asm,
96	Keyword_break,
97	Keyword_coldcc,
98	Keyword_comptime,
99	Keyword_const,
100	Keyword_continue,
101	Keyword_defer,
102	Keyword_else,
103	Keyword_enum,
104	Keyword_error,
105	Keyword_export,
106	Keyword_extern,
107	Keyword_false,
108	Keyword_fn,
109	Keyword_for,
110	Keyword_goto,
111	Keyword_if,
112	Keyword_inline,
113	Keyword_nakedcc,
114	Keyword_noalias,
115	Keyword_null,
116	Keyword_or,
117	Keyword_packed,
118	Keyword_pub,
119	Keyword_return,
120	Keyword_stdcallcc,
121	Keyword_struct,
122	Keyword_switch,
123	Keyword_test,
124	Keyword_this,
125	Keyword_true,
126	Keyword_undefined,
127	Keyword_union,
128	Keyword_unreachable,
129	Keyword_use,
130	Keyword_var,
131	Keyword_volatile,
132	Keyword_while,
133	};
134	};
135
136	pub const Tokenizer = struct {
137	buffer: []const u8,
138	index: usize,
139
140	pub const Location = struct {
141	line: usize,
142	column: usize,
143	line_start: usize,
144	line_end: usize,
145	};
146
147	pub fn getTokenLocation(self: &Tokenizer, token: &const Token) -> Location {
148	var loc = Location {
149	.line = 0,
150	.column = 0,
151	.line_start = 0,
152	.line_end = 0,
153	};
154	for (self.buffer) \|c, i\| {
155	if (i == token.start) {
156	loc.line_end = i;
157	while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
158	return loc;
159	}
160	if (c == '\n') {
161	loc.line += 1;
162	loc.column = 0;
163	loc.line_start = i + 1;
164	} else {
165	loc.column += 1;
166	}
167	}
168	return loc;
169	}
170
171	/// For debugging purposes
172	pub fn dump(self: &Tokenizer, token: &const Token) {
173	std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
174	}
175
176	pub fn init(buffer: []const u8) -> Tokenizer {
177	return Tokenizer {
178	.buffer = buffer,
179	.index = 0,
180	};
181	}
182
183	const State = enum {
184	Start,
185	Identifier,
186	Builtin,
187	C,
188	StringLiteral,
189	StringLiteralBackslash,
190	Minus,
191	Slash,
192	LineComment,
193	Zero,
194	IntegerLiteral,
195	NumberDot,
196	FloatFraction,
197	FloatExponentUnsigned,
198	FloatExponentNumber,
199	Ampersand,
200	Period,
201	Period2,
202	};
203
204	pub fn next(self: &Tokenizer) -> Token {
205	var state = State.Start;
206	var result = Token {
207	.id = Token.Id.Eof,
208	.start = self.index,
209	.end = undefined,
210	};
211	while (self.index < self.buffer.len) : (self.index += 1) {
212	const c = self.buffer[self.index];
213	switch (state) {
214	State.Start => switch (c) {
215	' ', '\n' => {
216	result.start = self.index + 1;
217	},
218	'c' => {
219	state = State.C;
220	result.id = Token.Id.Identifier;
221	},
222	'"' => {
223	state = State.StringLiteral;
224	result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
225	},
226	'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
227	state = State.Identifier;
228	result.id = Token.Id.Identifier;
229	},
230	'@' => {
231	state = State.Builtin;
232	result.id = Token.Id.Builtin;
233	},
234	'=' => {
235	result.id = Token.Id.Equal;
236	self.index += 1;
237	break;
238	},
239	'(' => {
240	result.id = Token.Id.LParen;
241	self.index += 1;
242	break;
243	},
244	')' => {
245	result.id = Token.Id.RParen;
246	self.index += 1;
247	break;
248	},
249	';' => {
250	result.id = Token.Id.Semicolon;
251	self.index += 1;
252	break;
253	},
254	',' => {
255	result.id = Token.Id.Comma;
256	self.index += 1;
257	break;
258	},
259	':' => {
260	result.id = Token.Id.Colon;
261	self.index += 1;
262	break;
263	},
264	'%' => {
265	result.id = Token.Id.Percent;
266	self.index += 1;
267	break;
268	},
269	'{' => {
270	result.id = Token.Id.LBrace;
271	self.index += 1;
272	break;
273	},
274	'}' => {
275	result.id = Token.Id.RBrace;
276	self.index += 1;
277	break;
278	},
279	'.' => {
280	state = State.Period;
281	},
282	'-' => {
283	state = State.Minus;
284	},
285	'/' => {
286	state = State.Slash;
287	},
288	'&' => {
289	state = State.Ampersand;
290	},
291	'0' => {
292	state = State.Zero;
293	result.id = Token.Id.IntegerLiteral;
294	},
295	'1'...'9' => {
296	state = State.IntegerLiteral;
297	result.id = Token.Id.IntegerLiteral;
298	},
299	else => {
300	result.id = Token.Id.Invalid;
301	self.index += 1;
302	break;
303	},
304	},
305	State.Ampersand => switch (c) {
306	'=' => {
307	result.id = Token.Id.AmpersandEqual;
308	self.index += 1;
309	break;
310	},
311	else => {
312	result.id = Token.Id.Ampersand;
313	break;
314	},
315	},
316	State.Identifier => switch (c) {
317	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
318	else => {
319	if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
320	result.id = id;
321	}
322	break;
323	},
324	},
325	State.Builtin => switch (c) {
326	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
327	else => break,
328	},
329	State.C => switch (c) {
330	'\\' => @panic("TODO"),
331	'"' => {
332	state = State.StringLiteral;
333	result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
334	},
335	'a'...'z', 'A'...'Z', '_', '0'...'9' => {
336	state = State.Identifier;
337	},
338	else => break,
339	},
340	State.StringLiteral => switch (c) {
341	'\\' => {
342	state = State.StringLiteralBackslash;
343	},
344	'"' => {
345	self.index += 1;
346	break;
347	},
348	'\n' => break, // Look for this error later.
349	else => {},
350	},
351
352	State.StringLiteralBackslash => switch (c) {
353	'\n' => break, // Look for this error later.
354	else => {
355	state = State.StringLiteral;
356	},
357	},
358
359	State.Minus => switch (c) {
360	'>' => {
361	result.id = Token.Id.Arrow;
362	self.index += 1;
363	break;
364	},
365	else => {
366	result.id = Token.Id.Minus;
367	break;
368	},
369	},
370
371	State.Period => switch (c) {
372	'.' => {
373	state = State.Period2;
374	},
375	else => {
376	result.id = Token.Id.Period;
377	break;
378	},
379	},
380
381	State.Period2 => switch (c) {
382	'.' => {
383	result.id = Token.Id.Ellipsis3;
384	self.index += 1;
385	break;
386	},
387	else => {
388	result.id = Token.Id.Ellipsis2;
389	break;
390	},
391	},
392
393	State.Slash => switch (c) {
394	'/' => {
395	result.id = undefined;
396	state = State.LineComment;
397	},
398	else => {
399	result.id = Token.Id.Slash;
400	break;
401	},
402	},
403	State.LineComment => switch (c) {
404	'\n' => {
405	state = State.Start;
406	result = Token {
407	.id = Token.Id.Eof,
408	.start = self.index + 1,
409	.end = undefined,
410	};
411	},
412	else => {},
413	},
414	State.Zero => switch (c) {
415	'b', 'o', 'x' => {
416	state = State.IntegerLiteral;
417	},
418	else => {
419	// reinterpret as a normal number
420	self.index -= 1;
421	state = State.IntegerLiteral;
422	},
423	},
424	State.IntegerLiteral => switch (c) {
425	'.' => {
426	state = State.NumberDot;
427	},
428	'p', 'P', 'e', 'E' => {
429	state = State.FloatExponentUnsigned;
430	},
431	'0'...'9', 'a'...'f', 'A'...'F' => {},
432	else => break,
433	},
434	State.NumberDot => switch (c) {
435	'.' => {
436	self.index -= 1;
437	state = State.Start;
438	break;
439	},
440	else => {
441	self.index -= 1;
442	result.id = Token.Id.FloatLiteral;
443	state = State.FloatFraction;
444	},
445	},
446	State.FloatFraction => switch (c) {
447	'p', 'P', 'e', 'E' => {
448	state = State.FloatExponentUnsigned;
449	},
450	'0'...'9', 'a'...'f', 'A'...'F' => {},
451	else => break,
452	},
453	State.FloatExponentUnsigned => switch (c) {
454	'+', '-' => {
455	state = State.FloatExponentNumber;
456	},
457	else => {
458	// reinterpret as a normal exponent number
459	self.index -= 1;
460	state = State.FloatExponentNumber;
461	}
462	},
463	State.FloatExponentNumber => switch (c) {
464	'0'...'9', 'a'...'f', 'A'...'F' => {},
465	else => break,
466	},
467	}
468	}
469	result.end = self.index;
470	// TODO check state when returning EOF
471	return result;
472	}
473
474	pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) -> []const u8 {
475	return self.buffer[token.start..token.end];
476	}
477	};
478
479