authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 20:00:02+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 20:00:02+02:00
logec611bf8b47af0db5244af644040907bfcb63945
treeb16541e6263b411d2c2877b2862cf2a1de36110f
parentd602f12df8f12c3a363abec8c1976f8bc84eb2be

std.zig.parser now parses regular enums, unions and struct

* Still missing packed, and extern

2 files changed, 350 insertions(+), 59 deletions(-)

std/zig/ast.zig+99-7
......@@ -12,6 +12,9 @@ pub const Node = struct {
1212 Root,
1313 VarDecl,
1414 ContainerDecl,
15 StructField,
16 UnionTag,
17 EnumTag,
1518 Identifier,
1619 FnProto,
1720 ParamDecl,
......@@ -42,6 +45,9 @@ pub const Node = struct {
4245 Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index),
4346 Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index),
4447 Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index),
48 Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index),
49 Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index),
50 Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).iterate(index),
4551 Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index),
4652 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
4753 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
......@@ -73,6 +79,9 @@ pub const Node = struct {
7379 Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(),
7480 Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(),
7581 Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(),
82 Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(),
83 Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(),
84 Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).firstToken(),
7685 Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(),
7786 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(),
7887 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(),
......@@ -104,6 +113,9 @@ pub const Node = struct {
104113 Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(),
105114 Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(),
106115 Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(),
116 Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(),
117 Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(),
118 Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).lastToken(),
107119 Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(),
108120 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(),
109121 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(),
......@@ -203,16 +215,15 @@ pub const NodeVarDecl = struct {
203215pub const NodeContainerDecl = struct {
204216 base: Node,
205217 kind_token: Token,
206 init_arg_expr: InitArg,
207218 kind: Kind,
208 decls: ArrayList(&Node),
219 init_arg_expr: InitArg,
220 fields_and_decls: ArrayList(&Node),
209221 rbrace_token: Token,
210222
211 // TODO: Different array lists for each kind.
212 const Kind = union(enum) {
213 Struct: ArrayList(&Node),
214 Enum: ArrayList(&Node),
215 Union: ArrayList(&Node),
223 const Kind = enum {
224 Struct,
225 Enum,
226 Union,
216227 };
217228
218229 const InitArg = union(enum) {
......@@ -263,6 +274,87 @@ pub const NodeContainerDecl = struct {
263274 }
264275};
265276
277pub const NodeStructField = struct {
278 base: Node,
279 name_token: Token,
280 type_expr: &Node,
281
282 pub fn iterate(self: &NodeStructField, index: usize) ?&Node {
283 var i = index;
284
285 if (i < 1) return self.type_expr;
286 i -= 1;
287
288 return null;
289 }
290
291 pub fn firstToken(self: &NodeStructField) Token {
292 return self.name_token;
293 }
294
295 pub fn lastToken(self: &NodeStructField) Token {
296 return self.type_expr.lastToken();
297 }
298};
299
300pub const NodeUnionTag = struct {
301 base: Node,
302 name_token: Token,
303 type_expr: ?&Node,
304
305 pub fn iterate(self: &NodeUnionTag, index: usize) ?&Node {
306 var i = index;
307
308 if (self.type_expr) |type_expr| {
309 if (i < 1) return type_expr;
310 i -= 1;
311 }
312
313 return null;
314 }
315
316 pub fn firstToken(self: &NodeUnionTag) Token {
317 return self.name_token;
318 }
319
320 pub fn lastToken(self: &NodeUnionTag) Token {
321 if (self.type_expr) |type_expr| {
322 return type_expr.lastToken();
323 }
324
325 return self.name_token;
326 }
327};
328
329pub const NodeEnumTag = struct {
330 base: Node,
331 name_token: Token,
332 value: ?&Node,
333
334 pub fn iterate(self: &NodeEnumTag, index: usize) ?&Node {
335 var i = index;
336
337 if (self.value) |value| {
338 if (i < 1) return value;
339 i -= 1;
340 }
341
342 return null;
343 }
344
345 pub fn firstToken(self: &NodeEnumTag) Token {
346 return self.name_token;
347 }
348
349 pub fn lastToken(self: &NodeEnumTag) Token {
350 if (self.value) |value| {
351 return value.lastToken();
352 }
353
354 return self.name_token;
355 }
356};
357
266358pub const NodeIdentifier = struct {
267359 base: Node,
268360 name_token: Token,
std/zig/parser.zig+251-52
......@@ -90,6 +90,7 @@ pub const Parser = struct {
9090 TopLevel,
9191 TopLevelExtern: TopLevelDeclCtx,
9292 TopLevelDecl: TopLevelDeclCtx,
93 ContainerDecl: &ast.NodeContainerDecl,
9394 Expression: DestPtr,
9495 ExpectOperand,
9596 Operand: &ast.Node,
......@@ -117,6 +118,7 @@ pub const Parser = struct {
117118 ExprListCommaOrEnd: ListState(&ast.Node),
118119 FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer),
119120 FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer),
121 FieldListCommaOrEnd: &ast.NodeContainerDecl,
120122 };
121123
122124 /// Returns an AST tree, allocated with the parser's allocator.
......@@ -181,17 +183,6 @@ pub const Parser = struct {
181183 State.TopLevel => {
182184 const token = self.getNextToken();
183185 switch (token.id) {
184 Token.Id.Keyword_pub, Token.Id.Keyword_export => {
185 stack.append(State {
186 .TopLevelExtern = TopLevelDeclCtx {
187 .decls = &root_node.decls,
188 .visib_token = token,
189 .extern_token = null,
190 .lib_name = null,
191 }
192 }) catch unreachable;
193 continue;
194 },
195186 Token.Id.Keyword_test => {
196187 stack.append(State.TopLevel) catch unreachable;
197188
......@@ -213,16 +204,29 @@ pub const Parser = struct {
213204 root_node.eof_token = token;
214205 return Tree {.root_node = root_node, .arena_allocator = arena_allocator};
215206 },
207 Token.Id.Keyword_pub, Token.Id.Keyword_export => {
208 stack.append(State.TopLevel) catch unreachable;
209 try stack.append(State {
210 .TopLevelExtern = TopLevelDeclCtx {
211 .decls = &root_node.decls,
212 .visib_token = token,
213 .extern_token = null,
214 .lib_name = null,
215 }
216 });
217 continue;
218 },
216219 else => {
217220 self.putBackToken(token);
218 stack.append(State {
221 stack.append(State.TopLevel) catch unreachable;
222 try stack.append(State {
219223 .TopLevelExtern = TopLevelDeclCtx {
220224 .decls = &root_node.decls,
221225 .visib_token = null,
222226 .extern_token = null,
223227 .lib_name = null,
224228 }
225 }) catch unreachable;
229 });
226230 continue;
227231 },
228232 }
......@@ -259,7 +263,6 @@ pub const Parser = struct {
259263 const token = self.getNextToken();
260264 switch (token.id) {
261265 Token.Id.Keyword_var, Token.Id.Keyword_const => {
262 stack.append(State.TopLevel) catch unreachable;
263266 // TODO shouldn't need these casts
264267 const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token,
265268 token, (?Token)(null), ctx.extern_token, ctx.lib_name);
......@@ -267,7 +270,6 @@ pub const Parser = struct {
267270 continue;
268271 },
269272 Token.Id.Keyword_fn => {
270 stack.append(State.TopLevel) catch unreachable;
271273 // TODO shouldn't need these casts
272274 const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token,
273275 ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null));
......@@ -276,7 +278,6 @@ pub const Parser = struct {
276278 continue;
277279 },
278280 Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
279 stack.append(State.TopLevel) catch unreachable;
280281 const fn_token = try self.eatToken(Token.Id.Keyword_fn);
281282 // TODO shouldn't need this cast
282283 const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token,
......@@ -336,6 +337,101 @@ pub const Parser = struct {
336337 }
337338 return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
338339 },
340
341 State.ContainerDecl => |container_decl| {
342 const token = self.getNextToken();
343
344 switch (token.id) {
345 Token.Id.Identifier => {
346 switch (container_decl.kind) {
347 ast.NodeContainerDecl.Kind.Struct => {
348 const node = try arena.create(ast.NodeStructField);
349 *node = ast.NodeStructField {
350 .base = self.initNode(ast.Node.Id.StructField),
351 .name_token = token,
352 .type_expr = undefined,
353 };
354 try container_decl.fields_and_decls.append(&node.base);
355
356 try stack.append(State { .FieldListCommaOrEnd = container_decl });
357 try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } });
358 try stack.append(State { .ExpectToken = Token.Id.Colon });
359 continue;
360 },
361 ast.NodeContainerDecl.Kind.Union => {
362 const node = try arena.create(ast.NodeUnionTag);
363 *node = ast.NodeUnionTag {
364 .base = self.initNode(ast.Node.Id.UnionTag),
365 .name_token = token,
366 .type_expr = null,
367 };
368 try container_decl.fields_and_decls.append(&node.base);
369
370 try stack.append(State { .FieldListCommaOrEnd = container_decl });
371
372 const next = self.getNextToken();
373 if (next.id != Token.Id.Colon) {
374 self.putBackToken(next);
375 continue;
376 }
377
378 try stack.append(State { .Expression = DestPtr { .NullableField = &node.type_expr } });
379 continue;
380 },
381 ast.NodeContainerDecl.Kind.Enum => {
382 const node = try arena.create(ast.NodeEnumTag);
383 *node = ast.NodeEnumTag {
384 .base = self.initNode(ast.Node.Id.EnumTag),
385 .name_token = token,
386 .value = null,
387 };
388 try container_decl.fields_and_decls.append(&node.base);
389
390 try stack.append(State { .FieldListCommaOrEnd = container_decl });
391
392 const next = self.getNextToken();
393 if (next.id != Token.Id.Equal) {
394 self.putBackToken(next);
395 continue;
396 }
397
398 try stack.append(State { .Expression = DestPtr { .NullableField = &node.value } });
399 continue;
400 },
401 }
402 },
403 Token.Id.Keyword_pub, Token.Id.Keyword_export => {
404 stack.append(State{ .ContainerDecl = container_decl }) catch unreachable;
405 try stack.append(State {
406 .TopLevelExtern = TopLevelDeclCtx {
407 .decls = &container_decl.fields_and_decls,
408 .visib_token = token,
409 .extern_token = null,
410 .lib_name = null,
411 }
412 });
413 continue;
414 },
415 Token.Id.RBrace => {
416 container_decl.rbrace_token = token;
417 continue;
418 },
419 else => {
420 self.putBackToken(token);
421 stack.append(State{ .ContainerDecl = container_decl }) catch unreachable;
422 try stack.append(State {
423 .TopLevelExtern = TopLevelDeclCtx {
424 .decls = &container_decl.fields_and_decls,
425 .visib_token = null,
426 .extern_token = null,
427 .lib_name = null,
428 }
429 });
430 continue;
431 }
432 }
433 },
434
339435 State.ExpectToken => |token_id| {
340436 _ = try self.eatToken(token_id);
341437 continue;
......@@ -537,6 +633,53 @@ pub const Parser = struct {
537633 try stack.append(State.AfterOperand);
538634 continue;
539635 },
636 Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => {
637 const node = try arena.create(ast.NodeContainerDecl);
638 *node = ast.NodeContainerDecl {
639 .base = self.initNode(ast.Node.Id.ContainerDecl),
640 .kind_token = token,
641 .kind = switch (token.id) {
642 Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct,
643 Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union,
644 Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum,
645 else => unreachable,
646 },
647 .init_arg_expr = undefined,
648 .fields_and_decls = ArrayList(&ast.Node).init(arena),
649 .rbrace_token = undefined,
650 };
651
652 try stack.append(State { .Operand = &node.base });
653 try stack.append(State.AfterOperand);
654 try stack.append(State { .ContainerDecl = node });
655 try stack.append(State { .ExpectToken = Token.Id.LBrace });
656
657 const lparen = self.getNextToken();
658 if (lparen.id != Token.Id.LParen) {
659 self.putBackToken(lparen);
660 node.init_arg_expr = ast.NodeContainerDecl.InitArg.None;
661 continue;
662 }
663
664 try stack.append(State { .ExpectToken = Token.Id.RParen });
665
666 const init_arg_token = self.getNextToken();
667 switch (init_arg_token.id) {
668 Token.Id.Keyword_enum => {
669 node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum;
670 },
671 else => {
672 self.putBackToken(lparen);
673 node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined };
674 try stack.append(State {
675 .Expression = DestPtr {
676 .Field = &node.init_arg_expr.Type
677 }
678 });
679 },
680 }
681 continue;
682 },
540683 Token.Id.Builtin => {
541684 const node = try arena.create(ast.NodeBuiltinCall);
542685 *node = ast.NodeBuiltinCall {
......@@ -799,24 +942,6 @@ pub const Parser = struct {
799942 try stack.append(State { .Expression = DestPtr{.List = list_state.list} });
800943 },
801944
802 State.ExprListCommaOrEnd => |list_state| {
803 var token = self.getNextToken();
804 switch (token.id) {
805 Token.Id.Comma => {
806 stack.append(State { .ExprListItemOrEnd = list_state }) catch unreachable;
807 },
808 else => {
809 const IdTag = @TagType(Token.Id);
810 if (IdTag(list_state.end) == token.id) {
811 *list_state.ptr = token;
812 continue;
813 }
814
815 return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id));
816 },
817 }
818 },
819
820945 State.FieldInitListItemOrEnd => |list_state| {
821946 var token = self.getNextToken();
822947
......@@ -854,22 +979,20 @@ pub const Parser = struct {
854979 });
855980 },
856981
982 State.ExprListCommaOrEnd => |list_state| {
983 try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .ExprListItemOrEnd = list_state });
984 continue;
985 },
986
857987 State.FieldInitListCommaOrEnd => |list_state| {
858 var token = self.getNextToken();
859 switch (token.id) {
860 Token.Id.Comma => {
861 stack.append(State { .FieldInitListItemOrEnd = list_state }) catch unreachable;
862 },
863 else => {
864 const IdTag = @TagType(Token.Id);
865 if (IdTag(list_state.end) == token.id) {
866 *list_state.ptr = token;
867 continue;
868 }
988 try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .FieldInitListItemOrEnd = list_state });
989 continue;
990 },
869991
870 return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id));
871 },
872 }
992 State.FieldListCommaOrEnd => |container_decl| {
993 try self.commaOrEnd(&stack, Token.Id.RBrace, &container_decl.rbrace_token,
994 State { .ContainerDecl = container_decl });
995 continue;
873996 },
874997
875998 State.AddrOfModifiers => |addr_of_info| {
......@@ -1089,6 +1212,24 @@ pub const Parser = struct {
10891212 }
10901213 }
10911214
1215 fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void {
1216 var token = self.getNextToken();
1217 switch (token.id) {
1218 Token.Id.Comma => {
1219 stack.append(state_after_comma) catch unreachable;
1220 },
1221 else => {
1222 const IdTag = @TagType(Token.Id);
1223 if (IdTag(*end) == token.id) {
1224 *ptr = token;
1225 return;
1226 }
1227
1228 return self.parseError(token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id));
1229 },
1230 }
1231 }
1232
10921233 fn popSuffixOp(stack: &ArrayList(State)) &ast.Node {
10931234 var expression: &ast.Node = undefined;
10941235 var left_leaf_ptr: &&ast.Node = &expression;
......@@ -1806,10 +1947,6 @@ pub const Parser = struct {
18061947 try stack.append(RenderState { .Expression = grouped_expr.expr });
18071948 try stack.append(RenderState { .Text = "("});
18081949 },
1809 ast.Node.Id.ContainerDecl => {
1810 const container_decl = @fieldParentPtr(ast.NodeGroupedExpression, "base", base);
1811 @panic("TODO: ContainerDecl");
1812 },
18131950 ast.Node.Id.FieldInitializer => {
18141951 const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base);
18151952 try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token));
......@@ -1851,6 +1988,46 @@ pub const Parser = struct {
18511988 const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base);
18521989 try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token));
18531990 },
1991 ast.Node.Id.ContainerDecl => {
1992 const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base);
1993 try stream.print("{} {{", self.tokenizer.getTokenSlice(container_decl.kind_token));
1994 try stack.append(RenderState { .Text = "}"});
1995 try stack.append(RenderState.PrintIndent);
1996 try stack.append(RenderState { .Indent = indent });
1997
1998 const fields_and_decls = container_decl.fields_and_decls.toSliceConst();
1999 var i = fields_and_decls.len;
2000 while (i != 0) {
2001 i -= 1;
2002 const node = fields_and_decls[i];
2003 if (i != 0) {
2004 switch (node.id) {
2005 ast.Node.Id.StructField,
2006 ast.Node.Id.UnionTag,
2007 ast.Node.Id.EnumTag => {
2008 try stack.append(RenderState { .Text = "," });
2009 },
2010 else => { }
2011 }
2012 }
2013 try stack.append(RenderState { .Expression = node});
2014 try stack.append(RenderState.PrintIndent);
2015 try stack.append(RenderState {
2016 .Text = blk: {
2017 if (i != 0) {
2018 const prev_node = fields_and_decls[i - 1];
2019 const prev_line_index = prev_node.lastToken().line;
2020 const this_line_index = node.firstToken().line;
2021 if (this_line_index - prev_line_index >= 2) {
2022 break :blk "\n\n";
2023 }
2024 }
2025 break :blk "\n";
2026 },
2027 });
2028 }
2029 try stack.append(RenderState { .Indent = indent + indent_delta});
2030 },
18542031 ast.Node.Id.MultilineStringLiteral => {
18552032 const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base);
18562033 try stream.print("\n");
......@@ -1883,6 +2060,28 @@ pub const Parser = struct {
18832060 }
18842061 }
18852062 },
2063 ast.Node.Id.StructField => {
2064 const field = @fieldParentPtr(ast.NodeStructField, "base", base);
2065 try stream.print("{}:", self.tokenizer.getTokenSlice(field.name_token));
2066 },
2067 ast.Node.Id.UnionTag => {
2068 const tag = @fieldParentPtr(ast.NodeUnionTag, "base", base);
2069 try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token));
2070
2071 if (tag.type_expr) |type_expr| {
2072 try stream.print(": ");
2073 try stack.append(RenderState { .Expression = type_expr});
2074 }
2075 },
2076 ast.Node.Id.EnumTag => {
2077 const tag = @fieldParentPtr(ast.NodeEnumTag, "base", base);
2078 try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token));
2079
2080 if (tag.value) |value| {
2081 try stream.print(" = ");
2082 try stack.append(RenderState { .Expression = value});
2083 }
2084 },
18862085 ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"),
18872086 ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),
18882087