authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-07 16:05:13+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-07 16:05:13+02:00
logdf12c1328eb9c2af006dfd9e5cf69046ad6fc235
treeeaa87964d638275e34258684085e2a033cbd4823
parent3ed6d7d24589aa295409880239833fdc8d6be9d6
signaturelock-open Commit is signed but in an unrecognized format.

std-c parser typing improvements

3 files changed, 254 insertions(+), 57 deletions(-)

lib/std/c/ast.zig+184-28
...@@ -23,6 +23,11 @@ pub const Tree = struct {...@@ -23,6 +23,11 @@ pub const Tree = struct {
23 arena_allocator.deinit();23 arena_allocator.deinit();
24 // self is destroyed24 // self is destroyed
25 }25 }
26
27 pub fn slice(tree: *Tree, token: TokenIndex) []const u8 {
28 const tok = tree.tokens.at(token);
29 return tok.source.buffer[tok.start..tok.end];
30 }
26};31};
2732
28pub const Msg = struct {33pub const Msg = struct {
...@@ -47,19 +52,19 @@ pub const Error = union(enum) {...@@ -47,19 +52,19 @@ pub const Error = union(enum) {
47 DuplicateQualifier: SingleTokenError("duplicate type qualifier '{}'"),52 DuplicateQualifier: SingleTokenError("duplicate type qualifier '{}'"),
48 DuplicateSpecifier: SingleTokenError("duplicate declaration specifier '{}'"),53 DuplicateSpecifier: SingleTokenError("duplicate declaration specifier '{}'"),
4954
50 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {55 pub fn render(self: *const Error, tree: *Tree, stream: var) !void {
51 switch (self.*) {56 switch (self.*) {
52 .InvalidToken => |*x| return x.render(tokens, stream),57 .InvalidToken => |*x| return x.render(tree, stream),
53 .ExpectedToken => |*x| return x.render(tokens, stream),58 .ExpectedToken => |*x| return x.render(tree, stream),
54 .ExpectedExpr => |*x| return x.render(tokens, stream),59 .ExpectedExpr => |*x| return x.render(tree, stream),
55 .ExpectedStmt => |*x| return x.render(tokens, stream),60 .ExpectedStmt => |*x| return x.render(tree, stream),
56 .ExpectedTypeName => |*x| return x.render(tokens, stream),61 .ExpectedTypeName => |*x| return x.render(tree, stream),
57 .ExpectedDeclarator => |*x| return x.render(tokens, stream),62 .ExpectedDeclarator => |*x| return x.render(tree, stream),
58 .ExpectedFnBody => |*x| return x.render(tokens, stream),63 .ExpectedFnBody => |*x| return x.render(tree, stream),
59 .ExpectedInitializer => |*x| return x.render(tokens, stream),64 .ExpectedInitializer => |*x| return x.render(tree, stream),
60 .InvalidTypeSpecifier => |*x| return x.render(tokens, stream),65 .InvalidTypeSpecifier => |*x| return x.render(tree, stream),
61 .DuplicateQualifier => |*x| return x.render(tokens, stream),66 .DuplicateQualifier => |*x| return x.render(tree, stream),
62 .DuplicateSpecifier => |*x| return x.render(tokens, stream),67 .DuplicateSpecifier => |*x| return x.render(tree, stream),
63 }68 }
64 }69 }
6570
...@@ -83,8 +88,8 @@ pub const Error = union(enum) {...@@ -83,8 +88,8 @@ pub const Error = union(enum) {
83 token: TokenIndex,88 token: TokenIndex,
84 expected_id: @TagType(Token.Id),89 expected_id: @TagType(Token.Id),
8590
86 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {91 pub fn render(self: *const ExpectedToken, tree: *Tree, stream: var) !void {
87 const found_token = tokens.at(self.token);92 const found_token = tree.tokens.at(self.token);
88 if (found_token.id == .Invalid) {93 if (found_token.id == .Invalid) {
89 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});94 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
90 } else {95 } else {
...@@ -98,10 +103,10 @@ pub const Error = union(enum) {...@@ -98,10 +103,10 @@ pub const Error = union(enum) {
98 token: TokenIndex,103 token: TokenIndex,
99 type_spec: *Node.TypeSpec,104 type_spec: *Node.TypeSpec,
100105
101 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {106 pub fn render(self: *const ExpectedToken, tree: *Tree, stream: var) !void {
102 try stream.write("invalid type specifier '");107 try stream.write("invalid type specifier '");
103 try type_spec.spec.print(tokens, stream);108 try type_spec.spec.print(tree, stream);
104 const token_name = tokens.at(self.token).id.symbol();109 const token_name = tree.tokens.at(self.token).id.symbol();
105 return stream.print("{}'", .{token_name});110 return stream.print("{}'", .{token_name});
106 }111 }
107 };112 };
...@@ -110,14 +115,59 @@ pub const Error = union(enum) {...@@ -110,14 +115,59 @@ pub const Error = union(enum) {
110 return struct {115 return struct {
111 token: TokenIndex,116 token: TokenIndex,
112117
113 pub fn render(self: *const @This(), tokens: *Tree.TokenList, stream: var) !void {118 pub fn render(self: *const @This(), tree: *Tree, stream: var) !void {
114 const actual_token = tokens.at(self.token);119 const actual_token = tree.tokens.at(self.token);
115 return stream.print(msg, .{actual_token.id.symbol()});120 return stream.print(msg, .{actual_token.id.symbol()});
116 }121 }
117 };122 };
118 }123 }
119};124};
120125
126pub const Type = struct {
127 pub const TypeList = std.SegmentedList(*Type, 4);
128 @"const": bool,
129 atomic: bool,
130 @"volatile": bool,
131 restrict: bool,
132
133 id: union(enum) {
134 Int: struct {
135 quals: Qualifiers,
136 id: Id,
137 is_signed: bool,
138
139 pub const Id = enum {
140 Char,
141 Short,
142 Int,
143 Long,
144 LongLong,
145 };
146 },
147 Float: struct {
148 quals: Qualifiers,
149 id: Id,
150
151 pub const Id = enum {
152 Float,
153 Double,
154 LongDouble,
155 };
156 },
157 Pointer: struct {
158 quals: Qualifiers,
159 child_type: *Type,
160 },
161 Function: struct {
162 return_type: *Type,
163 param_types: TypeList,
164 },
165 Typedef: *Type,
166 Record: *Node.RecordType,
167 Enum: *Node.EnumType,
168 },
169};
170
121pub const Node = struct {171pub const Node = struct {
122 id: Id,172 id: Id,
123173
...@@ -205,22 +255,128 @@ pub const Node = struct {...@@ -205,22 +255,128 @@ pub const Node = struct {
205 typename: *Node,255 typename: *Node,
206 rparen: TokenIndex,256 rparen: TokenIndex,
207 },257 },
258 Enum: *EnumType,
259 Record: *RecordType,
260 Typedef: struct {
261 sym: TokenIndex,
262 sym_type: *Type,
263 },
208264
209 //todo265 pub fn print(self: *@This(), self: *const @This(), tree: *Tree, stream: var) !void {
210 // @"enum",266 switch (self.spec) {
211 // record,
212
213 Typedef: TokenIndex,
214
215 pub fn print(self: *@This(), self: *const @This(), tokens: *Tree.TokenList, stream: var) !void {
216 switch (self) {
217 .None => unreachable,267 .None => unreachable,
218 else => @panic("TODO print type specifier"),268 .Void => |index| try stream.write(tree.slice(index)),
269 .Char => |char| {
270 if (char.sign) |s| {
271 try stream.write(tree.slice(s));
272 try stream.writeByte(' ');
273 }
274 try stream.write(tree.slice(char.char));
275 },
276 .Short => |short| {
277 if (short.sign) |s| {
278 try stream.write(tree.slice(s));
279 try stream.writeByte(' ');
280 }
281 try stream.write(tree.slice(short.short));
282 if (short.int) |i| {
283 try stream.writeByte(' ');
284 try stream.write(tree.slice(i));
285 }
286 },
287 .Int => |int| {
288 if (int.sign) |s| {
289 try stream.write(tree.slice(s));
290 try stream.writeByte(' ');
291 }
292 if (int.int) |i| {
293 try stream.writeByte(' ');
294 try stream.write(tree.slice(i));
295 }
296 },
297 .Long => |long| {
298 if (long.sign) |s| {
299 try stream.write(tree.slice(s));
300 try stream.writeByte(' ');
301 }
302 try stream.write(tree.slice(long.long));
303 if (long.longlong) |l| {
304 try stream.writeByte(' ');
305 try stream.write(tree.slice(l));
306 }
307 if (long.int) |i| {
308 try stream.writeByte(' ');
309 try stream.write(tree.slice(i));
310 }
311 },
312 .Float => |float| {
313 try stream.write(tree.slice(float.float));
314 if (float.complex) |c| {
315 try stream.writeByte(' ');
316 try stream.write(tree.slice(c));
317 }
318 },
319 .Double => |double| {
320 if (double.long) |l| {
321 try stream.write(tree.slice(l));
322 try stream.writeByte(' ');
323 }
324 try stream.write(tree.slice(double.double));
325 if (double.complex) |c| {
326 try stream.writeByte(' ');
327 try stream.write(tree.slice(c));
328 }
329 },
330 .Bool => |index| try stream.write(tree.slice(index)),
331 .Typedef => |typedef| try stream.write(tree.slice(typedef.sym)),
332 else => try stream.print("TODO print {}", self.spec),
219 }333 }
220 }334 }
221 } = .None,335 } = .None,
222 };336 };
223337
338 pub const EnumType = struct {
339 tok: TokenIndex,
340 name: ?TokenIndex,
341 body: ?struct {
342 lbrace: TokenIndex,
343
344 /// always EnumField
345 fields: FieldList,
346 rbrace: TokenIndex,
347 },
348
349 pub const FieldList = Root.DeclList;
350 };
351
352 pub const EnumField = struct {
353 base: Node = Node{ .id = EnumField },
354 name: TokenIndex,
355 value: ?*Node,
356 };
357
358 pub const RecordType = struct {
359 kind: union(enum) {
360 Struct: TokenIndex,
361 Union: TokenIndex,
362 },
363 name: ?TokenIndex,
364 body: ?struct {
365 lbrace: TokenIndex,
366
367 /// RecordField or StaticAssert
368 fields: FieldList,
369 rbrace: TokenIndex,
370 },
371
372 pub const FieldList = Root.DeclList;
373 };
374
375 pub const RecordField = struct {
376 base: Node = Node{ .id = RecordField },
377 // TODO
378 };
379
224 pub const TypeQual = struct {380 pub const TypeQual = struct {
225 @"const": ?TokenIndex = null,381 @"const": ?TokenIndex = null,
226 atomic: ?TokenIndex = null,382 atomic: ?TokenIndex = null,
lib/std/c/parse.zig+69-25
...@@ -3,6 +3,7 @@ const assert = std.debug.assert;...@@ -3,6 +3,7 @@ const assert = std.debug.assert;
3const Allocator = std.mem.Allocator;3const Allocator = std.mem.Allocator;
4const ast = std.c.ast;4const ast = std.c.ast;
5const Node = ast.Node;5const Node = ast.Node;
6const Type = ast.Type;
6const Tree = ast.Tree;7const Tree = ast.Tree;
7const TokenIndex = ast.TokenIndex;8const TokenIndex = ast.TokenIndex;
8const Token = std.c.Token;9const Token = std.c.Token;
...@@ -57,10 +58,12 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {...@@ -57,10 +58,12 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
57 }58 }
5859
59 var parser = Parser{60 var parser = Parser{
61 .symbols = Parser.SymbolList.init(allocator),
60 .arena = arena,62 .arena = arena,
61 .it = &it,63 .it = &it,
62 .tree = tree,64 .tree = tree,
63 };65 };
66 defer parser.symbols.deinit();
6467
65 tree.root_node = try parser.root();68 tree.root_node = try parser.root();
66 return tree;69 return tree;
...@@ -72,19 +75,35 @@ const Parser = struct {...@@ -72,19 +75,35 @@ const Parser = struct {
72 tree: *Tree,75 tree: *Tree,
7376
74 /// only used for scopes77 /// only used for scopes
75 arena_allocator: std.heap.ArenaAllocator,78 symbols: SymbolList,
76 // scopes: std.SegmentedLists(Scope),
77 warnings: bool = true,79 warnings: bool = true,
7880
79 // const Scope = struct {81 const SymbolList = std.ArrayList(Symbol);
80 // types:
81 // syms:
82 // };
8382
84 fn getTypeDef(parser: *Parser, tok: TokenIndex) bool {83 const Symbol = struct {
85 return false; // TODO84 name: []const u8,
86 // const token = parser.it.list.at(tok);85 ty: *Type,
87 // return parser.typedefs.contains(token.slice());86 };
87
88 fn pushScope(parser: *Parser) usize {
89 return parser.symbols.len;
90 }
91
92 fn popScope(parser: *Parser, len: usize) void {
93 parser.symbols.resize(len) catch unreachable;
94 }
95
96 fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type {
97 const token = parser.it.list.at(tok);
98 const name = parser.tree.slice(token);
99 const syms = parser.symbols.toSliceConst();
100 var i = syms.len;
101 while (i > 0) : (i -= 1) {
102 if (mem.eql(u8, name, syms[i].name)) {
103 return syms[i].ty;
104 }
105 }
106 return null;
88 }107 }
89108
90 /// Root <- ExternalDeclaration* eof109 /// Root <- ExternalDeclaration* eof
...@@ -264,8 +283,8 @@ const Parser = struct {...@@ -264,8 +283,8 @@ const Parser = struct {
264 /// <- Keyword_void / Keyword_char / Keyword_short / Keyword_int / Keyword_long / Keyword_float / Keyword_double283 /// <- Keyword_void / Keyword_char / Keyword_short / Keyword_int / Keyword_long / Keyword_float / Keyword_double
265 /// / Keyword_signed / Keyword_unsigned / Keyword_bool / Keyword_complex / Keyword_imaginary /284 /// / Keyword_signed / Keyword_unsigned / Keyword_bool / Keyword_complex / Keyword_imaginary /
266 /// / Keyword_atomic LPAREN TypeName RPAREN285 /// / Keyword_atomic LPAREN TypeName RPAREN
267 /// / EnumSpecifier286 /// / EnumSpec
268 /// / RecordSpecifier287 /// / RecordSpec
269 /// / IDENTIFIER // typedef name288 /// / IDENTIFIER // typedef name
270 /// / TypeQual289 /// / TypeQual
271 fn typeSpec(parser: *Parser, type_spec: *Node.TypeSpec) !bool {290 fn typeSpec(parser: *Parser, type_spec: *Node.TypeSpec) !bool {
...@@ -473,22 +492,48 @@ const Parser = struct {...@@ -473,22 +492,48 @@ const Parser = struct {
473 } else if (parser.eatToken(.Keyword_enum)) |tok| {492 } else if (parser.eatToken(.Keyword_enum)) |tok| {
474 if (type_spec.spec != .None)493 if (type_spec.spec != .None)
475 break :blk;494 break :blk;
476 @panic("TODO enum type");495 type_spec.Enum = try parser.enumSpec(tok);
477 // return true;496 return true;
478 } else if (parser.eatToken(.Keyword_union) orelse parser.eatToken(.Keyword_struct)) |tok| {497 } else if (parser.eatToken(.Keyword_union) orelse parser.eatToken(.Keyword_struct)) |tok| {
479 if (type_spec.spec != .None)498 if (type_spec.spec != .None)
480 break :blk;499 break :blk;
481 @panic("TODO record type");500 type_spec.Record = try parser.recordSpec();
482 // return true;501 return true;
483 } else if (parser.eatToken(.Identifier)) |tok| {502 } else if (parser.eatToken(.Identifier)) |tok| {
484 if (!parser.getTypeDef(tok)) {503 const ty = parser.getSymbol(tok) orelse {
485 parser.putBackToken(tok);504 parser.putBackToken(tok);
486 return false;505 return false;
487 }
488 type_spec.spec = .{
489 .Typedef = tok,
490 };506 };
491 return true;507 switch (ty) {
508 .Enum => |e| {
509 return parser.err(.{
510 .MustUseKwToRefer = .{ .kw = e.identifier, .sym = tok },
511 });
512 },
513 .Record => |r| {
514 return parser.err(.{
515 .MustUseKwToRefer = .{
516 .kw = switch (r.kind) {
517 .Struct, .Union => |kw| kw,
518 },
519 .sym = tok,
520 },
521 });
522 },
523 .Typedef => {
524 type_spec.spec = .{
525 .Typedef = .{
526 .sym = tok,
527 .sym_type = ty,
528 },
529 };
530 return true;
531 },
532 else => {
533 parser.putBackToken(tok);
534 return false;
535 },
536 }
492 }537 }
493 }538 }
494 return parser.err(.{539 return parser.err(.{
...@@ -567,13 +612,13 @@ const Parser = struct {...@@ -567,13 +612,13 @@ const Parser = struct {
567 return false;612 return false;
568 }613 }
569614
570 /// EnumSpecifier <- Keyword_enum IDENTIFIER? (LBRACE EnumField RBRACE)?615 /// EnumSpec <- Keyword_enum IDENTIFIER? (LBRACE EnumField RBRACE)?
571 fn enumSpecifier(parser: *Parser) !*Node {}616 fn enumSpecifier(parser: *Parser) !*Node {}
572617
573 /// EnumField <- IDENTIFIER (EQUAL ConstExpr)? (COMMA EnumField) COMMA?618 /// EnumField <- IDENTIFIER (EQUAL ConstExpr)? (COMMA EnumField) COMMA?
574 fn enumField(parser: *Parser) !*Node {}619 fn enumField(parser: *Parser) !*Node {}
575620
576 /// RecordSpecifier <- (Keyword_struct / Keyword_union) IDENTIFIER? (LBRACE RecordField+ RBRACE)?621 /// RecordSpec <- (Keyword_struct / Keyword_union) IDENTIFIER? (LBRACE RecordField+ RBRACE)?
577 fn recordSpecifier(parser: *Parser) !*Node {}622 fn recordSpecifier(parser: *Parser) !*Node {}
578623
579 /// RecordField624 /// RecordField
...@@ -581,8 +626,7 @@ const Parser = struct {...@@ -581,8 +626,7 @@ const Parser = struct {
581 /// \ StaticAssert626 /// \ StaticAssert
582 fn recordField(parser: *Parser) !*Node {}627 fn recordField(parser: *Parser) !*Node {}
583628
584 /// TypeName629 /// TypeName <- TypeSpec* AbstractDeclarator?
585 /// <- TypeSpec* AbstractDeclarator?
586 fn typeName(parser: *Parser) !*Node {630 fn typeName(parser: *Parser) !*Node {
587631
588 /// RecordDeclarator <- Declarator? (COLON ConstExpr)?632 /// RecordDeclarator <- Declarator? (COLON ConstExpr)?
lib/std/c/tokenizer.zig+1-4
...@@ -327,6 +327,7 @@ pub const Token = struct {...@@ -327,6 +327,7 @@ pub const Token = struct {
327 };327 };
328328
329 // TODO perfect hash at comptime329 // TODO perfect hash at comptime
330 // TODO do this in the preprocessor
330 pub fn getKeyword(bytes: []const u8, pp_directive: bool) ?Id {331 pub fn getKeyword(bytes: []const u8, pp_directive: bool) ?Id {
331 var hash = std.hash_map.hashString(bytes);332 var hash = std.hash_map.hashString(bytes);
332 for (keywords) |kw| {333 for (keywords) |kw| {
...@@ -347,10 +348,6 @@ pub const Token = struct {...@@ -347,10 +348,6 @@ pub const Token = struct {
347 return null;348 return null;
348 }349 }
349350
350 pub fn slice(tok: Token) []const u8 {
351 return tok.source.buffer[tok.start..tok.end];
352 }
353
354 pub const NumSuffix = enum {351 pub const NumSuffix = enum {
355 None,352 None,
356 F,353 F,