authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-07 16:05:13+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-07 16:05:13+02:00
logdf12c1328eb9c2af006dfd9e5cf69046ad6fc235
treeeaa87964d638275e34258684085e2a033cbd4823
parent3ed6d7d24589aa295409880239833fdc8d6be9d6
signaturelock-open Commit is signed but in an unrecognized format.

std-c parser typing improvements

3 files changed, 254 insertions(+), 57 deletions(-)

lib/std/c/ast.zig+184-28
......@@ -23,6 +23,11 @@ pub const Tree = struct {
2323 arena_allocator.deinit();
2424 // self is destroyed
2525 }
26
27 pub fn slice(tree: *Tree, token: TokenIndex) []const u8 {
28 const tok = tree.tokens.at(token);
29 return tok.source.buffer[tok.start..tok.end];
30 }
2631};
2732
2833pub const Msg = struct {
......@@ -47,19 +52,19 @@ pub const Error = union(enum) {
4752 DuplicateQualifier: SingleTokenError("duplicate type qualifier '{}'"),
4853 DuplicateSpecifier: SingleTokenError("duplicate declaration specifier '{}'"),
4954
50 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {
55 pub fn render(self: *const Error, tree: *Tree, stream: var) !void {
5156 switch (self.*) {
52 .InvalidToken => |*x| return x.render(tokens, stream),
53 .ExpectedToken => |*x| return x.render(tokens, stream),
54 .ExpectedExpr => |*x| return x.render(tokens, stream),
55 .ExpectedStmt => |*x| return x.render(tokens, stream),
56 .ExpectedTypeName => |*x| return x.render(tokens, stream),
57 .ExpectedDeclarator => |*x| return x.render(tokens, stream),
58 .ExpectedFnBody => |*x| return x.render(tokens, stream),
59 .ExpectedInitializer => |*x| return x.render(tokens, stream),
60 .InvalidTypeSpecifier => |*x| return x.render(tokens, stream),
61 .DuplicateQualifier => |*x| return x.render(tokens, stream),
62 .DuplicateSpecifier => |*x| return x.render(tokens, stream),
57 .InvalidToken => |*x| return x.render(tree, stream),
58 .ExpectedToken => |*x| return x.render(tree, stream),
59 .ExpectedExpr => |*x| return x.render(tree, stream),
60 .ExpectedStmt => |*x| return x.render(tree, stream),
61 .ExpectedTypeName => |*x| return x.render(tree, stream),
62 .ExpectedDeclarator => |*x| return x.render(tree, stream),
63 .ExpectedFnBody => |*x| return x.render(tree, stream),
64 .ExpectedInitializer => |*x| return x.render(tree, stream),
65 .InvalidTypeSpecifier => |*x| return x.render(tree, stream),
66 .DuplicateQualifier => |*x| return x.render(tree, stream),
67 .DuplicateSpecifier => |*x| return x.render(tree, stream),
6368 }
6469 }
6570
......@@ -83,8 +88,8 @@ pub const Error = union(enum) {
8388 token: TokenIndex,
8489 expected_id: @TagType(Token.Id),
8590
86 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
87 const found_token = tokens.at(self.token);
91 pub fn render(self: *const ExpectedToken, tree: *Tree, stream: var) !void {
92 const found_token = tree.tokens.at(self.token);
8893 if (found_token.id == .Invalid) {
8994 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
9095 } else {
......@@ -98,10 +103,10 @@ pub const Error = union(enum) {
98103 token: TokenIndex,
99104 type_spec: *Node.TypeSpec,
100105
101 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
106 pub fn render(self: *const ExpectedToken, tree: *Tree, stream: var) !void {
102107 try stream.write("invalid type specifier '");
103 try type_spec.spec.print(tokens, stream);
104 const token_name = tokens.at(self.token).id.symbol();
108 try type_spec.spec.print(tree, stream);
109 const token_name = tree.tokens.at(self.token).id.symbol();
105110 return stream.print("{}'", .{token_name});
106111 }
107112 };
......@@ -110,14 +115,59 @@ pub const Error = union(enum) {
110115 return struct {
111116 token: TokenIndex,
112117
113 pub fn render(self: *const @This(), tokens: *Tree.TokenList, stream: var) !void {
114 const actual_token = tokens.at(self.token);
118 pub fn render(self: *const @This(), tree: *Tree, stream: var) !void {
119 const actual_token = tree.tokens.at(self.token);
115120 return stream.print(msg, .{actual_token.id.symbol()});
116121 }
117122 };
118123 }
119124};
120125
126pub const Type = struct {
127 pub const TypeList = std.SegmentedList(*Type, 4);
128 @"const": bool,
129 atomic: bool,
130 @"volatile": bool,
131 restrict: bool,
132
133 id: union(enum) {
134 Int: struct {
135 quals: Qualifiers,
136 id: Id,
137 is_signed: bool,
138
139 pub const Id = enum {
140 Char,
141 Short,
142 Int,
143 Long,
144 LongLong,
145 };
146 },
147 Float: struct {
148 quals: Qualifiers,
149 id: Id,
150
151 pub const Id = enum {
152 Float,
153 Double,
154 LongDouble,
155 };
156 },
157 Pointer: struct {
158 quals: Qualifiers,
159 child_type: *Type,
160 },
161 Function: struct {
162 return_type: *Type,
163 param_types: TypeList,
164 },
165 Typedef: *Type,
166 Record: *Node.RecordType,
167 Enum: *Node.EnumType,
168 },
169};
170
121171pub const Node = struct {
122172 id: Id,
123173
......@@ -205,22 +255,128 @@ pub const Node = struct {
205255 typename: *Node,
206256 rparen: TokenIndex,
207257 },
258 Enum: *EnumType,
259 Record: *RecordType,
260 Typedef: struct {
261 sym: TokenIndex,
262 sym_type: *Type,
263 },
208264
209 //todo
210 // @"enum",
211 // record,
212
213 Typedef: TokenIndex,
214
215 pub fn print(self: *@This(), self: *const @This(), tokens: *Tree.TokenList, stream: var) !void {
216 switch (self) {
265 pub fn print(self: *@This(), self: *const @This(), tree: *Tree, stream: var) !void {
266 switch (self.spec) {
217267 .None => unreachable,
218 else => @panic("TODO print type specifier"),
268 .Void => |index| try stream.write(tree.slice(index)),
269 .Char => |char| {
270 if (char.sign) |s| {
271 try stream.write(tree.slice(s));
272 try stream.writeByte(' ');
273 }
274 try stream.write(tree.slice(char.char));
275 },
276 .Short => |short| {
277 if (short.sign) |s| {
278 try stream.write(tree.slice(s));
279 try stream.writeByte(' ');
280 }
281 try stream.write(tree.slice(short.short));
282 if (short.int) |i| {
283 try stream.writeByte(' ');
284 try stream.write(tree.slice(i));
285 }
286 },
287 .Int => |int| {
288 if (int.sign) |s| {
289 try stream.write(tree.slice(s));
290 try stream.writeByte(' ');
291 }
292 if (int.int) |i| {
293 try stream.writeByte(' ');
294 try stream.write(tree.slice(i));
295 }
296 },
297 .Long => |long| {
298 if (long.sign) |s| {
299 try stream.write(tree.slice(s));
300 try stream.writeByte(' ');
301 }
302 try stream.write(tree.slice(long.long));
303 if (long.longlong) |l| {
304 try stream.writeByte(' ');
305 try stream.write(tree.slice(l));
306 }
307 if (long.int) |i| {
308 try stream.writeByte(' ');
309 try stream.write(tree.slice(i));
310 }
311 },
312 .Float => |float| {
313 try stream.write(tree.slice(float.float));
314 if (float.complex) |c| {
315 try stream.writeByte(' ');
316 try stream.write(tree.slice(c));
317 }
318 },
319 .Double => |double| {
320 if (double.long) |l| {
321 try stream.write(tree.slice(l));
322 try stream.writeByte(' ');
323 }
324 try stream.write(tree.slice(double.double));
325 if (double.complex) |c| {
326 try stream.writeByte(' ');
327 try stream.write(tree.slice(c));
328 }
329 },
330 .Bool => |index| try stream.write(tree.slice(index)),
331 .Typedef => |typedef| try stream.write(tree.slice(typedef.sym)),
332 else => try stream.print("TODO print {}", self.spec),
219333 }
220334 }
221335 } = .None,
222336 };
223337
338 pub const EnumType = struct {
339 tok: TokenIndex,
340 name: ?TokenIndex,
341 body: ?struct {
342 lbrace: TokenIndex,
343
344 /// always EnumField
345 fields: FieldList,
346 rbrace: TokenIndex,
347 },
348
349 pub const FieldList = Root.DeclList;
350 };
351
352 pub const EnumField = struct {
353 base: Node = Node{ .id = EnumField },
354 name: TokenIndex,
355 value: ?*Node,
356 };
357
358 pub const RecordType = struct {
359 kind: union(enum) {
360 Struct: TokenIndex,
361 Union: TokenIndex,
362 },
363 name: ?TokenIndex,
364 body: ?struct {
365 lbrace: TokenIndex,
366
367 /// RecordField or StaticAssert
368 fields: FieldList,
369 rbrace: TokenIndex,
370 },
371
372 pub const FieldList = Root.DeclList;
373 };
374
375 pub const RecordField = struct {
376 base: Node = Node{ .id = RecordField },
377 // TODO
378 };
379
224380 pub const TypeQual = struct {
225381 @"const": ?TokenIndex = null,
226382 atomic: ?TokenIndex = null,
lib/std/c/parse.zig+69-25
......@@ -3,6 +3,7 @@ const assert = std.debug.assert;
33const Allocator = std.mem.Allocator;
44const ast = std.c.ast;
55const Node = ast.Node;
6const Type = ast.Type;
67const Tree = ast.Tree;
78const TokenIndex = ast.TokenIndex;
89const Token = std.c.Token;
......@@ -57,10 +58,12 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
5758 }
5859
5960 var parser = Parser{
61 .symbols = Parser.SymbolList.init(allocator),
6062 .arena = arena,
6163 .it = &it,
6264 .tree = tree,
6365 };
66 defer parser.symbols.deinit();
6467
6568 tree.root_node = try parser.root();
6669 return tree;
......@@ -72,19 +75,35 @@ const Parser = struct {
7275 tree: *Tree,
7376
7477 /// only used for scopes
75 arena_allocator: std.heap.ArenaAllocator,
76 // scopes: std.SegmentedLists(Scope),
78 symbols: SymbolList,
7779 warnings: bool = true,
7880
79 // const Scope = struct {
80 // types:
81 // syms:
82 // };
81 const SymbolList = std.ArrayList(Symbol);
8382
84 fn getTypeDef(parser: *Parser, tok: TokenIndex) bool {
85 return false; // TODO
86 // const token = parser.it.list.at(tok);
87 // return parser.typedefs.contains(token.slice());
83 const Symbol = struct {
84 name: []const u8,
85 ty: *Type,
86 };
87
88 fn pushScope(parser: *Parser) usize {
89 return parser.symbols.len;
90 }
91
92 fn popScope(parser: *Parser, len: usize) void {
93 parser.symbols.resize(len) catch unreachable;
94 }
95
96 fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type {
97 const token = parser.it.list.at(tok);
98 const name = parser.tree.slice(token);
99 const syms = parser.symbols.toSliceConst();
100 var i = syms.len;
101 while (i > 0) : (i -= 1) {
102 if (mem.eql(u8, name, syms[i].name)) {
103 return syms[i].ty;
104 }
105 }
106 return null;
88107 }
89108
90109 /// Root <- ExternalDeclaration* eof
......@@ -264,8 +283,8 @@ const Parser = struct {
264283 /// <- Keyword_void / Keyword_char / Keyword_short / Keyword_int / Keyword_long / Keyword_float / Keyword_double
265284 /// / Keyword_signed / Keyword_unsigned / Keyword_bool / Keyword_complex / Keyword_imaginary /
266285 /// / Keyword_atomic LPAREN TypeName RPAREN
267 /// / EnumSpecifier
268 /// / RecordSpecifier
286 /// / EnumSpec
287 /// / RecordSpec
269288 /// / IDENTIFIER // typedef name
270289 /// / TypeQual
271290 fn typeSpec(parser: *Parser, type_spec: *Node.TypeSpec) !bool {
......@@ -473,22 +492,48 @@ const Parser = struct {
473492 } else if (parser.eatToken(.Keyword_enum)) |tok| {
474493 if (type_spec.spec != .None)
475494 break :blk;
476 @panic("TODO enum type");
477 // return true;
495 type_spec.Enum = try parser.enumSpec(tok);
496 return true;
478497 } else if (parser.eatToken(.Keyword_union) orelse parser.eatToken(.Keyword_struct)) |tok| {
479498 if (type_spec.spec != .None)
480499 break :blk;
481 @panic("TODO record type");
482 // return true;
500 type_spec.Record = try parser.recordSpec();
501 return true;
483502 } else if (parser.eatToken(.Identifier)) |tok| {
484 if (!parser.getTypeDef(tok)) {
503 const ty = parser.getSymbol(tok) orelse {
485504 parser.putBackToken(tok);
486505 return false;
487 }
488 type_spec.spec = .{
489 .Typedef = tok,
490506 };
491 return true;
507 switch (ty) {
508 .Enum => |e| {
509 return parser.err(.{
510 .MustUseKwToRefer = .{ .kw = e.identifier, .sym = tok },
511 });
512 },
513 .Record => |r| {
514 return parser.err(.{
515 .MustUseKwToRefer = .{
516 .kw = switch (r.kind) {
517 .Struct, .Union => |kw| kw,
518 },
519 .sym = tok,
520 },
521 });
522 },
523 .Typedef => {
524 type_spec.spec = .{
525 .Typedef = .{
526 .sym = tok,
527 .sym_type = ty,
528 },
529 };
530 return true;
531 },
532 else => {
533 parser.putBackToken(tok);
534 return false;
535 },
536 }
492537 }
493538 }
494539 return parser.err(.{
......@@ -567,13 +612,13 @@ const Parser = struct {
567612 return false;
568613 }
569614
570 /// EnumSpecifier <- Keyword_enum IDENTIFIER? (LBRACE EnumField RBRACE)?
615 /// EnumSpec <- Keyword_enum IDENTIFIER? (LBRACE EnumField RBRACE)?
571616 fn enumSpecifier(parser: *Parser) !*Node {}
572617
573618 /// EnumField <- IDENTIFIER (EQUAL ConstExpr)? (COMMA EnumField) COMMA?
574619 fn enumField(parser: *Parser) !*Node {}
575620
576 /// RecordSpecifier <- (Keyword_struct / Keyword_union) IDENTIFIER? (LBRACE RecordField+ RBRACE)?
621 /// RecordSpec <- (Keyword_struct / Keyword_union) IDENTIFIER? (LBRACE RecordField+ RBRACE)?
577622 fn recordSpecifier(parser: *Parser) !*Node {}
578623
579624 /// RecordField
......@@ -581,8 +626,7 @@ const Parser = struct {
581626 /// \ StaticAssert
582627 fn recordField(parser: *Parser) !*Node {}
583628
584 /// TypeName
585 /// <- TypeSpec* AbstractDeclarator?
629 /// TypeName <- TypeSpec* AbstractDeclarator?
586630 fn typeName(parser: *Parser) !*Node {
587631
588632 /// RecordDeclarator <- Declarator? (COLON ConstExpr)?
lib/std/c/tokenizer.zig+1-4
......@@ -327,6 +327,7 @@ pub const Token = struct {
327327 };
328328
329329 // TODO perfect hash at comptime
330 // TODO do this in the preprocessor
330331 pub fn getKeyword(bytes: []const u8, pp_directive: bool) ?Id {
331332 var hash = std.hash_map.hashString(bytes);
332333 for (keywords) |kw| {
......@@ -347,10 +348,6 @@ pub const Token = struct {
347348 return null;
348349 }
349350
350 pub fn slice(tok: Token) []const u8 {
351 return tok.source.buffer[tok.start..tok.end];
352 }
353
354351 pub const NumSuffix = enum {
355352 None,
356353 F,