authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-07 22:43:44+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-07 22:43:44+02:00
log8b713ce88959a953c6d41b5d1372e9b3e666512f
treeedf8a8e3fecf0244256231ddba9e7bb62aa80fbc
parentdbc045706809e7135fdb55ebd7c4a7383f40bf0f
signaturelock-open Commit is signed but in an unrecognized format.

std-c parser add options

3 files changed, 79 insertions(+), 19 deletions(-)

lib/std/c/ast.zig+24-3
...@@ -24,9 +24,14 @@ pub const Tree = struct {...@@ -24,9 +24,14 @@ pub const Tree = struct {
24 // self is destroyed24 // self is destroyed
25 }25 }
2626
27 pub fn slice(tree: *Tree, token: TokenIndex) []const u8 {27 pub fn tokenSlice(tree: *Tree, token: TokenIndex) []const u8 {
28 const tok = tree.tokens.at(token);28 return tree.tokens.at(token).slice();
29 return tok.source.buffer[tok.start..tok.end];29 }
30
31 pub fn tokenEql(tree: *Tree, a: TokenIndex, b: TokenIndex) bool {
32 const atok = tree.tokens.at(a);
33 const btok = tree.tokens.at(b);
34 return atok.eql(btok.*);
30 }35 }
31};36};
3237
...@@ -205,6 +210,10 @@ pub const Type = struct {...@@ -205,6 +210,10 @@ pub const Type = struct {
205 Typedef: *Type,210 Typedef: *Type,
206 Record: *Node.RecordType,211 Record: *Node.RecordType,
207 Enum: *Node.EnumType,212 Enum: *Node.EnumType,
213
214 /// Special case for macro parameters that can be any type.
215 /// Only present if `retain_macros == true`.
216 Macro,
208 },217 },
209};218};
210219
...@@ -586,4 +595,16 @@ pub const Node = struct {...@@ -586,4 +595,16 @@ pub const Node = struct {
586 expr: *Expr,595 expr: *Expr,
587 pub const InitializerList = std.SegmentedList(*Initializer, 4);596 pub const InitializerList = std.SegmentedList(*Initializer, 4);
588 };597 };
598
599 pub const Macro = struct {
600 base: Node = Node{ .id = Macro },
601 kind: union(enum) {
602 Undef: []const u8,
603 Fn: struct {
604 params: []const []const u8,
605 expr: *Expr,
606 },
607 Expr: *Expr,
608 },
609 };
589};610};
lib/std/c/parse.zig+45-16
...@@ -1,4 +1,5 @@...@@ -1,4 +1,5 @@
1const std = @import("std");1const std = @import("std");
2const mem = std.mem;
2const assert = std.debug.assert;3const assert = std.debug.assert;
3const Allocator = std.mem.Allocator;4const Allocator = std.mem.Allocator;
4const ast = std.c.ast;5const ast = std.c.ast;
...@@ -11,9 +12,26 @@ const TokenIterator = ast.Tree.TokenList.Iterator;...@@ -11,9 +12,26 @@ const TokenIterator = ast.Tree.TokenList.Iterator;
1112
12pub const Error = error{ParseError} || Allocator.Error;13pub const Error = error{ParseError} || Allocator.Error;
1314
15pub const Options = struct {
16 /// Keep simple macros unexpanded and add the definitions to the ast
17 retain_macros: bool = false,
18
19 /// Warning or error
20 warn_as_err: union(enum) {
21 /// All warnings are warnings
22 None,
23
24 /// Some warnings are errors
25 Some: []@TagType(ast.Error),
26
27 /// All warnings are errors
28 All,
29 } = .All,
30};
31
14/// Result should be freed with tree.deinit() when there are32/// Result should be freed with tree.deinit() when there are
15/// no more references to any of the tokens or nodes.33/// no more references to any of the tokens or nodes.
16pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {34pub fn parse(allocator: *Allocator, source: []const u8, options: Options) !*Tree {
17 const tree = blk: {35 const tree = blk: {
18 // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists36 // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists
19 // from being initialized with a pointer to this `arena`, which is created on37 // from being initialized with a pointer to this `arena`, which is created on
...@@ -62,6 +80,7 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {...@@ -62,6 +80,7 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
62 .arena = arena,80 .arena = arena,
63 .it = &it,81 .it = &it,
64 .tree = tree,82 .tree = tree,
83 .options = options,
65 };84 };
66 defer parser.symbols.deinit();85 defer parser.symbols.deinit();
6786
...@@ -76,7 +95,7 @@ const Parser = struct {...@@ -76,7 +95,7 @@ const Parser = struct {
7695
77 /// only used for scopes96 /// only used for scopes
78 symbols: SymbolList,97 symbols: SymbolList,
79 warnings: bool = true,98 options: Options,
8099
81 const SymbolList = std.ArrayList(Symbol);100 const SymbolList = std.ArrayList(Symbol);
82101
...@@ -94,11 +113,11 @@ const Parser = struct {...@@ -94,11 +113,11 @@ const Parser = struct {
94 }113 }
95114
96 fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type {115 fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type {
97 const name = parser.tree.slice(tok);116 const name = parser.tree.tokenSlice(tok);
98 const syms = parser.symbols.toSliceConst();117 const syms = parser.symbols.toSliceConst();
99 var i = syms.len;118 var i = syms.len;
100 while (i > 0) : (i -= 1) {119 while (i > 0) : (i -= 1) {
101 if (std.mem.eql(u8, name, syms[i].name)) {120 if (mem.eql(u8, name, syms[i].name)) {
102 return syms[i].ty;121 return syms[i].ty;
103 }122 }
104 }123 }
...@@ -249,7 +268,7 @@ const Parser = struct {...@@ -249,7 +268,7 @@ const Parser = struct {
249 try node.initializers.push((try parser.initializer(dr)) orelse return parser.err(.{268 try node.initializers.push((try parser.initializer(dr)) orelse return parser.err(.{
250 .ExpectedInitializer = .{ .token = parser.it.index },269 .ExpectedInitializer = .{ .token = parser.it.index },
251 }));270 }));
252 } else 271 } else
253 try node.initializers.push(&dr.base);272 try node.initializers.push(&dr.base);
254 if (parser.eatToken(.Comma) != null) break;273 if (parser.eatToken(.Comma) != null) break;
255 dr = @fieldParentPtr(Node.Declarator, "base", (try parser.declarator(.Must)) orelse return parser.err(.{274 dr = @fieldParentPtr(Node.Declarator, "base", (try parser.declarator(.Must)) orelse return parser.err(.{
...@@ -558,12 +577,18 @@ const Parser = struct {...@@ -558,12 +577,18 @@ const Parser = struct {
558 return false;577 return false;
559 };578 };
560 switch (ty.id) {579 switch (ty.id) {
561 .Enum => |e| {580 .Enum => |e| blk: {
581 if (e.name) |some|
582 if (!parser.tree.tokenEql(some, tok))
583 break :blk;
562 return parser.err(.{584 return parser.err(.{
563 .MustUseKwToRefer = .{ .kw = e.tok, .name = tok },585 .MustUseKwToRefer = .{ .kw = e.tok, .name = tok },
564 });586 });
565 },587 },
566 .Record => |r| {588 .Record => |r| blk: {
589 if (r.name) |some|
590 if (!parser.tree.tokenEql(some, tok))
591 break :blk;
567 return parser.err(.{592 return parser.err(.{
568 .MustUseKwToRefer = .{593 .MustUseKwToRefer = .{
569 .kw = r.tok,594 .kw = r.tok,
...@@ -580,11 +605,10 @@ const Parser = struct {...@@ -580,11 +605,10 @@ const Parser = struct {
580 };605 };
581 return true;606 return true;
582 },607 },
583 else => {608 else => {},
584 parser.putBackToken(tok);
585 return false;
586 },
587 }609 }
610 parser.putBackToken(tok);
611 return false;
588 }612 }
589 }613 }
590 return parser.err(.{614 return parser.err(.{
...@@ -680,7 +704,7 @@ const Parser = struct {...@@ -680,7 +704,7 @@ const Parser = struct {
680 };704 };
681 if (name) |some|705 if (name) |some|
682 try parser.symbols.append(.{706 try parser.symbols.append(.{
683 .name = parser.tree.slice(some),707 .name = parser.tree.tokenSlice(some),
684 .ty = ty,708 .ty = ty,
685 });709 });
686 if (parser.eatToken(.LBrace)) |lbrace| {710 if (parser.eatToken(.LBrace)) |lbrace| {
...@@ -718,7 +742,7 @@ const Parser = struct {...@@ -718,7 +742,7 @@ const Parser = struct {
718 fn recordSpec(parser: *Parser, tok: TokenIndex) !*Node.RecordType {742 fn recordSpec(parser: *Parser, tok: TokenIndex) !*Node.RecordType {
719 const node = try parser.arena.create(Node.RecordType);743 const node = try parser.arena.create(Node.RecordType);
720 const name = parser.eatToken(.Identifier);744 const name = parser.eatToken(.Identifier);
721 const is_struct = parser.tree.slice(tok)[0] == 's';745 const is_struct = parser.tree.tokenSlice(tok)[0] == 's';
722 node.* = .{746 node.* = .{
723 .tok = tok,747 .tok = tok,
724 .kind = if (is_struct) .Struct else .Union,748 .kind = if (is_struct) .Struct else .Union,
...@@ -733,7 +757,7 @@ const Parser = struct {...@@ -733,7 +757,7 @@ const Parser = struct {
733 };757 };
734 if (name) |some|758 if (name) |some|
735 try parser.symbols.append(.{759 try parser.symbols.append(.{
736 .name = parser.tree.slice(some),760 .name = parser.tree.tokenSlice(some),
737 .ty = ty,761 .ty = ty,
738 });762 });
739 if (parser.eatToken(.LBrace)) |lbrace| {763 if (parser.eatToken(.LBrace)) |lbrace| {
...@@ -1195,11 +1219,16 @@ const Parser = struct {...@@ -1195,11 +1219,16 @@ const Parser = struct {
1195 }1219 }
11961220
1197 fn warn(parser: *Parser, msg: ast.Error) Error!void {1221 fn warn(parser: *Parser, msg: ast.Error) Error!void {
1222 const is_warning = switch (parser.options.warn_as_err) {
1223 .None => true,
1224 .Some => |list| for (list) |item| (if (item == msg) break false) else true,
1225 .All => false,
1226 };
1198 try parser.tree.msgs.push(.{1227 try parser.tree.msgs.push(.{
1199 .kind = if (parser.warnings) .Warning else .Error,1228 .kind = if (is_warning) .Warning else .Error,
1200 .inner = msg,1229 .inner = msg,
1201 });1230 });
1202 if (!parser.warnings) return error.ParseError;1231 if (!is_warning) return error.ParseError;
1203 }1232 }
12041233
1205 fn note(parser: *Parser, msg: ast.Error) Error!void {1234 fn note(parser: *Parser, msg: ast.Error) Error!void {
lib/std/c/tokenizer.zig+10
...@@ -251,6 +251,16 @@ pub const Token = struct {...@@ -251,6 +251,16 @@ pub const Token = struct {
251 }251 }
252 };252 };
253253
254 pub fn eql(a: Token, b: Token) bool {
255 // do we really need this cast here
256 if (@as(@TagType(Id), a.id) != b.id) return false;
257 return mem.eql(u8, a.slice(), b.slice());
258 }
259
260 pub fn slice(tok: Token) []const u8 {
261 return tok.source.buffer[tok.start..tok.end];
262 }
263
254 pub const Keyword = struct {264 pub const Keyword = struct {
255 bytes: []const u8,265 bytes: []const u8,
256 id: Id,266 id: Id,