authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-14 23:00:53-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-14 23:00:53-05:00
logca597e2bfb3c39aecdf3dea2718e84deb749ed07
tree569a7e0b30ddcb5f4be1e14cb01dcfcf1e1951e6
parent9fa35adbd4772f55e3e43dd7cc69823415661153

std.zig.parser understands try. zig fmt respects a double line break.

6 files changed, 296 insertions(+), 40 deletions(-)

src/tokenizer.cpp-2
...@@ -125,7 +125,6 @@ static const struct ZigKeyword zig_keywords[] = {...@@ -125,7 +125,6 @@ static const struct ZigKeyword zig_keywords[] = {
125 {"false", TokenIdKeywordFalse},125 {"false", TokenIdKeywordFalse},
126 {"fn", TokenIdKeywordFn},126 {"fn", TokenIdKeywordFn},
127 {"for", TokenIdKeywordFor},127 {"for", TokenIdKeywordFor},
128 {"goto", TokenIdKeywordGoto},
129 {"if", TokenIdKeywordIf},128 {"if", TokenIdKeywordIf},
130 {"inline", TokenIdKeywordInline},129 {"inline", TokenIdKeywordInline},
131 {"nakedcc", TokenIdKeywordNakedCC},130 {"nakedcc", TokenIdKeywordNakedCC},
...@@ -1542,7 +1541,6 @@ const char * token_name(TokenId id) {...@@ -1542,7 +1541,6 @@ const char * token_name(TokenId id) {
1542 case TokenIdKeywordFalse: return "false";1541 case TokenIdKeywordFalse: return "false";
1543 case TokenIdKeywordFn: return "fn";1542 case TokenIdKeywordFn: return "fn";
1544 case TokenIdKeywordFor: return "for";1543 case TokenIdKeywordFor: return "for";
1545 case TokenIdKeywordGoto: return "goto";
1546 case TokenIdKeywordIf: return "if";1544 case TokenIdKeywordIf: return "if";
1547 case TokenIdKeywordInline: return "inline";1545 case TokenIdKeywordInline: return "inline";
1548 case TokenIdKeywordNakedCC: return "nakedcc";1546 case TokenIdKeywordNakedCC: return "nakedcc";
src/tokenizer.hpp-1
...@@ -66,7 +66,6 @@ enum TokenId {...@@ -66,7 +66,6 @@ enum TokenId {
66 TokenIdKeywordFalse,66 TokenIdKeywordFalse,
67 TokenIdKeywordFn,67 TokenIdKeywordFn,
68 TokenIdKeywordFor,68 TokenIdKeywordFor,
69 TokenIdKeywordGoto,
70 TokenIdKeywordIf,69 TokenIdKeywordIf,
71 TokenIdKeywordInline,70 TokenIdKeywordInline,
72 TokenIdKeywordNakedCC,71 TokenIdKeywordNakedCC,
std/debug/index.zig+2-2
...@@ -47,7 +47,7 @@ pub fn getSelfDebugInfo() !&ElfStackTrace {...@@ -47,7 +47,7 @@ pub fn getSelfDebugInfo() !&ElfStackTrace {
47pub fn dumpCurrentStackTrace() void {47pub fn dumpCurrentStackTrace() void {
48 const stderr = getStderrStream() catch return;48 const stderr = getStderrStream() catch return;
49 const debug_info = getSelfDebugInfo() catch |err| {49 const debug_info = getSelfDebugInfo() catch |err| {
50 stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return;50 stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return;
51 return;51 return;
52 };52 };
53 defer debug_info.close();53 defer debug_info.close();
...@@ -61,7 +61,7 @@ pub fn dumpCurrentStackTrace() void {...@@ -61,7 +61,7 @@ pub fn dumpCurrentStackTrace() void {
61pub fn dumpStackTrace(stack_trace: &const builtin.StackTrace) void {61pub fn dumpStackTrace(stack_trace: &const builtin.StackTrace) void {
62 const stderr = getStderrStream() catch return;62 const stderr = getStderrStream() catch return;
63 const debug_info = getSelfDebugInfo() catch |err| {63 const debug_info = getSelfDebugInfo() catch |err| {
64 stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return;64 stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return;
65 return;65 return;
66 };66 };
67 defer debug_info.close();67 defer debug_info.close();
std/zig/ast.zig+160-4
...@@ -38,11 +38,46 @@ pub const Node = struct {...@@ -38,11 +38,46 @@ pub const Node = struct {
38 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),38 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
39 };39 };
40 }40 }
41
42 pub fn firstToken(base: &Node) Token {
43 return switch (base.id) {
44 Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(),
45 Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(),
46 Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(),
47 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(),
48 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(),
49 Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(),
50 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(),
51 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(),
52 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
53 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
54 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
55 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
56 };
57 }
58
59 pub fn lastToken(base: &Node) Token {
60 return switch (base.id) {
61 Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(),
62 Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(),
63 Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(),
64 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(),
65 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(),
66 Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(),
67 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(),
68 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(),
69 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
70 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
71 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
72 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
73 };
74 }
41};75};
4276
43pub const NodeRoot = struct {77pub const NodeRoot = struct {
44 base: Node,78 base: Node,
45 decls: ArrayList(&Node),79 decls: ArrayList(&Node),
80 eof_token: Token,
4681
47 pub fn iterate(self: &NodeRoot, index: usize) ?&Node {82 pub fn iterate(self: &NodeRoot, index: usize) ?&Node {
48 if (index < self.decls.len) {83 if (index < self.decls.len) {
...@@ -50,6 +85,14 @@ pub const NodeRoot = struct {...@@ -50,6 +85,14 @@ pub const NodeRoot = struct {
50 }85 }
51 return null;86 return null;
52 }87 }
88
89 pub fn firstToken(self: &NodeRoot) Token {
90 return if (self.decls.len == 0) self.eof_token else self.decls.at(0).firstToken();
91 }
92
93 pub fn lastToken(self: &NodeRoot) Token {
94 return if (self.decls.len == 0) self.eof_token else self.decls.at(self.decls.len - 1).lastToken();
95 }
53};96};
5497
55pub const NodeVarDecl = struct {98pub const NodeVarDecl = struct {
...@@ -64,6 +107,7 @@ pub const NodeVarDecl = struct {...@@ -64,6 +107,7 @@ pub const NodeVarDecl = struct {
64 type_node: ?&Node,107 type_node: ?&Node,
65 align_node: ?&Node,108 align_node: ?&Node,
66 init_node: ?&Node,109 init_node: ?&Node,
110 semicolon_token: Token,
67111
68 pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node {112 pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node {
69 var i = index;113 var i = index;
...@@ -85,6 +129,18 @@ pub const NodeVarDecl = struct {...@@ -85,6 +129,18 @@ pub const NodeVarDecl = struct {
85129
86 return null;130 return null;
87 }131 }
132
133 pub fn firstToken(self: &NodeVarDecl) Token {
134 if (self.visib_token) |visib_token| return visib_token;
135 if (self.comptime_token) |comptime_token| return comptime_token;
136 if (self.extern_token) |extern_token| return extern_token;
137 assert(self.lib_name == null);
138 return self.mut_token;
139 }
140
141 pub fn lastToken(self: &NodeVarDecl) Token {
142 return self.semicolon_token;
143 }
88};144};
89145
90pub const NodeIdentifier = struct {146pub const NodeIdentifier = struct {
...@@ -94,6 +150,14 @@ pub const NodeIdentifier = struct {...@@ -94,6 +150,14 @@ pub const NodeIdentifier = struct {
94 pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node {150 pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node {
95 return null;151 return null;
96 }152 }
153
154 pub fn firstToken(self: &NodeIdentifier) Token {
155 return self.name_token;
156 }
157
158 pub fn lastToken(self: &NodeIdentifier) Token {
159 return self.name_token;
160 }
97};161};
98162
99pub const NodeFnProto = struct {163pub const NodeFnProto = struct {
...@@ -113,7 +177,7 @@ pub const NodeFnProto = struct {...@@ -113,7 +177,7 @@ pub const NodeFnProto = struct {
113177
114 pub const ReturnType = union(enum) {178 pub const ReturnType = union(enum) {
115 Explicit: &Node,179 Explicit: &Node,
116 Infer,180 Infer: Token,
117 InferErrorSet: &Node,181 InferErrorSet: &Node,
118 };182 };
119183
...@@ -153,6 +217,25 @@ pub const NodeFnProto = struct {...@@ -153,6 +217,25 @@ pub const NodeFnProto = struct {
153217
154 return null;218 return null;
155 }219 }
220
221 pub fn firstToken(self: &NodeFnProto) Token {
222 if (self.visib_token) |visib_token| return visib_token;
223 if (self.extern_token) |extern_token| return extern_token;
224 assert(self.lib_name == null);
225 if (self.inline_token) |inline_token| return inline_token;
226 if (self.cc_token) |cc_token| return cc_token;
227 return self.fn_token;
228 }
229
230 pub fn lastToken(self: &NodeFnProto) Token {
231 if (self.body_node) |body_node| return body_node.lastToken();
232 switch (self.return_type) {
233 // TODO allow this and next prong to share bodies since the types are the same
234 ReturnType.Explicit => |node| return node.lastToken(),
235 ReturnType.InferErrorSet => |node| return node.lastToken(),
236 ReturnType.Infer => |token| return token,
237 }
238 }
156};239};
157240
158pub const NodeParamDecl = struct {241pub const NodeParamDecl = struct {
...@@ -171,6 +254,18 @@ pub const NodeParamDecl = struct {...@@ -171,6 +254,18 @@ pub const NodeParamDecl = struct {
171254
172 return null;255 return null;
173 }256 }
257
258 pub fn firstToken(self: &NodeParamDecl) Token {
259 if (self.comptime_token) |comptime_token| return comptime_token;
260 if (self.noalias_token) |noalias_token| return noalias_token;
261 if (self.name_token) |name_token| return name_token;
262 return self.type_node.firstToken();
263 }
264
265 pub fn lastToken(self: &NodeParamDecl) Token {
266 if (self.var_args_token) |var_args_token| return var_args_token;
267 return self.type_node.lastToken();
268 }
174};269};
175270
176pub const NodeBlock = struct {271pub const NodeBlock = struct {
...@@ -187,6 +282,14 @@ pub const NodeBlock = struct {...@@ -187,6 +282,14 @@ pub const NodeBlock = struct {
187282
188 return null;283 return null;
189 }284 }
285
286 pub fn firstToken(self: &NodeBlock) Token {
287 return self.begin_token;
288 }
289
290 pub fn lastToken(self: &NodeBlock) Token {
291 return self.end_token;
292 }
190};293};
191294
192pub const NodeInfixOp = struct {295pub const NodeInfixOp = struct {
...@@ -199,6 +302,7 @@ pub const NodeInfixOp = struct {...@@ -199,6 +302,7 @@ pub const NodeInfixOp = struct {
199 const InfixOp = enum {302 const InfixOp = enum {
200 EqualEqual,303 EqualEqual,
201 BangEqual,304 BangEqual,
305 Period,
202 };306 };
203307
204 pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {308 pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
...@@ -208,8 +312,9 @@ pub const NodeInfixOp = struct {...@@ -208,8 +312,9 @@ pub const NodeInfixOp = struct {
208 i -= 1;312 i -= 1;
209313
210 switch (self.op) {314 switch (self.op) {
211 InfixOp.EqualEqual => {},315 InfixOp.EqualEqual,
212 InfixOp.BangEqual => {},316 InfixOp.BangEqual,
317 InfixOp.Period => {},
213 }318 }
214319
215 if (i < 1) return self.rhs;320 if (i < 1) return self.rhs;
...@@ -217,6 +322,14 @@ pub const NodeInfixOp = struct {...@@ -217,6 +322,14 @@ pub const NodeInfixOp = struct {
217322
218 return null;323 return null;
219 }324 }
325
326 pub fn firstToken(self: &NodeInfixOp) Token {
327 return self.lhs.firstToken();
328 }
329
330 pub fn lastToken(self: &NodeInfixOp) Token {
331 return self.rhs.lastToken();
332 }
220};333};
221334
222pub const NodePrefixOp = struct {335pub const NodePrefixOp = struct {
...@@ -227,6 +340,7 @@ pub const NodePrefixOp = struct {...@@ -227,6 +340,7 @@ pub const NodePrefixOp = struct {
227340
228 const PrefixOp = union(enum) {341 const PrefixOp = union(enum) {
229 Return,342 Return,
343 Try,
230 AddrOf: AddrOfInfo,344 AddrOf: AddrOfInfo,
231 };345 };
232 const AddrOfInfo = struct {346 const AddrOfInfo = struct {
...@@ -241,7 +355,8 @@ pub const NodePrefixOp = struct {...@@ -241,7 +355,8 @@ pub const NodePrefixOp = struct {
241 var i = index;355 var i = index;
242356
243 switch (self.op) {357 switch (self.op) {
244 PrefixOp.Return => {},358 PrefixOp.Return,
359 PrefixOp.Try => {},
245 PrefixOp.AddrOf => |addr_of_info| {360 PrefixOp.AddrOf => |addr_of_info| {
246 if (addr_of_info.align_expr) |align_expr| {361 if (addr_of_info.align_expr) |align_expr| {
247 if (i < 1) return align_expr;362 if (i < 1) return align_expr;
...@@ -255,6 +370,14 @@ pub const NodePrefixOp = struct {...@@ -255,6 +370,14 @@ pub const NodePrefixOp = struct {
255370
256 return null;371 return null;
257 }372 }
373
374 pub fn firstToken(self: &NodePrefixOp) Token {
375 return self.op_token;
376 }
377
378 pub fn lastToken(self: &NodePrefixOp) Token {
379 return self.rhs.lastToken();
380 }
258};381};
259382
260pub const NodeIntegerLiteral = struct {383pub const NodeIntegerLiteral = struct {
...@@ -264,6 +387,14 @@ pub const NodeIntegerLiteral = struct {...@@ -264,6 +387,14 @@ pub const NodeIntegerLiteral = struct {
264 pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node {387 pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node {
265 return null;388 return null;
266 }389 }
390
391 pub fn firstToken(self: &NodeIntegerLiteral) Token {
392 return self.token;
393 }
394
395 pub fn lastToken(self: &NodeIntegerLiteral) Token {
396 return self.token;
397 }
267};398};
268399
269pub const NodeFloatLiteral = struct {400pub const NodeFloatLiteral = struct {
...@@ -273,12 +404,21 @@ pub const NodeFloatLiteral = struct {...@@ -273,12 +404,21 @@ pub const NodeFloatLiteral = struct {
273 pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node {404 pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node {
274 return null;405 return null;
275 }406 }
407
408 pub fn firstToken(self: &NodeFloatLiteral) Token {
409 return self.token;
410 }
411
412 pub fn lastToken(self: &NodeFloatLiteral) Token {
413 return self.token;
414 }
276};415};
277416
278pub const NodeBuiltinCall = struct {417pub const NodeBuiltinCall = struct {
279 base: Node,418 base: Node,
280 builtin_token: Token,419 builtin_token: Token,
281 params: ArrayList(&Node),420 params: ArrayList(&Node),
421 rparen_token: Token,
282422
283 pub fn iterate(self: &NodeBuiltinCall, index: usize) ?&Node {423 pub fn iterate(self: &NodeBuiltinCall, index: usize) ?&Node {
284 var i = index;424 var i = index;
...@@ -288,6 +428,14 @@ pub const NodeBuiltinCall = struct {...@@ -288,6 +428,14 @@ pub const NodeBuiltinCall = struct {
288428
289 return null;429 return null;
290 }430 }
431
432 pub fn firstToken(self: &NodeBuiltinCall) Token {
433 return self.builtin_token;
434 }
435
436 pub fn lastToken(self: &NodeBuiltinCall) Token {
437 return self.rparen_token;
438 }
291};439};
292440
293pub const NodeStringLiteral = struct {441pub const NodeStringLiteral = struct {
...@@ -297,4 +445,12 @@ pub const NodeStringLiteral = struct {...@@ -297,4 +445,12 @@ pub const NodeStringLiteral = struct {
297 pub fn iterate(self: &NodeStringLiteral, index: usize) ?&Node {445 pub fn iterate(self: &NodeStringLiteral, index: usize) ?&Node {
298 return null;446 return null;
299 }447 }
448
449 pub fn firstToken(self: &NodeStringLiteral) Token {
450 return self.token;
451 }
452
453 pub fn lastToken(self: &NodeStringLiteral) Token {
454 return self.token;
455 }
300};456};
std/zig/parser.zig+96-15
...@@ -69,6 +69,11 @@ pub const Parser = struct {...@@ -69,6 +69,11 @@ pub const Parser = struct {
69 }69 }
70 };70 };
7171
72 const ExpectTokenSave = struct {
73 id: Token.Id,
74 ptr: &Token,
75 };
76
72 const State = union(enum) {77 const State = union(enum) {
73 TopLevel,78 TopLevel,
74 TopLevelExtern: ?Token,79 TopLevelExtern: ?Token,
...@@ -85,6 +90,7 @@ pub const Parser = struct {...@@ -85,6 +90,7 @@ pub const Parser = struct {
85 VarDeclAlign: &ast.NodeVarDecl,90 VarDeclAlign: &ast.NodeVarDecl,
86 VarDeclEq: &ast.NodeVarDecl,91 VarDeclEq: &ast.NodeVarDecl,
87 ExpectToken: @TagType(Token.Id),92 ExpectToken: @TagType(Token.Id),
93 ExpectTokenSave: ExpectTokenSave,
88 FnProto: &ast.NodeFnProto,94 FnProto: &ast.NodeFnProto,
89 FnProtoAlign: &ast.NodeFnProto,95 FnProtoAlign: &ast.NodeFnProto,
90 FnProtoReturnType: &ast.NodeFnProto,96 FnProtoReturnType: &ast.NodeFnProto,
...@@ -136,7 +142,10 @@ pub const Parser = struct {...@@ -136,7 +142,10 @@ pub const Parser = struct {
136 stack.append(State { .TopLevelExtern = token }) catch unreachable;142 stack.append(State { .TopLevelExtern = token }) catch unreachable;
137 continue;143 continue;
138 },144 },
139 Token.Id.Eof => return Tree {.root_node = root_node, .arena_allocator = arena_allocator},145 Token.Id.Eof => {
146 root_node.eof_token = token;
147 return Tree {.root_node = root_node, .arena_allocator = arena_allocator};
148 },
140 else => {149 else => {
141 self.putBackToken(token);150 self.putBackToken(token);
142 stack.append(State { .TopLevelExtern = null }) catch unreachable;151 stack.append(State { .TopLevelExtern = null }) catch unreachable;
...@@ -231,13 +240,19 @@ pub const Parser = struct {...@@ -231,13 +240,19 @@ pub const Parser = struct {
231 const token = self.getNextToken();240 const token = self.getNextToken();
232 if (token.id == Token.Id.Equal) {241 if (token.id == Token.Id.Equal) {
233 var_decl.eq_token = token;242 var_decl.eq_token = token;
234 stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;243 stack.append(State {
244 .ExpectTokenSave = ExpectTokenSave {
245 .id = Token.Id.Semicolon,
246 .ptr = &var_decl.semicolon_token,
247 },
248 }) catch unreachable;
235 try stack.append(State {249 try stack.append(State {
236 .Expression = DestPtr {.NullableField = &var_decl.init_node},250 .Expression = DestPtr {.NullableField = &var_decl.init_node},
237 });251 });
238 continue;252 continue;
239 }253 }
240 if (token.id == Token.Id.Semicolon) {254 if (token.id == Token.Id.Semicolon) {
255 var_decl.semicolon_token = token;
241 continue;256 continue;
242 }257 }
243 return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));258 return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
...@@ -247,6 +262,11 @@ pub const Parser = struct {...@@ -247,6 +262,11 @@ pub const Parser = struct {
247 continue;262 continue;
248 },263 },
249264
265 State.ExpectTokenSave => |expect_token_save| {
266 *expect_token_save.ptr = try self.eatToken(expect_token_save.id);
267 continue;
268 },
269
250 State.Expression => |dest_ptr| {270 State.Expression => |dest_ptr| {
251 // save the dest_ptr for later271 // save the dest_ptr for later
252 stack.append(state) catch unreachable;272 stack.append(state) catch unreachable;
...@@ -264,6 +284,12 @@ pub const Parser = struct {...@@ -264,6 +284,12 @@ pub const Parser = struct {
264 try stack.append(State.ExpectOperand);284 try stack.append(State.ExpectOperand);
265 continue;285 continue;
266 },286 },
287 Token.Id.Keyword_try => {
288 try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token,
289 ast.NodePrefixOp.PrefixOp.Try) });
290 try stack.append(State.ExpectOperand);
291 continue;
292 },
267 Token.Id.Ampersand => {293 Token.Id.Ampersand => {
268 const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{294 const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{
269 .AddrOf = ast.NodePrefixOp.AddrOfInfo {295 .AddrOf = ast.NodePrefixOp.AddrOfInfo {
...@@ -306,13 +332,19 @@ pub const Parser = struct {...@@ -306,13 +332,19 @@ pub const Parser = struct {
306 .base = ast.Node {.id = ast.Node.Id.BuiltinCall},332 .base = ast.Node {.id = ast.Node.Id.BuiltinCall},
307 .builtin_token = token,333 .builtin_token = token,
308 .params = ArrayList(&ast.Node).init(arena),334 .params = ArrayList(&ast.Node).init(arena),
335 .rparen_token = undefined,
309 };336 };
310 try stack.append(State {337 try stack.append(State {
311 .Operand = &node.base338 .Operand = &node.base
312 });339 });
313 try stack.append(State.AfterOperand);340 try stack.append(State.AfterOperand);
314 try stack.append(State {.ExprListItemOrEnd = &node.params });341 try stack.append(State {.ExprListItemOrEnd = &node.params });
315 try stack.append(State {.ExpectToken = Token.Id.LParen });342 try stack.append(State {
343 .ExpectTokenSave = ExpectTokenSave {
344 .id = Token.Id.LParen,
345 .ptr = &node.rparen_token,
346 },
347 });
316 continue;348 continue;
317 },349 },
318 Token.Id.StringLiteral => {350 Token.Id.StringLiteral => {
...@@ -351,6 +383,13 @@ pub const Parser = struct {...@@ -351,6 +383,13 @@ pub const Parser = struct {
351 try stack.append(State.ExpectOperand);383 try stack.append(State.ExpectOperand);
352 continue;384 continue;
353 },385 },
386 Token.Id.Period => {
387 try stack.append(State {
388 .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period)
389 });
390 try stack.append(State.ExpectOperand);
391 continue;
392 },
354 else => {393 else => {
355 // no postfix/infix operator after this operand.394 // no postfix/infix operator after this operand.
356 self.putBackToken(token);395 self.putBackToken(token);
...@@ -476,7 +515,7 @@ pub const Parser = struct {...@@ -476,7 +515,7 @@ pub const Parser = struct {
476 const token = self.getNextToken();515 const token = self.getNextToken();
477 switch (token.id) {516 switch (token.id) {
478 Token.Id.Keyword_var => {517 Token.Id.Keyword_var => {
479 fn_proto.return_type = ast.NodeFnProto.ReturnType.Infer;518 fn_proto.return_type = ast.NodeFnProto.ReturnType { .Infer = token };
480 },519 },
481 Token.Id.Bang => {520 Token.Id.Bang => {
482 fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined };521 fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined };
...@@ -627,6 +666,8 @@ pub const Parser = struct {...@@ -627,6 +666,8 @@ pub const Parser = struct {
627 *node = ast.NodeRoot {666 *node = ast.NodeRoot {
628 .base = ast.Node {.id = ast.Node.Id.Root},667 .base = ast.Node {.id = ast.Node.Id.Root},
629 .decls = ArrayList(&ast.Node).init(arena),668 .decls = ArrayList(&ast.Node).init(arena),
669 // initialized when we get the eof token
670 .eof_token = undefined,
630 };671 };
631 return node;672 return node;
632 }673 }
...@@ -649,6 +690,7 @@ pub const Parser = struct {...@@ -649,6 +690,7 @@ pub const Parser = struct {
649 // initialized later690 // initialized later
650 .name_token = undefined,691 .name_token = undefined,
651 .eq_token = undefined,692 .eq_token = undefined,
693 .semicolon_token = undefined,
652 };694 };
653 return node;695 return node;
654 }696 }
...@@ -789,11 +831,11 @@ pub const Parser = struct {...@@ -789,11 +831,11 @@ pub const Parser = struct {
789831
790 fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) {832 fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) {
791 const loc = self.tokenizer.getTokenLocation(token);833 const loc = self.tokenizer.getTokenLocation(token);
792 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);834 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args);
793 warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);835 warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
794 {836 {
795 var i: usize = 0;837 var i: usize = 0;
796 while (i < loc.column) : (i += 1) {838 while (i < token.column) : (i += 1) {
797 warn(" ");839 warn(" ");
798 }840 }
799 }841 }
...@@ -885,11 +927,26 @@ pub const Parser = struct {...@@ -885,11 +927,26 @@ pub const Parser = struct {
885 defer self.deinitUtilityArrayList(stack);927 defer self.deinitUtilityArrayList(stack);
886928
887 {929 {
930 try stack.append(RenderState { .Text = "\n"});
931
888 var i = root_node.decls.len;932 var i = root_node.decls.len;
889 while (i != 0) {933 while (i != 0) {
890 i -= 1;934 i -= 1;
891 const decl = root_node.decls.items[i];935 const decl = root_node.decls.items[i];
892 try stack.append(RenderState {.TopLevelDecl = decl});936 try stack.append(RenderState {.TopLevelDecl = decl});
937 if (i != 0) {
938 try stack.append(RenderState {
939 .Text = blk: {
940 const prev_node = root_node.decls.at(i - 1);
941 const prev_line_index = prev_node.lastToken().line;
942 const this_line_index = decl.firstToken().line;
943 if (this_line_index - prev_line_index >= 2) {
944 break :blk "\n\n";
945 }
946 break :blk "\n";
947 },
948 });
949 }
893 }950 }
894 }951 }
895952
...@@ -919,7 +976,6 @@ pub const Parser = struct {...@@ -919,7 +976,6 @@ pub const Parser = struct {
919976
920 try stream.print("(");977 try stream.print("(");
921978
922 try stack.append(RenderState { .Text = "\n" });
923 if (fn_proto.body_node == null) {979 if (fn_proto.body_node == null) {
924 try stack.append(RenderState { .Text = ";" });980 try stack.append(RenderState { .Text = ";" });
925 }981 }
...@@ -937,7 +993,6 @@ pub const Parser = struct {...@@ -937,7 +993,6 @@ pub const Parser = struct {
937 },993 },
938 ast.Node.Id.VarDecl => {994 ast.Node.Id.VarDecl => {
939 const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);995 const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
940 try stack.append(RenderState { .Text = "\n"});
941 try stack.append(RenderState { .VarDecl = var_decl});996 try stack.append(RenderState { .VarDecl = var_decl});
942997
943 },998 },
...@@ -1019,7 +1074,19 @@ pub const Parser = struct {...@@ -1019,7 +1074,19 @@ pub const Parser = struct {
1019 try stack.append(RenderState { .Statement = statement_node});1074 try stack.append(RenderState { .Statement = statement_node});
1020 try stack.append(RenderState.PrintIndent);1075 try stack.append(RenderState.PrintIndent);
1021 try stack.append(RenderState { .Indent = indent + indent_delta});1076 try stack.append(RenderState { .Indent = indent + indent_delta});
1022 try stack.append(RenderState { .Text = "\n" });1077 try stack.append(RenderState {
1078 .Text = blk: {
1079 if (i != 0) {
1080 const prev_statement_node = block.statements.items[i - 1];
1081 const prev_line_index = prev_statement_node.lastToken().line;
1082 const this_line_index = statement_node.firstToken().line;
1083 if (this_line_index - prev_line_index >= 2) {
1084 break :blk "\n\n";
1085 }
1086 }
1087 break :blk "\n";
1088 },
1089 });
1023 }1090 }
1024 }1091 }
1025 },1092 },
...@@ -1033,7 +1100,9 @@ pub const Parser = struct {...@@ -1033,7 +1100,9 @@ pub const Parser = struct {
1033 ast.NodeInfixOp.InfixOp.BangEqual => {1100 ast.NodeInfixOp.InfixOp.BangEqual => {
1034 try stack.append(RenderState { .Text = " != "});1101 try stack.append(RenderState { .Text = " != "});
1035 },1102 },
1036 else => unreachable,1103 ast.NodeInfixOp.InfixOp.Period => {
1104 try stack.append(RenderState { .Text = "."});
1105 },
1037 }1106 }
1038 try stack.append(RenderState { .Expression = prefix_op_node.lhs });1107 try stack.append(RenderState { .Expression = prefix_op_node.lhs });
1039 },1108 },
...@@ -1044,6 +1113,9 @@ pub const Parser = struct {...@@ -1044,6 +1113,9 @@ pub const Parser = struct {
1044 ast.NodePrefixOp.PrefixOp.Return => {1113 ast.NodePrefixOp.PrefixOp.Return => {
1045 try stream.write("return ");1114 try stream.write("return ");
1046 },1115 },
1116 ast.NodePrefixOp.PrefixOp.Try => {
1117 try stream.write("try ");
1118 },
1047 ast.NodePrefixOp.PrefixOp.AddrOf => |addr_of_info| {1119 ast.NodePrefixOp.PrefixOp.AddrOf => |addr_of_info| {
1048 try stream.write("&");1120 try stream.write("&");
1049 if (addr_of_info.volatile_token != null) {1121 if (addr_of_info.volatile_token != null) {
...@@ -1058,7 +1130,6 @@ pub const Parser = struct {...@@ -1058,7 +1130,6 @@ pub const Parser = struct {
1058 try stack.append(RenderState { .Expression = align_expr});1130 try stack.append(RenderState { .Expression = align_expr});
1059 }1131 }
1060 },1132 },
1061 else => unreachable,
1062 }1133 }
1063 },1134 },
1064 ast.Node.Id.IntegerLiteral => {1135 ast.Node.Id.IntegerLiteral => {
...@@ -1153,10 +1224,7 @@ pub const Parser = struct {...@@ -1153,10 +1224,7 @@ pub const Parser = struct {
1153var fixed_buffer_mem: [100 * 1024]u8 = undefined;1224var fixed_buffer_mem: [100 * 1024]u8 = undefined;
11541225
1155fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {1226fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {
1156 var padded_source: [0x100]u8 = undefined;1227 var tokenizer = Tokenizer.init(source);
1157 std.mem.copy(u8, padded_source[0..source.len], source);
1158
1159 var tokenizer = Tokenizer.init(padded_source[0..source.len]);
1160 var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");1228 var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
1161 defer parser.deinit();1229 defer parser.deinit();
11621230
...@@ -1211,6 +1279,19 @@ fn testCanonical(source: []const u8) !void {...@@ -1211,6 +1279,19 @@ fn testCanonical(source: []const u8) !void {
1211}1279}
12121280
1213test "zig fmt" {1281test "zig fmt" {
1282 try testCanonical(
1283 \\const std = @import("std");
1284 \\
1285 \\pub fn main() !void {
1286 \\ var stdout_file = try std.io.getStdOut;
1287 \\ var stdout_file = try std.io.getStdOut;
1288 \\
1289 \\ var stdout_file = try std.io.getStdOut;
1290 \\ var stdout_file = try std.io.getStdOut;
1291 \\}
1292 \\
1293 );
1294
1214 try testCanonical(1295 try testCanonical(
1215 \\pub fn main() !void {}1296 \\pub fn main() !void {}
1216 \\pub fn main() var {}1297 \\pub fn main() var {}
std/zig/tokenizer.zig+38-16
...@@ -5,6 +5,8 @@ pub const Token = struct {...@@ -5,6 +5,8 @@ pub const Token = struct {
5 id: Id,5 id: Id,
6 start: usize,6 start: usize,
7 end: usize,7 end: usize,
8 line: usize,
9 column: usize,
810
9 const KeywordId = struct {11 const KeywordId = struct {
10 bytes: []const u8,12 bytes: []const u8,
...@@ -16,6 +18,7 @@ pub const Token = struct {...@@ -16,6 +18,7 @@ pub const Token = struct {
16 KeywordId{.bytes="and", .id = Id.Keyword_and},18 KeywordId{.bytes="and", .id = Id.Keyword_and},
17 KeywordId{.bytes="asm", .id = Id.Keyword_asm},19 KeywordId{.bytes="asm", .id = Id.Keyword_asm},
18 KeywordId{.bytes="break", .id = Id.Keyword_break},20 KeywordId{.bytes="break", .id = Id.Keyword_break},
21 KeywordId{.bytes="catch", .id = Id.Keyword_catch},
19 KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},22 KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
20 KeywordId{.bytes="const", .id = Id.Keyword_const},23 KeywordId{.bytes="const", .id = Id.Keyword_const},
21 KeywordId{.bytes="continue", .id = Id.Keyword_continue},24 KeywordId{.bytes="continue", .id = Id.Keyword_continue},
...@@ -28,7 +31,6 @@ pub const Token = struct {...@@ -28,7 +31,6 @@ pub const Token = struct {
28 KeywordId{.bytes="false", .id = Id.Keyword_false},31 KeywordId{.bytes="false", .id = Id.Keyword_false},
29 KeywordId{.bytes="fn", .id = Id.Keyword_fn},32 KeywordId{.bytes="fn", .id = Id.Keyword_fn},
30 KeywordId{.bytes="for", .id = Id.Keyword_for},33 KeywordId{.bytes="for", .id = Id.Keyword_for},
31 KeywordId{.bytes="goto", .id = Id.Keyword_goto},
32 KeywordId{.bytes="if", .id = Id.Keyword_if},34 KeywordId{.bytes="if", .id = Id.Keyword_if},
33 KeywordId{.bytes="inline", .id = Id.Keyword_inline},35 KeywordId{.bytes="inline", .id = Id.Keyword_inline},
34 KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},36 KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
...@@ -38,12 +40,14 @@ pub const Token = struct {...@@ -38,12 +40,14 @@ pub const Token = struct {
38 KeywordId{.bytes="packed", .id = Id.Keyword_packed},40 KeywordId{.bytes="packed", .id = Id.Keyword_packed},
39 KeywordId{.bytes="pub", .id = Id.Keyword_pub},41 KeywordId{.bytes="pub", .id = Id.Keyword_pub},
40 KeywordId{.bytes="return", .id = Id.Keyword_return},42 KeywordId{.bytes="return", .id = Id.Keyword_return},
43 KeywordId{.bytes="section", .id = Id.Keyword_section},
41 KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},44 KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
42 KeywordId{.bytes="struct", .id = Id.Keyword_struct},45 KeywordId{.bytes="struct", .id = Id.Keyword_struct},
43 KeywordId{.bytes="switch", .id = Id.Keyword_switch},46 KeywordId{.bytes="switch", .id = Id.Keyword_switch},
44 KeywordId{.bytes="test", .id = Id.Keyword_test},47 KeywordId{.bytes="test", .id = Id.Keyword_test},
45 KeywordId{.bytes="this", .id = Id.Keyword_this},48 KeywordId{.bytes="this", .id = Id.Keyword_this},
46 KeywordId{.bytes="true", .id = Id.Keyword_true},49 KeywordId{.bytes="true", .id = Id.Keyword_true},
50 KeywordId{.bytes="try", .id = Id.Keyword_try},
47 KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},51 KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
48 KeywordId{.bytes="union", .id = Id.Keyword_union},52 KeywordId{.bytes="union", .id = Id.Keyword_union},
49 KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},53 KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
...@@ -99,6 +103,7 @@ pub const Token = struct {...@@ -99,6 +103,7 @@ pub const Token = struct {
99 Keyword_and,103 Keyword_and,
100 Keyword_asm,104 Keyword_asm,
101 Keyword_break,105 Keyword_break,
106 Keyword_catch,
102 Keyword_comptime,107 Keyword_comptime,
103 Keyword_const,108 Keyword_const,
104 Keyword_continue,109 Keyword_continue,
...@@ -111,7 +116,6 @@ pub const Token = struct {...@@ -111,7 +116,6 @@ pub const Token = struct {
111 Keyword_false,116 Keyword_false,
112 Keyword_fn,117 Keyword_fn,
113 Keyword_for,118 Keyword_for,
114 Keyword_goto,
115 Keyword_if,119 Keyword_if,
116 Keyword_inline,120 Keyword_inline,
117 Keyword_nakedcc,121 Keyword_nakedcc,
...@@ -121,12 +125,14 @@ pub const Token = struct {...@@ -121,12 +125,14 @@ pub const Token = struct {
121 Keyword_packed,125 Keyword_packed,
122 Keyword_pub,126 Keyword_pub,
123 Keyword_return,127 Keyword_return,
128 Keyword_section,
124 Keyword_stdcallcc,129 Keyword_stdcallcc,
125 Keyword_struct,130 Keyword_struct,
126 Keyword_switch,131 Keyword_switch,
127 Keyword_test,132 Keyword_test,
128 Keyword_this,133 Keyword_this,
129 Keyword_true,134 Keyword_true,
135 Keyword_try,
130 Keyword_undefined,136 Keyword_undefined,
131 Keyword_union,137 Keyword_union,
132 Keyword_unreachable,138 Keyword_unreachable,
...@@ -140,21 +146,19 @@ pub const Token = struct {...@@ -140,21 +146,19 @@ pub const Token = struct {
140pub const Tokenizer = struct {146pub const Tokenizer = struct {
141 buffer: []const u8,147 buffer: []const u8,
142 index: usize,148 index: usize,
149 line: usize,
150 column: usize,
143 pending_invalid_token: ?Token,151 pending_invalid_token: ?Token,
144152
145 pub const Location = struct {153 pub const LineLocation = struct {
146 line: usize,
147 column: usize,
148 line_start: usize,154 line_start: usize,
149 line_end: usize,155 line_end: usize,
150 };156 };
151157
152 pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location {158 pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation {
153 var loc = Location {159 var loc = LineLocation {
154 .line = 0,
155 .column = 0,
156 .line_start = 0,160 .line_start = 0,
157 .line_end = 0,161 .line_end = self.buffer.len,
158 };162 };
159 for (self.buffer) |c, i| {163 for (self.buffer) |c, i| {
160 if (i == token.start) {164 if (i == token.start) {
...@@ -163,11 +167,7 @@ pub const Tokenizer = struct {...@@ -163,11 +167,7 @@ pub const Tokenizer = struct {
163 return loc;167 return loc;
164 }168 }
165 if (c == '\n') {169 if (c == '\n') {
166 loc.line += 1;
167 loc.column = 0;
168 loc.line_start = i + 1;170 loc.line_start = i + 1;
169 } else {
170 loc.column += 1;
171 }171 }
172 }172 }
173 return loc;173 return loc;
...@@ -182,6 +182,8 @@ pub const Tokenizer = struct {...@@ -182,6 +182,8 @@ pub const Tokenizer = struct {
182 return Tokenizer {182 return Tokenizer {
183 .buffer = buffer,183 .buffer = buffer,
184 .index = 0,184 .index = 0,
185 .line = 0,
186 .column = 0,
185 .pending_invalid_token = null,187 .pending_invalid_token = null,
186 };188 };
187 }189 }
...@@ -222,13 +224,21 @@ pub const Tokenizer = struct {...@@ -222,13 +224,21 @@ pub const Tokenizer = struct {
222 .id = Token.Id.Eof,224 .id = Token.Id.Eof,
223 .start = self.index,225 .start = self.index,
224 .end = undefined,226 .end = undefined,
227 .line = self.line,
228 .column = self.column,
225 };229 };
226 while (self.index < self.buffer.len) : (self.index += 1) {230 while (self.index < self.buffer.len) {
227 const c = self.buffer[self.index];231 const c = self.buffer[self.index];
228 switch (state) {232 switch (state) {
229 State.Start => switch (c) {233 State.Start => switch (c) {
230 ' ', '\n' => {234 ' ' => {
235 result.start = self.index + 1;
236 result.column += 1;
237 },
238 '\n' => {
231 result.start = self.index + 1;239 result.start = self.index + 1;
240 result.line += 1;
241 result.column = 0;
232 },242 },
233 'c' => {243 'c' => {
234 state = State.C;244 state = State.C;
...@@ -474,6 +484,8 @@ pub const Tokenizer = struct {...@@ -474,6 +484,8 @@ pub const Tokenizer = struct {
474 result = Token {484 result = Token {
475 .id = Token.Id.Eof,485 .id = Token.Id.Eof,
476 .start = self.index + 1,486 .start = self.index + 1,
487 .column = 0,
488 .line = self.line + 1,
477 .end = undefined,489 .end = undefined,
478 };490 };
479 },491 },
...@@ -543,6 +555,14 @@ pub const Tokenizer = struct {...@@ -543,6 +555,14 @@ pub const Tokenizer = struct {
543 else => break,555 else => break,
544 },556 },
545 }557 }
558
559 self.index += 1;
560 if (c == '\n') {
561 self.line += 1;
562 self.column = 0;
563 } else {
564 self.column += 1;
565 }
546 } else if (self.index == self.buffer.len) {566 } else if (self.index == self.buffer.len) {
547 switch (state) {567 switch (state) {
548 State.Start,568 State.Start,
...@@ -622,6 +642,8 @@ pub const Tokenizer = struct {...@@ -622,6 +642,8 @@ pub const Tokenizer = struct {
622 .id = Token.Id.Invalid,642 .id = Token.Id.Invalid,
623 .start = self.index,643 .start = self.index,
624 .end = self.index + invalid_length,644 .end = self.index + invalid_length,
645 .line = self.line,
646 .column = self.column,
625 };647 };
626 }648 }
627649