authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-22 12:34:12-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-22 12:34:12-04:00
log8df0841d6ab964e2aec750a8ceeda51912ae448b
treefa2a7f30a4eaa8f94f6d5c251ff109cd4bb0427c
parent295bca9b5f397ff98e5a0162fcb2a9f5e0a3e35c

stage2 parser: token ids in their own array

To prevent cache misses, token ids go in their own array, and the start/end offsets go in a different one. perf measurement before: 2,667,914 cache-misses:u 2,139,139,935 instructions:u 894,167,331 cycles:u perf measurement after: 1,757,723 cache-misses:u 2,069,932,298 instructions:u 858,105,570 cycles:u

6 files changed, 217 insertions(+), 240 deletions(-)

lib/std/zig/ast.zig+26-24
...@@ -10,7 +10,8 @@ pub const NodeIndex = usize;...@@ -10,7 +10,8 @@ pub const NodeIndex = usize;
10pub const Tree = struct {10pub const Tree = struct {
11 /// Reference to externally-owned data.11 /// Reference to externally-owned data.
12 source: []const u8,12 source: []const u8,
13 tokens: []const Token,13 token_ids: []const Token.Id,
14 token_locs: []const Token.Loc,
14 errors: []const Error,15 errors: []const Error,
15 /// undefined on parse error (when errors field is not empty)16 /// undefined on parse error (when errors field is not empty)
16 root_node: *Node.Root,17 root_node: *Node.Root,
...@@ -23,26 +24,27 @@ pub const Tree = struct {...@@ -23,26 +24,27 @@ pub const Tree = struct {
23 generated: bool = false,24 generated: bool = false,
2425
25 pub fn deinit(self: *Tree) void {26 pub fn deinit(self: *Tree) void {
26 self.gpa.free(self.tokens);27 self.gpa.free(self.token_ids);
28 self.gpa.free(self.token_locs);
27 self.gpa.free(self.errors);29 self.gpa.free(self.errors);
28 self.arena.promote(self.gpa).deinit();30 self.arena.promote(self.gpa).deinit();
29 }31 }
3032
31 pub fn renderError(self: *Tree, parse_error: *const Error, stream: var) !void {33 pub fn renderError(self: *Tree, parse_error: *const Error, stream: var) !void {
32 return parse_error.render(self.tokens, stream);34 return parse_error.render(self.token_ids, stream);
33 }35 }
3436
35 pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 {37 pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 {
36 return self.tokenSlicePtr(self.tokens[token_index]);38 return self.tokenSliceLoc(self.token_locs[token_index]);
37 }39 }
3840
39 pub fn tokenSlicePtr(self: *Tree, token: Token) []const u8 {41 pub fn tokenSliceLoc(self: *Tree, token: Token.Loc) []const u8 {
40 return self.source[token.start..token.end];42 return self.source[token.start..token.end];
41 }43 }
4244
43 pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 {45 pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 {
44 const first_token = self.tokens[node.firstToken()];46 const first_token = self.token_locs[node.firstToken()];
45 const last_token = self.tokens[node.lastToken()];47 const last_token = self.token_locs[node.lastToken()];
46 return self.source[first_token.start..last_token.end];48 return self.source[first_token.start..last_token.end];
47 }49 }
4850
...@@ -54,7 +56,7 @@ pub const Tree = struct {...@@ -54,7 +56,7 @@ pub const Tree = struct {
54 };56 };
5557
56 /// Return the Location of the token relative to the offset specified by `start_index`.58 /// Return the Location of the token relative to the offset specified by `start_index`.
57 pub fn tokenLocationPtr(self: *Tree, start_index: usize, token: Token) Location {59 pub fn tokenLocationLoc(self: *Tree, start_index: usize, token: Token.Loc) Location {
58 var loc = Location{60 var loc = Location{
59 .line = 0,61 .line = 0,
60 .column = 0,62 .column = 0,
...@@ -82,14 +84,14 @@ pub const Tree = struct {...@@ -82,14 +84,14 @@ pub const Tree = struct {
82 }84 }
8385
84 pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location {86 pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location {
85 return self.tokenLocationPtr(start_index, self.tokens[token_index]);87 return self.tokenLocationLoc(start_index, self.token_locs[token_index]);
86 }88 }
8789
88 pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool {90 pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool {
89 return self.tokensOnSameLinePtr(self.tokens[token1_index], self.tokens[token2_index]);91 return self.tokensOnSameLineLoc(self.token_locs[token1_index], self.token_locs[token2_index]);
90 }92 }
9193
92 pub fn tokensOnSameLinePtr(self: *Tree, token1: Token, token2: Token) bool {94 pub fn tokensOnSameLineLoc(self: *Tree, token1: Token.Loc, token2: Token.Loc) bool {
93 return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null;95 return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null;
94 }96 }
9597
...@@ -100,7 +102,7 @@ pub const Tree = struct {...@@ -100,7 +102,7 @@ pub const Tree = struct {
100 /// Skips over comments102 /// Skips over comments
101 pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex {103 pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex {
102 var index = token_index - 1;104 var index = token_index - 1;
103 while (self.tokens[index].id == Token.Id.LineComment) {105 while (self.token_ids[index] == Token.Id.LineComment) {
104 index -= 1;106 index -= 1;
105 }107 }
106 return index;108 return index;
...@@ -109,7 +111,7 @@ pub const Tree = struct {...@@ -109,7 +111,7 @@ pub const Tree = struct {
109 /// Skips over comments111 /// Skips over comments
110 pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex {112 pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex {
111 var index = token_index + 1;113 var index = token_index + 1;
112 while (self.tokens[index].id == Token.Id.LineComment) {114 while (self.token_ids[index] == Token.Id.LineComment) {
113 index += 1;115 index += 1;
114 }116 }
115 return index;117 return index;
...@@ -166,7 +168,7 @@ pub const Error = union(enum) {...@@ -166,7 +168,7 @@ pub const Error = union(enum) {
166 DeclBetweenFields: DeclBetweenFields,168 DeclBetweenFields: DeclBetweenFields,
167 InvalidAnd: InvalidAnd,169 InvalidAnd: InvalidAnd,
168170
169 pub fn render(self: *const Error, tokens: []const Token, stream: var) !void {171 pub fn render(self: *const Error, tokens: []const Token.Id, stream: var) !void {
170 switch (self.*) {172 switch (self.*) {
171 .InvalidToken => |*x| return x.render(tokens, stream),173 .InvalidToken => |*x| return x.render(tokens, stream),
172 .ExpectedContainerMembers => |*x| return x.render(tokens, stream),174 .ExpectedContainerMembers => |*x| return x.render(tokens, stream),
...@@ -321,7 +323,7 @@ pub const Error = union(enum) {...@@ -321,7 +323,7 @@ pub const Error = union(enum) {
321 pub const ExpectedCall = struct {323 pub const ExpectedCall = struct {
322 node: *Node,324 node: *Node,
323325
324 pub fn render(self: *const ExpectedCall, tokens: []const Token, stream: var) !void {326 pub fn render(self: *const ExpectedCall, tokens: []const Token.Id, stream: var) !void {
325 return stream.print("expected " ++ @tagName(Node.Id.Call) ++ ", found {}", .{327 return stream.print("expected " ++ @tagName(Node.Id.Call) ++ ", found {}", .{
326 @tagName(self.node.id),328 @tagName(self.node.id),
327 });329 });
...@@ -331,7 +333,7 @@ pub const Error = union(enum) {...@@ -331,7 +333,7 @@ pub const Error = union(enum) {
331 pub const ExpectedCallOrFnProto = struct {333 pub const ExpectedCallOrFnProto = struct {
332 node: *Node,334 node: *Node,
333335
334 pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token, stream: var) !void {336 pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token.Id, stream: var) !void {
335 return stream.print("expected " ++ @tagName(Node.Id.Call) ++ " or " ++337 return stream.print("expected " ++ @tagName(Node.Id.Call) ++ " or " ++
336 @tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)});338 @tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)});
337 }339 }
...@@ -341,14 +343,14 @@ pub const Error = union(enum) {...@@ -341,14 +343,14 @@ pub const Error = union(enum) {
341 token: TokenIndex,343 token: TokenIndex,
342 expected_id: Token.Id,344 expected_id: Token.Id,
343345
344 pub fn render(self: *const ExpectedToken, tokens: []const Token, stream: var) !void {346 pub fn render(self: *const ExpectedToken, tokens: []const Token.Id, stream: var) !void {
345 const found_token = tokens[self.token];347 const found_token = tokens[self.token];
346 switch (found_token.id) {348 switch (found_token) {
347 .Invalid => {349 .Invalid => {
348 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});350 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
349 },351 },
350 else => {352 else => {
351 const token_name = found_token.id.symbol();353 const token_name = found_token.symbol();
352 return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name });354 return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name });
353 },355 },
354 }356 }
...@@ -359,11 +361,11 @@ pub const Error = union(enum) {...@@ -359,11 +361,11 @@ pub const Error = union(enum) {
359 token: TokenIndex,361 token: TokenIndex,
360 end_id: Token.Id,362 end_id: Token.Id,
361363
362 pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token, stream: var) !void {364 pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token.Id, stream: var) !void {
363 const actual_token = tokens[self.token];365 const actual_token = tokens[self.token];
364 return stream.print("expected ',' or '{}', found '{}'", .{366 return stream.print("expected ',' or '{}', found '{}'", .{
365 self.end_id.symbol(),367 self.end_id.symbol(),
366 actual_token.id.symbol(),368 actual_token.symbol(),
367 });369 });
368 }370 }
369 };371 };
...@@ -374,9 +376,9 @@ pub const Error = union(enum) {...@@ -374,9 +376,9 @@ pub const Error = union(enum) {
374376
375 token: TokenIndex,377 token: TokenIndex,
376378
377 pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void {379 pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void {
378 const actual_token = tokens[self.token];380 const actual_token = tokens[self.token];
379 return stream.print(msg, .{actual_token.id.symbol()});381 return stream.print(msg, .{actual_token.symbol()});
380 }382 }
381 };383 };
382 }384 }
...@@ -387,7 +389,7 @@ pub const Error = union(enum) {...@@ -387,7 +389,7 @@ pub const Error = union(enum) {
387389
388 token: TokenIndex,390 token: TokenIndex,
389391
390 pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void {392 pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void {
391 return stream.writeAll(msg);393 return stream.writeAll(msg);
392 }394 }
393 };395 };
lib/std/zig/parse.zig+73-95
...@@ -16,28 +16,32 @@ pub const Error = error{ParseError} || Allocator.Error;...@@ -16,28 +16,32 @@ pub const Error = error{ParseError} || Allocator.Error;
16pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {16pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
17 // TODO optimization idea: ensureCapacity on the tokens list and17 // TODO optimization idea: ensureCapacity on the tokens list and
18 // then appendAssumeCapacity inside the loop.18 // then appendAssumeCapacity inside the loop.
19 var tokens = std.ArrayList(Token).init(gpa);19 var token_ids = std.ArrayList(Token.Id).init(gpa);
20 defer tokens.deinit();20 defer token_ids.deinit();
21 var token_locs = std.ArrayList(Token.Loc).init(gpa);
22 defer token_locs.deinit();
2123
22 var tokenizer = std.zig.Tokenizer.init(source);24 var tokenizer = std.zig.Tokenizer.init(source);
23 while (true) {25 while (true) {
24 const tree_token = try tokens.addOne();26 const token = tokenizer.next();
25 tree_token.* = tokenizer.next();27 try token_ids.append(token.id);
26 if (tree_token.id == .Eof) break;28 try token_locs.append(token.loc);
29 if (token.id == .Eof) break;
27 }30 }
2831
29 var parser: Parser = .{32 var parser: Parser = .{
30 .source = source,33 .source = source,
31 .arena = std.heap.ArenaAllocator.init(gpa),34 .arena = std.heap.ArenaAllocator.init(gpa),
32 .gpa = gpa,35 .gpa = gpa,
33 .tokens = tokens.items,36 .token_ids = token_ids.items,
37 .token_locs = token_locs.items,
34 .errors = .{},38 .errors = .{},
35 .tok_i = 0,39 .tok_i = 0,
36 };40 };
37 defer parser.errors.deinit(gpa);41 defer parser.errors.deinit(gpa);
38 errdefer parser.arena.deinit();42 errdefer parser.arena.deinit();
3943
40 while (tokens.items[parser.tok_i].id == .LineComment) parser.tok_i += 1;44 while (token_ids.items[parser.tok_i] == .LineComment) parser.tok_i += 1;
4145
42 const root_node = try parser.parseRoot();46 const root_node = try parser.parseRoot();
4347
...@@ -45,7 +49,8 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {...@@ -45,7 +49,8 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
45 tree.* = .{49 tree.* = .{
46 .gpa = gpa,50 .gpa = gpa,
47 .source = source,51 .source = source,
48 .tokens = tokens.toOwnedSlice(),52 .token_ids = token_ids.toOwnedSlice(),
53 .token_locs = token_locs.toOwnedSlice(),
49 .errors = parser.errors.toOwnedSlice(gpa),54 .errors = parser.errors.toOwnedSlice(gpa),
50 .root_node = root_node,55 .root_node = root_node,
51 .arena = parser.arena.state,56 .arena = parser.arena.state,
...@@ -58,9 +63,8 @@ const Parser = struct {...@@ -58,9 +63,8 @@ const Parser = struct {
58 arena: std.heap.ArenaAllocator,63 arena: std.heap.ArenaAllocator,
59 gpa: *Allocator,64 gpa: *Allocator,
60 source: []const u8,65 source: []const u8,
61 /// TODO: Optimization idea: have this be several arrays of the token fields rather66 token_ids: []const Token.Id,
62 /// than an array of structs.67 token_locs: []const Token.Loc,
63 tokens: []const Token,
64 tok_i: TokenIndex,68 tok_i: TokenIndex,
65 errors: std.ArrayListUnmanaged(AstError),69 errors: std.ArrayListUnmanaged(AstError),
6670
...@@ -80,19 +84,6 @@ const Parser = struct {...@@ -80,19 +84,6 @@ const Parser = struct {
80 return node;84 return node;
81 }85 }
8286
83 /// Helper function for appending elements to a singly linked list.
84 fn llpush(
85 p: *Parser,
86 comptime T: type,
87 it: *?*std.SinglyLinkedList(T).Node,
88 data: T,
89 ) !*?*std.SinglyLinkedList(T).Node {
90 const llnode = try p.arena.allocator.create(std.SinglyLinkedList(T).Node);
91 llnode.* = .{ .data = data };
92 it.* = llnode;
93 return &llnode.next;
94 }
95
96 /// ContainerMembers87 /// ContainerMembers
97 /// <- TestDecl ContainerMembers88 /// <- TestDecl ContainerMembers
98 /// / TopLevelComptime ContainerMembers89 /// / TopLevelComptime ContainerMembers
...@@ -228,7 +219,7 @@ const Parser = struct {...@@ -228,7 +219,7 @@ const Parser = struct {
228 // try to continue parsing219 // try to continue parsing
229 const index = p.tok_i;220 const index = p.tok_i;
230 p.findNextContainerMember();221 p.findNextContainerMember();
231 const next = p.tokens[p.tok_i].id;222 const next = p.token_ids[p.tok_i];
232 switch (next) {223 switch (next) {
233 .Eof => break,224 .Eof => break,
234 else => {225 else => {
...@@ -257,7 +248,7 @@ const Parser = struct {...@@ -257,7 +248,7 @@ const Parser = struct {
257 });248 });
258 }249 }
259250
260 const next = p.tokens[p.tok_i].id;251 const next = p.token_ids[p.tok_i];
261 switch (next) {252 switch (next) {
262 .Eof => break,253 .Eof => break,
263 .Keyword_comptime => {254 .Keyword_comptime => {
...@@ -291,7 +282,7 @@ const Parser = struct {...@@ -291,7 +282,7 @@ const Parser = struct {
291 var level: u32 = 0;282 var level: u32 = 0;
292 while (true) {283 while (true) {
293 const tok = p.nextToken();284 const tok = p.nextToken();
294 switch (tok.ptr.id) {285 switch (p.token_ids[tok]) {
295 // any of these can start a new top level declaration286 // any of these can start a new top level declaration
296 .Keyword_test,287 .Keyword_test,
297 .Keyword_comptime,288 .Keyword_comptime,
...@@ -308,7 +299,7 @@ const Parser = struct {...@@ -308,7 +299,7 @@ const Parser = struct {
308 .Identifier,299 .Identifier,
309 => {300 => {
310 if (level == 0) {301 if (level == 0) {
311 p.putBackToken(tok.index);302 p.putBackToken(tok);
312 return;303 return;
313 }304 }
314 },305 },
...@@ -325,13 +316,13 @@ const Parser = struct {...@@ -325,13 +316,13 @@ const Parser = struct {
325 .RBrace => {316 .RBrace => {
326 if (level == 0) {317 if (level == 0) {
327 // end of container, exit318 // end of container, exit
328 p.putBackToken(tok.index);319 p.putBackToken(tok);
329 return;320 return;
330 }321 }
331 level -= 1;322 level -= 1;
332 },323 },
333 .Eof => {324 .Eof => {
334 p.putBackToken(tok.index);325 p.putBackToken(tok);
335 return;326 return;
336 },327 },
337 else => {},328 else => {},
...@@ -344,11 +335,11 @@ const Parser = struct {...@@ -344,11 +335,11 @@ const Parser = struct {
344 var level: u32 = 0;335 var level: u32 = 0;
345 while (true) {336 while (true) {
346 const tok = p.nextToken();337 const tok = p.nextToken();
347 switch (tok.ptr.id) {338 switch (p.token_ids[tok]) {
348 .LBrace => level += 1,339 .LBrace => level += 1,
349 .RBrace => {340 .RBrace => {
350 if (level == 0) {341 if (level == 0) {
351 p.putBackToken(tok.index);342 p.putBackToken(tok);
352 return;343 return;
353 }344 }
354 level -= 1;345 level -= 1;
...@@ -359,7 +350,7 @@ const Parser = struct {...@@ -359,7 +350,7 @@ const Parser = struct {
359 }350 }
360 },351 },
361 .Eof => {352 .Eof => {
362 p.putBackToken(tok.index);353 p.putBackToken(tok);
363 return;354 return;
364 },355 },
365 else => {},356 else => {},
...@@ -454,8 +445,8 @@ const Parser = struct {...@@ -454,8 +445,8 @@ const Parser = struct {
454 }445 }
455446
456 if (extern_export_inline_token) |token| {447 if (extern_export_inline_token) |token| {
457 if (p.tokens[token].id == .Keyword_inline or448 if (p.token_ids[token] == .Keyword_inline or
458 p.tokens[token].id == .Keyword_noinline)449 p.token_ids[token] == .Keyword_noinline)
459 {450 {
460 try p.errors.append(p.gpa, .{451 try p.errors.append(p.gpa, .{
461 .ExpectedFn = .{ .token = p.tok_i },452 .ExpectedFn = .{ .token = p.tok_i },
...@@ -722,7 +713,7 @@ const Parser = struct {...@@ -722,7 +713,7 @@ const Parser = struct {
722713
723 const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer);714 const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer);
724 if (defer_token) |token| {715 if (defer_token) |token| {
725 const payload = if (p.tokens[token].id == .Keyword_errdefer)716 const payload = if (p.token_ids[token] == .Keyword_errdefer)
726 try p.parsePayload()717 try p.parsePayload()
727 else718 else
728 null;719 null;
...@@ -2269,7 +2260,7 @@ const Parser = struct {...@@ -2269,7 +2260,7 @@ const Parser = struct {
2269 /// / EQUAL2260 /// / EQUAL
2270 fn parseAssignOp(p: *Parser) !?*Node {2261 fn parseAssignOp(p: *Parser) !?*Node {
2271 const token = p.nextToken();2262 const token = p.nextToken();
2272 const op: Node.InfixOp.Op = switch (token.ptr.id) {2263 const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2273 .AsteriskEqual => .AssignMul,2264 .AsteriskEqual => .AssignMul,
2274 .SlashEqual => .AssignDiv,2265 .SlashEqual => .AssignDiv,
2275 .PercentEqual => .AssignMod,2266 .PercentEqual => .AssignMod,
...@@ -2285,14 +2276,14 @@ const Parser = struct {...@@ -2285,14 +2276,14 @@ const Parser = struct {
2285 .MinusPercentEqual => .AssignSubWrap,2276 .MinusPercentEqual => .AssignSubWrap,
2286 .Equal => .Assign,2277 .Equal => .Assign,
2287 else => {2278 else => {
2288 p.putBackToken(token.index);2279 p.putBackToken(token);
2289 return null;2280 return null;
2290 },2281 },
2291 };2282 };
22922283
2293 const node = try p.arena.allocator.create(Node.InfixOp);2284 const node = try p.arena.allocator.create(Node.InfixOp);
2294 node.* = .{2285 node.* = .{
2295 .op_token = token.index,2286 .op_token = token,
2296 .lhs = undefined, // set by caller2287 .lhs = undefined, // set by caller
2297 .op = op,2288 .op = op,
2298 .rhs = undefined, // set by caller2289 .rhs = undefined, // set by caller
...@@ -2309,7 +2300,7 @@ const Parser = struct {...@@ -2309,7 +2300,7 @@ const Parser = struct {
2309 /// / RARROWEQUAL2300 /// / RARROWEQUAL
2310 fn parseCompareOp(p: *Parser) !?*Node {2301 fn parseCompareOp(p: *Parser) !?*Node {
2311 const token = p.nextToken();2302 const token = p.nextToken();
2312 const op: Node.InfixOp.Op = switch (token.ptr.id) {2303 const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2313 .EqualEqual => .EqualEqual,2304 .EqualEqual => .EqualEqual,
2314 .BangEqual => .BangEqual,2305 .BangEqual => .BangEqual,
2315 .AngleBracketLeft => .LessThan,2306 .AngleBracketLeft => .LessThan,
...@@ -2317,12 +2308,12 @@ const Parser = struct {...@@ -2317,12 +2308,12 @@ const Parser = struct {
2317 .AngleBracketLeftEqual => .LessOrEqual,2308 .AngleBracketLeftEqual => .LessOrEqual,
2318 .AngleBracketRightEqual => .GreaterOrEqual,2309 .AngleBracketRightEqual => .GreaterOrEqual,
2319 else => {2310 else => {
2320 p.putBackToken(token.index);2311 p.putBackToken(token);
2321 return null;2312 return null;
2322 },2313 },
2323 };2314 };
23242315
2325 return p.createInfixOp(token.index, op);2316 return p.createInfixOp(token, op);
2326 }2317 }
23272318
2328 /// BitwiseOp2319 /// BitwiseOp
...@@ -2333,19 +2324,19 @@ const Parser = struct {...@@ -2333,19 +2324,19 @@ const Parser = struct {
2333 /// / KEYWORD_catch Payload?2324 /// / KEYWORD_catch Payload?
2334 fn parseBitwiseOp(p: *Parser) !?*Node {2325 fn parseBitwiseOp(p: *Parser) !?*Node {
2335 const token = p.nextToken();2326 const token = p.nextToken();
2336 const op: Node.InfixOp.Op = switch (token.ptr.id) {2327 const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2337 .Ampersand => .BitAnd,2328 .Ampersand => .BitAnd,
2338 .Caret => .BitXor,2329 .Caret => .BitXor,
2339 .Pipe => .BitOr,2330 .Pipe => .BitOr,
2340 .Keyword_orelse => .UnwrapOptional,2331 .Keyword_orelse => .UnwrapOptional,
2341 .Keyword_catch => .{ .Catch = try p.parsePayload() },2332 .Keyword_catch => .{ .Catch = try p.parsePayload() },
2342 else => {2333 else => {
2343 p.putBackToken(token.index);2334 p.putBackToken(token);
2344 return null;2335 return null;
2345 },2336 },
2346 };2337 };
23472338
2348 return p.createInfixOp(token.index, op);2339 return p.createInfixOp(token, op);
2349 }2340 }
23502341
2351 /// BitShiftOp2342 /// BitShiftOp
...@@ -2353,16 +2344,16 @@ const Parser = struct {...@@ -2353,16 +2344,16 @@ const Parser = struct {
2353 /// / RARROW22344 /// / RARROW2
2354 fn parseBitShiftOp(p: *Parser) !?*Node {2345 fn parseBitShiftOp(p: *Parser) !?*Node {
2355 const token = p.nextToken();2346 const token = p.nextToken();
2356 const op: Node.InfixOp.Op = switch (token.ptr.id) {2347 const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2357 .AngleBracketAngleBracketLeft => .BitShiftLeft,2348 .AngleBracketAngleBracketLeft => .BitShiftLeft,
2358 .AngleBracketAngleBracketRight => .BitShiftRight,2349 .AngleBracketAngleBracketRight => .BitShiftRight,
2359 else => {2350 else => {
2360 p.putBackToken(token.index);2351 p.putBackToken(token);
2361 return null;2352 return null;
2362 },2353 },
2363 };2354 };
23642355
2365 return p.createInfixOp(token.index, op);2356 return p.createInfixOp(token, op);
2366 }2357 }
23672358
2368 /// AdditionOp2359 /// AdditionOp
...@@ -2373,19 +2364,19 @@ const Parser = struct {...@@ -2373,19 +2364,19 @@ const Parser = struct {
2373 /// / MINUSPERCENT2364 /// / MINUSPERCENT
2374 fn parseAdditionOp(p: *Parser) !?*Node {2365 fn parseAdditionOp(p: *Parser) !?*Node {
2375 const token = p.nextToken();2366 const token = p.nextToken();
2376 const op: Node.InfixOp.Op = switch (token.ptr.id) {2367 const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2377 .Plus => .Add,2368 .Plus => .Add,
2378 .Minus => .Sub,2369 .Minus => .Sub,
2379 .PlusPlus => .ArrayCat,2370 .PlusPlus => .ArrayCat,
2380 .PlusPercent => .AddWrap,2371 .PlusPercent => .AddWrap,
2381 .MinusPercent => .SubWrap,2372 .MinusPercent => .SubWrap,
2382 else => {2373 else => {
2383 p.putBackToken(token.index);2374 p.putBackToken(token);
2384 return null;2375 return null;
2385 },2376 },
2386 };2377 };
23872378
2388 return p.createInfixOp(token.index, op);2379 return p.createInfixOp(token, op);
2389 }2380 }
23902381
2391 /// MultiplyOp2382 /// MultiplyOp
...@@ -2397,7 +2388,7 @@ const Parser = struct {...@@ -2397,7 +2388,7 @@ const Parser = struct {
2397 /// / ASTERISKPERCENT2388 /// / ASTERISKPERCENT
2398 fn parseMultiplyOp(p: *Parser) !?*Node {2389 fn parseMultiplyOp(p: *Parser) !?*Node {
2399 const token = p.nextToken();2390 const token = p.nextToken();
2400 const op: Node.InfixOp.Op = switch (token.ptr.id) {2391 const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2401 .PipePipe => .MergeErrorSets,2392 .PipePipe => .MergeErrorSets,
2402 .Asterisk => .Mul,2393 .Asterisk => .Mul,
2403 .Slash => .Div,2394 .Slash => .Div,
...@@ -2405,12 +2396,12 @@ const Parser = struct {...@@ -2405,12 +2396,12 @@ const Parser = struct {
2405 .AsteriskAsterisk => .ArrayMult,2396 .AsteriskAsterisk => .ArrayMult,
2406 .AsteriskPercent => .MulWrap,2397 .AsteriskPercent => .MulWrap,
2407 else => {2398 else => {
2408 p.putBackToken(token.index);2399 p.putBackToken(token);
2409 return null;2400 return null;
2410 },2401 },
2411 };2402 };
24122403
2413 return p.createInfixOp(token.index, op);2404 return p.createInfixOp(token, op);
2414 }2405 }
24152406
2416 /// PrefixOp2407 /// PrefixOp
...@@ -2423,7 +2414,7 @@ const Parser = struct {...@@ -2423,7 +2414,7 @@ const Parser = struct {
2423 /// / KEYWORD_await2414 /// / KEYWORD_await
2424 fn parsePrefixOp(p: *Parser) !?*Node {2415 fn parsePrefixOp(p: *Parser) !?*Node {
2425 const token = p.nextToken();2416 const token = p.nextToken();
2426 const op: Node.PrefixOp.Op = switch (token.ptr.id) {2417 const op: Node.PrefixOp.Op = switch (p.token_ids[token]) {
2427 .Bang => .BoolNot,2418 .Bang => .BoolNot,
2428 .Minus => .Negation,2419 .Minus => .Negation,
2429 .Tilde => .BitNot,2420 .Tilde => .BitNot,
...@@ -2432,14 +2423,14 @@ const Parser = struct {...@@ -2432,14 +2423,14 @@ const Parser = struct {
2432 .Keyword_try => .Try,2423 .Keyword_try => .Try,
2433 .Keyword_await => .Await,2424 .Keyword_await => .Await,
2434 else => {2425 else => {
2435 p.putBackToken(token.index);2426 p.putBackToken(token);
2436 return null;2427 return null;
2437 },2428 },
2438 };2429 };
24392430
2440 const node = try p.arena.allocator.create(Node.PrefixOp);2431 const node = try p.arena.allocator.create(Node.PrefixOp);
2441 node.* = .{2432 node.* = .{
2442 .op_token = token.index,2433 .op_token = token,
2443 .op = op,2434 .op = op,
2444 .rhs = undefined, // set by caller2435 .rhs = undefined, // set by caller
2445 };2436 };
...@@ -2493,7 +2484,7 @@ const Parser = struct {...@@ -2493,7 +2484,7 @@ const Parser = struct {
2493 // If the token encountered was **, there will be two nodes instead of one.2484 // If the token encountered was **, there will be two nodes instead of one.
2494 // The attributes should be applied to the rightmost operator.2485 // The attributes should be applied to the rightmost operator.
2495 const prefix_op = node.cast(Node.PrefixOp).?;2486 const prefix_op = node.cast(Node.PrefixOp).?;
2496 var ptr_info = if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk)2487 var ptr_info = if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk)
2497 &prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType2488 &prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType
2498 else2489 else
2499 &prefix_op.op.PtrType;2490 &prefix_op.op.PtrType;
...@@ -2812,7 +2803,8 @@ const Parser = struct {...@@ -2812,7 +2803,8 @@ const Parser = struct {
2812 return null;2803 return null;
2813 };2804 };
2814 if (p.eatToken(.Identifier)) |ident| {2805 if (p.eatToken(.Identifier)) |ident| {
2815 const token_slice = p.source[p.tokens[ident].start..p.tokens[ident].end];2806 const token_loc = p.token_locs[ident];
2807 const token_slice = p.source[token_loc.start..token_loc.end];
2816 if (!std.mem.eql(u8, token_slice, "c")) {2808 if (!std.mem.eql(u8, token_slice, "c")) {
2817 p.putBackToken(ident);2809 p.putBackToken(ident);
2818 } else {2810 } else {
...@@ -2879,7 +2871,7 @@ const Parser = struct {...@@ -2879,7 +2871,7 @@ const Parser = struct {
2879 fn parseContainerDeclType(p: *Parser) !?ContainerDeclType {2871 fn parseContainerDeclType(p: *Parser) !?ContainerDeclType {
2880 const kind_token = p.nextToken();2872 const kind_token = p.nextToken();
28812873
2882 const init_arg_expr = switch (kind_token.ptr.id) {2874 const init_arg_expr = switch (p.token_ids[kind_token]) {
2883 .Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} },2875 .Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} },
2884 .Keyword_enum => blk: {2876 .Keyword_enum => blk: {
2885 if (p.eatToken(.LParen) != null) {2877 if (p.eatToken(.LParen) != null) {
...@@ -2914,13 +2906,13 @@ const Parser = struct {...@@ -2914,13 +2906,13 @@ const Parser = struct {
2914 break :blk Node.ContainerDecl.InitArg{ .None = {} };2906 break :blk Node.ContainerDecl.InitArg{ .None = {} };
2915 },2907 },
2916 else => {2908 else => {
2917 p.putBackToken(kind_token.index);2909 p.putBackToken(kind_token);
2918 return null;2910 return null;
2919 },2911 },
2920 };2912 };
29212913
2922 return ContainerDeclType{2914 return ContainerDeclType{
2923 .kind_token = kind_token.index,2915 .kind_token = kind_token,
2924 .init_arg_expr = init_arg_expr,2916 .init_arg_expr = init_arg_expr,
2925 };2917 };
2926 }2918 }
...@@ -2973,7 +2965,7 @@ const Parser = struct {...@@ -2973,7 +2965,7 @@ const Parser = struct {
2973 while (try nodeParseFn(p)) |item| {2965 while (try nodeParseFn(p)) |item| {
2974 try list.append(item);2966 try list.append(item);
29752967
2976 switch (p.tokens[p.tok_i].id) {2968 switch (p.token_ids[p.tok_i]) {
2977 .Comma => _ = p.nextToken(),2969 .Comma => _ = p.nextToken(),
2978 // all possible delimiters2970 // all possible delimiters
2979 .Colon, .RParen, .RBrace, .RBracket => break,2971 .Colon, .RParen, .RBrace, .RBracket => break,
...@@ -2994,13 +2986,13 @@ const Parser = struct {...@@ -2994,13 +2986,13 @@ const Parser = struct {
2994 fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {2986 fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {
2995 return struct {2987 return struct {
2996 pub fn parse(p: *Parser) Error!?*Node {2988 pub fn parse(p: *Parser) Error!?*Node {
2997 const op_token = if (token == .Keyword_and) switch (p.tokens[p.tok_i].id) {2989 const op_token = if (token == .Keyword_and) switch (p.token_ids[p.tok_i]) {
2998 .Keyword_and => p.nextToken().index,2990 .Keyword_and => p.nextToken(),
2999 .Invalid_ampersands => blk: {2991 .Invalid_ampersands => blk: {
3000 try p.errors.append(p.gpa, .{2992 try p.errors.append(p.gpa, .{
3001 .InvalidAnd = .{ .token = p.tok_i },2993 .InvalidAnd = .{ .token = p.tok_i },
3002 });2994 });
3003 break :blk p.nextToken().index;2995 break :blk p.nextToken();
3004 },2996 },
3005 else => return null,2997 else => return null,
3006 } else p.eatToken(token) orelse return null;2998 } else p.eatToken(token) orelse return null;
...@@ -3104,7 +3096,7 @@ const Parser = struct {...@@ -3104,7 +3096,7 @@ const Parser = struct {
3104 var tok_i = start_tok_i;3096 var tok_i = start_tok_i;
3105 var count: usize = 1; // including first_line3097 var count: usize = 1; // including first_line
3106 while (true) : (tok_i += 1) {3098 while (true) : (tok_i += 1) {
3107 switch (p.tokens[tok_i].id) {3099 switch (p.token_ids[tok_i]) {
3108 .LineComment => continue,3100 .LineComment => continue,
3109 .MultilineStringLiteralLine => count += 1,3101 .MultilineStringLiteralLine => count += 1,
3110 else => break,3102 else => break,
...@@ -3118,7 +3110,7 @@ const Parser = struct {...@@ -3118,7 +3110,7 @@ const Parser = struct {
3118 lines[0] = first_line;3110 lines[0] = first_line;
3119 count = 1;3111 count = 1;
3120 while (true) : (tok_i += 1) {3112 while (true) : (tok_i += 1) {
3121 switch (p.tokens[tok_i].id) {3113 switch (p.token_ids[tok_i]) {
3122 .LineComment => continue,3114 .LineComment => continue,
3123 .MultilineStringLiteralLine => {3115 .MultilineStringLiteralLine => {
3124 lines[count] = tok_i;3116 lines[count] = tok_i;
...@@ -3215,7 +3207,7 @@ const Parser = struct {...@@ -3215,7 +3207,7 @@ const Parser = struct {
3215 }3207 }
32163208
3217 fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {3209 fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {
3218 return std.mem.indexOfScalar(u8, p.source[p.tokens[token1].end..p.tokens[token2].start], '\n') == null;3210 return std.mem.indexOfScalar(u8, p.source[p.token_locs[token1].end..p.token_locs[token2].start], '\n') == null;
3219 }3211 }
32203212
3221 /// Eat a single-line doc comment on the same line as another node3213 /// Eat a single-line doc comment on the same line as another node
...@@ -3239,7 +3231,7 @@ const Parser = struct {...@@ -3239,7 +3231,7 @@ const Parser = struct {
3239 .PrefixOp => {3231 .PrefixOp => {
3240 var prefix_op = rightmost_op.cast(Node.PrefixOp).?;3232 var prefix_op = rightmost_op.cast(Node.PrefixOp).?;
3241 // If the token encountered was **, there will be two nodes3233 // If the token encountered was **, there will be two nodes
3242 if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) {3234 if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk) {
3243 rightmost_op = prefix_op.rhs;3235 rightmost_op = prefix_op.rhs;
3244 prefix_op = rightmost_op.cast(Node.PrefixOp).?;3236 prefix_op = rightmost_op.cast(Node.PrefixOp).?;
3245 }3237 }
...@@ -3328,11 +3320,7 @@ const Parser = struct {...@@ -3328,11 +3320,7 @@ const Parser = struct {
3328 }3320 }
33293321
3330 fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex {3322 fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex {
3331 return if (p.eatAnnotatedToken(id)) |token| token.index else null;3323 return if (p.token_ids[p.tok_i] == id) p.nextToken() else null;
3332 }
3333
3334 fn eatAnnotatedToken(p: *Parser, id: Token.Id) ?AnnotatedToken {
3335 return if (p.tokens[p.tok_i].id == id) p.nextToken() else null;
3336 }3324 }
33373325
3338 fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex {3326 fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex {
...@@ -3341,29 +3329,25 @@ const Parser = struct {...@@ -3341,29 +3329,25 @@ const Parser = struct {
33413329
3342 fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex {3330 fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex {
3343 const token = p.nextToken();3331 const token = p.nextToken();
3344 if (token.ptr.id != id) {3332 if (p.token_ids[token] != id) {
3345 try p.errors.append(p.gpa, .{3333 try p.errors.append(p.gpa, .{
3346 .ExpectedToken = .{ .token = token.index, .expected_id = id },3334 .ExpectedToken = .{ .token = token, .expected_id = id },
3347 });3335 });
3348 // go back so that we can recover properly3336 // go back so that we can recover properly
3349 p.putBackToken(token.index);3337 p.putBackToken(token);
3350 return null;3338 return null;
3351 }3339 }
3352 return token.index;3340 return token;
3353 }3341 }
33543342
3355 fn nextToken(p: *Parser) AnnotatedToken {3343 fn nextToken(p: *Parser) TokenIndex {
3356 const result = AnnotatedToken{3344 const result = p.tok_i;
3357 .index = p.tok_i,
3358 .ptr = &p.tokens[p.tok_i],
3359 };
3360 p.tok_i += 1;3345 p.tok_i += 1;
3361 assert(result.ptr.id != .LineComment);3346 assert(p.token_ids[result] != .LineComment);
3362 if (p.tok_i >= p.tokens.len) return result;3347 if (p.tok_i >= p.token_ids.len) return result;
33633348
3364 while (true) {3349 while (true) {
3365 const next_tok = p.tokens[p.tok_i];3350 if (p.token_ids[p.tok_i] != .LineComment) return result;
3366 if (next_tok.id != .LineComment) return result;
3367 p.tok_i += 1;3351 p.tok_i += 1;
3368 }3352 }
3369 }3353 }
...@@ -3371,18 +3355,12 @@ const Parser = struct {...@@ -3371,18 +3355,12 @@ const Parser = struct {
3371 fn putBackToken(p: *Parser, putting_back: TokenIndex) void {3355 fn putBackToken(p: *Parser, putting_back: TokenIndex) void {
3372 while (p.tok_i > 0) {3356 while (p.tok_i > 0) {
3373 p.tok_i -= 1;3357 p.tok_i -= 1;
3374 const prev_tok = p.tokens[p.tok_i];3358 if (p.token_ids[p.tok_i] == .LineComment) continue;
3375 if (prev_tok.id == .LineComment) continue;
3376 assert(putting_back == p.tok_i);3359 assert(putting_back == p.tok_i);
3377 return;3360 return;
3378 }3361 }
3379 }3362 }
33803363
3381 const AnnotatedToken = struct {
3382 index: TokenIndex,
3383 ptr: *const Token,
3384 };
3385
3386 fn expectNode(3364 fn expectNode(
3387 p: *Parser,3365 p: *Parser,
3388 parseFn: NodeParseFn,3366 parseFn: NodeParseFn,
lib/std/zig/parser_test.zig+1-1
...@@ -3181,7 +3181,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b...@@ -3181,7 +3181,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b
3181 defer tree.deinit();3181 defer tree.deinit();
31823182
3183 for (tree.errors) |*parse_error| {3183 for (tree.errors) |*parse_error| {
3184 const token = tree.tokens[parse_error.loc()];3184 const token = tree.token_locs[parse_error.loc()];
3185 const loc = tree.tokenLocation(0, parse_error.loc());3185 const loc = tree.tokenLocation(0, parse_error.loc());
3186 try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 });3186 try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 });
3187 try tree.renderError(parse_error, stderr);3187 try tree.renderError(parse_error, stderr);
lib/std/zig/render.zig+99-89
...@@ -68,11 +68,12 @@ fn renderRoot(...@@ -68,11 +68,12 @@ fn renderRoot(
68 tree: *ast.Tree,68 tree: *ast.Tree,
69) (@TypeOf(stream).Error || Error)!void {69) (@TypeOf(stream).Error || Error)!void {
70 // render all the line comments at the beginning of the file70 // render all the line comments at the beginning of the file
71 for (tree.tokens) |token, i| {71 for (tree.token_ids) |token_id, i| {
72 if (token.id != .LineComment) break;72 if (token_id != .LineComment) break;
73 try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSlicePtr(token), " ")});73 const token_loc = tree.token_locs[i];
74 const next_token = &tree.tokens[i + 1];74 try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSliceLoc(token_loc), " ")});
75 const loc = tree.tokenLocationPtr(token.end, next_token.*);75 const next_token = tree.token_locs[i + 1];
76 const loc = tree.tokenLocationLoc(token_loc.end, next_token);
76 if (loc.line >= 2) {77 if (loc.line >= 2) {
77 try stream.writeByte('\n');78 try stream.writeByte('\n');
78 }79 }
...@@ -101,8 +102,8 @@ fn renderRoot(...@@ -101,8 +102,8 @@ fn renderRoot(
101102
102 while (token_index != 0) {103 while (token_index != 0) {
103 token_index -= 1;104 token_index -= 1;
104 const token = tree.tokens[token_index];105 const token_id = tree.token_ids[token_index];
105 switch (token.id) {106 switch (token_id) {
106 .LineComment => {},107 .LineComment => {},
107 .DocComment => {108 .DocComment => {
108 copy_start_token_index = token_index;109 copy_start_token_index = token_index;
...@@ -111,12 +112,13 @@ fn renderRoot(...@@ -111,12 +112,13 @@ fn renderRoot(
111 else => break,112 else => break,
112 }113 }
113114
114 if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) {115 const token_loc = tree.token_locs[token_index];
116 if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) {
115 if (!found_fmt_directive) {117 if (!found_fmt_directive) {
116 fmt_active = false;118 fmt_active = false;
117 found_fmt_directive = true;119 found_fmt_directive = true;
118 }120 }
119 } else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) {121 } else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) {
120 if (!found_fmt_directive) {122 if (!found_fmt_directive) {
121 fmt_active = true;123 fmt_active = true;
122 found_fmt_directive = true;124 found_fmt_directive = true;
...@@ -135,7 +137,7 @@ fn renderRoot(...@@ -135,7 +137,7 @@ fn renderRoot(
135 if (decl_i >= root_decls.len) {137 if (decl_i >= root_decls.len) {
136 // If there's no next reformatted `decl`, just copy the138 // If there's no next reformatted `decl`, just copy the
137 // remaining input tokens and bail out.139 // remaining input tokens and bail out.
138 const start = tree.tokens[copy_start_token_index].start;140 const start = tree.token_locs[copy_start_token_index].start;
139 try copyFixingWhitespace(stream, tree.source[start..]);141 try copyFixingWhitespace(stream, tree.source[start..]);
140 return;142 return;
141 }143 }
...@@ -143,15 +145,16 @@ fn renderRoot(...@@ -143,15 +145,16 @@ fn renderRoot(
143 var decl_first_token_index = decl.firstToken();145 var decl_first_token_index = decl.firstToken();
144146
145 while (token_index < decl_first_token_index) : (token_index += 1) {147 while (token_index < decl_first_token_index) : (token_index += 1) {
146 const token = tree.tokens[token_index];148 const token_id = tree.token_ids[token_index];
147 switch (token.id) {149 switch (token_id) {
148 .LineComment => {},150 .LineComment => {},
149 .Eof => unreachable,151 .Eof => unreachable,
150 else => continue,152 else => continue,
151 }153 }
152 if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) {154 const token_loc = tree.token_locs[token_index];
155 if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) {
153 fmt_active = true;156 fmt_active = true;
154 } else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) {157 } else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) {
155 fmt_active = false;158 fmt_active = false;
156 }159 }
157 }160 }
...@@ -163,8 +166,8 @@ fn renderRoot(...@@ -163,8 +166,8 @@ fn renderRoot(
163 token_index = copy_end_token_index;166 token_index = copy_end_token_index;
164 while (token_index != 0) {167 while (token_index != 0) {
165 token_index -= 1;168 token_index -= 1;
166 const token = tree.tokens[token_index];169 const token_id = tree.token_ids[token_index];
167 switch (token.id) {170 switch (token_id) {
168 .LineComment => {},171 .LineComment => {},
169 .DocComment => {172 .DocComment => {
170 copy_end_token_index = token_index;173 copy_end_token_index = token_index;
...@@ -174,8 +177,8 @@ fn renderRoot(...@@ -174,8 +177,8 @@ fn renderRoot(
174 }177 }
175 }178 }
176179
177 const start = tree.tokens[copy_start_token_index].start;180 const start = tree.token_locs[copy_start_token_index].start;
178 const end = tree.tokens[copy_end_token_index].start;181 const end = tree.token_locs[copy_end_token_index].start;
179 try copyFixingWhitespace(stream, tree.source[start..end]);182 try copyFixingWhitespace(stream, tree.source[start..end]);
180 }183 }
181184
...@@ -194,13 +197,13 @@ fn renderExtraNewlineToken(tree: *ast.Tree, stream: var, start_col: *usize, firs...@@ -194,13 +197,13 @@ fn renderExtraNewlineToken(tree: *ast.Tree, stream: var, start_col: *usize, firs
194 var prev_token = first_token;197 var prev_token = first_token;
195 if (prev_token == 0) return;198 if (prev_token == 0) return;
196 var newline_threshold: usize = 2;199 var newline_threshold: usize = 2;
197 while (tree.tokens[prev_token - 1].id == .DocComment) {200 while (tree.token_ids[prev_token - 1] == .DocComment) {
198 if (tree.tokenLocation(tree.tokens[prev_token - 1].end, prev_token).line == 1) {201 if (tree.tokenLocation(tree.token_locs[prev_token - 1].end, prev_token).line == 1) {
199 newline_threshold += 1;202 newline_threshold += 1;
200 }203 }
201 prev_token -= 1;204 prev_token -= 1;
202 }205 }
203 const prev_token_end = tree.tokens[prev_token - 1].end;206 const prev_token_end = tree.token_locs[prev_token - 1].end;
204 const loc = tree.tokenLocation(prev_token_end, first_token);207 const loc = tree.tokenLocation(prev_token_end, first_token);
205 if (loc.line >= newline_threshold) {208 if (loc.line >= newline_threshold) {
206 try stream.writeByte('\n');209 try stream.writeByte('\n');
...@@ -265,7 +268,7 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,...@@ -265,7 +268,7 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,
265268
266 const src_has_trailing_comma = blk: {269 const src_has_trailing_comma = blk: {
267 const maybe_comma = tree.nextToken(field.lastToken());270 const maybe_comma = tree.nextToken(field.lastToken());
268 break :blk tree.tokens[maybe_comma].id == .Comma;271 break :blk tree.token_ids[maybe_comma] == .Comma;
269 };272 };
270273
271 // The trailing comma is emitted at the end, but if it's not present274 // The trailing comma is emitted at the end, but if it's not present
...@@ -327,11 +330,11 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,...@@ -327,11 +330,11 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,
327330
328 .DocComment => {331 .DocComment => {
329 const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);332 const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
330 const kind = tree.tokens[comment.first_line].id;333 const kind = tree.token_ids[comment.first_line];
331 try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline);334 try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline);
332 var tok_i = comment.first_line + 1;335 var tok_i = comment.first_line + 1;
333 while (true) : (tok_i += 1) {336 while (true) : (tok_i += 1) {
334 const tok_id = tree.tokens[tok_i].id;337 const tok_id = tree.token_ids[tok_i];
335 if (tok_id == kind) {338 if (tok_id == kind) {
336 try stream.writeByteNTimes(' ', indent);339 try stream.writeByteNTimes(' ', indent);
337 try renderToken(tree, stream, tok_i, indent, start_col, .Newline);340 try renderToken(tree, stream, tok_i, indent, start_col, .Newline);
...@@ -436,13 +439,13 @@ fn renderExpression(...@@ -436,13 +439,13 @@ fn renderExpression(
436 try renderExpression(allocator, stream, tree, indent, start_col, infix_op_node.lhs, op_space);439 try renderExpression(allocator, stream, tree, indent, start_col, infix_op_node.lhs, op_space);
437440
438 const after_op_space = blk: {441 const after_op_space = blk: {
439 const loc = tree.tokenLocation(tree.tokens[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token));442 const loc = tree.tokenLocation(tree.token_locs[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token));
440 break :blk if (loc.line == 0) op_space else Space.Newline;443 break :blk if (loc.line == 0) op_space else Space.Newline;
441 };444 };
442445
443 try renderToken(tree, stream, infix_op_node.op_token, indent, start_col, after_op_space);446 try renderToken(tree, stream, infix_op_node.op_token, indent, start_col, after_op_space);
444 if (after_op_space == Space.Newline and447 if (after_op_space == Space.Newline and
445 tree.tokens[tree.nextToken(infix_op_node.op_token)].id != .MultilineStringLiteralLine)448 tree.token_ids[tree.nextToken(infix_op_node.op_token)] != .MultilineStringLiteralLine)
446 {449 {
447 try stream.writeByteNTimes(' ', indent + indent_delta);450 try stream.writeByteNTimes(' ', indent + indent_delta);
448 start_col.* = indent + indent_delta;451 start_col.* = indent + indent_delta;
...@@ -463,10 +466,10 @@ fn renderExpression(...@@ -463,10 +466,10 @@ fn renderExpression(
463466
464 switch (prefix_op_node.op) {467 switch (prefix_op_node.op) {
465 .PtrType => |ptr_info| {468 .PtrType => |ptr_info| {
466 const op_tok_id = tree.tokens[prefix_op_node.op_token].id;469 const op_tok_id = tree.token_ids[prefix_op_node.op_token];
467 switch (op_tok_id) {470 switch (op_tok_id) {
468 .Asterisk, .AsteriskAsterisk => try stream.writeByte('*'),471 .Asterisk, .AsteriskAsterisk => try stream.writeByte('*'),
469 .LBracket => if (tree.tokens[prefix_op_node.op_token + 2].id == .Identifier)472 .LBracket => if (tree.token_ids[prefix_op_node.op_token + 2] == .Identifier)
470 try stream.writeAll("[*c")473 try stream.writeAll("[*c")
471 else474 else
472 try stream.writeAll("[*"),475 try stream.writeAll("[*"),
...@@ -578,8 +581,8 @@ fn renderExpression(...@@ -578,8 +581,8 @@ fn renderExpression(
578581
579 try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [582 try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
580583
581 const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment;584 const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment;
582 const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment;585 const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment;
583 const new_indent = if (ends_with_comment) indent + indent_delta else indent;586 const new_indent = if (ends_with_comment) indent + indent_delta else indent;
584 const new_space = if (ends_with_comment) Space.Newline else Space.None;587 const new_space = if (ends_with_comment) Space.Newline else Space.None;
585 try renderExpression(allocator, stream, tree, new_indent, start_col, array_info.len_expr, new_space);588 try renderExpression(allocator, stream, tree, new_indent, start_col, array_info.len_expr, new_space);
...@@ -653,7 +656,7 @@ fn renderExpression(...@@ -653,7 +656,7 @@ fn renderExpression(
653 return renderToken(tree, stream, rtoken, indent, start_col, space);656 return renderToken(tree, stream, rtoken, indent, start_col, space);
654 }657 }
655658
656 if (exprs.len == 1 and tree.tokens[exprs[0].lastToken() + 1].id == .RBrace) {659 if (exprs.len == 1 and tree.token_ids[exprs[0].lastToken() + 1] == .RBrace) {
657 const expr = exprs[0];660 const expr = exprs[0];
658 switch (lhs) {661 switch (lhs) {
659 .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None),662 .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None),
...@@ -675,17 +678,17 @@ fn renderExpression(...@@ -675,17 +678,17 @@ fn renderExpression(
675 for (exprs) |expr, i| {678 for (exprs) |expr, i| {
676 if (i + 1 < exprs.len) {679 if (i + 1 < exprs.len) {
677 const expr_last_token = expr.lastToken() + 1;680 const expr_last_token = expr.lastToken() + 1;
678 const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, exprs[i+1].firstToken());681 const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i+1].firstToken());
679 if (loc.line != 0) break :blk count;682 if (loc.line != 0) break :blk count;
680 count += 1;683 count += 1;
681 } else {684 } else {
682 const expr_last_token = expr.lastToken();685 const expr_last_token = expr.lastToken();
683 const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, rtoken);686 const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken);
684 if (loc.line == 0) {687 if (loc.line == 0) {
685 // all on one line688 // all on one line
686 const src_has_trailing_comma = trailblk: {689 const src_has_trailing_comma = trailblk: {
687 const maybe_comma = tree.prevToken(rtoken);690 const maybe_comma = tree.prevToken(rtoken);
688 break :trailblk tree.tokens[maybe_comma].id == .Comma;691 break :trailblk tree.token_ids[maybe_comma] == .Comma;
689 };692 };
690 if (src_has_trailing_comma) {693 if (src_has_trailing_comma) {
691 break :blk 1; // force row size 1694 break :blk 1; // force row size 1
...@@ -723,7 +726,7 @@ fn renderExpression(...@@ -723,7 +726,7 @@ fn renderExpression(
723726
724 var new_indent = indent + indent_delta;727 var new_indent = indent + indent_delta;
725728
726 if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) {729 if (tree.token_ids[tree.nextToken(lbrace)] != .MultilineStringLiteralLine) {
727 try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline);730 try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline);
728 try stream.writeByteNTimes(' ', new_indent);731 try stream.writeByteNTimes(' ', new_indent);
729 } else {732 } else {
...@@ -750,7 +753,7 @@ fn renderExpression(...@@ -750,7 +753,7 @@ fn renderExpression(
750 }753 }
751 col = 1;754 col = 1;
752755
753 if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) {756 if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) {
754 try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // ,757 try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // ,
755 } else {758 } else {
756 try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // ,759 try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // ,
...@@ -819,11 +822,11 @@ fn renderExpression(...@@ -819,11 +822,11 @@ fn renderExpression(
819822
820 const src_has_trailing_comma = blk: {823 const src_has_trailing_comma = blk: {
821 const maybe_comma = tree.prevToken(rtoken);824 const maybe_comma = tree.prevToken(rtoken);
822 break :blk tree.tokens[maybe_comma].id == .Comma;825 break :blk tree.token_ids[maybe_comma] == .Comma;
823 };826 };
824827
825 const src_same_line = blk: {828 const src_same_line = blk: {
826 const loc = tree.tokenLocation(tree.tokens[lbrace].end, rtoken);829 const loc = tree.tokenLocation(tree.token_locs[lbrace].end, rtoken);
827 break :blk loc.line == 0;830 break :blk loc.line == 0;
828 };831 };
829832
...@@ -929,7 +932,7 @@ fn renderExpression(...@@ -929,7 +932,7 @@ fn renderExpression(
929932
930 const src_has_trailing_comma = blk: {933 const src_has_trailing_comma = blk: {
931 const maybe_comma = tree.prevToken(call.rtoken);934 const maybe_comma = tree.prevToken(call.rtoken);
932 break :blk tree.tokens[maybe_comma].id == .Comma;935 break :blk tree.token_ids[maybe_comma] == .Comma;
933 };936 };
934937
935 if (src_has_trailing_comma) {938 if (src_has_trailing_comma) {
...@@ -983,8 +986,8 @@ fn renderExpression(...@@ -983,8 +986,8 @@ fn renderExpression(
983 try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);986 try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);
984 try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [987 try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
985988
986 const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment;989 const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment;
987 const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment;990 const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment;
988 const new_indent = if (ends_with_comment) indent + indent_delta else indent;991 const new_indent = if (ends_with_comment) indent + indent_delta else indent;
989 const new_space = if (ends_with_comment) Space.Newline else Space.None;992 const new_space = if (ends_with_comment) Space.Newline else Space.None;
990 try renderExpression(allocator, stream, tree, new_indent, start_col, index_expr, new_space);993 try renderExpression(allocator, stream, tree, new_indent, start_col, index_expr, new_space);
...@@ -1226,9 +1229,9 @@ fn renderExpression(...@@ -1226,9 +1229,9 @@ fn renderExpression(
1226 var maybe_comma = tree.prevToken(container_decl.lastToken());1229 var maybe_comma = tree.prevToken(container_decl.lastToken());
1227 // Doc comments for a field may also appear after the comma, eg.1230 // Doc comments for a field may also appear after the comma, eg.
1228 // field_name: T, // comment attached to field_name1231 // field_name: T, // comment attached to field_name
1229 if (tree.tokens[maybe_comma].id == .DocComment)1232 if (tree.token_ids[maybe_comma] == .DocComment)
1230 maybe_comma = tree.prevToken(maybe_comma);1233 maybe_comma = tree.prevToken(maybe_comma);
1231 break :blk tree.tokens[maybe_comma].id == .Comma;1234 break :blk tree.token_ids[maybe_comma] == .Comma;
1232 };1235 };
12331236
1234 const fields_and_decls = container_decl.fieldsAndDecls();1237 const fields_and_decls = container_decl.fieldsAndDecls();
...@@ -1321,7 +1324,7 @@ fn renderExpression(...@@ -1321,7 +1324,7 @@ fn renderExpression(
13211324
1322 const src_has_trailing_comma = blk: {1325 const src_has_trailing_comma = blk: {
1323 const maybe_comma = tree.prevToken(err_set_decl.rbrace_token);1326 const maybe_comma = tree.prevToken(err_set_decl.rbrace_token);
1324 break :blk tree.tokens[maybe_comma].id == .Comma;1327 break :blk tree.token_ids[maybe_comma] == .Comma;
1325 };1328 };
13261329
1327 if (src_has_trailing_comma) {1330 if (src_has_trailing_comma) {
...@@ -1353,7 +1356,7 @@ fn renderExpression(...@@ -1353,7 +1356,7 @@ fn renderExpression(
1353 try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None);1356 try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None);
13541357
1355 const comma_token = tree.nextToken(node.lastToken());1358 const comma_token = tree.nextToken(node.lastToken());
1356 assert(tree.tokens[comma_token].id == .Comma);1359 assert(tree.token_ids[comma_token] == .Comma);
1357 try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // ,1360 try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // ,
1358 try renderExtraNewline(tree, stream, start_col, decls[i + 1]);1361 try renderExtraNewline(tree, stream, start_col, decls[i + 1]);
1359 } else {1362 } else {
...@@ -1378,7 +1381,7 @@ fn renderExpression(...@@ -1378,7 +1381,7 @@ fn renderExpression(
1378 const multiline_str_literal = @fieldParentPtr(ast.Node.MultilineStringLiteral, "base", base);1381 const multiline_str_literal = @fieldParentPtr(ast.Node.MultilineStringLiteral, "base", base);
13791382
1380 var skip_first_indent = true;1383 var skip_first_indent = true;
1381 if (tree.tokens[multiline_str_literal.firstToken() - 1].id != .LineComment) {1384 if (tree.token_ids[multiline_str_literal.firstToken() - 1] != .LineComment) {
1382 try stream.print("\n", .{});1385 try stream.print("\n", .{});
1383 skip_first_indent = false;1386 skip_first_indent = false;
1384 }1387 }
...@@ -1406,7 +1409,7 @@ fn renderExpression(...@@ -1406,7 +1409,7 @@ fn renderExpression(
1406 if (builtin_call.params_len < 2) break :blk false;1409 if (builtin_call.params_len < 2) break :blk false;
1407 const last_node = builtin_call.params()[builtin_call.params_len - 1];1410 const last_node = builtin_call.params()[builtin_call.params_len - 1];
1408 const maybe_comma = tree.nextToken(last_node.lastToken());1411 const maybe_comma = tree.nextToken(last_node.lastToken());
1409 break :blk tree.tokens[maybe_comma].id == .Comma;1412 break :blk tree.token_ids[maybe_comma] == .Comma;
1410 };1413 };
14111414
1412 const lparen = tree.nextToken(builtin_call.builtin_token);1415 const lparen = tree.nextToken(builtin_call.builtin_token);
...@@ -1443,8 +1446,8 @@ fn renderExpression(...@@ -1443,8 +1446,8 @@ fn renderExpression(
1443 const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", base);1446 const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", base);
14441447
1445 if (fn_proto.visib_token) |visib_token_index| {1448 if (fn_proto.visib_token) |visib_token_index| {
1446 const visib_token = tree.tokens[visib_token_index];1449 const visib_token = tree.token_ids[visib_token_index];
1447 assert(visib_token.id == .Keyword_pub or visib_token.id == .Keyword_export);1450 assert(visib_token == .Keyword_pub or visib_token == .Keyword_export);
14481451
1449 try renderToken(tree, stream, visib_token_index, indent, start_col, Space.Space); // pub1452 try renderToken(tree, stream, visib_token_index, indent, start_col, Space.Space); // pub
1450 }1453 }
...@@ -1466,7 +1469,7 @@ fn renderExpression(...@@ -1466,7 +1469,7 @@ fn renderExpression(
1466 try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn1469 try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn
1467 break :blk tree.nextToken(fn_proto.fn_token);1470 break :blk tree.nextToken(fn_proto.fn_token);
1468 };1471 };
1469 assert(tree.tokens[lparen].id == .LParen);1472 assert(tree.token_ids[lparen] == .LParen);
14701473
1471 const rparen = tree.prevToken(1474 const rparen = tree.prevToken(
1472 // the first token for the annotation expressions is the left1475 // the first token for the annotation expressions is the left
...@@ -1482,10 +1485,10 @@ fn renderExpression(...@@ -1482,10 +1485,10 @@ fn renderExpression(
1482 .InferErrorSet => |node| tree.prevToken(node.firstToken()),1485 .InferErrorSet => |node| tree.prevToken(node.firstToken()),
1483 .Invalid => unreachable,1486 .Invalid => unreachable,
1484 });1487 });
1485 assert(tree.tokens[rparen].id == .RParen);1488 assert(tree.token_ids[rparen] == .RParen);
14861489
1487 const src_params_trailing_comma = blk: {1490 const src_params_trailing_comma = blk: {
1488 const maybe_comma = tree.tokens[rparen - 1].id;1491 const maybe_comma = tree.token_ids[rparen - 1];
1489 break :blk maybe_comma == .Comma or maybe_comma == .LineComment;1492 break :blk maybe_comma == .Comma or maybe_comma == .LineComment;
1490 };1493 };
14911494
...@@ -1622,7 +1625,7 @@ fn renderExpression(...@@ -1622,7 +1625,7 @@ fn renderExpression(
1622 const src_has_trailing_comma = blk: {1625 const src_has_trailing_comma = blk: {
1623 const last_node = switch_case.items()[switch_case.items_len - 1];1626 const last_node = switch_case.items()[switch_case.items_len - 1];
1624 const maybe_comma = tree.nextToken(last_node.lastToken());1627 const maybe_comma = tree.nextToken(last_node.lastToken());
1625 break :blk tree.tokens[maybe_comma].id == .Comma;1628 break :blk tree.token_ids[maybe_comma] == .Comma;
1626 };1629 };
16271630
1628 if (switch_case.items_len == 1 or !src_has_trailing_comma) {1631 if (switch_case.items_len == 1 or !src_has_trailing_comma) {
...@@ -1967,7 +1970,7 @@ fn renderExpression(...@@ -1967,7 +1970,7 @@ fn renderExpression(
1967 try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline);1970 try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline);
1968 try stream.writeByteNTimes(' ', indent_once);1971 try stream.writeByteNTimes(' ', indent_once);
1969 const comma_or_colon = tree.nextToken(asm_output.lastToken());1972 const comma_or_colon = tree.nextToken(asm_output.lastToken());
1970 break :blk switch (tree.tokens[comma_or_colon].id) {1973 break :blk switch (tree.token_ids[comma_or_colon]) {
1971 .Comma => tree.nextToken(comma_or_colon),1974 .Comma => tree.nextToken(comma_or_colon),
1972 else => comma_or_colon,1975 else => comma_or_colon,
1973 };1976 };
...@@ -2002,7 +2005,7 @@ fn renderExpression(...@@ -2002,7 +2005,7 @@ fn renderExpression(
2002 try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline);2005 try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline);
2003 try stream.writeByteNTimes(' ', indent_once);2006 try stream.writeByteNTimes(' ', indent_once);
2004 const comma_or_colon = tree.nextToken(asm_input.lastToken());2007 const comma_or_colon = tree.nextToken(asm_input.lastToken());
2005 break :blk switch (tree.tokens[comma_or_colon].id) {2008 break :blk switch (tree.token_ids[comma_or_colon]) {
2006 .Comma => tree.nextToken(comma_or_colon),2009 .Comma => tree.nextToken(comma_or_colon),
2007 else => comma_or_colon,2010 else => comma_or_colon,
2008 };2011 };
...@@ -2205,7 +2208,7 @@ fn renderStatement(...@@ -2205,7 +2208,7 @@ fn renderStatement(
2205 try renderExpression(allocator, stream, tree, indent, start_col, base, Space.None);2208 try renderExpression(allocator, stream, tree, indent, start_col, base, Space.None);
22062209
2207 const semicolon_index = tree.nextToken(base.lastToken());2210 const semicolon_index = tree.nextToken(base.lastToken());
2208 assert(tree.tokens[semicolon_index].id == .Semicolon);2211 assert(tree.token_ids[semicolon_index] == .Semicolon);
2209 try renderToken(tree, stream, semicolon_index, indent, start_col, Space.Newline);2212 try renderToken(tree, stream, semicolon_index, indent, start_col, Space.Newline);
2210 } else {2213 } else {
2211 try renderExpression(allocator, stream, tree, indent, start_col, base, Space.Newline);2214 try renderExpression(allocator, stream, tree, indent, start_col, base, Space.Newline);
...@@ -2243,22 +2246,25 @@ fn renderTokenOffset(...@@ -2243,22 +2246,25 @@ fn renderTokenOffset(
2243 return;2246 return;
2244 }2247 }
22452248
2246 var token = tree.tokens[token_index];2249 var token_loc = tree.token_locs[token_index];
2247 try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " "));2250 try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(token_loc)[token_skip_bytes..], " "));
22482251
2249 if (space == Space.NoComment)2252 if (space == Space.NoComment)
2250 return;2253 return;
22512254
2252 var next_token = tree.tokens[token_index + 1];2255 var next_token_id = tree.token_ids[token_index + 1];
2256 var next_token_loc = tree.token_locs[token_index + 1];
22532257
2254 if (space == Space.Comma) switch (next_token.id) {2258 if (space == Space.Comma) switch (next_token_id) {
2255 .Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline),2259 .Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline),
2256 .LineComment => {2260 .LineComment => {
2257 try stream.writeAll(", ");2261 try stream.writeAll(", ");
2258 return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline);2262 return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline);
2259 },2263 },
2260 else => {2264 else => {
2261 if (token_index + 2 < tree.tokens.len and tree.tokens[token_index + 2].id == .MultilineStringLiteralLine) {2265 if (token_index + 2 < tree.token_ids.len and
2266 tree.token_ids[token_index + 2] == .MultilineStringLiteralLine)
2267 {
2262 try stream.writeAll(",");2268 try stream.writeAll(",");
2263 return;2269 return;
2264 } else {2270 } else {
...@@ -2271,19 +2277,20 @@ fn renderTokenOffset(...@@ -2271,19 +2277,20 @@ fn renderTokenOffset(
22712277
2272 // Skip over same line doc comments2278 // Skip over same line doc comments
2273 var offset: usize = 1;2279 var offset: usize = 1;
2274 if (next_token.id == .DocComment) {2280 if (next_token_id == .DocComment) {
2275 const loc = tree.tokenLocationPtr(token.end, next_token);2281 const loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
2276 if (loc.line == 0) {2282 if (loc.line == 0) {
2277 offset += 1;2283 offset += 1;
2278 next_token = tree.tokens[token_index + offset];2284 next_token_id = tree.token_ids[token_index + offset];
2285 next_token_loc = tree.token_locs[token_index + offset];
2279 }2286 }
2280 }2287 }
22812288
2282 if (next_token.id != .LineComment) blk: {2289 if (next_token_id != .LineComment) blk: {
2283 switch (space) {2290 switch (space) {
2284 Space.None, Space.NoNewline => return,2291 Space.None, Space.NoNewline => return,
2285 Space.Newline => {2292 Space.Newline => {
2286 if (next_token.id == .MultilineStringLiteralLine) {2293 if (next_token_id == .MultilineStringLiteralLine) {
2287 return;2294 return;
2288 } else {2295 } else {
2289 try stream.writeAll("\n");2296 try stream.writeAll("\n");
...@@ -2292,7 +2299,7 @@ fn renderTokenOffset(...@@ -2292,7 +2299,7 @@ fn renderTokenOffset(
2292 }2299 }
2293 },2300 },
2294 Space.Space, Space.SpaceOrOutdent => {2301 Space.Space, Space.SpaceOrOutdent => {
2295 if (next_token.id == .MultilineStringLiteralLine)2302 if (next_token_id == .MultilineStringLiteralLine)
2296 return;2303 return;
2297 try stream.writeByte(' ');2304 try stream.writeByte(' ');
2298 return;2305 return;
...@@ -2302,14 +2309,15 @@ fn renderTokenOffset(...@@ -2302,14 +2309,15 @@ fn renderTokenOffset(
2302 }2309 }
23032310
2304 while (true) {2311 while (true) {
2305 const comment_is_empty = mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ").len == 2;2312 const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ").len == 2;
2306 if (comment_is_empty) {2313 if (comment_is_empty) {
2307 switch (space) {2314 switch (space) {
2308 Space.Newline => {2315 Space.Newline => {
2309 offset += 1;2316 offset += 1;
2310 token = next_token;2317 token_loc = next_token_loc;
2311 next_token = tree.tokens[token_index + offset];2318 next_token_id = tree.token_ids[token_index + offset];
2312 if (next_token.id != .LineComment) {2319 next_token_loc = tree.token_locs[token_index + offset];
2320 if (next_token_id != .LineComment) {
2313 try stream.writeByte('\n');2321 try stream.writeByte('\n');
2314 start_col.* = 0;2322 start_col.* = 0;
2315 return;2323 return;
...@@ -2322,18 +2330,19 @@ fn renderTokenOffset(...@@ -2322,18 +2330,19 @@ fn renderTokenOffset(
2322 }2330 }
2323 }2331 }
23242332
2325 var loc = tree.tokenLocationPtr(token.end, next_token);2333 var loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
2326 if (loc.line == 0) {2334 if (loc.line == 0) {
2327 try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")});2335 try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ")});
2328 offset = 2;2336 offset = 2;
2329 token = next_token;2337 token_loc = next_token_loc;
2330 next_token = tree.tokens[token_index + offset];2338 next_token_loc = tree.token_locs[token_index + offset];
2331 if (next_token.id != .LineComment) {2339 next_token_id = tree.token_ids[token_index + offset];
2340 if (next_token_id != .LineComment) {
2332 switch (space) {2341 switch (space) {
2333 Space.None, Space.Space => {2342 Space.None, Space.Space => {
2334 try stream.writeByte('\n');2343 try stream.writeByte('\n');
2335 const after_comment_token = tree.tokens[token_index + offset];2344 const after_comment_token = tree.token_ids[token_index + offset];
2336 const next_line_indent = switch (after_comment_token.id) {2345 const next_line_indent = switch (after_comment_token) {
2337 .RParen, .RBrace, .RBracket => indent,2346 .RParen, .RBrace, .RBracket => indent,
2338 else => indent + indent_delta,2347 else => indent + indent_delta,
2339 };2348 };
...@@ -2346,7 +2355,7 @@ fn renderTokenOffset(...@@ -2346,7 +2355,7 @@ fn renderTokenOffset(
2346 start_col.* = indent;2355 start_col.* = indent;
2347 },2356 },
2348 Space.Newline => {2357 Space.Newline => {
2349 if (next_token.id == .MultilineStringLiteralLine) {2358 if (next_token_id == .MultilineStringLiteralLine) {
2350 return;2359 return;
2351 } else {2360 } else {
2352 try stream.writeAll("\n");2361 try stream.writeAll("\n");
...@@ -2359,7 +2368,7 @@ fn renderTokenOffset(...@@ -2359,7 +2368,7 @@ fn renderTokenOffset(
2359 }2368 }
2360 return;2369 return;
2361 }2370 }
2362 loc = tree.tokenLocationPtr(token.end, next_token);2371 loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
2363 }2372 }
23642373
2365 while (true) {2374 while (true) {
...@@ -2369,15 +2378,16 @@ fn renderTokenOffset(...@@ -2369,15 +2378,16 @@ fn renderTokenOffset(
2369 const newline_count = if (loc.line <= 1) @as(u8, 1) else @as(u8, 2);2378 const newline_count = if (loc.line <= 1) @as(u8, 1) else @as(u8, 2);
2370 try stream.writeByteNTimes('\n', newline_count);2379 try stream.writeByteNTimes('\n', newline_count);
2371 try stream.writeByteNTimes(' ', indent);2380 try stream.writeByteNTimes(' ', indent);
2372 try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " "));2381 try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " "));
23732382
2374 offset += 1;2383 offset += 1;
2375 token = next_token;2384 token_loc = next_token_loc;
2376 next_token = tree.tokens[token_index + offset];2385 next_token_loc = tree.token_locs[token_index + offset];
2377 if (next_token.id != .LineComment) {2386 next_token_id = tree.token_ids[token_index + offset];
2387 if (next_token_id != .LineComment) {
2378 switch (space) {2388 switch (space) {
2379 Space.Newline => {2389 Space.Newline => {
2380 if (next_token.id == .MultilineStringLiteralLine) {2390 if (next_token_id == .MultilineStringLiteralLine) {
2381 return;2391 return;
2382 } else {2392 } else {
2383 try stream.writeAll("\n");2393 try stream.writeAll("\n");
...@@ -2388,8 +2398,8 @@ fn renderTokenOffset(...@@ -2388,8 +2398,8 @@ fn renderTokenOffset(
2388 Space.None, Space.Space => {2398 Space.None, Space.Space => {
2389 try stream.writeByte('\n');2399 try stream.writeByte('\n');
23902400
2391 const after_comment_token = tree.tokens[token_index + offset];2401 const after_comment_token = tree.token_ids[token_index + offset];
2392 const next_line_indent = switch (after_comment_token.id) {2402 const next_line_indent = switch (after_comment_token) {
2393 .RParen, .RBrace, .RBracket => blk: {2403 .RParen, .RBrace, .RBracket => blk: {
2394 if (indent > indent_delta) {2404 if (indent > indent_delta) {
2395 break :blk indent - indent_delta;2405 break :blk indent - indent_delta;
...@@ -2412,7 +2422,7 @@ fn renderTokenOffset(...@@ -2412,7 +2422,7 @@ fn renderTokenOffset(
2412 }2422 }
2413 return;2423 return;
2414 }2424 }
2415 loc = tree.tokenLocationPtr(token.end, next_token);2425 loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
2416 }2426 }
2417}2427}
24182428
...@@ -2448,7 +2458,7 @@ fn renderDocCommentsToken(...@@ -2448,7 +2458,7 @@ fn renderDocCommentsToken(
2448) (@TypeOf(stream).Error || Error)!void {2458) (@TypeOf(stream).Error || Error)!void {
2449 var tok_i = comment.first_line;2459 var tok_i = comment.first_line;
2450 while (true) : (tok_i += 1) {2460 while (true) : (tok_i += 1) {
2451 switch (tree.tokens[tok_i].id) {2461 switch (tree.token_ids[tok_i]) {
2452 .DocComment, .ContainerDocComment => {2462 .DocComment, .ContainerDocComment => {
2453 if (comment.first_line < first_token) {2463 if (comment.first_line < first_token) {
2454 try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline);2464 try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline);
lib/std/zig/tokenizer.zig+18-10
...@@ -3,8 +3,12 @@ const mem = std.mem;...@@ -3,8 +3,12 @@ const mem = std.mem;
33
4pub const Token = struct {4pub const Token = struct {
5 id: Id,5 id: Id,
6 start: usize,6 loc: Loc,
7 end: usize,7
8 pub const Loc = struct {
9 start: usize,
10 end: usize,
11 };
812
9 pub const Keyword = struct {13 pub const Keyword = struct {
10 bytes: []const u8,14 bytes: []const u8,
...@@ -426,8 +430,10 @@ pub const Tokenizer = struct {...@@ -426,8 +430,10 @@ pub const Tokenizer = struct {
426 var state: State = .start;430 var state: State = .start;
427 var result = Token{431 var result = Token{
428 .id = .Eof,432 .id = .Eof,
429 .start = self.index,433 .loc = .{
430 .end = undefined,434 .start = self.index,
435 .end = undefined,
436 },
431 };437 };
432 var seen_escape_digits: usize = undefined;438 var seen_escape_digits: usize = undefined;
433 var remaining_code_units: usize = undefined;439 var remaining_code_units: usize = undefined;
...@@ -436,7 +442,7 @@ pub const Tokenizer = struct {...@@ -436,7 +442,7 @@ pub const Tokenizer = struct {
436 switch (state) {442 switch (state) {
437 .start => switch (c) {443 .start => switch (c) {
438 ' ', '\n', '\t', '\r' => {444 ' ', '\n', '\t', '\r' => {
439 result.start = self.index + 1;445 result.loc.start = self.index + 1;
440 },446 },
441 '"' => {447 '"' => {
442 state = .string_literal;448 state = .string_literal;
...@@ -686,7 +692,7 @@ pub const Tokenizer = struct {...@@ -686,7 +692,7 @@ pub const Tokenizer = struct {
686 .identifier => switch (c) {692 .identifier => switch (c) {
687 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},693 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
688 else => {694 else => {
689 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {695 if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| {
690 result.id = id;696 result.id = id;
691 }697 }
692 break;698 break;
...@@ -1313,7 +1319,7 @@ pub const Tokenizer = struct {...@@ -1313,7 +1319,7 @@ pub const Tokenizer = struct {
1313 => {},1319 => {},
13141320
1315 .identifier => {1321 .identifier => {
1316 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {1322 if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| {
1317 result.id = id;1323 result.id = id;
1318 }1324 }
1319 },1325 },
...@@ -1420,7 +1426,7 @@ pub const Tokenizer = struct {...@@ -1420,7 +1426,7 @@ pub const Tokenizer = struct {
1420 }1426 }
1421 }1427 }
14221428
1423 result.end = self.index;1429 result.loc.end = self.index;
1424 return result;1430 return result;
1425 }1431 }
14261432
...@@ -1430,8 +1436,10 @@ pub const Tokenizer = struct {...@@ -1430,8 +1436,10 @@ pub const Tokenizer = struct {
1430 if (invalid_length == 0) return;1436 if (invalid_length == 0) return;
1431 self.pending_invalid_token = .{1437 self.pending_invalid_token = .{
1432 .id = .Invalid,1438 .id = .Invalid,
1433 .start = self.index,1439 .loc = .{
1434 .end = self.index + invalid_length,1440 .start = self.index,
1441 .end = self.index + invalid_length,
1442 },
1435 };1443 };
1436 }1444 }
14371445
src-self-hosted/translate_c.zig-21
...@@ -247,27 +247,6 @@ pub const Context = struct {...@@ -247,27 +247,6 @@ pub const Context = struct {
247 }247 }
248 };248 };
249249
250 /// Helper function to append items to a singly linked list.
251 fn llpusher(c: *Context, list: *std.SinglyLinkedList(*ast.Node)) LinkedListPusher {
252 assert(list.first == null);
253 return .{
254 .c = c,
255 .it = &list.first,
256 };
257 }
258
259 fn llpush(
260 c: *Context,
261 comptime T: type,
262 it: *?*std.SinglyLinkedList(T).Node,
263 data: T,
264 ) !*?*std.SinglyLinkedList(T).Node {
265 const llnode = try c.arena.create(std.SinglyLinkedList(T).Node);
266 llnode.* = .{ .data = data };
267 it.* = llnode;
268 return &llnode.next;
269 }
270
271 fn getMangle(c: *Context) u32 {250 fn getMangle(c: *Context) u32 {
272 c.mangle_count += 1;251 c.mangle_count += 1;
273 return c.mangle_count;252 return c.mangle_count;