Commit 8df0841d6ab964e2aec750a8ceeda51912ae448b | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-05-22 12:34:12-04:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-05-22 12:34:12-04:00
log	8df0841d6ab964e2aec750a8ceeda51912ae448b
tree	fa2a7f30a4eaa8f94f6d5c251ff109cd4bb0427c
parent	295bca9b5f397ff98e5a0162fcb2a9f5e0a3e35c

stage2 parser: token ids in their own array

To prevent cache misses, token ids go in their own array, and the
start/end offsets go in a different one.

perf measurement before:
         2,667,914      cache-misses:u
     2,139,139,935      instructions:u
       894,167,331      cycles:u

perf measurement after:
         1,757,723      cache-misses:u
     2,069,932,298      instructions:u
       858,105,570      cycles:u

6 files changed, 217 insertions(+), 240 deletions(-)

lib
- std
  - zig
    ast.zig
    parse.zig
    parser_test.zig
    render.zig
    tokenizer.zig
src-self-hosted
- translate_c.zig

lib/std/zig/ast.zig+26-24

...	...	@@ -10,7 +10,8 @@ pub const NodeIndex = usize;
10	10	pub const Tree = struct {
11	11	/// Reference to externally-owned data.
12	12	source: []const u8,
13		tokens: []const Token,
	13	token_ids: []const Token.Id,
	14	token_locs: []const Token.Loc,
14	15	errors: []const Error,
15	16	/// undefined on parse error (when errors field is not empty)
16	17	root_node: *Node.Root,
...	...	@@ -23,26 +24,27 @@ pub const Tree = struct {
23	24	generated: bool = false,
24	25
25	26	pub fn deinit(self: *Tree) void {
26		self.gpa.free(self.tokens);
	27	self.gpa.free(self.token_ids);
	28	self.gpa.free(self.token_locs);
27	29	self.gpa.free(self.errors);
28	30	self.arena.promote(self.gpa).deinit();
29	31	}
30	32
31	33	pub fn renderError(self: Tree, parse_error: const Error, stream: var) !void {
32		return parse_error.render(self.tokens, stream);
	34	return parse_error.render(self.token_ids, stream);
33	35	}
34	36
35	37	pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 {
36		return self.tokenSlicePtr(self.tokens[token_index]);
	38	return self.tokenSliceLoc(self.token_locs[token_index]);
37	39	}
38	40
39		pub fn tokenSlicePtr(self: *Tree, token: Token) []const u8 {
	41	pub fn tokenSliceLoc(self: *Tree, token: Token.Loc) []const u8 {
40	42	return self.source[token.start..token.end];
41	43	}
42	44
43	45	pub fn getNodeSource(self: const Tree, node: const Node) []const u8 {
44		const first_token = self.tokens[node.firstToken()];
45		const last_token = self.tokens[node.lastToken()];
	46	const first_token = self.token_locs[node.firstToken()];
	47	const last_token = self.token_locs[node.lastToken()];
46	48	return self.source[first_token.start..last_token.end];
47	49	}
48	50
...	...	@@ -54,7 +56,7 @@ pub const Tree = struct {
54	56	};
55	57
56	58	/// Return the Location of the token relative to the offset specified by `start_index`.
57		pub fn tokenLocationPtr(self: *Tree, start_index: usize, token: Token) Location {
	59	pub fn tokenLocationLoc(self: *Tree, start_index: usize, token: Token.Loc) Location {
58	60	var loc = Location{
59	61	.line = 0,
60	62	.column = 0,
...	...	@@ -82,14 +84,14 @@ pub const Tree = struct {
82	84	}
83	85
84	86	pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location {
85		return self.tokenLocationPtr(start_index, self.tokens[token_index]);
	87	return self.tokenLocationLoc(start_index, self.token_locs[token_index]);
86	88	}
87	89
88	90	pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool {
89		return self.tokensOnSameLinePtr(self.tokens[token1_index], self.tokens[token2_index]);
	91	return self.tokensOnSameLineLoc(self.token_locs[token1_index], self.token_locs[token2_index]);
90	92	}
91	93
92		pub fn tokensOnSameLinePtr(self: *Tree, token1: Token, token2: Token) bool {
	94	pub fn tokensOnSameLineLoc(self: *Tree, token1: Token.Loc, token2: Token.Loc) bool {
93	95	return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null;
94	96	}
95	97
...	...	@@ -100,7 +102,7 @@ pub const Tree = struct {
100	102	/// Skips over comments
101	103	pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex {
102	104	var index = token_index - 1;
103		while (self.tokens[index].id == Token.Id.LineComment) {
	105	while (self.token_ids[index] == Token.Id.LineComment) {
104	106	index -= 1;
105	107	}
106	108	return index;
...	...	@@ -109,7 +111,7 @@ pub const Tree = struct {
109	111	/// Skips over comments
110	112	pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex {
111	113	var index = token_index + 1;
112		while (self.tokens[index].id == Token.Id.LineComment) {
	114	while (self.token_ids[index] == Token.Id.LineComment) {
113	115	index += 1;
114	116	}
115	117	return index;
...	...	@@ -166,7 +168,7 @@ pub const Error = union(enum) {
166	168	DeclBetweenFields: DeclBetweenFields,
167	169	InvalidAnd: InvalidAnd,
168	170
169		pub fn render(self: *const Error, tokens: []const Token, stream: var) !void {
	171	pub fn render(self: *const Error, tokens: []const Token.Id, stream: var) !void {
170	172	switch (self.*) {
171	173	.InvalidToken => \|*x\| return x.render(tokens, stream),
172	174	.ExpectedContainerMembers => \|*x\| return x.render(tokens, stream),
...	...	@@ -321,7 +323,7 @@ pub const Error = union(enum) {
321	323	pub const ExpectedCall = struct {
322	324	node: *Node,
323	325
324		pub fn render(self: *const ExpectedCall, tokens: []const Token, stream: var) !void {
	326	pub fn render(self: *const ExpectedCall, tokens: []const Token.Id, stream: var) !void {
325	327	return stream.print("expected " ++ @tagName(Node.Id.Call) ++ ", found {}", .{
326	328	@tagName(self.node.id),
327	329	});
...	...	@@ -331,7 +333,7 @@ pub const Error = union(enum) {
331	333	pub const ExpectedCallOrFnProto = struct {
332	334	node: *Node,
333	335
334		pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token, stream: var) !void {
	336	pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token.Id, stream: var) !void {
335	337	return stream.print("expected " ++ @tagName(Node.Id.Call) ++ " or " ++
336	338	@tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)});
337	339	}
...	...	@@ -341,14 +343,14 @@ pub const Error = union(enum) {
341	343	token: TokenIndex,
342	344	expected_id: Token.Id,
343	345
344		pub fn render(self: *const ExpectedToken, tokens: []const Token, stream: var) !void {
	346	pub fn render(self: *const ExpectedToken, tokens: []const Token.Id, stream: var) !void {
345	347	const found_token = tokens[self.token];
346		switch (found_token.id) {
	348	switch (found_token) {
347	349	.Invalid => {
348	350	return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
349	351	},
350	352	else => {
351		const token_name = found_token.id.symbol();
	353	const token_name = found_token.symbol();
352	354	return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name });
353	355	},
354	356	}
...	...	@@ -359,11 +361,11 @@ pub const Error = union(enum) {
359	361	token: TokenIndex,
360	362	end_id: Token.Id,
361	363
362		pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token, stream: var) !void {
	364	pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token.Id, stream: var) !void {
363	365	const actual_token = tokens[self.token];
364	366	return stream.print("expected ',' or '{}', found '{}'", .{
365	367	self.end_id.symbol(),
366		actual_token.id.symbol(),
	368	actual_token.symbol(),
367	369	});
368	370	}
369	371	};
...	...	@@ -374,9 +376,9 @@ pub const Error = union(enum) {
374	376
375	377	token: TokenIndex,
376	378
377		pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void {
	379	pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void {
378	380	const actual_token = tokens[self.token];
379		return stream.print(msg, .{actual_token.id.symbol()});
	381	return stream.print(msg, .{actual_token.symbol()});
380	382	}
381	383	};
382	384	}
...	...	@@ -387,7 +389,7 @@ pub const Error = union(enum) {
387	389
388	390	token: TokenIndex,
389	391
390		pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void {
	392	pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void {
391	393	return stream.writeAll(msg);
392	394	}
393	395	};

lib/std/zig/parse.zig+73-95

...	...	@@ -16,28 +16,32 @@ pub const Error = error{ParseError} \|\| Allocator.Error;
16	16	pub fn parse(gpa: Allocator, source: []const u8) Allocator.Error!Tree {
17	17	// TODO optimization idea: ensureCapacity on the tokens list and
18	18	// then appendAssumeCapacity inside the loop.
19		var tokens = std.ArrayList(Token).init(gpa);
20		defer tokens.deinit();
	19	var token_ids = std.ArrayList(Token.Id).init(gpa);
	20	defer token_ids.deinit();
	21	var token_locs = std.ArrayList(Token.Loc).init(gpa);
	22	defer token_locs.deinit();
21	23
22	24	var tokenizer = std.zig.Tokenizer.init(source);
23	25	while (true) {
24		const tree_token = try tokens.addOne();
25		tree_token.* = tokenizer.next();
26		if (tree_token.id == .Eof) break;
	26	const token = tokenizer.next();
	27	try token_ids.append(token.id);
	28	try token_locs.append(token.loc);
	29	if (token.id == .Eof) break;
27	30	}
28	31
29	32	var parser: Parser = .{
30	33	.source = source,
31	34	.arena = std.heap.ArenaAllocator.init(gpa),
32	35	.gpa = gpa,
33		.tokens = tokens.items,
	36	.token_ids = token_ids.items,
	37	.token_locs = token_locs.items,
34	38	.errors = .{},
35	39	.tok_i = 0,
36	40	};
37	41	defer parser.errors.deinit(gpa);
38	42	errdefer parser.arena.deinit();
39	43
40		while (tokens.items[parser.tok_i].id == .LineComment) parser.tok_i += 1;
	44	while (token_ids.items[parser.tok_i] == .LineComment) parser.tok_i += 1;
41	45
42	46	const root_node = try parser.parseRoot();
43	47
...	...	@@ -45,7 +49,8 @@ pub fn parse(gpa: Allocator, source: []const u8) Allocator.Error!Tree {
45	49	tree.* = .{
46	50	.gpa = gpa,
47	51	.source = source,
48		.tokens = tokens.toOwnedSlice(),
	52	.token_ids = token_ids.toOwnedSlice(),
	53	.token_locs = token_locs.toOwnedSlice(),
49	54	.errors = parser.errors.toOwnedSlice(gpa),
50	55	.root_node = root_node,
51	56	.arena = parser.arena.state,
...	...	@@ -58,9 +63,8 @@ const Parser = struct {
58	63	arena: std.heap.ArenaAllocator,
59	64	gpa: *Allocator,
60	65	source: []const u8,
61		/// TODO: Optimization idea: have this be several arrays of the token fields rather
62		/// than an array of structs.
63		tokens: []const Token,
	66	token_ids: []const Token.Id,
	67	token_locs: []const Token.Loc,
64	68	tok_i: TokenIndex,
65	69	errors: std.ArrayListUnmanaged(AstError),
66	70
...	...	@@ -80,19 +84,6 @@ const Parser = struct {
80	84	return node;
81	85	}
82	86
83		/// Helper function for appending elements to a singly linked list.
84		fn llpush(
85		p: *Parser,
86		comptime T: type,
87		it: ?std.SinglyLinkedList(T).Node,
88		data: T,
89		) !?std.SinglyLinkedList(T).Node {
90		const llnode = try p.arena.allocator.create(std.SinglyLinkedList(T).Node);
91		llnode.* = .{ .data = data };
92		it.* = llnode;
93		return &llnode.next;
94		}
95
96	87	/// ContainerMembers
97	88	/// <- TestDecl ContainerMembers
98	89	/// / TopLevelComptime ContainerMembers
...	...	@@ -228,7 +219,7 @@ const Parser = struct {
228	219	// try to continue parsing
229	220	const index = p.tok_i;
230	221	p.findNextContainerMember();
231		const next = p.tokens[p.tok_i].id;
	222	const next = p.token_ids[p.tok_i];
232	223	switch (next) {
233	224	.Eof => break,
234	225	else => {
...	...	@@ -257,7 +248,7 @@ const Parser = struct {
257	248	});
258	249	}
259	250
260		const next = p.tokens[p.tok_i].id;
	251	const next = p.token_ids[p.tok_i];
261	252	switch (next) {
262	253	.Eof => break,
263	254	.Keyword_comptime => {
...	...	@@ -291,7 +282,7 @@ const Parser = struct {
291	282	var level: u32 = 0;
292	283	while (true) {
293	284	const tok = p.nextToken();
294		switch (tok.ptr.id) {
	285	switch (p.token_ids[tok]) {
295	286	// any of these can start a new top level declaration
296	287	.Keyword_test,
297	288	.Keyword_comptime,
...	...	@@ -308,7 +299,7 @@ const Parser = struct {
308	299	.Identifier,
309	300	=> {
310	301	if (level == 0) {
311		p.putBackToken(tok.index);
	302	p.putBackToken(tok);
312	303	return;
313	304	}
314	305	},
...	...	@@ -325,13 +316,13 @@ const Parser = struct {
325	316	.RBrace => {
326	317	if (level == 0) {
327	318	// end of container, exit
328		p.putBackToken(tok.index);
	319	p.putBackToken(tok);
329	320	return;
330	321	}
331	322	level -= 1;
332	323	},
333	324	.Eof => {
334		p.putBackToken(tok.index);
	325	p.putBackToken(tok);
335	326	return;
336	327	},
337	328	else => {},
...	...	@@ -344,11 +335,11 @@ const Parser = struct {
344	335	var level: u32 = 0;
345	336	while (true) {
346	337	const tok = p.nextToken();
347		switch (tok.ptr.id) {
	338	switch (p.token_ids[tok]) {
348	339	.LBrace => level += 1,
349	340	.RBrace => {
350	341	if (level == 0) {
351		p.putBackToken(tok.index);
	342	p.putBackToken(tok);
352	343	return;
353	344	}
354	345	level -= 1;
...	...	@@ -359,7 +350,7 @@ const Parser = struct {
359	350	}
360	351	},
361	352	.Eof => {
362		p.putBackToken(tok.index);
	353	p.putBackToken(tok);
363	354	return;
364	355	},
365	356	else => {},
...	...	@@ -454,8 +445,8 @@ const Parser = struct {
454	445	}
455	446
456	447	if (extern_export_inline_token) \|token\| {
457		if (p.tokens[token].id == .Keyword_inline or
458		p.tokens[token].id == .Keyword_noinline)
	448	if (p.token_ids[token] == .Keyword_inline or
	449	p.token_ids[token] == .Keyword_noinline)
459	450	{
460	451	try p.errors.append(p.gpa, .{
461	452	.ExpectedFn = .{ .token = p.tok_i },
...	...	@@ -722,7 +713,7 @@ const Parser = struct {
722	713
723	714	const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer);
724	715	if (defer_token) \|token\| {
725		const payload = if (p.tokens[token].id == .Keyword_errdefer)
	716	const payload = if (p.token_ids[token] == .Keyword_errdefer)
726	717	try p.parsePayload()
727	718	else
728	719	null;
...	...	@@ -2269,7 +2260,7 @@ const Parser = struct {
2269	2260	/// / EQUAL
2270	2261	fn parseAssignOp(p: Parser) !?Node {
2271	2262	const token = p.nextToken();
2272		const op: Node.InfixOp.Op = switch (token.ptr.id) {
	2263	const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2273	2264	.AsteriskEqual => .AssignMul,
2274	2265	.SlashEqual => .AssignDiv,
2275	2266	.PercentEqual => .AssignMod,
...	...	@@ -2285,14 +2276,14 @@ const Parser = struct {
2285	2276	.MinusPercentEqual => .AssignSubWrap,
2286	2277	.Equal => .Assign,
2287	2278	else => {
2288		p.putBackToken(token.index);
	2279	p.putBackToken(token);
2289	2280	return null;
2290	2281	},
2291	2282	};
2292	2283
2293	2284	const node = try p.arena.allocator.create(Node.InfixOp);
2294	2285	node.* = .{
2295		.op_token = token.index,
	2286	.op_token = token,
2296	2287	.lhs = undefined, // set by caller
2297	2288	.op = op,
2298	2289	.rhs = undefined, // set by caller
...	...	@@ -2309,7 +2300,7 @@ const Parser = struct {
2309	2300	/// / RARROWEQUAL
2310	2301	fn parseCompareOp(p: Parser) !?Node {
2311	2302	const token = p.nextToken();
2312		const op: Node.InfixOp.Op = switch (token.ptr.id) {
	2303	const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2313	2304	.EqualEqual => .EqualEqual,
2314	2305	.BangEqual => .BangEqual,
2315	2306	.AngleBracketLeft => .LessThan,
...	...	@@ -2317,12 +2308,12 @@ const Parser = struct {
2317	2308	.AngleBracketLeftEqual => .LessOrEqual,
2318	2309	.AngleBracketRightEqual => .GreaterOrEqual,
2319	2310	else => {
2320		p.putBackToken(token.index);
	2311	p.putBackToken(token);
2321	2312	return null;
2322	2313	},
2323	2314	};
2324	2315
2325		return p.createInfixOp(token.index, op);
	2316	return p.createInfixOp(token, op);
2326	2317	}
2327	2318
2328	2319	/// BitwiseOp
...	...	@@ -2333,19 +2324,19 @@ const Parser = struct {
2333	2324	/// / KEYWORD_catch Payload?
2334	2325	fn parseBitwiseOp(p: Parser) !?Node {
2335	2326	const token = p.nextToken();
2336		const op: Node.InfixOp.Op = switch (token.ptr.id) {
	2327	const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2337	2328	.Ampersand => .BitAnd,
2338	2329	.Caret => .BitXor,
2339	2330	.Pipe => .BitOr,
2340	2331	.Keyword_orelse => .UnwrapOptional,
2341	2332	.Keyword_catch => .{ .Catch = try p.parsePayload() },
2342	2333	else => {
2343		p.putBackToken(token.index);
	2334	p.putBackToken(token);
2344	2335	return null;
2345	2336	},
2346	2337	};
2347	2338
2348		return p.createInfixOp(token.index, op);
	2339	return p.createInfixOp(token, op);
2349	2340	}
2350	2341
2351	2342	/// BitShiftOp
...	...	@@ -2353,16 +2344,16 @@ const Parser = struct {
2353	2344	/// / RARROW2
2354	2345	fn parseBitShiftOp(p: Parser) !?Node {
2355	2346	const token = p.nextToken();
2356		const op: Node.InfixOp.Op = switch (token.ptr.id) {
	2347	const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2357	2348	.AngleBracketAngleBracketLeft => .BitShiftLeft,
2358	2349	.AngleBracketAngleBracketRight => .BitShiftRight,
2359	2350	else => {
2360		p.putBackToken(token.index);
	2351	p.putBackToken(token);
2361	2352	return null;
2362	2353	},
2363	2354	};
2364	2355
2365		return p.createInfixOp(token.index, op);
	2356	return p.createInfixOp(token, op);
2366	2357	}
2367	2358
2368	2359	/// AdditionOp
...	...	@@ -2373,19 +2364,19 @@ const Parser = struct {
2373	2364	/// / MINUSPERCENT
2374	2365	fn parseAdditionOp(p: Parser) !?Node {
2375	2366	const token = p.nextToken();
2376		const op: Node.InfixOp.Op = switch (token.ptr.id) {
	2367	const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2377	2368	.Plus => .Add,
2378	2369	.Minus => .Sub,
2379	2370	.PlusPlus => .ArrayCat,
2380	2371	.PlusPercent => .AddWrap,
2381	2372	.MinusPercent => .SubWrap,
2382	2373	else => {
2383		p.putBackToken(token.index);
	2374	p.putBackToken(token);
2384	2375	return null;
2385	2376	},
2386	2377	};
2387	2378
2388		return p.createInfixOp(token.index, op);
	2379	return p.createInfixOp(token, op);
2389	2380	}
2390	2381
2391	2382	/// MultiplyOp
...	...	@@ -2397,7 +2388,7 @@ const Parser = struct {
2397	2388	/// / ASTERISKPERCENT
2398	2389	fn parseMultiplyOp(p: Parser) !?Node {
2399	2390	const token = p.nextToken();
2400		const op: Node.InfixOp.Op = switch (token.ptr.id) {
	2391	const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
2401	2392	.PipePipe => .MergeErrorSets,
2402	2393	.Asterisk => .Mul,
2403	2394	.Slash => .Div,
...	...	@@ -2405,12 +2396,12 @@ const Parser = struct {
2405	2396	.AsteriskAsterisk => .ArrayMult,
2406	2397	.AsteriskPercent => .MulWrap,
2407	2398	else => {
2408		p.putBackToken(token.index);
	2399	p.putBackToken(token);
2409	2400	return null;
2410	2401	},
2411	2402	};
2412	2403
2413		return p.createInfixOp(token.index, op);
	2404	return p.createInfixOp(token, op);
2414	2405	}
2415	2406
2416	2407	/// PrefixOp
...	...	@@ -2423,7 +2414,7 @@ const Parser = struct {
2423	2414	/// / KEYWORD_await
2424	2415	fn parsePrefixOp(p: Parser) !?Node {
2425	2416	const token = p.nextToken();
2426		const op: Node.PrefixOp.Op = switch (token.ptr.id) {
	2417	const op: Node.PrefixOp.Op = switch (p.token_ids[token]) {
2427	2418	.Bang => .BoolNot,
2428	2419	.Minus => .Negation,
2429	2420	.Tilde => .BitNot,
...	...	@@ -2432,14 +2423,14 @@ const Parser = struct {
2432	2423	.Keyword_try => .Try,
2433	2424	.Keyword_await => .Await,
2434	2425	else => {
2435		p.putBackToken(token.index);
	2426	p.putBackToken(token);
2436	2427	return null;
2437	2428	},
2438	2429	};
2439	2430
2440	2431	const node = try p.arena.allocator.create(Node.PrefixOp);
2441	2432	node.* = .{
2442		.op_token = token.index,
	2433	.op_token = token,
2443	2434	.op = op,
2444	2435	.rhs = undefined, // set by caller
2445	2436	};
...	...	@@ -2493,7 +2484,7 @@ const Parser = struct {
2493	2484	// If the token encountered was **, there will be two nodes instead of one.
2494	2485	// The attributes should be applied to the rightmost operator.
2495	2486	const prefix_op = node.cast(Node.PrefixOp).?;
2496		var ptr_info = if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk)
	2487	var ptr_info = if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk)
2497	2488	&prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType
2498	2489	else
2499	2490	&prefix_op.op.PtrType;
...	...	@@ -2812,7 +2803,8 @@ const Parser = struct {
2812	2803	return null;
2813	2804	};
2814	2805	if (p.eatToken(.Identifier)) \|ident\| {
2815		const token_slice = p.source[p.tokens[ident].start..p.tokens[ident].end];
	2806	const token_loc = p.token_locs[ident];
	2807	const token_slice = p.source[token_loc.start..token_loc.end];
2816	2808	if (!std.mem.eql(u8, token_slice, "c")) {
2817	2809	p.putBackToken(ident);
2818	2810	} else {
...	...	@@ -2879,7 +2871,7 @@ const Parser = struct {
2879	2871	fn parseContainerDeclType(p: *Parser) !?ContainerDeclType {
2880	2872	const kind_token = p.nextToken();
2881	2873
2882		const init_arg_expr = switch (kind_token.ptr.id) {
	2874	const init_arg_expr = switch (p.token_ids[kind_token]) {
2883	2875	.Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} },
2884	2876	.Keyword_enum => blk: {
2885	2877	if (p.eatToken(.LParen) != null) {
...	...	@@ -2914,13 +2906,13 @@ const Parser = struct {
2914	2906	break :blk Node.ContainerDecl.InitArg{ .None = {} };
2915	2907	},
2916	2908	else => {
2917		p.putBackToken(kind_token.index);
	2909	p.putBackToken(kind_token);
2918	2910	return null;
2919	2911	},
2920	2912	};
2921	2913
2922	2914	return ContainerDeclType{
2923		.kind_token = kind_token.index,
	2915	.kind_token = kind_token,
2924	2916	.init_arg_expr = init_arg_expr,
2925	2917	};
2926	2918	}
...	...	@@ -2973,7 +2965,7 @@ const Parser = struct {
2973	2965	while (try nodeParseFn(p)) \|item\| {
2974	2966	try list.append(item);
2975	2967
2976		switch (p.tokens[p.tok_i].id) {
	2968	switch (p.token_ids[p.tok_i]) {
2977	2969	.Comma => _ = p.nextToken(),
2978	2970	// all possible delimiters
2979	2971	.Colon, .RParen, .RBrace, .RBracket => break,
...	...	@@ -2994,13 +2986,13 @@ const Parser = struct {
2994	2986	fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {
2995	2987	return struct {
2996	2988	pub fn parse(p: Parser) Error!?Node {
2997		const op_token = if (token == .Keyword_and) switch (p.tokens[p.tok_i].id) {
2998		.Keyword_and => p.nextToken().index,
	2989	const op_token = if (token == .Keyword_and) switch (p.token_ids[p.tok_i]) {
	2990	.Keyword_and => p.nextToken(),
2999	2991	.Invalid_ampersands => blk: {
3000	2992	try p.errors.append(p.gpa, .{
3001	2993	.InvalidAnd = .{ .token = p.tok_i },
3002	2994	});
3003		break :blk p.nextToken().index;
	2995	break :blk p.nextToken();
3004	2996	},
3005	2997	else => return null,
3006	2998	} else p.eatToken(token) orelse return null;
...	...	@@ -3104,7 +3096,7 @@ const Parser = struct {
3104	3096	var tok_i = start_tok_i;
3105	3097	var count: usize = 1; // including first_line
3106	3098	while (true) : (tok_i += 1) {
3107		switch (p.tokens[tok_i].id) {
	3099	switch (p.token_ids[tok_i]) {
3108	3100	.LineComment => continue,
3109	3101	.MultilineStringLiteralLine => count += 1,
3110	3102	else => break,
...	...	@@ -3118,7 +3110,7 @@ const Parser = struct {
3118	3110	lines[0] = first_line;
3119	3111	count = 1;
3120	3112	while (true) : (tok_i += 1) {
3121		switch (p.tokens[tok_i].id) {
	3113	switch (p.token_ids[tok_i]) {
3122	3114	.LineComment => continue,
3123	3115	.MultilineStringLiteralLine => {
3124	3116	lines[count] = tok_i;
...	...	@@ -3215,7 +3207,7 @@ const Parser = struct {
3215	3207	}
3216	3208
3217	3209	fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {
3218		return std.mem.indexOfScalar(u8, p.source[p.tokens[token1].end..p.tokens[token2].start], '\n') == null;
	3210	return std.mem.indexOfScalar(u8, p.source[p.token_locs[token1].end..p.token_locs[token2].start], '\n') == null;
3219	3211	}
3220	3212
3221	3213	/// Eat a single-line doc comment on the same line as another node
...	...	@@ -3239,7 +3231,7 @@ const Parser = struct {
3239	3231	.PrefixOp => {
3240	3232	var prefix_op = rightmost_op.cast(Node.PrefixOp).?;
3241	3233	// If the token encountered was **, there will be two nodes
3242		if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) {
	3234	if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk) {
3243	3235	rightmost_op = prefix_op.rhs;
3244	3236	prefix_op = rightmost_op.cast(Node.PrefixOp).?;
3245	3237	}
...	...	@@ -3328,11 +3320,7 @@ const Parser = struct {
3328	3320	}
3329	3321
3330	3322	fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex {
3331		return if (p.eatAnnotatedToken(id)) \|token\| token.index else null;
3332		}
3333
3334		fn eatAnnotatedToken(p: *Parser, id: Token.Id) ?AnnotatedToken {
3335		return if (p.tokens[p.tok_i].id == id) p.nextToken() else null;
	3323	return if (p.token_ids[p.tok_i] == id) p.nextToken() else null;
3336	3324	}
3337	3325
3338	3326	fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex {
...	...	@@ -3341,29 +3329,25 @@ const Parser = struct {
3341	3329
3342	3330	fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex {
3343	3331	const token = p.nextToken();
3344		if (token.ptr.id != id) {
	3332	if (p.token_ids[token] != id) {
3345	3333	try p.errors.append(p.gpa, .{
3346		.ExpectedToken = .{ .token = token.index, .expected_id = id },
	3334	.ExpectedToken = .{ .token = token, .expected_id = id },
3347	3335	});
3348	3336	// go back so that we can recover properly
3349		p.putBackToken(token.index);
	3337	p.putBackToken(token);
3350	3338	return null;
3351	3339	}
3352		return token.index;
	3340	return token;
3353	3341	}
3354	3342
3355		fn nextToken(p: *Parser) AnnotatedToken {
3356		const result = AnnotatedToken{
3357		.index = p.tok_i,
3358		.ptr = &p.tokens[p.tok_i],
3359		};
	3343	fn nextToken(p: *Parser) TokenIndex {
	3344	const result = p.tok_i;
3360	3345	p.tok_i += 1;
3361		assert(result.ptr.id != .LineComment);
3362		if (p.tok_i >= p.tokens.len) return result;
	3346	assert(p.token_ids[result] != .LineComment);
	3347	if (p.tok_i >= p.token_ids.len) return result;
3363	3348
3364	3349	while (true) {
3365		const next_tok = p.tokens[p.tok_i];
3366		if (next_tok.id != .LineComment) return result;
	3350	if (p.token_ids[p.tok_i] != .LineComment) return result;
3367	3351	p.tok_i += 1;
3368	3352	}
3369	3353	}
...	...	@@ -3371,18 +3355,12 @@ const Parser = struct {
3371	3355	fn putBackToken(p: *Parser, putting_back: TokenIndex) void {
3372	3356	while (p.tok_i > 0) {
3373	3357	p.tok_i -= 1;
3374		const prev_tok = p.tokens[p.tok_i];
3375		if (prev_tok.id == .LineComment) continue;
	3358	if (p.token_ids[p.tok_i] == .LineComment) continue;
3376	3359	assert(putting_back == p.tok_i);
3377	3360	return;
3378	3361	}
3379	3362	}
3380	3363
3381		const AnnotatedToken = struct {
3382		index: TokenIndex,
3383		ptr: *const Token,
3384		};
3385
3386	3364	fn expectNode(
3387	3365	p: *Parser,
3388	3366	parseFn: NodeParseFn,

lib/std/zig/parser_test.zig+1-1

...	...	@@ -3181,7 +3181,7 @@ fn testParse(source: []const u8, allocator: mem.Allocator, anything_changed: b
3181	3181	defer tree.deinit();
3182	3182
3183	3183	for (tree.errors) \|*parse_error\| {
3184		const token = tree.tokens[parse_error.loc()];
	3184	const token = tree.token_locs[parse_error.loc()];
3185	3185	const loc = tree.tokenLocation(0, parse_error.loc());
3186	3186	try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 });
3187	3187	try tree.renderError(parse_error, stderr);

lib/std/zig/render.zig+99-89

...	...	@@ -68,11 +68,12 @@ fn renderRoot(
68	68	tree: *ast.Tree,
69	69	) (@TypeOf(stream).Error \|\| Error)!void {
70	70	// render all the line comments at the beginning of the file
71		for (tree.tokens) \|token, i\| {
72		if (token.id != .LineComment) break;
73		try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSlicePtr(token), " ")});
74		const next_token = &tree.tokens[i + 1];
75		const loc = tree.tokenLocationPtr(token.end, next_token.*);
	71	for (tree.token_ids) \|token_id, i\| {
	72	if (token_id != .LineComment) break;
	73	const token_loc = tree.token_locs[i];
	74	try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSliceLoc(token_loc), " ")});
	75	const next_token = tree.token_locs[i + 1];
	76	const loc = tree.tokenLocationLoc(token_loc.end, next_token);
76	77	if (loc.line >= 2) {
77	78	try stream.writeByte('\n');
78	79	}
...	...	@@ -101,8 +102,8 @@ fn renderRoot(
101	102
102	103	while (token_index != 0) {
103	104	token_index -= 1;
104		const token = tree.tokens[token_index];
105		switch (token.id) {
	105	const token_id = tree.token_ids[token_index];
	106	switch (token_id) {
106	107	.LineComment => {},
107	108	.DocComment => {
108	109	copy_start_token_index = token_index;
...	...	@@ -111,12 +112,13 @@ fn renderRoot(
111	112	else => break,
112	113	}
113	114
114		if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) {
	115	const token_loc = tree.token_locs[token_index];
	116	if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) {
115	117	if (!found_fmt_directive) {
116	118	fmt_active = false;
117	119	found_fmt_directive = true;
118	120	}
119		} else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) {
	121	} else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) {
120	122	if (!found_fmt_directive) {
121	123	fmt_active = true;
122	124	found_fmt_directive = true;
...	...	@@ -135,7 +137,7 @@ fn renderRoot(
135	137	if (decl_i >= root_decls.len) {
136	138	// If there's no next reformatted `decl`, just copy the
137	139	// remaining input tokens and bail out.
138		const start = tree.tokens[copy_start_token_index].start;
	140	const start = tree.token_locs[copy_start_token_index].start;
139	141	try copyFixingWhitespace(stream, tree.source[start..]);
140	142	return;
141	143	}
...	...	@@ -143,15 +145,16 @@ fn renderRoot(
143	145	var decl_first_token_index = decl.firstToken();
144	146
145	147	while (token_index < decl_first_token_index) : (token_index += 1) {
146		const token = tree.tokens[token_index];
147		switch (token.id) {
	148	const token_id = tree.token_ids[token_index];
	149	switch (token_id) {
148	150	.LineComment => {},
149	151	.Eof => unreachable,
150	152	else => continue,
151	153	}
152		if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) {
	154	const token_loc = tree.token_locs[token_index];
	155	if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) {
153	156	fmt_active = true;
154		} else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) {
	157	} else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) {
155	158	fmt_active = false;
156	159	}
157	160	}
...	...	@@ -163,8 +166,8 @@ fn renderRoot(
163	166	token_index = copy_end_token_index;
164	167	while (token_index != 0) {
165	168	token_index -= 1;
166		const token = tree.tokens[token_index];
167		switch (token.id) {
	169	const token_id = tree.token_ids[token_index];
	170	switch (token_id) {
168	171	.LineComment => {},
169	172	.DocComment => {
170	173	copy_end_token_index = token_index;
...	...	@@ -174,8 +177,8 @@ fn renderRoot(
174	177	}
175	178	}
176	179
177		const start = tree.tokens[copy_start_token_index].start;
178		const end = tree.tokens[copy_end_token_index].start;
	180	const start = tree.token_locs[copy_start_token_index].start;
	181	const end = tree.token_locs[copy_end_token_index].start;
179	182	try copyFixingWhitespace(stream, tree.source[start..end]);
180	183	}
181	184
...	...	@@ -194,13 +197,13 @@ fn renderExtraNewlineToken(tree: ast.Tree, stream: var, start_col: usize, firs
194	197	var prev_token = first_token;
195	198	if (prev_token == 0) return;
196	199	var newline_threshold: usize = 2;
197		while (tree.tokens[prev_token - 1].id == .DocComment) {
198		if (tree.tokenLocation(tree.tokens[prev_token - 1].end, prev_token).line == 1) {
	200	while (tree.token_ids[prev_token - 1] == .DocComment) {
	201	if (tree.tokenLocation(tree.token_locs[prev_token - 1].end, prev_token).line == 1) {
199	202	newline_threshold += 1;
200	203	}
201	204	prev_token -= 1;
202	205	}
203		const prev_token_end = tree.tokens[prev_token - 1].end;
	206	const prev_token_end = tree.token_locs[prev_token - 1].end;
204	207	const loc = tree.tokenLocation(prev_token_end, first_token);
205	208	if (loc.line >= newline_threshold) {
206	209	try stream.writeByte('\n');
...	...	@@ -265,7 +268,7 @@ fn renderContainerDecl(allocator: mem.Allocator, stream: var, tree: ast.Tree,
265	268
266	269	const src_has_trailing_comma = blk: {
267	270	const maybe_comma = tree.nextToken(field.lastToken());
268		break :blk tree.tokens[maybe_comma].id == .Comma;
	271	break :blk tree.token_ids[maybe_comma] == .Comma;
269	272	};
270	273
271	274	// The trailing comma is emitted at the end, but if it's not present
...	...	@@ -327,11 +330,11 @@ fn renderContainerDecl(allocator: mem.Allocator, stream: var, tree: ast.Tree,
327	330
328	331	.DocComment => {
329	332	const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
330		const kind = tree.tokens[comment.first_line].id;
	333	const kind = tree.token_ids[comment.first_line];
331	334	try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline);
332	335	var tok_i = comment.first_line + 1;
333	336	while (true) : (tok_i += 1) {
334		const tok_id = tree.tokens[tok_i].id;
	337	const tok_id = tree.token_ids[tok_i];
335	338	if (tok_id == kind) {
336	339	try stream.writeByteNTimes(' ', indent);
337	340	try renderToken(tree, stream, tok_i, indent, start_col, .Newline);
...	...	@@ -436,13 +439,13 @@ fn renderExpression(
436	439	try renderExpression(allocator, stream, tree, indent, start_col, infix_op_node.lhs, op_space);
437	440
438	441	const after_op_space = blk: {
439		const loc = tree.tokenLocation(tree.tokens[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token));
	442	const loc = tree.tokenLocation(tree.token_locs[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token));
440	443	break :blk if (loc.line == 0) op_space else Space.Newline;
441	444	};
442	445
443	446	try renderToken(tree, stream, infix_op_node.op_token, indent, start_col, after_op_space);
444	447	if (after_op_space == Space.Newline and
445		tree.tokens[tree.nextToken(infix_op_node.op_token)].id != .MultilineStringLiteralLine)
	448	tree.token_ids[tree.nextToken(infix_op_node.op_token)] != .MultilineStringLiteralLine)
446	449	{
447	450	try stream.writeByteNTimes(' ', indent + indent_delta);
448	451	start_col.* = indent + indent_delta;
...	...	@@ -463,10 +466,10 @@ fn renderExpression(
463	466
464	467	switch (prefix_op_node.op) {
465	468	.PtrType => \|ptr_info\| {
466		const op_tok_id = tree.tokens[prefix_op_node.op_token].id;
	469	const op_tok_id = tree.token_ids[prefix_op_node.op_token];
467	470	switch (op_tok_id) {
468	471	.Asterisk, .AsteriskAsterisk => try stream.writeByte('*'),
469		.LBracket => if (tree.tokens[prefix_op_node.op_token + 2].id == .Identifier)
	472	.LBracket => if (tree.token_ids[prefix_op_node.op_token + 2] == .Identifier)
470	473	try stream.writeAll("[*c")
471	474	else
472	475	try stream.writeAll("[*"),
...	...	@@ -578,8 +581,8 @@ fn renderExpression(
578	581
579	582	try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
580	583
581		const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment;
582		const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment;
	584	const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment;
	585	const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment;
583	586	const new_indent = if (ends_with_comment) indent + indent_delta else indent;
584	587	const new_space = if (ends_with_comment) Space.Newline else Space.None;
585	588	try renderExpression(allocator, stream, tree, new_indent, start_col, array_info.len_expr, new_space);
...	...	@@ -653,7 +656,7 @@ fn renderExpression(
653	656	return renderToken(tree, stream, rtoken, indent, start_col, space);
654	657	}
655	658
656		if (exprs.len == 1 and tree.tokens[exprs[0].lastToken() + 1].id == .RBrace) {
	659	if (exprs.len == 1 and tree.token_ids[exprs[0].lastToken() + 1] == .RBrace) {
657	660	const expr = exprs[0];
658	661	switch (lhs) {
659	662	.dot => \|dot\| try renderToken(tree, stream, dot, indent, start_col, Space.None),
...	...	@@ -675,17 +678,17 @@ fn renderExpression(
675	678	for (exprs) \|expr, i\| {
676	679	if (i + 1 < exprs.len) {
677	680	const expr_last_token = expr.lastToken() + 1;
678		const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, exprs[i+1].firstToken());
	681	const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i+1].firstToken());
679	682	if (loc.line != 0) break :blk count;
680	683	count += 1;
681	684	} else {
682	685	const expr_last_token = expr.lastToken();
683		const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, rtoken);
	686	const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken);
684	687	if (loc.line == 0) {
685	688	// all on one line
686	689	const src_has_trailing_comma = trailblk: {
687	690	const maybe_comma = tree.prevToken(rtoken);
688		break :trailblk tree.tokens[maybe_comma].id == .Comma;
	691	break :trailblk tree.token_ids[maybe_comma] == .Comma;
689	692	};
690	693	if (src_has_trailing_comma) {
691	694	break :blk 1; // force row size 1
...	...	@@ -723,7 +726,7 @@ fn renderExpression(
723	726
724	727	var new_indent = indent + indent_delta;
725	728
726		if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) {
	729	if (tree.token_ids[tree.nextToken(lbrace)] != .MultilineStringLiteralLine) {
727	730	try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline);
728	731	try stream.writeByteNTimes(' ', new_indent);
729	732	} else {
...	...	@@ -750,7 +753,7 @@ fn renderExpression(
750	753	}
751	754	col = 1;
752	755
753		if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) {
	756	if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) {
754	757	try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // ,
755	758	} else {
756	759	try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // ,
...	...	@@ -819,11 +822,11 @@ fn renderExpression(
819	822
820	823	const src_has_trailing_comma = blk: {
821	824	const maybe_comma = tree.prevToken(rtoken);
822		break :blk tree.tokens[maybe_comma].id == .Comma;
	825	break :blk tree.token_ids[maybe_comma] == .Comma;
823	826	};
824	827
825	828	const src_same_line = blk: {
826		const loc = tree.tokenLocation(tree.tokens[lbrace].end, rtoken);
	829	const loc = tree.tokenLocation(tree.token_locs[lbrace].end, rtoken);
827	830	break :blk loc.line == 0;
828	831	};
829	832
...	...	@@ -929,7 +932,7 @@ fn renderExpression(
929	932
930	933	const src_has_trailing_comma = blk: {
931	934	const maybe_comma = tree.prevToken(call.rtoken);
932		break :blk tree.tokens[maybe_comma].id == .Comma;
	935	break :blk tree.token_ids[maybe_comma] == .Comma;
933	936	};
934	937
935	938	if (src_has_trailing_comma) {
...	...	@@ -983,8 +986,8 @@ fn renderExpression(
983	986	try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);
984	987	try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
985	988
986		const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment;
987		const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment;
	989	const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment;
	990	const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment;
988	991	const new_indent = if (ends_with_comment) indent + indent_delta else indent;
989	992	const new_space = if (ends_with_comment) Space.Newline else Space.None;
990	993	try renderExpression(allocator, stream, tree, new_indent, start_col, index_expr, new_space);
...	...	@@ -1226,9 +1229,9 @@ fn renderExpression(
1226	1229	var maybe_comma = tree.prevToken(container_decl.lastToken());
1227	1230	// Doc comments for a field may also appear after the comma, eg.
1228	1231	// field_name: T, // comment attached to field_name
1229		if (tree.tokens[maybe_comma].id == .DocComment)
	1232	if (tree.token_ids[maybe_comma] == .DocComment)
1230	1233	maybe_comma = tree.prevToken(maybe_comma);
1231		break :blk tree.tokens[maybe_comma].id == .Comma;
	1234	break :blk tree.token_ids[maybe_comma] == .Comma;
1232	1235	};
1233	1236
1234	1237	const fields_and_decls = container_decl.fieldsAndDecls();
...	...	@@ -1321,7 +1324,7 @@ fn renderExpression(
1321	1324
1322	1325	const src_has_trailing_comma = blk: {
1323	1326	const maybe_comma = tree.prevToken(err_set_decl.rbrace_token);
1324		break :blk tree.tokens[maybe_comma].id == .Comma;
	1327	break :blk tree.token_ids[maybe_comma] == .Comma;
1325	1328	};
1326	1329
1327	1330	if (src_has_trailing_comma) {
...	...	@@ -1353,7 +1356,7 @@ fn renderExpression(
1353	1356	try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None);
1354	1357
1355	1358	const comma_token = tree.nextToken(node.lastToken());
1356		assert(tree.tokens[comma_token].id == .Comma);
	1359	assert(tree.token_ids[comma_token] == .Comma);
1357	1360	try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // ,
1358	1361	try renderExtraNewline(tree, stream, start_col, decls[i + 1]);
1359	1362	} else {
...	...	@@ -1378,7 +1381,7 @@ fn renderExpression(
1378	1381	const multiline_str_literal = @fieldParentPtr(ast.Node.MultilineStringLiteral, "base", base);
1379	1382
1380	1383	var skip_first_indent = true;
1381		if (tree.tokens[multiline_str_literal.firstToken() - 1].id != .LineComment) {
	1384	if (tree.token_ids[multiline_str_literal.firstToken() - 1] != .LineComment) {
1382	1385	try stream.print("\n", .{});
1383	1386	skip_first_indent = false;
1384	1387	}
...	...	@@ -1406,7 +1409,7 @@ fn renderExpression(
1406	1409	if (builtin_call.params_len < 2) break :blk false;
1407	1410	const last_node = builtin_call.params()[builtin_call.params_len - 1];
1408	1411	const maybe_comma = tree.nextToken(last_node.lastToken());
1409		break :blk tree.tokens[maybe_comma].id == .Comma;
	1412	break :blk tree.token_ids[maybe_comma] == .Comma;
1410	1413	};
1411	1414
1412	1415	const lparen = tree.nextToken(builtin_call.builtin_token);
...	...	@@ -1443,8 +1446,8 @@ fn renderExpression(
1443	1446	const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", base);
1444	1447
1445	1448	if (fn_proto.visib_token) \|visib_token_index\| {
1446		const visib_token = tree.tokens[visib_token_index];
1447		assert(visib_token.id == .Keyword_pub or visib_token.id == .Keyword_export);
	1449	const visib_token = tree.token_ids[visib_token_index];
	1450	assert(visib_token == .Keyword_pub or visib_token == .Keyword_export);
1448	1451
1449	1452	try renderToken(tree, stream, visib_token_index, indent, start_col, Space.Space); // pub
1450	1453	}
...	...	@@ -1466,7 +1469,7 @@ fn renderExpression(
1466	1469	try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn
1467	1470	break :blk tree.nextToken(fn_proto.fn_token);
1468	1471	};
1469		assert(tree.tokens[lparen].id == .LParen);
	1472	assert(tree.token_ids[lparen] == .LParen);
1470	1473
1471	1474	const rparen = tree.prevToken(
1472	1475	// the first token for the annotation expressions is the left
...	...	@@ -1482,10 +1485,10 @@ fn renderExpression(
1482	1485	.InferErrorSet => \|node\| tree.prevToken(node.firstToken()),
1483	1486	.Invalid => unreachable,
1484	1487	});
1485		assert(tree.tokens[rparen].id == .RParen);
	1488	assert(tree.token_ids[rparen] == .RParen);
1486	1489
1487	1490	const src_params_trailing_comma = blk: {
1488		const maybe_comma = tree.tokens[rparen - 1].id;
	1491	const maybe_comma = tree.token_ids[rparen - 1];
1489	1492	break :blk maybe_comma == .Comma or maybe_comma == .LineComment;
1490	1493	};
1491	1494
...	...	@@ -1622,7 +1625,7 @@ fn renderExpression(
1622	1625	const src_has_trailing_comma = blk: {
1623	1626	const last_node = switch_case.items()[switch_case.items_len - 1];
1624	1627	const maybe_comma = tree.nextToken(last_node.lastToken());
1625		break :blk tree.tokens[maybe_comma].id == .Comma;
	1628	break :blk tree.token_ids[maybe_comma] == .Comma;
1626	1629	};
1627	1630
1628	1631	if (switch_case.items_len == 1 or !src_has_trailing_comma) {
...	...	@@ -1967,7 +1970,7 @@ fn renderExpression(
1967	1970	try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline);
1968	1971	try stream.writeByteNTimes(' ', indent_once);
1969	1972	const comma_or_colon = tree.nextToken(asm_output.lastToken());
1970		break :blk switch (tree.tokens[comma_or_colon].id) {
	1973	break :blk switch (tree.token_ids[comma_or_colon]) {
1971	1974	.Comma => tree.nextToken(comma_or_colon),
1972	1975	else => comma_or_colon,
1973	1976	};
...	...	@@ -2002,7 +2005,7 @@ fn renderExpression(
2002	2005	try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline);
2003	2006	try stream.writeByteNTimes(' ', indent_once);
2004	2007	const comma_or_colon = tree.nextToken(asm_input.lastToken());
2005		break :blk switch (tree.tokens[comma_or_colon].id) {
	2008	break :blk switch (tree.token_ids[comma_or_colon]) {
2006	2009	.Comma => tree.nextToken(comma_or_colon),
2007	2010	else => comma_or_colon,
2008	2011	};
...	...	@@ -2205,7 +2208,7 @@ fn renderStatement(
2205	2208	try renderExpression(allocator, stream, tree, indent, start_col, base, Space.None);
2206	2209
2207	2210	const semicolon_index = tree.nextToken(base.lastToken());
2208		assert(tree.tokens[semicolon_index].id == .Semicolon);
	2211	assert(tree.token_ids[semicolon_index] == .Semicolon);
2209	2212	try renderToken(tree, stream, semicolon_index, indent, start_col, Space.Newline);
2210	2213	} else {
2211	2214	try renderExpression(allocator, stream, tree, indent, start_col, base, Space.Newline);
...	...	@@ -2243,22 +2246,25 @@ fn renderTokenOffset(
2243	2246	return;
2244	2247	}
2245	2248
2246		var token = tree.tokens[token_index];
2247		try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " "));
	2249	var token_loc = tree.token_locs[token_index];
	2250	try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(token_loc)[token_skip_bytes..], " "));
2248	2251
2249	2252	if (space == Space.NoComment)
2250	2253	return;
2251	2254
2252		var next_token = tree.tokens[token_index + 1];
	2255	var next_token_id = tree.token_ids[token_index + 1];
	2256	var next_token_loc = tree.token_locs[token_index + 1];
2253	2257
2254		if (space == Space.Comma) switch (next_token.id) {
	2258	if (space == Space.Comma) switch (next_token_id) {
2255	2259	.Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline),
2256	2260	.LineComment => {
2257	2261	try stream.writeAll(", ");
2258	2262	return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline);
2259	2263	},
2260	2264	else => {
2261		if (token_index + 2 < tree.tokens.len and tree.tokens[token_index + 2].id == .MultilineStringLiteralLine) {
	2265	if (token_index + 2 < tree.token_ids.len and
	2266	tree.token_ids[token_index + 2] == .MultilineStringLiteralLine)
	2267	{
2262	2268	try stream.writeAll(",");
2263	2269	return;
2264	2270	} else {
...	...	@@ -2271,19 +2277,20 @@ fn renderTokenOffset(
2271	2277
2272	2278	// Skip over same line doc comments
2273	2279	var offset: usize = 1;
2274		if (next_token.id == .DocComment) {
2275		const loc = tree.tokenLocationPtr(token.end, next_token);
	2280	if (next_token_id == .DocComment) {
	2281	const loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
2276	2282	if (loc.line == 0) {
2277	2283	offset += 1;
2278		next_token = tree.tokens[token_index + offset];
	2284	next_token_id = tree.token_ids[token_index + offset];
	2285	next_token_loc = tree.token_locs[token_index + offset];
2279	2286	}
2280	2287	}
2281	2288
2282		if (next_token.id != .LineComment) blk: {
	2289	if (next_token_id != .LineComment) blk: {
2283	2290	switch (space) {
2284	2291	Space.None, Space.NoNewline => return,
2285	2292	Space.Newline => {
2286		if (next_token.id == .MultilineStringLiteralLine) {
	2293	if (next_token_id == .MultilineStringLiteralLine) {
2287	2294	return;
2288	2295	} else {
2289	2296	try stream.writeAll("\n");
...	...	@@ -2292,7 +2299,7 @@ fn renderTokenOffset(
2292	2299	}
2293	2300	},
2294	2301	Space.Space, Space.SpaceOrOutdent => {
2295		if (next_token.id == .MultilineStringLiteralLine)
	2302	if (next_token_id == .MultilineStringLiteralLine)
2296	2303	return;
2297	2304	try stream.writeByte(' ');
2298	2305	return;
...	...	@@ -2302,14 +2309,15 @@ fn renderTokenOffset(
2302	2309	}
2303	2310
2304	2311	while (true) {
2305		const comment_is_empty = mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ").len == 2;
	2312	const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ").len == 2;
2306	2313	if (comment_is_empty) {
2307	2314	switch (space) {
2308	2315	Space.Newline => {
2309	2316	offset += 1;
2310		token = next_token;
2311		next_token = tree.tokens[token_index + offset];
2312		if (next_token.id != .LineComment) {
	2317	token_loc = next_token_loc;
	2318	next_token_id = tree.token_ids[token_index + offset];
	2319	next_token_loc = tree.token_locs[token_index + offset];
	2320	if (next_token_id != .LineComment) {
2313	2321	try stream.writeByte('\n');
2314	2322	start_col.* = 0;
2315	2323	return;
...	...	@@ -2322,18 +2330,19 @@ fn renderTokenOffset(
2322	2330	}
2323	2331	}
2324	2332
2325		var loc = tree.tokenLocationPtr(token.end, next_token);
	2333	var loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
2326	2334	if (loc.line == 0) {
2327		try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")});
	2335	try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ")});
2328	2336	offset = 2;
2329		token = next_token;
2330		next_token = tree.tokens[token_index + offset];
2331		if (next_token.id != .LineComment) {
	2337	token_loc = next_token_loc;
	2338	next_token_loc = tree.token_locs[token_index + offset];
	2339	next_token_id = tree.token_ids[token_index + offset];
	2340	if (next_token_id != .LineComment) {
2332	2341	switch (space) {
2333	2342	Space.None, Space.Space => {
2334	2343	try stream.writeByte('\n');
2335		const after_comment_token = tree.tokens[token_index + offset];
2336		const next_line_indent = switch (after_comment_token.id) {
	2344	const after_comment_token = tree.token_ids[token_index + offset];
	2345	const next_line_indent = switch (after_comment_token) {
2337	2346	.RParen, .RBrace, .RBracket => indent,
2338	2347	else => indent + indent_delta,
2339	2348	};
...	...	@@ -2346,7 +2355,7 @@ fn renderTokenOffset(
2346	2355	start_col.* = indent;
2347	2356	},
2348	2357	Space.Newline => {
2349		if (next_token.id == .MultilineStringLiteralLine) {
	2358	if (next_token_id == .MultilineStringLiteralLine) {
2350	2359	return;
2351	2360	} else {
2352	2361	try stream.writeAll("\n");
...	...	@@ -2359,7 +2368,7 @@ fn renderTokenOffset(
2359	2368	}
2360	2369	return;
2361	2370	}
2362		loc = tree.tokenLocationPtr(token.end, next_token);
	2371	loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
2363	2372	}
2364	2373
2365	2374	while (true) {
...	...	@@ -2369,15 +2378,16 @@ fn renderTokenOffset(
2369	2378	const newline_count = if (loc.line <= 1) @as(u8, 1) else @as(u8, 2);
2370	2379	try stream.writeByteNTimes('\n', newline_count);
2371	2380	try stream.writeByteNTimes(' ', indent);
2372		try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " "));
	2381	try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " "));
2373	2382
2374	2383	offset += 1;
2375		token = next_token;
2376		next_token = tree.tokens[token_index + offset];
2377		if (next_token.id != .LineComment) {
	2384	token_loc = next_token_loc;
	2385	next_token_loc = tree.token_locs[token_index + offset];
	2386	next_token_id = tree.token_ids[token_index + offset];
	2387	if (next_token_id != .LineComment) {
2378	2388	switch (space) {
2379	2389	Space.Newline => {
2380		if (next_token.id == .MultilineStringLiteralLine) {
	2390	if (next_token_id == .MultilineStringLiteralLine) {
2381	2391	return;
2382	2392	} else {
2383	2393	try stream.writeAll("\n");
...	...	@@ -2388,8 +2398,8 @@ fn renderTokenOffset(
2388	2398	Space.None, Space.Space => {
2389	2399	try stream.writeByte('\n');
2390	2400
2391		const after_comment_token = tree.tokens[token_index + offset];
2392		const next_line_indent = switch (after_comment_token.id) {
	2401	const after_comment_token = tree.token_ids[token_index + offset];
	2402	const next_line_indent = switch (after_comment_token) {
2393	2403	.RParen, .RBrace, .RBracket => blk: {
2394	2404	if (indent > indent_delta) {
2395	2405	break :blk indent - indent_delta;
...	...	@@ -2412,7 +2422,7 @@ fn renderTokenOffset(
2412	2422	}
2413	2423	return;
2414	2424	}
2415		loc = tree.tokenLocationPtr(token.end, next_token);
	2425	loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
2416	2426	}
2417	2427	}
2418	2428
...	...	@@ -2448,7 +2458,7 @@ fn renderDocCommentsToken(
2448	2458	) (@TypeOf(stream).Error \|\| Error)!void {
2449	2459	var tok_i = comment.first_line;
2450	2460	while (true) : (tok_i += 1) {
2451		switch (tree.tokens[tok_i].id) {
	2461	switch (tree.token_ids[tok_i]) {
2452	2462	.DocComment, .ContainerDocComment => {
2453	2463	if (comment.first_line < first_token) {
2454	2464	try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline);

lib/std/zig/tokenizer.zig+18-10

...	...	@@ -3,8 +3,12 @@ const mem = std.mem;
3	3
4	4	pub const Token = struct {
5	5	id: Id,
6		start: usize,
7		end: usize,
	6	loc: Loc,
	7
	8	pub const Loc = struct {
	9	start: usize,
	10	end: usize,
	11	};
8	12
9	13	pub const Keyword = struct {
10	14	bytes: []const u8,
...	...	@@ -426,8 +430,10 @@ pub const Tokenizer = struct {
426	430	var state: State = .start;
427	431	var result = Token{
428	432	.id = .Eof,
429		.start = self.index,
430		.end = undefined,
	433	.loc = .{
	434	.start = self.index,
	435	.end = undefined,
	436	},
431	437	};
432	438	var seen_escape_digits: usize = undefined;
433	439	var remaining_code_units: usize = undefined;
...	...	@@ -436,7 +442,7 @@ pub const Tokenizer = struct {
436	442	switch (state) {
437	443	.start => switch (c) {
438	444	' ', '\n', '\t', '\r' => {
439		result.start = self.index + 1;
	445	result.loc.start = self.index + 1;
440	446	},
441	447	'"' => {
442	448	state = .string_literal;
...	...	@@ -686,7 +692,7 @@ pub const Tokenizer = struct {
686	692	.identifier => switch (c) {
687	693	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
688	694	else => {
689		if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
	695	if (Token.getKeyword(self.buffer[result.loc.start..self.index])) \|id\| {
690	696	result.id = id;
691	697	}
692	698	break;
...	...	@@ -1313,7 +1319,7 @@ pub const Tokenizer = struct {
1313	1319	=> {},
1314	1320
1315	1321	.identifier => {
1316		if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
	1322	if (Token.getKeyword(self.buffer[result.loc.start..self.index])) \|id\| {
1317	1323	result.id = id;
1318	1324	}
1319	1325	},
...	...	@@ -1420,7 +1426,7 @@ pub const Tokenizer = struct {
1420	1426	}
1421	1427	}
1422	1428
1423		result.end = self.index;
	1429	result.loc.end = self.index;
1424	1430	return result;
1425	1431	}
1426	1432
...	...	@@ -1430,8 +1436,10 @@ pub const Tokenizer = struct {
1430	1436	if (invalid_length == 0) return;
1431	1437	self.pending_invalid_token = .{
1432	1438	.id = .Invalid,
1433		.start = self.index,
1434		.end = self.index + invalid_length,
	1439	.loc = .{
	1440	.start = self.index,
	1441	.end = self.index + invalid_length,
	1442	},
1435	1443	};
1436	1444	}
1437	1445

src-self-hosted/translate_c.zig-21

...	...	@@ -247,27 +247,6 @@ pub const Context = struct {
247	247	}
248	248	};
249	249
250		/// Helper function to append items to a singly linked list.
251		fn llpusher(c: Context, list: std.SinglyLinkedList(*ast.Node)) LinkedListPusher {
252		assert(list.first == null);
253		return .{
254		.c = c,
255		.it = &list.first,
256		};
257		}
258
259		fn llpush(
260		c: *Context,
261		comptime T: type,
262		it: ?std.SinglyLinkedList(T).Node,
263		data: T,
264		) !?std.SinglyLinkedList(T).Node {
265		const llnode = try c.arena.create(std.SinglyLinkedList(T).Node);
266		llnode.* = .{ .data = data };
267		it.* = llnode;
268		return &llnode.next;
269		}
270
271	250	fn getMangle(c: *Context) u32 {
272	251	c.mangle_count += 1;
273	252	return c.mangle_count;