authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-30 18:03:54-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-07-02 13:27:35-07:00
log3f680abbe2c4d2eeefd0eb73b8af25d1768e6ceb
tree06d67e5aa40cede2686d492445d4d32c2f69a33c
parent8ce880ca753ce95138bf03d956cf363ea2dfde5a

stage2: tokenizer: require null terminated source

By requiring the source file to be null-terminated, we avoid extra branching while simplifying the logic at the same time. Running ast-check on a large zig source file (udivmodti4_test.zig), master branch compared to this commit: * 4% faster wall clock * 7% fewer cache misses * 1% fewer branches

6 files changed, 56 insertions(+), 181 deletions(-)

lib/std/zig/ast.zig+1-1
......@@ -20,7 +20,7 @@ pub const NodeList = std.MultiArrayList(Node);
2020
2121pub const Tree = struct {
2222 /// Reference to externally-owned data.
23 source: []const u8,
23 source: [:0]const u8,
2424
2525 tokens: TokenList.Slice,
2626 /// The root AST node is assumed to be index 0. Since there can be no
lib/std/zig/parse.zig+1-1
......@@ -17,7 +17,7 @@ pub const Error = error{ParseError} || Allocator.Error;
1717
1818/// Result should be freed with tree.deinit() when there are
1919/// no more references to any of the tokens or nodes.
20pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!Tree {
20pub fn parse(gpa: *Allocator, source: [:0]const u8) Allocator.Error!Tree {
2121 var tokens = ast.TokenList{};
2222 defer tokens.deinit(gpa);
2323
lib/std/zig/parser_test.zig+4-4
......@@ -5194,7 +5194,7 @@ const maxInt = std.math.maxInt;
51945194
51955195var fixed_buffer_mem: [100 * 1024]u8 = undefined;
51965196
5197fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *bool) ![]u8 {
5197fn testParse(source: [:0]const u8, allocator: *mem.Allocator, anything_changed: *bool) ![]u8 {
51985198 const stderr = io.getStdErr().writer();
51995199
52005200 var tree = try std.zig.parse(allocator, source);
......@@ -5222,7 +5222,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b
52225222 anything_changed.* = !mem.eql(u8, formatted, source);
52235223 return formatted;
52245224}
5225fn testTransform(source: []const u8, expected_source: []const u8) !void {
5225fn testTransform(source: [:0]const u8, expected_source: []const u8) !void {
52265226 const needed_alloc_count = x: {
52275227 // Try it once with unlimited memory, make sure it works
52285228 var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
......@@ -5268,13 +5268,13 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
52685268 }
52695269 }
52705270}
5271fn testCanonical(source: []const u8) !void {
5271fn testCanonical(source: [:0]const u8) !void {
52725272 return testTransform(source, source);
52735273}
52745274
52755275const Error = std.zig.ast.Error.Tag;
52765276
5277fn testError(source: []const u8, expected_errors: []const Error) !void {
5277fn testError(source: [:0]const u8, expected_errors: []const Error) !void {
52785278 var tree = try std.zig.parse(std.testing.allocator, source);
52795279 defer tree.deinit(std.testing.allocator);
52805280
lib/std/zig/tokenizer.zig+48-173
......@@ -326,7 +326,7 @@ pub const Token = struct {
326326};
327327
328328pub const Tokenizer = struct {
329 buffer: []const u8,
329 buffer: [:0]const u8,
330330 index: usize,
331331 pending_invalid_token: ?Token,
332332
......@@ -335,7 +335,7 @@ pub const Tokenizer = struct {
335335 std.debug.warn("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.start..token.end] });
336336 }
337337
338 pub fn init(buffer: []const u8) Tokenizer {
338 pub fn init(buffer: [:0]const u8) Tokenizer {
339339 // Skip the UTF-8 BOM if present
340340 const src_start = if (mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else @as(usize, 0);
341341 return Tokenizer{
......@@ -373,7 +373,6 @@ pub const Tokenizer = struct {
373373 line_comment,
374374 doc_comment_start,
375375 doc_comment,
376 container_doc_comment,
377376 zero,
378377 int_literal_dec,
379378 int_literal_dec_no_underscore,
......@@ -407,10 +406,6 @@ pub const Tokenizer = struct {
407406 saw_at_sign,
408407 };
409408
410 fn isIdentifierChar(char: u8) bool {
411 return std.ascii.isAlNum(char) or char == '_';
412 }
413
414409 pub fn next(self: *Tokenizer) Token {
415410 if (self.pending_invalid_token) |token| {
416411 self.pending_invalid_token = null;
......@@ -426,10 +421,11 @@ pub const Tokenizer = struct {
426421 };
427422 var seen_escape_digits: usize = undefined;
428423 var remaining_code_units: usize = undefined;
429 while (self.index < self.buffer.len) : (self.index += 1) {
424 while (true) : (self.index += 1) {
430425 const c = self.buffer[self.index];
431426 switch (state) {
432427 .start => switch (c) {
428 0 => break,
433429 ' ', '\n', '\t', '\r' => {
434430 result.loc.start = self.index + 1;
435431 },
......@@ -705,18 +701,22 @@ pub const Tokenizer = struct {
705701 self.index += 1;
706702 break;
707703 },
708 '\n', '\r' => break, // Look for this error later.
704 0, '\n', '\r' => break, // Look for this error later.
709705 else => self.checkLiteralCharacter(),
710706 },
711707
712708 .string_literal_backslash => switch (c) {
713 '\n', '\r' => break, // Look for this error later.
709 0, '\n', '\r' => break, // Look for this error later.
714710 else => {
715711 state = .string_literal;
716712 },
717713 },
718714
719715 .char_literal => switch (c) {
716 0 => {
717 result.tag = .invalid;
718 break;
719 },
720720 '\\' => {
721721 state = .char_literal_backslash;
722722 },
......@@ -742,7 +742,7 @@ pub const Tokenizer = struct {
742742 },
743743
744744 .char_literal_backslash => switch (c) {
745 '\n' => {
745 0, '\n' => {
746746 result.tag = .invalid;
747747 break;
748748 },
......@@ -834,6 +834,7 @@ pub const Tokenizer = struct {
834834 },
835835
836836 .multiline_string_literal_line => switch (c) {
837 0 => break,
837838 '\n' => {
838839 self.index += 1;
839840 break;
......@@ -1025,12 +1026,13 @@ pub const Tokenizer = struct {
10251026 },
10261027 },
10271028 .line_comment_start => switch (c) {
1029 0 => break,
10281030 '/' => {
10291031 state = .doc_comment_start;
10301032 },
10311033 '!' => {
10321034 result.tag = .container_doc_comment;
1033 state = .container_doc_comment;
1035 state = .doc_comment;
10341036 },
10351037 '\n' => {
10361038 state = .start;
......@@ -1046,7 +1048,7 @@ pub const Tokenizer = struct {
10461048 '/' => {
10471049 state = .line_comment;
10481050 },
1049 '\n' => {
1051 0, '\n' => {
10501052 result.tag = .doc_comment;
10511053 break;
10521054 },
......@@ -1061,6 +1063,7 @@ pub const Tokenizer = struct {
10611063 },
10621064 },
10631065 .line_comment => switch (c) {
1066 0 => break,
10641067 '\n' => {
10651068 state = .start;
10661069 result.loc.start = self.index + 1;
......@@ -1068,8 +1071,8 @@ pub const Tokenizer = struct {
10681071 '\t', '\r' => {},
10691072 else => self.checkLiteralCharacter(),
10701073 },
1071 .doc_comment, .container_doc_comment => switch (c) {
1072 '\n' => break,
1074 .doc_comment => switch (c) {
1075 0, '\n' => break,
10731076 '\t', '\r' => {},
10741077 else => self.checkLiteralCharacter(),
10751078 },
......@@ -1088,12 +1091,11 @@ pub const Tokenizer = struct {
10881091 self.index -= 1;
10891092 state = .int_literal_dec;
10901093 },
1091 else => {
1092 if (isIdentifierChar(c)) {
1093 result.tag = .invalid;
1094 }
1094 'a', 'c', 'd', 'f'...'n', 'p'...'w', 'y', 'z', 'A'...'D', 'F'...'Z' => {
1095 result.tag = .invalid;
10951096 break;
10961097 },
1098 else => break,
10971099 },
10981100 .int_literal_bin_no_underscore => switch (c) {
10991101 '0'...'1' => {
......@@ -1109,12 +1111,11 @@ pub const Tokenizer = struct {
11091111 state = .int_literal_bin_no_underscore;
11101112 },
11111113 '0'...'1' => {},
1112 else => {
1113 if (isIdentifierChar(c)) {
1114 result.tag = .invalid;
1115 }
1114 '2'...'9', 'a'...'z', 'A'...'Z' => {
1115 result.tag = .invalid;
11161116 break;
11171117 },
1118 else => break,
11181119 },
11191120 .int_literal_oct_no_underscore => switch (c) {
11201121 '0'...'7' => {
......@@ -1130,12 +1131,11 @@ pub const Tokenizer = struct {
11301131 state = .int_literal_oct_no_underscore;
11311132 },
11321133 '0'...'7' => {},
1133 else => {
1134 if (isIdentifierChar(c)) {
1135 result.tag = .invalid;
1136 }
1134 '8', '9', 'a'...'z', 'A'...'Z' => {
1135 result.tag = .invalid;
11371136 break;
11381137 },
1138 else => break,
11391139 },
11401140 .int_literal_dec_no_underscore => switch (c) {
11411141 '0'...'9' => {
......@@ -1159,12 +1159,11 @@ pub const Tokenizer = struct {
11591159 result.tag = .float_literal;
11601160 },
11611161 '0'...'9' => {},
1162 else => {
1163 if (isIdentifierChar(c)) {
1164 result.tag = .invalid;
1165 }
1162 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
1163 result.tag = .invalid;
11661164 break;
11671165 },
1166 else => break,
11681167 },
11691168 .int_literal_hex_no_underscore => switch (c) {
11701169 '0'...'9', 'a'...'f', 'A'...'F' => {
......@@ -1188,12 +1187,11 @@ pub const Tokenizer = struct {
11881187 result.tag = .float_literal;
11891188 },
11901189 '0'...'9', 'a'...'f', 'A'...'F' => {},
1191 else => {
1192 if (isIdentifierChar(c)) {
1193 result.tag = .invalid;
1194 }
1190 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
1191 result.tag = .invalid;
11951192 break;
11961193 },
1194 else => break,
11971195 },
11981196 .num_dot_dec => switch (c) {
11991197 '.' => {
......@@ -1206,12 +1204,11 @@ pub const Tokenizer = struct {
12061204 result.tag = .float_literal;
12071205 state = .float_fraction_dec;
12081206 },
1209 else => {
1210 if (isIdentifierChar(c)) {
1211 result.tag = .invalid;
1212 }
1207 '_', 'a'...'z', 'A'...'Z' => {
1208 result.tag = .invalid;
12131209 break;
12141210 },
1211 else => break,
12151212 },
12161213 .num_dot_hex => switch (c) {
12171214 '.' => {
......@@ -1224,12 +1221,11 @@ pub const Tokenizer = struct {
12241221 result.tag = .float_literal;
12251222 state = .float_fraction_hex;
12261223 },
1227 else => {
1228 if (isIdentifierChar(c)) {
1229 result.tag = .invalid;
1230 }
1224 '_', 'g'...'z', 'G'...'Z' => {
1225 result.tag = .invalid;
12311226 break;
12321227 },
1228 else => break,
12331229 },
12341230 .float_fraction_dec_no_underscore => switch (c) {
12351231 '0'...'9' => {
......@@ -1248,12 +1244,11 @@ pub const Tokenizer = struct {
12481244 state = .float_exponent_unsigned;
12491245 },
12501246 '0'...'9' => {},
1251 else => {
1252 if (isIdentifierChar(c)) {
1253 result.tag = .invalid;
1254 }
1247 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
1248 result.tag = .invalid;
12551249 break;
12561250 },
1251 else => break,
12571252 },
12581253 .float_fraction_hex_no_underscore => switch (c) {
12591254 '0'...'9', 'a'...'f', 'A'...'F' => {
......@@ -1272,12 +1267,11 @@ pub const Tokenizer = struct {
12721267 state = .float_exponent_unsigned;
12731268 },
12741269 '0'...'9', 'a'...'f', 'A'...'F' => {},
1275 else => {
1276 if (isIdentifierChar(c)) {
1277 result.tag = .invalid;
1278 }
1270 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
1271 result.tag = .invalid;
12791272 break;
12801273 },
1274 else => break,
12811275 },
12821276 .float_exponent_unsigned => switch (c) {
12831277 '+', '-' => {
......@@ -1303,130 +1297,11 @@ pub const Tokenizer = struct {
13031297 state = .float_exponent_num_no_underscore;
13041298 },
13051299 '0'...'9' => {},
1306 else => {
1307 if (isIdentifierChar(c)) {
1308 result.tag = .invalid;
1309 }
1300 'a'...'z', 'A'...'Z' => {
1301 result.tag = .invalid;
13101302 break;
13111303 },
1312 },
1313 }
1314 } else if (self.index == self.buffer.len) {
1315 switch (state) {
1316 .start,
1317 .int_literal_dec,
1318 .int_literal_bin,
1319 .int_literal_oct,
1320 .int_literal_hex,
1321 .num_dot_dec,
1322 .num_dot_hex,
1323 .float_fraction_dec,
1324 .float_fraction_hex,
1325 .float_exponent_num,
1326 .string_literal, // find this error later
1327 .multiline_string_literal_line,
1328 .builtin,
1329 .line_comment,
1330 .line_comment_start,
1331 => {},
1332
1333 .identifier => {
1334 if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {
1335 result.tag = tag;
1336 }
1337 },
1338 .doc_comment, .doc_comment_start => {
1339 result.tag = .doc_comment;
1340 },
1341 .container_doc_comment => {
1342 result.tag = .container_doc_comment;
1343 },
1344
1345 .int_literal_dec_no_underscore,
1346 .int_literal_bin_no_underscore,
1347 .int_literal_oct_no_underscore,
1348 .int_literal_hex_no_underscore,
1349 .float_fraction_dec_no_underscore,
1350 .float_fraction_hex_no_underscore,
1351 .float_exponent_num_no_underscore,
1352 .float_exponent_unsigned,
1353 .saw_at_sign,
1354 .backslash,
1355 .char_literal,
1356 .char_literal_backslash,
1357 .char_literal_hex_escape,
1358 .char_literal_unicode_escape_saw_u,
1359 .char_literal_unicode_escape,
1360 .char_literal_unicode_invalid,
1361 .char_literal_end,
1362 .char_literal_unicode,
1363 .string_literal_backslash,
1364 => {
1365 result.tag = .invalid;
1366 },
1367
1368 .equal => {
1369 result.tag = .equal;
1370 },
1371 .bang => {
1372 result.tag = .bang;
1373 },
1374 .minus => {
1375 result.tag = .minus;
1376 },
1377 .slash => {
1378 result.tag = .slash;
1379 },
1380 .zero => {
1381 result.tag = .integer_literal;
1382 },
1383 .ampersand => {
1384 result.tag = .ampersand;
1385 },
1386 .period => {
1387 result.tag = .period;
1388 },
1389 .period_2 => {
1390 result.tag = .ellipsis2;
1391 },
1392 .period_asterisk => {
1393 result.tag = .period_asterisk;
1394 },
1395 .pipe => {
1396 result.tag = .pipe;
1397 },
1398 .angle_bracket_angle_bracket_right => {
1399 result.tag = .angle_bracket_angle_bracket_right;
1400 },
1401 .angle_bracket_right => {
1402 result.tag = .angle_bracket_right;
1403 },
1404 .angle_bracket_angle_bracket_left => {
1405 result.tag = .angle_bracket_angle_bracket_left;
1406 },
1407 .angle_bracket_left => {
1408 result.tag = .angle_bracket_left;
1409 },
1410 .plus_percent => {
1411 result.tag = .plus_percent;
1412 },
1413 .plus => {
1414 result.tag = .plus;
1415 },
1416 .percent => {
1417 result.tag = .percent;
1418 },
1419 .caret => {
1420 result.tag = .caret;
1421 },
1422 .asterisk_percent => {
1423 result.tag = .asterisk_percent;
1424 },
1425 .asterisk => {
1426 result.tag = .asterisk;
1427 },
1428 .minus_percent => {
1429 result.tag = .minus_percent;
1304 else => break,
14301305 },
14311306 }
14321307 }
src/Compilation.zig+1-1
......@@ -1541,7 +1541,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
15411541 }
15421542 }
15431543
1544 if (comp.bin_file.options.use_stage1) {
1544 if (comp.bin_file.options.use_stage1 and comp.bin_file.options.module != null) {
15451545 try comp.work_queue.writeItem(.{ .stage1_module = {} });
15461546 }
15471547
src/translate_c/ast.zig+1-1
......@@ -754,7 +754,7 @@ pub fn render(gpa: *Allocator, nodes: []const Node) !std.zig.ast.Tree {
754754 });
755755
756756 return std.zig.ast.Tree{
757 .source = ctx.buf.toOwnedSlice(),
757 .source = try ctx.buf.toOwnedSliceSentinel(0),
758758 .tokens = ctx.tokens.toOwnedSlice(),
759759 .nodes = ctx.nodes.toOwnedSlice(),
760760 .extra_data = ctx.extra_data.toOwnedSlice(gpa),