authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-03-20 21:59:28-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-03-20 21:59:28-07:00
logd8692b8bdb4630f2bb2763cdbe609de7d73b28d8
treeffc6b276760424460cfe620f7f2f7887b9ac013c
parent8bad5dfa72a33dec3919c3c3cb7590e51d03723b

astgen: implement string literals

2 files changed, 19 insertions(+), 58 deletions(-)

BRANCH_TODO+1
......@@ -30,3 +30,4 @@ Performance optimizations to look into:
3030 * use a smaller encoding for the auto generated return void at the end of
3131 function ZIR.
3232 * enum literals can use small strings
33 * string literals can use small strings
src/astgen.zig+18-58
......@@ -265,7 +265,6 @@ pub fn expr(mod: *Module, scope: *Scope, rl: ResultLoc, node: ast.Node.Index) In
265265 const token_tags = tree.tokens.items(.tag);
266266 const node_datas = tree.nodes.items(.data);
267267 const node_tags = tree.nodes.items(.tag);
268 const token_starts = tree.tokens.items(.start);
269268
270269 const gz = scope.getGenZir();
271270
......@@ -695,7 +694,6 @@ pub fn comptimeExpr(
695694
696695 const gz = parent_scope.getGenZir();
697696 const tree = parent_scope.tree();
698 const token_starts = tree.tokens.items(.start);
699697
700698 // Make a scope to collect generated instructions in the sub-expression.
701699 var block_scope: Scope.GenZir = .{
......@@ -727,7 +725,6 @@ fn breakExpr(
727725 const tree = parent_scope.tree();
728726 const node_datas = tree.nodes.items(.data);
729727 const main_tokens = tree.nodes.items(.main_token);
730 const token_starts = tree.tokens.items(.start);
731728
732729 const break_label = node_datas[node].lhs;
733730 const rhs = node_datas[node].rhs;
......@@ -805,7 +802,6 @@ fn continueExpr(
805802 const tree = parent_scope.tree();
806803 const node_datas = tree.nodes.items(.data);
807804 const main_tokens = tree.nodes.items(.main_token);
808 const token_starts = tree.tokens.items(.start);
809805
810806 const break_label = node_datas[node].lhs;
811807
......@@ -886,7 +882,6 @@ fn checkLabelRedefinition(mod: *Module, parent_scope: *Scope, label: ast.TokenIn
886882 if (try tokenIdentEql(mod, parent_scope, label, prev_label.token)) {
887883 const tree = parent_scope.tree();
888884 const main_tokens = tree.nodes.items(.main_token);
889 const token_starts = tree.tokens.items(.start);
890885
891886 const label_name = try mod.identifierTokenString(parent_scope, label);
892887 const msg = msg: {
......@@ -935,7 +930,6 @@ fn labeledBlockExpr(
935930
936931 const tree = parent_scope.tree();
937932 const main_tokens = tree.nodes.items(.main_token);
938 const token_starts = tree.tokens.items(.start);
939933 const token_tags = tree.tokens.items(.tag);
940934
941935 const lbrace = main_tokens[block_node];
......@@ -1018,7 +1012,6 @@ fn blockExprStmts(
10181012) !void {
10191013 const tree = parent_scope.tree();
10201014 const main_tokens = tree.nodes.items(.main_token);
1021 const token_starts = tree.tokens.items(.start);
10221015 const node_tags = tree.nodes.items(.tag);
10231016
10241017 var block_arena = std.heap.ArenaAllocator.init(mod.gpa);
......@@ -1297,7 +1290,6 @@ fn assignOp(
12971290 const tree = scope.tree();
12981291 const node_datas = tree.nodes.items(.data);
12991292 const main_tokens = tree.nodes.items(.main_token);
1300 const token_starts = tree.tokens.items(.start);
13011293
13021294 const lhs_ptr = try lvalExpr(mod, scope, node_datas[infix_node].lhs);
13031295 const lhs = try addZIRUnOp(mod, scope, lhs_ptr.src, .deref, lhs_ptr);
......@@ -1336,7 +1328,6 @@ fn negation(
13361328 const tree = scope.tree();
13371329 const node_datas = tree.nodes.items(.data);
13381330 const main_tokens = tree.nodes.items(.main_token);
1339 const token_starts = tree.tokens.items(.start);
13401331
13411332 const lhs = try addZIRInstConst(mod, scope, src, .{
13421333 .ty = Type.initTag(.comptime_int),
......@@ -1354,7 +1345,6 @@ fn ptrType(
13541345) InnerError!zir.Inst.Ref {
13551346 if (true) @panic("TODO update for zir-memory-layout");
13561347 const tree = scope.tree();
1357 const token_starts = tree.tokens.items(.start);
13581348
13591349 const simple = ptr_info.allowzero_token == null and
13601350 ptr_info.ast.align_node == 0 and
......@@ -1399,7 +1389,6 @@ fn arrayType(mod: *Module, scope: *Scope, rl: ResultLoc, node: ast.Node.Index) !
13991389 const tree = scope.tree();
14001390 const main_tokens = tree.nodes.items(.main_token);
14011391 const node_datas = tree.nodes.items(.data);
1402 const token_starts = tree.tokens.items(.start);
14031392
14041393 const usize_type = try addZIRInstConst(mod, scope, src, .{
14051394 .ty = Type.initTag(.type),
......@@ -1425,7 +1414,6 @@ fn arrayTypeSentinel(mod: *Module, scope: *Scope, rl: ResultLoc, node: ast.Node.
14251414 if (true) @panic("TODO update for zir-memory-layout");
14261415 const tree = scope.tree();
14271416 const main_tokens = tree.nodes.items(.main_token);
1428 const token_starts = tree.tokens.items(.start);
14291417 const node_datas = tree.nodes.items(.data);
14301418
14311419 const len_node = node_datas[node].lhs;
......@@ -1469,7 +1457,6 @@ fn errorSetDecl(
14691457 const tree = scope.tree();
14701458 const main_tokens = tree.nodes.items(.main_token);
14711459 const token_tags = tree.tokens.items(.tag);
1472 const token_starts = tree.tokens.items(.start);
14731460
14741461 // Count how many fields there are.
14751462 const error_token = main_tokens[node];
......@@ -1521,7 +1508,6 @@ fn orelseCatchExpr(
15211508 if (true) @panic("TODO update for zir-memory-layout");
15221509
15231510 const tree = scope.tree();
1524 const token_starts = tree.tokens.items(.start);
15251511
15261512 var block_scope: Scope.GenZir = .{
15271513 .parent = scope,
......@@ -1703,7 +1689,6 @@ fn tokenIdentEql(mod: *Module, scope: *Scope, token1: ast.TokenIndex, token2: as
17031689pub fn fieldAccess(mod: *Module, scope: *Scope, rl: ResultLoc, node: ast.Node.Index) InnerError!zir.Inst.Ref {
17041690 if (true) @panic("TODO update for zir-memory-layout");
17051691 const tree = scope.tree();
1706 const token_starts = tree.tokens.items(.start);
17071692 const main_tokens = tree.nodes.items(.main_token);
17081693 const node_datas = tree.nodes.items(.data);
17091694
......@@ -1732,7 +1717,6 @@ fn arrayAccess(
17321717 if (true) @panic("TODO update for zir-memory-layout");
17331718 const tree = scope.tree();
17341719 const main_tokens = tree.nodes.items(.main_token);
1735 const token_starts = tree.tokens.items(.start);
17361720 const node_datas = tree.nodes.items(.data);
17371721
17381722 const usize_type = try addZIRInstConst(mod, scope, src, .{
......@@ -1760,7 +1744,6 @@ fn sliceExpr(
17601744) InnerError!zir.Inst.Ref {
17611745 if (true) @panic("TODO update for zir-memory-layout");
17621746 const tree = scope.tree();
1763 const token_starts = tree.tokens.items(.start);
17641747
17651748 const usize_type = try addZIRInstConst(mod, scope, src, .{
17661749 .ty = Type.initTag(.type),
......@@ -1810,7 +1793,6 @@ fn simpleBinOp(
18101793 const tree = scope.tree();
18111794 const node_datas = tree.nodes.items(.data);
18121795 const main_tokens = tree.nodes.items(.main_token);
1813 const token_starts = tree.tokens.items(.start);
18141796
18151797 const lhs = try expr(mod, scope, .none, node_datas[infix_node].lhs);
18161798 const rhs = try expr(mod, scope, .none, node_datas[infix_node].rhs);
......@@ -1829,7 +1811,6 @@ fn boolBinOp(
18291811 const tree = scope.tree();
18301812 const node_datas = tree.nodes.items(.data);
18311813 const main_tokens = tree.nodes.items(.main_token);
1832 const token_starts = tree.tokens.items(.start);
18331814
18341815 const bool_type = try addZIRInstConst(mod, scope, src, .{
18351816 .ty = Type.initTag(.type),
......@@ -1926,7 +1907,6 @@ fn ifExpr(
19261907
19271908 const tree = scope.tree();
19281909 const main_tokens = tree.nodes.items(.main_token);
1929 const token_starts = tree.tokens.items(.start);
19301910
19311911 const if_src = token_starts[if_full.ast.if_token];
19321912
......@@ -2070,7 +2050,6 @@ fn whileExpr(
20702050
20712051 const tree = scope.tree();
20722052 const main_tokens = tree.nodes.items(.main_token);
2073 const token_starts = tree.tokens.items(.start);
20742053
20752054 const while_src = token_starts[while_full.ast.while_token];
20762055 const void_type = try addZIRInstConst(mod, scope, while_src, .{
......@@ -2210,7 +2189,6 @@ fn forExpr(
22102189 // Set up variables and constants.
22112190 const tree = scope.tree();
22122191 const main_tokens = tree.nodes.items(.main_token);
2213 const token_starts = tree.tokens.items(.start);
22142192 const token_tags = tree.tokens.items(.tag);
22152193
22162194 const for_src = token_starts[for_full.ast.while_token];
......@@ -2416,7 +2394,6 @@ fn switchExpr(
24162394 const node_datas = tree.nodes.items(.data);
24172395 const main_tokens = tree.nodes.items(.main_token);
24182396 const token_tags = tree.tokens.items(.tag);
2419 const token_starts = tree.tokens.items(.start);
24202397 const node_tags = tree.nodes.items(.tag);
24212398
24222399 const switch_token = main_tokens[switch_node];
......@@ -2707,7 +2684,6 @@ fn switchCaseExpr(
27072684 const tree = scope.tree();
27082685 const node_datas = tree.nodes.items(.data);
27092686 const main_tokens = tree.nodes.items(.main_token);
2710 const token_starts = tree.tokens.items(.start);
27112687 const token_tags = tree.tokens.items(.tag);
27122688
27132689 const case_src = token_starts[case.ast.arrow_token];
......@@ -2833,41 +2809,29 @@ fn identifier(
28332809 return mod.failNode(scope, ident, "use of undeclared identifier '{s}'", .{ident_name});
28342810}
28352811
2836fn parseStringLiteral(mod: *Module, scope: *Scope, token: ast.TokenIndex) ![]u8 {
2837 const tree = scope.tree();
2838 const token_tags = tree.tokens.items(.tag);
2839 const token_starts = tree.tokens.items(.start);
2840 assert(token_tags[token] == .string_literal);
2841 const unparsed = tree.tokenSlice(token);
2842 const arena = scope.arena();
2843 var bad_index: usize = undefined;
2844 const bytes = std.zig.parseStringLiteral(arena, unparsed, &bad_index) catch |err| switch (err) {
2845 error.InvalidCharacter => {
2846 const bad_byte = unparsed[bad_index];
2847 return mod.fail(scope, src + bad_index, "invalid string literal character: '{c}'", .{
2848 bad_byte,
2849 });
2850 },
2851 else => |e| return e,
2852 };
2853 return bytes;
2854}
2855
28562812fn stringLiteral(
28572813 mod: *Module,
28582814 scope: *Scope,
28592815 rl: ResultLoc,
2860 str_lit: ast.Node.Index,
2816 node: ast.Node.Index,
28612817) InnerError!zir.Inst.Ref {
2862 if (true) @panic("TODO update for zir-memory-layout");
28632818 const tree = scope.tree();
28642819 const main_tokens = tree.nodes.items(.main_token);
2865 const token_starts = tree.tokens.items(.start);
2866
2867 const str_lit_token = main_tokens[str_lit];
2868 const bytes = try parseStringLiteral(mod, scope, str_lit_token);
2869 const str_inst = try addZIRInst(mod, scope, src, zir.Inst.Str, .{ .bytes = bytes }, .{});
2870 return rvalue(mod, scope, rl, str_inst);
2820 const gz = scope.getGenZir();
2821 const string_bytes = &gz.zir_code.string_bytes;
2822 const str_index = string_bytes.items.len;
2823 const str_lit_token = main_tokens[node];
2824 const token_bytes = tree.tokenSlice(str_lit_token);
2825 try mod.parseStrLit(scope, str_lit_token, string_bytes, token_bytes, 0);
2826 const str_len = string_bytes.items.len - str_index;
2827 const result = try gz.add(.{
2828 .tag = .str,
2829 .data = .{ .str = .{
2830 .start = @intCast(u32, str_index),
2831 .len = @intCast(u32, str_len),
2832 } },
2833 });
2834 return rvalue(mod, scope, rl, result, node);
28712835}
28722836
28732837fn multilineStringLiteral(
......@@ -2880,7 +2844,6 @@ fn multilineStringLiteral(
28802844 const tree = scope.tree();
28812845 const node_datas = tree.nodes.items(.data);
28822846 const main_tokens = tree.nodes.items(.main_token);
2883 const token_starts = tree.tokens.items(.start);
28842847
28852848 const start = node_datas[str_lit].lhs;
28862849 const end = node_datas[str_lit].rhs;
......@@ -2924,7 +2887,6 @@ fn charLiteral(mod: *Module, scope: *Scope, rl: ResultLoc, node: ast.Node.Index)
29242887 const tree = scope.tree();
29252888 const main_tokens = tree.nodes.items(.main_token);
29262889 const main_token = main_tokens[node];
2927 const token_starts = tree.tokens.items(.start);
29282890
29292891 const slice = tree.tokenSlice(main_token);
29302892
......@@ -2975,7 +2937,6 @@ fn floatLiteral(
29752937 const arena = scope.arena();
29762938 const tree = scope.tree();
29772939 const main_tokens = tree.nodes.items(.main_token);
2978 const token_starts = tree.tokens.items(.start);
29792940
29802941 const main_token = main_tokens[float_lit];
29812942 const bytes = tree.tokenSlice(main_token);
......@@ -3002,7 +2963,6 @@ fn asmExpr(
30022963 const arena = scope.arena();
30032964 const tree = scope.tree();
30042965 const main_tokens = tree.nodes.items(.main_token);
3005 const token_starts = tree.tokens.items(.start);
30062966 const node_datas = tree.nodes.items(.data);
30072967 const gz = scope.getGenZir();
30082968
......@@ -3022,7 +2982,8 @@ fn asmExpr(
30222982 const constraint_token = main_tokens[input] + 2;
30232983 const string_bytes = &gz.zir_code.string_bytes;
30242984 constraints[i] = @intCast(u32, string_bytes.items.len);
3025 try mod.appendIdentStr(scope, constraint_token, string_bytes);
2985 const token_bytes = tree.tokenSlice(constraint_token);
2986 try mod.parseStrLit(scope, constraint_token, string_bytes, token_bytes, 0);
30262987 try string_bytes.append(mod.gpa, 0);
30272988
30282989 const usize_rl: ResultLoc = .{ .ty = @enumToInt(zir.Const.usize_type) };
......@@ -3203,7 +3164,6 @@ fn builtinCall(
32033164 if (true) @panic("TODO update for zir-memory-layout");
32043165 const tree = scope.tree();
32053166 const main_tokens = tree.nodes.items(.main_token);
3206 const token_starts = tree.tokens.items(.start);
32073167
32083168 const builtin_token = main_tokens[call];
32093169 const builtin_name = tree.tokenSlice(builtin_token);