authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-11-04 15:48:28-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-11-04 15:48:28-07:00
logfc1e7a56441aea4a9394c4e8c0142110891b4a1a
tree8d7fc370a62874bcfe779df04a67fdd8fa17e0a4
parent802c82b0720ff2b7c59ba96effbdd1eed7491df4

zig reduce: rename identifiers when inlining an `@import`

3 files changed, 112 insertions(+), 26 deletions(-)

lib/std/zig/render.zig+17-1
...@@ -26,6 +26,9 @@ pub const Fixups = struct {...@@ -26,6 +26,9 @@ pub const Fixups = struct {
26 omit_nodes: std.AutoHashMapUnmanaged(Ast.Node.Index, void) = .{},26 omit_nodes: std.AutoHashMapUnmanaged(Ast.Node.Index, void) = .{},
27 /// These expressions will be replaced with the string value.27 /// These expressions will be replaced with the string value.
28 replace_nodes: std.AutoHashMapUnmanaged(Ast.Node.Index, []const u8) = .{},28 replace_nodes: std.AutoHashMapUnmanaged(Ast.Node.Index, []const u8) = .{},
29 /// Change all identifier names matching the key to be value instead.
30 rename_identifiers: std.StringArrayHashMapUnmanaged([]const u8) = .{},
31
29 /// All `@import` builtin calls which refer to a file path will be prefixed32 /// All `@import` builtin calls which refer to a file path will be prefixed
30 /// with this path.33 /// with this path.
31 rebase_imported_paths: ?[]const u8 = null,34 rebase_imported_paths: ?[]const u8 = null,
...@@ -34,7 +37,9 @@ pub const Fixups = struct {...@@ -34,7 +37,9 @@ pub const Fixups = struct {
34 return f.unused_var_decls.count() +37 return f.unused_var_decls.count() +
35 f.gut_functions.count() +38 f.gut_functions.count() +
36 f.omit_nodes.count() +39 f.omit_nodes.count() +
37 f.replace_nodes.count();40 f.replace_nodes.count() +
41 f.rename_identifiers.count() +
42 @intFromBool(f.rebase_imported_paths != null);
38 }43 }
3944
40 pub fn clearRetainingCapacity(f: *Fixups) void {45 pub fn clearRetainingCapacity(f: *Fixups) void {
...@@ -42,6 +47,9 @@ pub const Fixups = struct {...@@ -42,6 +47,9 @@ pub const Fixups = struct {
42 f.gut_functions.clearRetainingCapacity();47 f.gut_functions.clearRetainingCapacity();
43 f.omit_nodes.clearRetainingCapacity();48 f.omit_nodes.clearRetainingCapacity();
44 f.replace_nodes.clearRetainingCapacity();49 f.replace_nodes.clearRetainingCapacity();
50 f.rename_identifiers.clearRetainingCapacity();
51
52 f.rebase_imported_paths = null;
45 }53 }
4654
47 pub fn deinit(f: *Fixups, gpa: Allocator) void {55 pub fn deinit(f: *Fixups, gpa: Allocator) void {
...@@ -49,6 +57,7 @@ pub const Fixups = struct {...@@ -49,6 +57,7 @@ pub const Fixups = struct {
49 f.gut_functions.deinit(gpa);57 f.gut_functions.deinit(gpa);
50 f.omit_nodes.deinit(gpa);58 f.omit_nodes.deinit(gpa);
51 f.replace_nodes.deinit(gpa);59 f.replace_nodes.deinit(gpa);
60 f.rename_identifiers.deinit(gpa);
52 f.* = undefined;61 f.* = undefined;
53 }62 }
54};63};
...@@ -2833,6 +2842,13 @@ fn renderIdentifier(r: *Render, token_index: Ast.TokenIndex, space: Space, quote...@@ -2833,6 +2842,13 @@ fn renderIdentifier(r: *Render, token_index: Ast.TokenIndex, space: Space, quote
2833 const token_tags = tree.tokens.items(.tag);2842 const token_tags = tree.tokens.items(.tag);
2834 assert(token_tags[token_index] == .identifier);2843 assert(token_tags[token_index] == .identifier);
2835 const lexeme = tokenSliceForRender(tree, token_index);2844 const lexeme = tokenSliceForRender(tree, token_index);
2845
2846 if (r.fixups.rename_identifiers.get(lexeme)) |mangled| {
2847 try r.ais.writer().writeAll(mangled);
2848 try renderSpace(r, token_index, lexeme.len, space);
2849 return;
2850 }
2851
2836 if (lexeme[0] != '@') {2852 if (lexeme[0] != '@') {
2837 return renderToken(r, token_index, space);2853 return renderToken(r, token_index, space);
2838 }2854 }
src/reduce.zig+34-8
...@@ -148,7 +148,7 @@ pub fn main(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -148,7 +148,7 @@ pub fn main(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
148148
149 var transformations = std.ArrayList(Walk.Transformation).init(gpa);149 var transformations = std.ArrayList(Walk.Transformation).init(gpa);
150 defer transformations.deinit();150 defer transformations.deinit();
151 try Walk.findTransformations(&tree, &transformations);151 try Walk.findTransformations(arena, &tree, &transformations);
152 sortTransformations(transformations.items, rng.random());152 sortTransformations(transformations.items, rng.random());
153153
154 fresh: while (transformations.items.len > 0) {154 fresh: while (transformations.items.len > 0) {
...@@ -162,12 +162,19 @@ pub fn main(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -162,12 +162,19 @@ pub fn main(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
162 subset_size = @max(1, subset_size / 2);162 subset_size = @max(1, subset_size / 2);
163163
164 const this_set = transformations.items[start_index..][0..subset_size];164 const this_set = transformations.items[start_index..][0..subset_size];
165 std.debug.print("trying {d} random transformations: ", .{subset_size});
166 for (this_set[0..@min(this_set.len, 20)]) |t| {
167 std.debug.print("{s} ", .{@tagName(t)});
168 }
169 std.debug.print("\n", .{});
165 try transformationsToFixups(gpa, arena, root_source_file_path, this_set, &fixups);170 try transformationsToFixups(gpa, arena, root_source_file_path, this_set, &fixups);
166171
167 rendered.clearRetainingCapacity();172 rendered.clearRetainingCapacity();
168 try tree.renderToArrayList(&rendered, fixups);173 try tree.renderToArrayList(&rendered, fixups);
169 try std.fs.cwd().writeFile(root_source_file_path, rendered.items);174 try std.fs.cwd().writeFile(root_source_file_path, rendered.items);
170175
176 //std.debug.print("trying this code:\n{s}\n", .{rendered.items});
177
171 const interestingness = try runCheck(arena, interestingness_argv.items);178 const interestingness = try runCheck(arena, interestingness_argv.items);
172 std.debug.print("{d} random transformations: {s}. {d} remaining\n", .{179 std.debug.print("{d} random transformations: {s}. {d} remaining\n", .{
173 subset_size, @tagName(interestingness), transformations.items.len - start_index,180 subset_size, @tagName(interestingness), transformations.items.len - start_index,
...@@ -179,7 +186,7 @@ pub fn main(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -179,7 +186,7 @@ pub fn main(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
179 tree.deinit(gpa);186 tree.deinit(gpa);
180 tree = new_tree;187 tree = new_tree;
181188
182 try Walk.findTransformations(&tree, &transformations);189 try Walk.findTransformations(arena, &tree, &transformations);
183 // Resetting based on the seed again means we will get the same190 // Resetting based on the seed again means we will get the same
184 // results if restarting the reduction process from this new point.191 // results if restarting the reduction process from this new point.
185 rng = std.rand.DefaultPrng.init(seed);192 rng = std.rand.DefaultPrng.init(seed);
...@@ -263,7 +270,6 @@ fn transformationsToFixups(...@@ -263,7 +270,6 @@ fn transformationsToFixups(
263 try fixups.replace_nodes.put(gpa, node, "undefined");270 try fixups.replace_nodes.put(gpa, node, "undefined");
264 },271 },
265 .inline_imported_file => |inline_imported_file| {272 .inline_imported_file => |inline_imported_file| {
266 defer gpa.free(inline_imported_file.imported_string);
267 const full_imported_path = try std.fs.path.join(gpa, &.{273 const full_imported_path = try std.fs.path.join(gpa, &.{
268 std.fs.path.dirname(root_source_file_path) orelse ".",274 std.fs.path.dirname(root_source_file_path) orelse ".",
269 inline_imported_file.imported_string,275 inline_imported_file.imported_string,
...@@ -274,14 +280,34 @@ fn transformationsToFixups(...@@ -274,14 +280,34 @@ fn transformationsToFixups(
274 gpa.free(other_file_ast.source);280 gpa.free(other_file_ast.source);
275 other_file_ast.deinit(gpa);281 other_file_ast.deinit(gpa);
276 }282 }
277 var other_source = std.ArrayList(u8).init(gpa);283
278 defer other_source.deinit();
279 var inlined_fixups: Ast.Fixups = .{};284 var inlined_fixups: Ast.Fixups = .{};
280 defer inlined_fixups.deinit(gpa);285 defer inlined_fixups.deinit(gpa);
286 if (std.fs.path.dirname(inline_imported_file.imported_string)) |dirname| {
287 inlined_fixups.rebase_imported_paths = dirname;
288 }
289 for (inline_imported_file.in_scope_names.keys()) |name| {
290 // This name needs to be mangled in order to not cause an
291 // ambiguous reference error.
292 var i: u32 = 2;
293 const mangled = while (true) : (i += 1) {
294 const mangled = try std.fmt.allocPrint(gpa, "{s}{d}", .{ name, i });
295 if (!inline_imported_file.in_scope_names.contains(mangled))
296 break mangled;
297 gpa.free(mangled);
298 };
299 try inlined_fixups.rename_identifiers.put(gpa, name, mangled);
300 }
301 defer {
302 for (inlined_fixups.rename_identifiers.values()) |v| {
303 gpa.free(v);
304 }
305 }
306
307 var other_source = std.ArrayList(u8).init(gpa);
308 defer other_source.deinit();
281 try other_source.appendSlice("struct {\n");309 try other_source.appendSlice("struct {\n");
282 try other_file_ast.renderToArrayList(&other_source, .{310 try other_file_ast.renderToArrayList(&other_source, inlined_fixups);
283 .rebase_imported_paths = std.fs.path.dirname(inline_imported_file.imported_string),
284 });
285 try other_source.appendSlice("}");311 try other_source.appendSlice("}");
286312
287 try fixups.replace_nodes.put(313 try fixups.replace_nodes.put(
src/reduce/Walk.zig+61-17
...@@ -7,7 +7,9 @@ const BuiltinFn = @import("../BuiltinFn.zig");...@@ -7,7 +7,9 @@ const BuiltinFn = @import("../BuiltinFn.zig");
7ast: *const Ast,7ast: *const Ast,
8transformations: *std.ArrayList(Transformation),8transformations: *std.ArrayList(Transformation),
9unreferenced_globals: std.StringArrayHashMapUnmanaged(Ast.Node.Index),9unreferenced_globals: std.StringArrayHashMapUnmanaged(Ast.Node.Index),
10in_scope_names: std.StringArrayHashMapUnmanaged(u32),
10gpa: std.mem.Allocator,11gpa: std.mem.Allocator,
12arena: std.mem.Allocator,
1113
12pub const Transformation = union(enum) {14pub const Transformation = union(enum) {
13 /// Replace the fn decl AST Node with one whose body is only `@trap()` with15 /// Replace the fn decl AST Node with one whose body is only `@trap()` with
...@@ -18,25 +20,39 @@ pub const Transformation = union(enum) {...@@ -18,25 +20,39 @@ pub const Transformation = union(enum) {
18 /// Replace an expression with `undefined`.20 /// Replace an expression with `undefined`.
19 replace_with_undef: Ast.Node.Index,21 replace_with_undef: Ast.Node.Index,
20 /// Replace an `@import` with the imported file contents wrapped in a struct.22 /// Replace an `@import` with the imported file contents wrapped in a struct.
21 inline_imported_file: struct {23 inline_imported_file: InlineImportedFile,
24
25 pub const InlineImportedFile = struct {
22 builtin_call_node: Ast.Node.Index,26 builtin_call_node: Ast.Node.Index,
23 imported_string: []const u8,27 imported_string: []const u8,
24 },28 /// Identifier names that must be renamed in the inlined code or else
29 /// will cause ambiguous reference errors.
30 in_scope_names: std.StringArrayHashMapUnmanaged(void),
31 };
25};32};
2633
27pub const Error = error{OutOfMemory};34pub const Error = error{OutOfMemory};
2835
29/// The result will be priority shuffled.36/// The result will be priority shuffled.
30pub fn findTransformations(ast: *const Ast, transformations: *std.ArrayList(Transformation)) !void {37pub fn findTransformations(
38 arena: std.mem.Allocator,
39 ast: *const Ast,
40 transformations: *std.ArrayList(Transformation),
41) !void {
31 transformations.clearRetainingCapacity();42 transformations.clearRetainingCapacity();
3243
33 var walk: Walk = .{44 var walk: Walk = .{
34 .ast = ast,45 .ast = ast,
35 .transformations = transformations,46 .transformations = transformations,
36 .gpa = transformations.allocator,47 .gpa = transformations.allocator,
48 .arena = arena,
37 .unreferenced_globals = .{},49 .unreferenced_globals = .{},
50 .in_scope_names = .{},
38 };51 };
39 defer walk.unreferenced_globals.deinit(walk.gpa);52 defer {
53 walk.unreferenced_globals.deinit(walk.gpa);
54 walk.in_scope_names.deinit(walk.gpa);
55 }
4056
41 try walkMembers(&walk, walk.ast.rootDecls());57 try walkMembers(&walk, walk.ast.rootDecls());
4258
...@@ -49,14 +65,18 @@ pub fn findTransformations(ast: *const Ast, transformations: *std.ArrayList(Tran...@@ -49,14 +65,18 @@ pub fn findTransformations(ast: *const Ast, transformations: *std.ArrayList(Tran
4965
50fn walkMembers(w: *Walk, members: []const Ast.Node.Index) Error!void {66fn walkMembers(w: *Walk, members: []const Ast.Node.Index) Error!void {
51 // First we scan for globals so that we can delete them while walking.67 // First we scan for globals so that we can delete them while walking.
52 try scanDecls(w, members);68 try scanDecls(w, members, .add);
5369
54 for (members) |member| {70 for (members) |member| {
55 try walkMember(w, member);71 try walkMember(w, member);
56 }72 }
73
74 try scanDecls(w, members, .remove);
57}75}
5876
59fn scanDecls(w: *Walk, members: []const Ast.Node.Index) Error!void {77const ScanDeclsAction = enum { add, remove };
78
79fn scanDecls(w: *Walk, members: []const Ast.Node.Index, action: ScanDeclsAction) Error!void {
60 const ast = w.ast;80 const ast = w.ast;
61 const gpa = w.gpa;81 const gpa = w.gpa;
62 const node_tags = ast.nodes.items(.tag);82 const node_tags = ast.nodes.items(.tag);
...@@ -80,9 +100,27 @@ fn scanDecls(w: *Walk, members: []const Ast.Node.Index) Error!void {...@@ -80,9 +100,27 @@ fn scanDecls(w: *Walk, members: []const Ast.Node.Index) Error!void {
80100
81 else => continue,101 else => continue,
82 };102 };
103
83 assert(token_tags[name_token] == .identifier);104 assert(token_tags[name_token] == .identifier);
84 const name_bytes = ast.tokenSlice(name_token);105 const name_bytes = ast.tokenSlice(name_token);
85 try w.unreferenced_globals.put(gpa, name_bytes, member_node);106
107 switch (action) {
108 .add => {
109 try w.unreferenced_globals.put(gpa, name_bytes, member_node);
110
111 const gop = try w.in_scope_names.getOrPut(gpa, name_bytes);
112 if (!gop.found_existing) gop.value_ptr.* = 0;
113 gop.value_ptr.* += 1;
114 },
115 .remove => {
116 const entry = w.in_scope_names.getEntry(name_bytes).?;
117 if (entry.value_ptr.* <= 1) {
118 assert(w.in_scope_names.swapRemove(name_bytes));
119 } else {
120 entry.value_ptr.* -= 1;
121 }
122 },
123 }
86 }124 }
87}125}
88126
...@@ -567,12 +605,12 @@ fn walkLocalVarDecl(w: *Walk, var_decl: Ast.full.VarDecl) Error!void {...@@ -567,12 +605,12 @@ fn walkLocalVarDecl(w: *Walk, var_decl: Ast.full.VarDecl) Error!void {
567 try walkExpression(w, var_decl.ast.section_node);605 try walkExpression(w, var_decl.ast.section_node);
568 }606 }
569607
570 assert(var_decl.ast.init_node != 0);608 if (var_decl.ast.init_node != 0) {
571 if (!isUndefinedIdent(w.ast, var_decl.ast.init_node)) {609 if (!isUndefinedIdent(w.ast, var_decl.ast.init_node)) {
572 try w.transformations.append(.{ .replace_with_undef = var_decl.ast.init_node });610 try w.transformations.append(.{ .replace_with_undef = var_decl.ast.init_node });
611 }
612 try walkExpression(w, var_decl.ast.init_node);
573 }613 }
574
575 return walkExpression(w, var_decl.ast.init_node);
576}614}
577615
578fn walkContainerField(w: *Walk, field: Ast.full.ContainerField) Error!void {616fn walkContainerField(w: *Walk, field: Ast.full.ContainerField) Error!void {
...@@ -582,7 +620,9 @@ fn walkContainerField(w: *Walk, field: Ast.full.ContainerField) Error!void {...@@ -582,7 +620,9 @@ fn walkContainerField(w: *Walk, field: Ast.full.ContainerField) Error!void {
582 if (field.ast.align_expr != 0) {620 if (field.ast.align_expr != 0) {
583 try walkExpression(w, field.ast.align_expr); // alignment621 try walkExpression(w, field.ast.align_expr); // alignment
584 }622 }
585 try walkExpression(w, field.ast.value_expr); // value623 if (field.ast.value_expr != 0) {
624 try walkExpression(w, field.ast.value_expr); // value
625 }
586}626}
587627
588fn walkBlock(628fn walkBlock(
...@@ -690,7 +730,6 @@ fn walkBuiltinCall(...@@ -690,7 +730,6 @@ fn walkBuiltinCall(
690 params: []const Ast.Node.Index,730 params: []const Ast.Node.Index,
691) Error!void {731) Error!void {
692 const ast = w.ast;732 const ast = w.ast;
693 const gpa = w.gpa;
694 const main_tokens = ast.nodes.items(.main_token);733 const main_tokens = ast.nodes.items(.main_token);
695 const builtin_token = main_tokens[call_node];734 const builtin_token = main_tokens[call_node];
696 const builtin_name = ast.tokenSlice(builtin_token);735 const builtin_name = ast.tokenSlice(builtin_token);
...@@ -700,12 +739,17 @@ fn walkBuiltinCall(...@@ -700,12 +739,17 @@ fn walkBuiltinCall(
700 const operand_node = params[0];739 const operand_node = params[0];
701 const str_lit_token = main_tokens[operand_node];740 const str_lit_token = main_tokens[operand_node];
702 const token_bytes = ast.tokenSlice(str_lit_token);741 const token_bytes = ast.tokenSlice(str_lit_token);
703 const imported_string = std.zig.string_literal.parseAlloc(gpa, token_bytes) catch742 if (std.mem.endsWith(u8, token_bytes, ".zig\"")) {
704 unreachable;743 const imported_string = std.zig.string_literal.parseAlloc(w.arena, token_bytes) catch
705 if (std.mem.endsWith(u8, imported_string, ".zig")) {744 unreachable;
706 try w.transformations.append(.{ .inline_imported_file = .{745 try w.transformations.append(.{ .inline_imported_file = .{
707 .builtin_call_node = call_node,746 .builtin_call_node = call_node,
708 .imported_string = imported_string,747 .imported_string = imported_string,
748 .in_scope_names = try std.StringArrayHashMapUnmanaged(void).init(
749 w.arena,
750 w.in_scope_names.keys(),
751 &.{},
752 ),
709 } });753 } });
710 }754 }
711 },755 },