authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-28 21:15:08-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-28 22:42:13-04:00
log0bd89979fdc42a7fd14fe127ac8a586d7c170444
tree7c9376195fdc9ab68e0a447e64c4cea09156bb14
parent3eed7a4dea3b66bf236278caba7f96228b13214f

stage2: handle deletions and better dependency resolution

* Deleted decls are deleted; unused decls are also detected as deleted. Cycles are not yet detected. * Re-analysis is smarter and will not cause a re-analysis of dependants when only a function body is changed.

4 files changed, 367 insertions(+), 198 deletions(-)

src-self-hosted/Module.zig+299-174
...@@ -55,9 +55,20 @@ failed_files: std.AutoHashMap(*Scope.ZIRModule, *ErrorMsg),...@@ -55,9 +55,20 @@ failed_files: std.AutoHashMap(*Scope.ZIRModule, *ErrorMsg),
55/// The ErrorMsg memory is owned by the `Export`, using Module's allocator.55/// The ErrorMsg memory is owned by the `Export`, using Module's allocator.
56failed_exports: std.AutoHashMap(*Export, *ErrorMsg),56failed_exports: std.AutoHashMap(*Export, *ErrorMsg),
5757
58/// Incrementing integer used to compare against the corresponding Decl
59/// field to determine whether a Decl's status applies to an ongoing update, or a
60/// previous analysis.
61generation: u32 = 0,
62
63/// Candidates for deletion. After a semantic analysis update completes, this list
64/// contains Decls that need to be deleted if they end up having no references to them.
65deletion_set: std.ArrayListUnmanaged(*Decl) = std.ArrayListUnmanaged(*Decl){},
66
58pub const WorkItem = union(enum) {67pub const WorkItem = union(enum) {
59 /// Write the machine code for a Decl to the output file.68 /// Write the machine code for a Decl to the output file.
60 codegen_decl: *Decl,69 codegen_decl: *Decl,
70 /// Decl has been determined to be outdated; perform semantic analysis again.
71 re_analyze_decl: *Decl,
61};72};
6273
63pub const Export = struct {74pub const Export = struct {
...@@ -68,6 +79,8 @@ pub const Export = struct {...@@ -68,6 +79,8 @@ pub const Export = struct {
68 link: link.ElfFile.Export,79 link: link.ElfFile.Export,
69 /// The Decl that performs the export. Note that this is *not* the Decl being exported.80 /// The Decl that performs the export. Note that this is *not* the Decl being exported.
70 owner_decl: *Decl,81 owner_decl: *Decl,
82 /// The Decl being exported. Note this is *not* the Decl performing the export.
83 exported_decl: *Decl,
71 status: enum {84 status: enum {
72 in_progress,85 in_progress,
73 failed,86 failed,
...@@ -94,8 +107,7 @@ pub const Decl = struct {...@@ -94,8 +107,7 @@ pub const Decl = struct {
94 /// This is the base offset that src offsets within this Decl are relative to.107 /// This is the base offset that src offsets within this Decl are relative to.
95 src: usize,108 src: usize,
96 /// The most recent value of the Decl after a successful semantic analysis.109 /// The most recent value of the Decl after a successful semantic analysis.
97 /// The tag for this union is determined by the tag value of the analysis field.110 typed_value: union(enum) {
98 typed_value: union {
99 never_succeeded: void,111 never_succeeded: void,
100 most_recent: TypedValue.Managed,112 most_recent: TypedValue.Managed,
101 },113 },
...@@ -104,36 +116,35 @@ pub const Decl = struct {...@@ -104,36 +116,35 @@ pub const Decl = struct {
104 /// analysis of the function body is performed with this value set to `success`. Functions116 /// analysis of the function body is performed with this value set to `success`. Functions
105 /// have their own analysis status field.117 /// have their own analysis status field.
106 analysis: enum {118 analysis: enum {
107 initial_in_progress,119 /// Semantic analysis for this Decl is running right now. This state detects dependency loops.
120 in_progress,
108 /// This Decl might be OK but it depends on another one which did not successfully complete121 /// This Decl might be OK but it depends on another one which did not successfully complete
109 /// semantic analysis. This Decl never had a value computed.122 /// semantic analysis.
110 initial_dependency_failure,123 dependency_failure,
111 /// Semantic analysis failure. This Decl never had a value computed.124 /// Semantic analysis failure.
112 /// There will be a corresponding ErrorMsg in Module.failed_decls.125 /// There will be a corresponding ErrorMsg in Module.failed_decls.
113 initial_sema_failure,126 sema_failure,
114 /// In this case the `typed_value.most_recent` can still be accessed.
115 /// There will be a corresponding ErrorMsg in Module.failed_decls.127 /// There will be a corresponding ErrorMsg in Module.failed_decls.
116 codegen_failure,128 codegen_failure,
117 /// In this case the `typed_value.most_recent` can still be accessed.
118 /// There will be a corresponding ErrorMsg in Module.failed_decls.129 /// There will be a corresponding ErrorMsg in Module.failed_decls.
119 /// This indicates the failure was something like running out of disk space,130 /// This indicates the failure was something like running out of disk space,
120 /// and attempting codegen again may succeed.131 /// and attempting codegen again may succeed.
121 codegen_failure_retryable,132 codegen_failure_retryable,
122 /// This Decl might be OK but it depends on another one which did not successfully complete133 /// Everything is done. During an update, this Decl may be out of date, depending
123 /// semantic analysis. There is a most recent value available.134 /// on its dependencies. The `generation` field can be used to determine if this
124 repeat_dependency_failure,135 /// completion status occurred before or after a given update.
125 /// Semantic anlaysis failure, but the `typed_value.most_recent` can be accessed.
126 /// There will be a corresponding ErrorMsg in Module.failed_decls.
127 repeat_sema_failure,
128 /// Completed successfully before; the `typed_value.most_recent` can be accessed, and
129 /// new semantic analysis is in progress.
130 repeat_in_progress,
131 /// Failed before; the `typed_value.most_recent` is not available, and
132 /// new semantic analysis is in progress.
133 repeat_in_progress_novalue,
134 /// Everything is done and updated.
135 complete,136 complete,
137 /// A Module update is in progress, and this Decl has been flagged as being known
138 /// to require re-analysis.
139 outdated,
136 },140 },
141 /// This flag is set when this Decl is added to a check_for_deletion set, and cleared
142 /// when removed.
143 deletion_flag: bool,
144 /// An integer that can be checked against the corresponding incrementing
145 /// generation field of Module. This is used to determine whether `complete` status
146 /// represents pre- or post- re-analysis.
147 generation: u32,
137148
138 /// Represents the position of the code in the output file.149 /// Represents the position of the code in the output file.
139 /// This is populated regardless of semantic analysis and code generation.150 /// This is populated regardless of semantic analysis and code generation.
...@@ -143,11 +154,9 @@ pub const Decl = struct {...@@ -143,11 +154,9 @@ pub const Decl = struct {
143154
144 /// The shallow set of other decls whose typed_value could possibly change if this Decl's155 /// The shallow set of other decls whose typed_value could possibly change if this Decl's
145 /// typed_value is modified.156 /// typed_value is modified.
146 /// TODO look into using a lightweight map/set data structure rather than a linear array.
147 dependants: ArrayListUnmanaged(*Decl) = ArrayListUnmanaged(*Decl){},157 dependants: ArrayListUnmanaged(*Decl) = ArrayListUnmanaged(*Decl){},
148 /// The shallow set of other decls whose typed_value changing indicates that this Decl's158 /// The shallow set of other decls whose typed_value changing indicates that this Decl's
149 /// typed_value may need to be regenerated.159 /// typed_value may need to be regenerated.
150 /// TODO look into using a lightweight map/set data structure rather than a linear array.
151 dependencies: ArrayListUnmanaged(*Decl) = ArrayListUnmanaged(*Decl){},160 dependencies: ArrayListUnmanaged(*Decl) = ArrayListUnmanaged(*Decl){},
152161
153 pub fn destroy(self: *Decl, allocator: *Allocator) void {162 pub fn destroy(self: *Decl, allocator: *Allocator) void {
...@@ -181,7 +190,7 @@ pub const Decl = struct {...@@ -181,7 +190,7 @@ pub const Decl = struct {
181 pub fn fullyQualifiedNameHash(self: Decl) Hash {190 pub fn fullyQualifiedNameHash(self: Decl) Hash {
182 // Right now we only have ZIRModule as the source. So this is simply the191 // Right now we only have ZIRModule as the source. So this is simply the
183 // relative name of the decl.192 // relative name of the decl.
184 return hashSimpleName(mem.spanZ(u8, self.name));193 return hashSimpleName(mem.spanZ(self.name));
185 }194 }
186195
187 pub fn typedValue(self: *Decl) error{AnalysisFail}!TypedValue {196 pub fn typedValue(self: *Decl) error{AnalysisFail}!TypedValue {
...@@ -209,37 +218,12 @@ pub const Decl = struct {...@@ -209,37 +218,12 @@ pub const Decl = struct {
209 }218 }
210219
211 fn typedValueManaged(self: *Decl) ?*TypedValue.Managed {220 fn typedValueManaged(self: *Decl) ?*TypedValue.Managed {
212 switch (self.analysis) {221 switch (self.typed_value) {
213 .initial_in_progress,222 .most_recent => |*x| return x,
214 .initial_dependency_failure,223 .never_succeeded => return null,
215 .initial_sema_failure,
216 .repeat_in_progress_novalue,
217 => return null,
218 .codegen_failure,
219 .codegen_failure_retryable,
220 .repeat_dependency_failure,
221 .repeat_sema_failure,
222 .repeat_in_progress,
223 .complete,
224 => return &self.typed_value.most_recent,
225 }
226 }
227
228 fn flagForRegeneration(self: *Decl) void {
229 if (self.typedValueManaged() == null) {
230 self.analysis = .repeat_in_progress_novalue;
231 } else {
232 self.analysis = .repeat_in_progress;
233 }224 }
234 }225 }
235226
236 fn isFlaggedForRegeneration(self: *Decl) bool {
237 return switch (self.analysis) {
238 .repeat_in_progress, .repeat_in_progress_novalue => true,
239 else => false,
240 };
241 }
242
243 fn removeDependant(self: *Decl, other: *Decl) void {227 fn removeDependant(self: *Decl, other: *Decl) void {
244 for (self.dependants.items) |item, i| {228 for (self.dependants.items) |item, i| {
245 if (item == other) {229 if (item == other) {
...@@ -249,6 +233,16 @@ pub const Decl = struct {...@@ -249,6 +233,16 @@ pub const Decl = struct {
249 }233 }
250 unreachable;234 unreachable;
251 }235 }
236
237 fn removeDependency(self: *Decl, other: *Decl) void {
238 for (self.dependencies.items) |item, i| {
239 if (item == other) {
240 _ = self.dependencies.swapRemove(i);
241 return;
242 }
243 }
244 unreachable;
245 }
252};246};
253247
254/// Fn struct memory is owned by the Decl's TypedValue.Managed arena allocator.248/// Fn struct memory is owned by the Decl's TypedValue.Managed arena allocator.
...@@ -512,6 +506,7 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module {...@@ -512,6 +506,7 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module {
512pub fn deinit(self: *Module) void {506pub fn deinit(self: *Module) void {
513 self.bin_file.deinit();507 self.bin_file.deinit();
514 const allocator = self.allocator;508 const allocator = self.allocator;
509 self.deletion_set.deinit(allocator);
515 self.work_queue.deinit();510 self.work_queue.deinit();
516 {511 {
517 var it = self.decl_table.iterator();512 var it = self.decl_table.iterator();
...@@ -576,6 +571,8 @@ pub fn target(self: Module) std.Target {...@@ -576,6 +571,8 @@ pub fn target(self: Module) std.Target {
576571
577/// Detect changes to source files, perform semantic analysis, and update the output files.572/// Detect changes to source files, perform semantic analysis, and update the output files.
578pub fn update(self: *Module) !void {573pub fn update(self: *Module) !void {
574 self.generation += 1;
575
579 // TODO Use the cache hash file system to detect which source files changed.576 // TODO Use the cache hash file system to detect which source files changed.
580 // Here we simulate a full cache miss.577 // Here we simulate a full cache miss.
581 // Analyze the root source file now.578 // Analyze the root source file now.
...@@ -588,6 +585,15 @@ pub fn update(self: *Module) !void {...@@ -588,6 +585,15 @@ pub fn update(self: *Module) !void {
588585
589 try self.performAllTheWork();586 try self.performAllTheWork();
590587
588 // Process the deletion set.
589 while (self.deletion_set.popOrNull()) |decl| {
590 if (decl.dependants.items.len != 0) {
591 decl.deletion_flag = false;
592 continue;
593 }
594 try self.deleteDecl(decl);
595 }
596
591 // Unload all the source files from memory.597 // Unload all the source files from memory.
592 self.root_scope.unload(self.allocator);598 self.root_scope.unload(self.allocator);
593599
...@@ -672,15 +678,12 @@ const InnerError = error{ OutOfMemory, AnalysisFail };...@@ -672,15 +678,12 @@ const InnerError = error{ OutOfMemory, AnalysisFail };
672pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void {678pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void {
673 while (self.work_queue.readItem()) |work_item| switch (work_item) {679 while (self.work_queue.readItem()) |work_item| switch (work_item) {
674 .codegen_decl => |decl| switch (decl.analysis) {680 .codegen_decl => |decl| switch (decl.analysis) {
675 .initial_in_progress => unreachable,681 .in_progress => unreachable,
676 .repeat_in_progress => unreachable,682 .outdated => unreachable,
677 .repeat_in_progress_novalue => unreachable,
678683
679 .initial_sema_failure,684 .sema_failure,
680 .repeat_sema_failure,
681 .codegen_failure,685 .codegen_failure,
682 .initial_dependency_failure,686 .dependency_failure,
683 .repeat_dependency_failure,
684 => continue,687 => continue,
685688
686 .complete, .codegen_failure_retryable => {689 .complete, .codegen_failure_retryable => {
...@@ -706,7 +709,7 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void {...@@ -706,7 +709,7 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void {
706 self.bin_file.updateDecl(self, decl) catch |err| switch (err) {709 self.bin_file.updateDecl(self, decl) catch |err| switch (err) {
707 error.OutOfMemory => return error.OutOfMemory,710 error.OutOfMemory => return error.OutOfMemory,
708 error.AnalysisFail => {711 error.AnalysisFail => {
709 decl.analysis = .repeat_dependency_failure;712 decl.analysis = .dependency_failure;
710 },713 },
711 else => {714 else => {
712 try self.failed_decls.ensureCapacity(self.failed_decls.size + 1);715 try self.failed_decls.ensureCapacity(self.failed_decls.size + 1);
...@@ -721,6 +724,40 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void {...@@ -721,6 +724,40 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void {
721 };724 };
722 },725 },
723 },726 },
727 .re_analyze_decl => |decl| switch (decl.analysis) {
728 .in_progress => unreachable,
729
730 .sema_failure,
731 .codegen_failure,
732 .dependency_failure,
733 .complete,
734 .codegen_failure_retryable,
735 => continue,
736
737 .outdated => {
738 const zir_module = self.getSrcModule(decl.scope) catch |err| switch (err) {
739 error.OutOfMemory => return error.OutOfMemory,
740 else => {
741 try self.failed_decls.ensureCapacity(self.failed_decls.size + 1);
742 self.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
743 self.allocator,
744 decl.src,
745 "unable to load source file '{}': {}",
746 .{decl.scope.sub_file_path, @errorName(err)},
747 ));
748 decl.analysis = .codegen_failure_retryable;
749 continue;
750 },
751 };
752 const decl_name = mem.spanZ(decl.name);
753 // We already detected deletions, so we know this will be found.
754 const src_decl = zir_module.findDecl(decl_name).?;
755 self.reAnalyzeDecl(decl, src_decl) catch |err| switch (err) {
756 error.OutOfMemory => return error.OutOfMemory,
757 error.AnalysisFail => continue,
758 };
759 }
760 },
724 };761 };
725}762}
726763
...@@ -797,13 +834,6 @@ fn getSrcModule(self: *Module, root_scope: *Scope.ZIRModule) !*zir.Module {...@@ -797,13 +834,6 @@ fn getSrcModule(self: *Module, root_scope: *Scope.ZIRModule) !*zir.Module {
797}834}
798835
799fn analyzeRoot(self: *Module, root_scope: *Scope.ZIRModule) !void {836fn analyzeRoot(self: *Module, root_scope: *Scope.ZIRModule) !void {
800 // TODO use the cache to identify, from the modified source files, the decls which have
801 // changed based on the span of memory that represents the decl in the re-parsed source file.
802 // Use the cached dependency graph to recursively determine the set of decls which need
803 // regeneration.
804 // Here we simulate adding a source file which was previously not part of the compilation,
805 // which means scanning the decls looking for exports.
806 // TODO also identify decls that need to be deleted.
807 switch (root_scope.status) {837 switch (root_scope.status) {
808 .never_loaded => {838 .never_loaded => {
809 const src_module = try self.getSrcModule(root_scope);839 const src_module = try self.getSrcModule(root_scope);
...@@ -814,7 +844,7 @@ fn analyzeRoot(self: *Module, root_scope: *Scope.ZIRModule) !void {...@@ -814,7 +844,7 @@ fn analyzeRoot(self: *Module, root_scope: *Scope.ZIRModule) !void {
814844
815 for (src_module.decls) |decl| {845 for (src_module.decls) |decl| {
816 if (decl.cast(zir.Inst.Export)) |export_inst| {846 if (decl.cast(zir.Inst.Export)) |export_inst| {
817 _ = try self.resolveDecl(&root_scope.base, &export_inst.base, link.ElfFile.TextBlock.empty);847 _ = try self.resolveDecl(&root_scope.base, &export_inst.base);
818 }848 }
819 }849 }
820 },850 },
...@@ -827,107 +857,110 @@ fn analyzeRoot(self: *Module, root_scope: *Scope.ZIRModule) !void {...@@ -827,107 +857,110 @@ fn analyzeRoot(self: *Module, root_scope: *Scope.ZIRModule) !void {
827 => {857 => {
828 const src_module = try self.getSrcModule(root_scope);858 const src_module = try self.getSrcModule(root_scope);
829859
830 // Look for changed decls. First we add all the decls that changed
831 // into the set.
832 var regen_decl_set = std.ArrayList(*Decl).init(self.allocator);
833 defer regen_decl_set.deinit();
834 try regen_decl_set.ensureCapacity(src_module.decls.len);
835
836 var exports_to_resolve = std.ArrayList(*zir.Inst).init(self.allocator);860 var exports_to_resolve = std.ArrayList(*zir.Inst).init(self.allocator);
837 defer exports_to_resolve.deinit();861 defer exports_to_resolve.deinit();
838862
863 // Keep track of the decls that we expect to see in this file so that
864 // we know which ones have been deleted.
865 var deleted_decls = std.AutoHashMap(*Decl, void).init(self.allocator);
866 defer deleted_decls.deinit();
867 try deleted_decls.ensureCapacity(self.decl_table.size);
868 {
869 var it = self.decl_table.iterator();
870 while (it.next()) |kv| {
871 deleted_decls.putAssumeCapacityNoClobber(kv.value, {});
872 }
873 }
874
839 for (src_module.decls) |src_decl| {875 for (src_module.decls) |src_decl| {
840 const name_hash = Decl.hashSimpleName(src_decl.name);876 const name_hash = Decl.hashSimpleName(src_decl.name);
841 if (self.decl_table.get(name_hash)) |kv| {877 if (self.decl_table.get(name_hash)) |kv| {
842 const decl = kv.value;878 const decl = kv.value;
879 deleted_decls.removeAssertDiscard(decl);
843 const new_contents_hash = Decl.hashSimpleName(src_decl.contents);880 const new_contents_hash = Decl.hashSimpleName(src_decl.contents);
844 if (!mem.eql(u8, &new_contents_hash, &decl.contents_hash)) {881 if (!mem.eql(u8, &new_contents_hash, &decl.contents_hash)) {
845 std.debug.warn("noticed that '{}' changed\n", .{src_decl.name});882 std.debug.warn("noticed '{}' source changed\n", .{src_decl.name});
846 regen_decl_set.appendAssumeCapacity(decl);883 decl.analysis = .outdated;
884 decl.contents_hash = new_contents_hash;
885 try self.work_queue.writeItem(.{ .re_analyze_decl = decl });
847 }886 }
848 } else if (src_decl.cast(zir.Inst.Export)) |export_inst| {887 } else if (src_decl.cast(zir.Inst.Export)) |export_inst| {
849 try exports_to_resolve.append(&export_inst.base);888 try exports_to_resolve.append(&export_inst.base);
850 }889 }
851 }890 }
852
853 // Next, recursively chase the dependency graph, to populate the set.
854 {891 {
855 var i: usize = 0;892 // Handle explicitly deleted decls from the source code. Not to be confused
856 while (i < regen_decl_set.items.len) : (i += 1) {893 // with when we delete decls because they are no longer referenced.
857 const decl = regen_decl_set.items[i];894 var it = deleted_decls.iterator();
858 if (decl.isFlaggedForRegeneration()) {895 while (it.next()) |kv| {
859 // We already looked at this decl's dependency graph.896 std.debug.warn("noticed '{}' deleted from source\n", .{kv.key.name});
860 continue;897 try self.deleteDecl(kv.key);
861 }
862 decl.flagForRegeneration();
863 // Remove itself from its dependencies, because we are about to destroy the
864 // decl pointer.
865 for (decl.dependencies.items) |dep| {
866 dep.removeDependant(decl);
867 }
868 // Populate the set with decls that need to get regenerated because they
869 // depend on this one.
870 // TODO If it is only a function body that is modified, it should break the chain
871 // and not cause its dependants to be regenerated.
872 for (decl.dependants.items) |dep| {
873 if (!dep.isFlaggedForRegeneration()) {
874 regen_decl_set.appendAssumeCapacity(dep);
875 }
876 }
877 }898 }
878 }899 }
879900 for (exports_to_resolve.items) |export_inst| {
880 // Remove them all from the decl_table.901 _ = try self.resolveDecl(&root_scope.base, export_inst);
881 for (regen_decl_set.items) |decl| {
882 const decl_name = mem.spanZ(decl.name);
883 const old_name_hash = Decl.hashSimpleName(decl_name);
884 self.decl_table.removeAssertDiscard(old_name_hash);
885
886 if (self.export_owners.remove(decl)) |kv| {
887 for (kv.value) |exp| {
888 self.bin_file.deleteExport(exp.link);
889 }
890 freeExportList(self.allocator, kv.value);
891 }
892 }902 }
903 },
904 }
905}
893906
894 // Regenerate the decls in the set.907fn deleteDecl(self: *Module, decl: *Decl) !void {
895 const zir_module = try self.getSrcModule(root_scope);908 std.debug.warn("deleting decl '{}'\n", .{decl.name});
896909 const name_hash = decl.fullyQualifiedNameHash();
897 while (regen_decl_set.popOrNull()) |decl| {910 self.decl_table.removeAssertDiscard(name_hash);
898 const decl_name = mem.spanZ(decl.name);911 // Remove itself from its dependencies, because we are about to destroy the decl pointer.
899 std.debug.warn("regenerating {}\n", .{decl_name});912 for (decl.dependencies.items) |dep| {
900 const saved_link = decl.link;913 dep.removeDependant(decl);
901 const decl_exports_entry = if (self.decl_exports.remove(decl)) |kv| kv.value else null;914 if (dep.dependants.items.len == 0) {
902 const src_decl = zir_module.findDecl(decl_name) orelse {915 // We don't recursively perform a deletion here, because during the update,
903 @panic("TODO treat this as a deleted decl");916 // another reference to it may turn up.
904 };917 assert(!dep.deletion_flag);
905918 dep.deletion_flag = true;
906 decl.destroy(self.allocator);919 try self.deletion_set.append(self.allocator, dep);
907920 }
908 const new_decl = self.resolveDecl(921 }
909 &root_scope.base,922 // Anything that depends on this deleted decl certainly needs to be re-analyzed.
910 src_decl,923 for (decl.dependants.items) |dep| {
911 saved_link,924 dep.removeDependency(decl);
912 ) catch |err| switch (err) {925 if (dep.analysis != .outdated) {
913 error.OutOfMemory => return error.OutOfMemory,926 dep.analysis = .outdated;
914 error.AnalysisFail => continue,927 try self.work_queue.writeItem(.{ .re_analyze_decl = dep });
915 };928 }
916 if (decl_exports_entry) |entry| {929 }
917 const gop = try self.decl_exports.getOrPut(new_decl);930 self.deleteDeclExports(decl);
918 if (gop.found_existing) {931 self.bin_file.freeDecl(decl);
919 self.allocator.free(entry);932 decl.destroy(self.allocator);
920 } else {933}
921 gop.kv.value = entry;934
922 }935/// Delete all the Export objects that are caused by this Decl. Re-analysis of
936/// this Decl will cause them to be re-created (or not).
937fn deleteDeclExports(self: *Module, decl: *Decl) void {
938 const kv = self.export_owners.remove(decl) orelse return;
939
940 for (kv.value) |exp| {
941 if (self.decl_exports.get(exp.exported_decl)) |decl_exports_kv| {
942 // Remove exports with owner_decl matching the regenerating decl.
943 const list = decl_exports_kv.value;
944 var i: usize = 0;
945 var new_len = list.len;
946 while (i < new_len) {
947 if (list[i].owner_decl == decl) {
948 mem.copyBackwards(*Export, list[i..], list[i + 1..new_len]);
949 new_len -= 1;
950 } else {
951 i += 1;
923 }952 }
924 }953 }
925954 decl_exports_kv.value = self.allocator.shrink(list, new_len);
926 for (exports_to_resolve.items) |export_inst| {955 if (new_len == 0) {
927 _ = try self.resolveDecl(&root_scope.base, export_inst, link.ElfFile.TextBlock.empty);956 self.decl_exports.removeAssertDiscard(exp.exported_decl);
928 }957 }
929 },958 }
959
960 self.bin_file.deleteExport(exp.link);
961 self.allocator.destroy(exp);
930 }962 }
963 self.allocator.free(kv.value);
931}964}
932965
933fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {966fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {
...@@ -959,15 +992,111 @@ fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {...@@ -959,15 +992,111 @@ fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {
959 };992 };
960}993}
961994
962fn resolveDecl(995fn reAnalyzeDecl(self: *Module, decl: *Decl, old_inst: *zir.Inst) InnerError!void {
963 self: *Module,996 switch (decl.analysis) {
964 scope: *Scope,997 .in_progress => unreachable,
965 old_inst: *zir.Inst,998 .dependency_failure,
966 bin_file_link: link.ElfFile.TextBlock,999 .sema_failure,
967) InnerError!*Decl {1000 .codegen_failure,
1001 .codegen_failure_retryable,
1002 .complete,
1003 => return,
1004
1005 .outdated => {}, // Decl re-analysis
1006 }
1007 std.debug.warn("re-analyzing {}\n", .{decl.name});
1008 decl.src = old_inst.src;
1009
1010 // The exports this Decl performs will be re-discovered, so we remove them here
1011 // prior to re-analysis.
1012 self.deleteDeclExports(decl);
1013 // Dependencies will be re-discovered, so we remove them here prior to re-analysis.
1014 for (decl.dependencies.items) |dep| {
1015 dep.removeDependant(decl);
1016 if (dep.dependants.items.len == 0) {
1017 // We don't perform a deletion here, because this Decl or another one
1018 // may end up referencing it before the update is complete.
1019 assert(!dep.deletion_flag);
1020 dep.deletion_flag = true;
1021 try self.deletion_set.append(self.allocator, dep);
1022 }
1023 }
1024 decl.dependencies.shrink(self.allocator, 0);
1025 var decl_scope: Scope.DeclAnalysis = .{
1026 .decl = decl,
1027 .arena = std.heap.ArenaAllocator.init(self.allocator),
1028 };
1029 errdefer decl_scope.arena.deinit();
1030
1031 const typed_value = self.analyzeInstConst(&decl_scope.base, old_inst) catch |err| switch (err) {
1032 error.OutOfMemory => return error.OutOfMemory,
1033 error.AnalysisFail => {
1034 switch (decl.analysis) {
1035 .in_progress => decl.analysis = .dependency_failure,
1036 else => {},
1037 }
1038 decl.generation = self.generation;
1039 return error.AnalysisFail;
1040 },
1041 };
1042 const arena_state = try decl_scope.arena.allocator.create(std.heap.ArenaAllocator.State);
1043 arena_state.* = decl_scope.arena.state;
1044
1045 var prev_type_has_bits = false;
1046 var type_changed = true;
1047
1048 if (decl.typedValueManaged()) |tvm| {
1049 prev_type_has_bits = tvm.typed_value.ty.hasCodeGenBits();
1050 type_changed = !tvm.typed_value.ty.eql(typed_value.ty);
1051
1052 tvm.deinit(self.allocator);
1053 }
1054 decl.typed_value = .{
1055 .most_recent = .{
1056 .typed_value = typed_value,
1057 .arena = arena_state,
1058 },
1059 };
1060 decl.analysis = .complete;
1061 decl.generation = self.generation;
1062 if (typed_value.ty.hasCodeGenBits()) {
1063 // We don't fully codegen the decl until later, but we do need to reserve a global
1064 // offset table index for it. This allows us to codegen decls out of dependency order,
1065 // increasing how many computations can be done in parallel.
1066 try self.bin_file.allocateDeclIndexes(decl);
1067 try self.work_queue.writeItem(.{ .codegen_decl = decl });
1068 } else if (prev_type_has_bits) {
1069 self.bin_file.freeDecl(decl);
1070 }
1071
1072 // If the decl is a function, and the type is the same, we do not need
1073 // to chase the dependants.
1074 if (type_changed or typed_value.val.tag() != .function) {
1075 for (decl.dependants.items) |dep| {
1076 switch (dep.analysis) {
1077 .in_progress => unreachable,
1078 .outdated => continue, // already queued for update
1079
1080 .dependency_failure,
1081 .sema_failure,
1082 .codegen_failure,
1083 .codegen_failure_retryable,
1084 .complete,
1085 => if (dep.generation != self.generation) {
1086 dep.analysis = .outdated;
1087 try self.work_queue.writeItem(.{ .re_analyze_decl = dep });
1088 },
1089 }
1090 }
1091 }
1092}
1093
1094fn resolveDecl(self: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!*Decl {
968 const hash = Decl.hashSimpleName(old_inst.name);1095 const hash = Decl.hashSimpleName(old_inst.name);
969 if (self.decl_table.get(hash)) |kv| {1096 if (self.decl_table.get(hash)) |kv| {
970 return kv.value;1097 const decl = kv.value;
1098 try self.reAnalyzeDecl(decl, old_inst);
1099 return decl;
971 } else {1100 } else {
972 const new_decl = blk: {1101 const new_decl = blk: {
973 try self.decl_table.ensureCapacity(self.decl_table.size + 1);1102 try self.decl_table.ensureCapacity(self.decl_table.size + 1);
...@@ -980,9 +1109,11 @@ fn resolveDecl(...@@ -980,9 +1109,11 @@ fn resolveDecl(
980 .scope = scope.namespace(),1109 .scope = scope.namespace(),
981 .src = old_inst.src,1110 .src = old_inst.src,
982 .typed_value = .{ .never_succeeded = {} },1111 .typed_value = .{ .never_succeeded = {} },
983 .analysis = .initial_in_progress,1112 .analysis = .in_progress,
1113 .deletion_flag = false,
984 .contents_hash = Decl.hashSimpleName(old_inst.contents),1114 .contents_hash = Decl.hashSimpleName(old_inst.contents),
985 .link = bin_file_link,1115 .link = link.ElfFile.TextBlock.empty,
1116 .generation = 0,
986 };1117 };
987 self.decl_table.putAssumeCapacityNoClobber(hash, new_decl);1118 self.decl_table.putAssumeCapacityNoClobber(hash, new_decl);
988 break :blk new_decl;1119 break :blk new_decl;
...@@ -998,10 +1129,10 @@ fn resolveDecl(...@@ -998,10 +1129,10 @@ fn resolveDecl(
998 error.OutOfMemory => return error.OutOfMemory,1129 error.OutOfMemory => return error.OutOfMemory,
999 error.AnalysisFail => {1130 error.AnalysisFail => {
1000 switch (new_decl.analysis) {1131 switch (new_decl.analysis) {
1001 .initial_in_progress => new_decl.analysis = .initial_dependency_failure,1132 .in_progress => new_decl.analysis = .dependency_failure,
1002 .repeat_in_progress => new_decl.analysis = .repeat_dependency_failure,
1003 else => {},1133 else => {},
1004 }1134 }
1135 new_decl.generation = self.generation;
1005 return error.AnalysisFail;1136 return error.AnalysisFail;
1006 },1137 },
1007 };1138 };
...@@ -1016,14 +1147,13 @@ fn resolveDecl(...@@ -1016,14 +1147,13 @@ fn resolveDecl(
1016 },1147 },
1017 };1148 };
1018 new_decl.analysis = .complete;1149 new_decl.analysis = .complete;
1150 new_decl.generation = self.generation;
1019 if (typed_value.ty.hasCodeGenBits()) {1151 if (typed_value.ty.hasCodeGenBits()) {
1020 // We don't fully codegen the decl until later, but we do need to reserve a global1152 // We don't fully codegen the decl until later, but we do need to reserve a global
1021 // offset table index for it. This allows us to codegen decls out of dependency order,1153 // offset table index for it. This allows us to codegen decls out of dependency order,
1022 // increasing how many computations can be done in parallel.1154 // increasing how many computations can be done in parallel.
1023 try self.bin_file.allocateDeclIndexes(new_decl);1155 try self.bin_file.allocateDeclIndexes(new_decl);
10241156 try self.work_queue.writeItem(.{ .codegen_decl = new_decl });
1025 // We ensureCapacity when scanning for decls.
1026 self.work_queue.writeItemAssumeCapacity(.{ .codegen_decl = new_decl });
1027 }1157 }
1028 return new_decl;1158 return new_decl;
1029 }1159 }
...@@ -1031,15 +1161,13 @@ fn resolveDecl(...@@ -1031,15 +1161,13 @@ fn resolveDecl(
10311161
1032/// Declares a dependency on the decl.1162/// Declares a dependency on the decl.
1033fn resolveCompleteDecl(self: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!*Decl {1163fn resolveCompleteDecl(self: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!*Decl {
1034 const decl = try self.resolveDecl(scope, old_inst, link.ElfFile.TextBlock.empty);1164 const decl = try self.resolveDecl(scope, old_inst);
1035 switch (decl.analysis) {1165 switch (decl.analysis) {
1036 .initial_in_progress => unreachable,1166 .in_progress => unreachable,
1037 .repeat_in_progress => unreachable,1167 .outdated => unreachable,
1038 .repeat_in_progress_novalue => unreachable,1168
1039 .initial_dependency_failure,1169 .dependency_failure,
1040 .repeat_dependency_failure,1170 .sema_failure,
1041 .initial_sema_failure,
1042 .repeat_sema_failure,
1043 .codegen_failure,1171 .codegen_failure,
1044 .codegen_failure_retryable,1172 .codegen_failure_retryable,
1045 => return error.AnalysisFail,1173 => return error.AnalysisFail,
...@@ -1134,6 +1262,7 @@ fn analyzeExport(self: *Module, scope: *Scope, export_inst: *zir.Inst.Export) In...@@ -1134,6 +1262,7 @@ fn analyzeExport(self: *Module, scope: *Scope, export_inst: *zir.Inst.Export) In
1134 .src = export_inst.base.src,1262 .src = export_inst.base.src,
1135 .link = .{},1263 .link = .{},
1136 .owner_decl = owner_decl,1264 .owner_decl = owner_decl,
1265 .exported_decl = exported_decl,
1137 .status = .in_progress,1266 .status = .in_progress,
1138 };1267 };
11391268
...@@ -2153,11 +2282,7 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Err...@@ -2153,11 +2282,7 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Err
2153 switch (scope.tag) {2282 switch (scope.tag) {
2154 .decl => {2283 .decl => {
2155 const decl = scope.cast(Scope.DeclAnalysis).?.decl;2284 const decl = scope.cast(Scope.DeclAnalysis).?.decl;
2156 switch (decl.analysis) {2285 decl.analysis = .sema_failure;
2157 .initial_in_progress => decl.analysis = .initial_sema_failure,
2158 .repeat_in_progress => decl.analysis = .repeat_sema_failure,
2159 else => unreachable,
2160 }
2161 self.failed_decls.putAssumeCapacityNoClobber(decl, err_msg);2286 self.failed_decls.putAssumeCapacityNoClobber(decl, err_msg);
2162 },2287 },
2163 .block => {2288 .block => {
src-self-hosted/link.zig+36-13
...@@ -126,7 +126,9 @@ pub const ElfFile = struct {...@@ -126,7 +126,9 @@ pub const ElfFile = struct {
126 local_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){},126 local_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){},
127 global_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){},127 global_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){},
128128
129 global_symbol_free_list: std.ArrayListUnmanaged(usize) = std.ArrayListUnmanaged(usize){},129 local_symbol_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){},
130 global_symbol_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){},
131 offset_table_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){},
130132
131 /// Same order as in the file. The value is the absolute vaddr value.133 /// Same order as in the file. The value is the absolute vaddr value.
132 /// If the vaddr of the executable program header changes, the entire134 /// If the vaddr of the executable program header changes, the entire
...@@ -232,6 +234,8 @@ pub const ElfFile = struct {...@@ -232,6 +234,8 @@ pub const ElfFile = struct {
232 self.local_symbols.deinit(self.allocator);234 self.local_symbols.deinit(self.allocator);
233 self.global_symbols.deinit(self.allocator);235 self.global_symbols.deinit(self.allocator);
234 self.global_symbol_free_list.deinit(self.allocator);236 self.global_symbol_free_list.deinit(self.allocator);
237 self.local_symbol_free_list.deinit(self.allocator);
238 self.offset_table_free_list.deinit(self.allocator);
235 self.text_block_free_list.deinit(self.allocator);239 self.text_block_free_list.deinit(self.allocator);
236 self.offset_table.deinit(self.allocator);240 self.offset_table.deinit(self.allocator);
237 if (self.owns_file_handle) {241 if (self.owns_file_handle) {
...@@ -792,6 +796,7 @@ pub const ElfFile = struct {...@@ -792,6 +796,7 @@ pub const ElfFile = struct {
792 }796 }
793797
794 if (self.last_text_block == text_block) {798 if (self.last_text_block == text_block) {
799 // TODO shrink the .text section size here
795 self.last_text_block = text_block.prev;800 self.last_text_block = text_block.prev;
796 }801 }
797802
...@@ -944,33 +949,51 @@ pub const ElfFile = struct {...@@ -944,33 +949,51 @@ pub const ElfFile = struct {
944 pub fn allocateDeclIndexes(self: *ElfFile, decl: *Module.Decl) !void {949 pub fn allocateDeclIndexes(self: *ElfFile, decl: *Module.Decl) !void {
945 if (decl.link.local_sym_index != 0) return;950 if (decl.link.local_sym_index != 0) return;
946951
952 // Here we also ensure capacity for the free lists so that they can be appended to without fail.
947 try self.local_symbols.ensureCapacity(self.allocator, self.local_symbols.items.len + 1);953 try self.local_symbols.ensureCapacity(self.allocator, self.local_symbols.items.len + 1);
954 try self.local_symbol_free_list.ensureCapacity(self.allocator, self.local_symbols.items.len);
948 try self.offset_table.ensureCapacity(self.allocator, self.offset_table.items.len + 1);955 try self.offset_table.ensureCapacity(self.allocator, self.offset_table.items.len + 1);
949 const local_sym_index = self.local_symbols.items.len;956 try self.offset_table_free_list.ensureCapacity(self.allocator, self.local_symbols.items.len);
950 const offset_table_index = self.offset_table.items.len;957
958 if (self.local_symbol_free_list.popOrNull()) |i| {
959 std.debug.warn("reusing symbol index {} for {}\n", .{i, decl.name});
960 decl.link.local_sym_index = i;
961 } else {
962 std.debug.warn("allocating symbol index {} for {}\n", .{self.local_symbols.items.len, decl.name});
963 decl.link.local_sym_index = @intCast(u32, self.local_symbols.items.len);
964 _ = self.local_symbols.addOneAssumeCapacity();
965 }
966
967 if (self.offset_table_free_list.popOrNull()) |i| {
968 decl.link.offset_table_index = i;
969 } else {
970 decl.link.offset_table_index = @intCast(u32, self.offset_table.items.len);
971 _ = self.offset_table.addOneAssumeCapacity();
972 self.offset_table_count_dirty = true;
973 }
974
951 const phdr = &self.program_headers.items[self.phdr_load_re_index.?];975 const phdr = &self.program_headers.items[self.phdr_load_re_index.?];
952976
953 self.local_symbols.appendAssumeCapacity(.{977 self.local_symbols.items[decl.link.local_sym_index] = .{
954 .st_name = 0,978 .st_name = 0,
955 .st_info = 0,979 .st_info = 0,
956 .st_other = 0,980 .st_other = 0,
957 .st_shndx = 0,981 .st_shndx = 0,
958 .st_value = phdr.p_vaddr,982 .st_value = phdr.p_vaddr,
959 .st_size = 0,983 .st_size = 0,
960 });984 };
961 self.offset_table.appendAssumeCapacity(0);985 self.offset_table.items[decl.link.offset_table_index] = 0;
962
963 self.offset_table_count_dirty = true;
964
965 std.debug.warn("allocating symbol index {} for {}\n", .{local_sym_index, decl.name});
966 decl.link.local_sym_index = @intCast(u32, local_sym_index);
967 decl.link.offset_table_index = @intCast(u32, offset_table_index);
968 }986 }
969987
970 pub fn freeDecl(self: *ElfFile, decl: *Module.Decl) void {988 pub fn freeDecl(self: *ElfFile, decl: *Module.Decl) void {
971 self.freeTextBlock(&decl.link);989 self.freeTextBlock(&decl.link);
972 if (decl.link.local_sym_index != 0) {990 if (decl.link.local_sym_index != 0) {
973 @panic("TODO free the symbol entry and offset table entry");991 self.local_symbol_free_list.appendAssumeCapacity(decl.link.local_sym_index);
992 self.offset_table_free_list.appendAssumeCapacity(decl.link.offset_table_index);
993
994 self.local_symbols.items[decl.link.local_sym_index].st_info = 0;
995
996 decl.link.local_sym_index = 0;
974 }997 }
975 }998 }
976999
src-self-hosted/type.zig+27-11
...@@ -92,13 +92,13 @@ pub const Type = extern union {...@@ -92,13 +92,13 @@ pub const Type = extern union {
92 return @fieldParentPtr(T, "base", self.ptr_otherwise);92 return @fieldParentPtr(T, "base", self.ptr_otherwise);
93 }93 }
9494
95 pub fn eql(self: Type, other: Type) bool {95 pub fn eql(a: Type, b: Type) bool {
96 //std.debug.warn("test {} == {}\n", .{ self, other });96 //std.debug.warn("test {} == {}\n", .{ a, b });
97 // As a shortcut, if the small tags / addresses match, we're done.97 // As a shortcut, if the small tags / addresses match, we're done.
98 if (self.tag_if_small_enough == other.tag_if_small_enough)98 if (a.tag_if_small_enough == b.tag_if_small_enough)
99 return true;99 return true;
100 const zig_tag_a = self.zigTypeTag();100 const zig_tag_a = a.zigTypeTag();
101 const zig_tag_b = self.zigTypeTag();101 const zig_tag_b = b.zigTypeTag();
102 if (zig_tag_a != zig_tag_b)102 if (zig_tag_a != zig_tag_b)
103 return false;103 return false;
104 switch (zig_tag_a) {104 switch (zig_tag_a) {
...@@ -111,24 +111,40 @@ pub const Type = extern union {...@@ -111,24 +111,40 @@ pub const Type = extern union {
111 .Undefined => return true,111 .Undefined => return true,
112 .Null => return true,112 .Null => return true,
113 .Pointer => {113 .Pointer => {
114 const is_slice_a = isSlice(self);114 const is_slice_a = isSlice(a);
115 const is_slice_b = isSlice(other);115 const is_slice_b = isSlice(b);
116 if (is_slice_a != is_slice_b)116 if (is_slice_a != is_slice_b)
117 return false;117 return false;
118 @panic("TODO implement more pointer Type equality comparison");118 @panic("TODO implement more pointer Type equality comparison");
119 },119 },
120 .Int => {120 .Int => {
121 if (self.tag() != other.tag()) {121 if (a.tag() != b.tag()) {
122 // Detect that e.g. u64 != usize, even if the bits match on a particular target.122 // Detect that e.g. u64 != usize, even if the bits match on a particular target.
123 return false;123 return false;
124 }124 }
125 // The target will not be branched upon, because we handled target-dependent cases above.125 // The target will not be branched upon, because we handled target-dependent cases above.
126 const info_a = self.intInfo(@as(Target, undefined));126 const info_a = a.intInfo(@as(Target, undefined));
127 const info_b = self.intInfo(@as(Target, undefined));127 const info_b = b.intInfo(@as(Target, undefined));
128 return info_a.signed == info_b.signed and info_a.bits == info_b.bits;128 return info_a.signed == info_b.signed and info_a.bits == info_b.bits;
129 },129 },
130 .Array => {
131 if (a.arrayLen() != b.arrayLen())
132 return false;
133 if (a.elemType().eql(b.elemType()))
134 return false;
135 const sentinel_a = a.arraySentinel();
136 const sentinel_b = b.arraySentinel();
137 if (sentinel_a) |sa| {
138 if (sentinel_b) |sb| {
139 return sa.eql(sb);
140 } else {
141 return false;
142 }
143 } else {
144 return sentinel_b == null;
145 }
146 },
130 .Float,147 .Float,
131 .Array,
132 .Struct,148 .Struct,
133 .Optional,149 .Optional,
134 .ErrorUnion,150 .ErrorUnion,
src-self-hosted/value.zig+5
...@@ -666,6 +666,11 @@ pub const Value = extern union {...@@ -666,6 +666,11 @@ pub const Value = extern union {
666 return orderAgainstZero(lhs).compare(op);666 return orderAgainstZero(lhs).compare(op);
667 }667 }
668668
669 pub fn eql(a: Value, b: Value) bool {
670 // TODO non numerical comparisons
671 return compare(a, .eq, b);
672 }
673
669 pub fn toBool(self: Value) bool {674 pub fn toBool(self: Value) bool {
670 return switch (self.tag()) {675 return switch (self.tag()) {
671 .bool_true => true,676 .bool_true => true,