| author |  Andrew Kelley <andrew@ziglang.org> 2023-09-27 18:20:00-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2023-09-28 16:28:41-07:00 |
| log | 8f2f12f94080ede762df4a5018249160eedc7f09 |
| tree | 40df4f059ae5c271102dcb2dc73be3be318e32d6 |
| parent | 077994abb66be93f38142ee55243fee8cdc3a5da |
Compilation: introduce saveState()
This commit introduces `--debug-incremental` so that we can start
playing around with incremental compilation while it is still being
developed, and before it is enabled by default.
Currently it saves InternPool data, and has TODO comments for the
remaining things. Deserialization is not implemented yet, which will
require some post-processing such as to build a string map out of
null-terminated string table bytes.
The saved compiler state is stored in a file called <root-name>.zcs
alongside <root-name>.o, <root-name>.pdb, <root-name>.exe, etc. In case
of using the zig build system, these files are all in a zig-cache
directory.
For the self-hosted compiler, here is one data point on the performance
penalty of saving this data:
```
Benchmark 1 (3 runs): zig build-exe ...
measurement mean ± σ min … max outliers delta
wall_time 51.1s ± 354ms 50.7s … 51.4s 0 ( 0%) 0%
peak_rss 3.91GB ± 354KB 3.91GB … 3.91GB 0 ( 0%) 0%
cpu_cycles 212G ± 3.17G 210G … 216G 0 ( 0%) 0%
instructions 274G ± 57.5M 274G … 275G 0 ( 0%) 0%
cache_references 13.1G ± 97.6M 13.0G … 13.2G 0 ( 0%) 0%
cache_misses 1.12G ± 24.6M 1.10G … 1.15G 0 ( 0%) 0%
branch_misses 1.53G ± 1.46M 1.53G … 1.53G 0 ( 0%) 0%
Benchmark 2 (3 runs): zig build-exe ... --debug-incremental
measurement mean ± σ min … max outliers delta
wall_time 51.8s ± 271ms 51.5s … 52.1s 0 ( 0%) + 1.3% ± 1.4%
peak_rss 3.91GB ± 317KB 3.91GB … 3.91GB 0 ( 0%) - 0.0% ± 0.0%
cpu_cycles 213G ± 398M 212G … 213G 0 ( 0%) + 0.3% ± 2.4%
instructions 275G ± 79.1M 275G … 275G 0 ( 0%) + 0.1% ± 0.1%
cache_references 13.1G ± 26.9M 13.0G … 13.1G 0 ( 0%) - 0.1% ± 1.2%
cache_misses 1.12G ± 5.66M 1.11G … 1.12G 0 ( 0%) - 0.6% ± 3.6%
branch_misses 1.53G ± 1.75M 1.53G … 1.54G 0 ( 0%) + 0.2% ± 0.2%
```
At the end of each compilation with `--debug-incremental`, we end up
with a 43 MiB `zig.zcs` file that contains all of the InternPool data
serialized.
Of course, it will necessarily be more expensive to save the state than
to not save the state. However, this data point shows just how cheap the
save state operation is, with all of the groundwork laid for using a
serialization-friendly in-memory data layout.2 files changed, 84 insertions(+), 1 deletions(-)
src/Compilation.zig+66
| ... | @@ -2702,6 +2702,72 @@ pub fn makeBinFileWritable(self: *Compilation) !void { | ... | @@ -2702,6 +2702,72 @@ pub fn makeBinFileWritable(self: *Compilation) !void { |
| 2702 | return self.bin_file.makeWritable(); | 2702 | return self.bin_file.makeWritable(); |
| 2703 | } | 2703 | } |
| 2704 | 2704 | ||
| 2705 | const Header = extern struct { | ||
| 2706 | intern_pool: extern struct { | ||
| 2707 | items_len: u32, | ||
| 2708 | extra_len: u32, | ||
| 2709 | limbs_len: u32, | ||
| 2710 | string_bytes_len: u32, | ||
| 2711 | }, | ||
| 2712 | }; | ||
| 2713 | |||
| 2714 | /// Note that all state that is included in the cache hash namespace is *not* | ||
| 2715 | /// saved, such as the target and most CLI flags. A cache hit will only occur | ||
| 2716 | /// when subsequent compiler invocations use the same set of flags. | ||
| 2717 | pub fn saveState(comp: *Compilation) !void { | ||
| 2718 | var bufs_list: [6]std.os.iovec_const = undefined; | ||
| 2719 | var bufs_len: usize = 0; | ||
| 2720 | |||
| 2721 | const emit = comp.bin_file.options.emit orelse return; | ||
| 2722 | |||
| 2723 | if (comp.bin_file.options.module) |mod| { | ||
| 2724 | const ip = &mod.intern_pool; | ||
| 2725 | const header: Header = .{ | ||
| 2726 | .intern_pool = .{ | ||
| 2727 | .items_len = @intCast(ip.items.len), | ||
| 2728 | .extra_len = @intCast(ip.extra.items.len), | ||
| 2729 | .limbs_len = @intCast(ip.limbs.items.len), | ||
| 2730 | .string_bytes_len = @intCast(ip.string_bytes.items.len), | ||
| 2731 | }, | ||
| 2732 | }; | ||
| 2733 | addBuf(&bufs_list, &bufs_len, mem.asBytes(&header)); | ||
| 2734 | addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.limbs.items)); | ||
| 2735 | addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.extra.items)); | ||
| 2736 | addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.items.items(.data))); | ||
| 2737 | addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.items.items(.tag))); | ||
| 2738 | addBuf(&bufs_list, &bufs_len, ip.string_bytes.items); | ||
| 2739 | |||
| 2740 | // TODO: compilation errors | ||
| 2741 | // TODO: files | ||
| 2742 | // TODO: namespaces | ||
| 2743 | // TODO: decls | ||
| 2744 | // TODO: linker state | ||
| 2745 | } | ||
| 2746 | var basename_buf: [255]u8 = undefined; | ||
| 2747 | const basename = std.fmt.bufPrint(&basename_buf, "{s}.zcs", .{ | ||
| 2748 | comp.bin_file.options.root_name, | ||
| 2749 | }) catch o: { | ||
| 2750 | basename_buf[basename_buf.len - 4 ..].* = ".zcs".*; | ||
| 2751 | break :o &basename_buf; | ||
| 2752 | }; | ||
| 2753 | |||
| 2754 | // Using an atomic file prevents a crash or power failure from corrupting | ||
| 2755 | // the previous incremental compilation state. | ||
| 2756 | var af = try emit.directory.handle.atomicFile(basename, .{}); | ||
| 2757 | defer af.deinit(); | ||
| 2758 | try af.file.pwritevAll(bufs_list[0..bufs_len], 0); | ||
| 2759 | try af.finish(); | ||
| 2760 | } | ||
| 2761 | |||
| 2762 | fn addBuf(bufs_list: []std.os.iovec_const, bufs_len: *usize, buf: []const u8) void { | ||
| 2763 | const i = bufs_len.*; | ||
| 2764 | bufs_len.* = i + 1; | ||
| 2765 | bufs_list[i] = .{ | ||
| 2766 | .iov_base = buf.ptr, | ||
| 2767 | .iov_len = buf.len, | ||
| 2768 | }; | ||
| 2769 | } | ||
| 2770 | |||
| 2705 | /// This function is temporally single-threaded. | 2771 | /// This function is temporally single-threaded. |
| 2706 | pub fn totalErrorCount(self: *Compilation) u32 { | 2772 | pub fn totalErrorCount(self: *Compilation) u32 { |
| 2707 | var total: usize = self.failed_c_objects.count() + | 2773 | var total: usize = self.failed_c_objects.count() + |
src/main.zig+18-1
| ... | @@ -594,6 +594,7 @@ const usage_build_generic = | ... | @@ -594,6 +594,7 @@ const usage_build_generic = |
| 594 | \\ --debug-log [scope] Enable printing debug/info log messages for scope | 594 | \\ --debug-log [scope] Enable printing debug/info log messages for scope |
| 595 | \\ --debug-compile-errors Crash with helpful diagnostics at the first compile error | 595 | \\ --debug-compile-errors Crash with helpful diagnostics at the first compile error |
| 596 | \\ --debug-link-snapshot Enable dumping of the linker's state in JSON format | 596 | \\ --debug-link-snapshot Enable dumping of the linker's state in JSON format |
| 597 | \\ --debug-incremental Enable experimental feature: incremental compilation | ||
| 597 | \\ | 598 | \\ |
| 598 | ; | 599 | ; |
| 599 | 600 | ||
| ... | @@ -904,6 +905,7 @@ fn buildOutputType( | ... | @@ -904,6 +905,7 @@ fn buildOutputType( |
| 904 | var minor_subsystem_version: ?u32 = null; | 905 | var minor_subsystem_version: ?u32 = null; |
| 905 | var wasi_exec_model: ?std.builtin.WasiExecModel = null; | 906 | var wasi_exec_model: ?std.builtin.WasiExecModel = null; |
| 906 | var enable_link_snapshots: bool = false; | 907 | var enable_link_snapshots: bool = false; |
| 908 | var debug_incremental: bool = false; | ||
| 907 | var install_name: ?[]const u8 = null; | 909 | var install_name: ?[]const u8 = null; |
| 908 | var hash_style: link.HashStyle = .both; | 910 | var hash_style: link.HashStyle = .both; |
| 909 | var entitlements: ?[]const u8 = null; | 911 | var entitlements: ?[]const u8 = null; |
| ... | @@ -1272,6 +1274,8 @@ fn buildOutputType( | ... | @@ -1272,6 +1274,8 @@ fn buildOutputType( |
| 1272 | } else { | 1274 | } else { |
| 1273 | enable_link_snapshots = true; | 1275 | enable_link_snapshots = true; |
| 1274 | } | 1276 | } |
| 1277 | } else if (mem.eql(u8, arg, "--debug-incremental")) { | ||
| 1278 | debug_incremental = true; | ||
| 1275 | } else if (mem.eql(u8, arg, "--entitlements")) { | 1279 | } else if (mem.eql(u8, arg, "--entitlements")) { |
| 1276 | entitlements = args_iter.nextOrFatal(); | 1280 | entitlements = args_iter.nextOrFatal(); |
| 1277 | } else if (mem.eql(u8, arg, "-fcompiler-rt")) { | 1281 | } else if (mem.eql(u8, arg, "-fcompiler-rt")) { |
| ... | @@ -3591,11 +3595,16 @@ fn buildOutputType( | ... | @@ -3591,11 +3595,16 @@ fn buildOutputType( |
| 3591 | } | 3595 | } |
| 3592 | 3596 | ||
| 3593 | updateModule(comp) catch |err| switch (err) { | 3597 | updateModule(comp) catch |err| switch (err) { |
| 3594 | error.SemanticAnalyzeFail => if (listen == .none) process.exit(1), | 3598 | error.SemanticAnalyzeFail => { |
| 3599 | assert(listen == .none); | ||
| 3600 | saveState(comp, debug_incremental); | ||
| 3601 | process.exit(1); | ||
| 3602 | }, | ||
| 3595 | else => |e| return e, | 3603 | else => |e| return e, |
| 3596 | }; | 3604 | }; |
| 3597 | if (build_options.only_c) return cleanExit(); | 3605 | if (build_options.only_c) return cleanExit(); |
| 3598 | try comp.makeBinFileExecutable(); | 3606 | try comp.makeBinFileExecutable(); |
| 3607 | saveState(comp, debug_incremental); | ||
| 3599 | 3608 | ||
| 3600 | if (test_exec_args.items.len == 0 and object_format == .c) default_exec_args: { | 3609 | if (test_exec_args.items.len == 0 and object_format == .c) default_exec_args: { |
| 3601 | // Default to using `zig run` to execute the produced .c code from `zig test`. | 3610 | // Default to using `zig run` to execute the produced .c code from `zig test`. |
| ... | @@ -3658,6 +3667,14 @@ fn buildOutputType( | ... | @@ -3658,6 +3667,14 @@ fn buildOutputType( |
| 3658 | return cleanExit(); | 3667 | return cleanExit(); |
| 3659 | } | 3668 | } |
| 3660 | 3669 | ||
| 3670 | fn saveState(comp: *Compilation, debug_incremental: bool) void { | ||
| 3671 | if (debug_incremental) { | ||
| 3672 | comp.saveState() catch |err| { | ||
| 3673 | warn("unable to save incremental compilation state: {s}", .{@errorName(err)}); | ||
| 3674 | }; | ||
| 3675 | } | ||
| 3676 | } | ||
| 3677 | |||
| 3661 | fn serve( | 3678 | fn serve( |
| 3662 | comp: *Compilation, | 3679 | comp: *Compilation, |
| 3663 | in: fs.File, | 3680 | in: fs.File, |