authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-22 17:23:03-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-23 01:08:23-05:00
logd3c9bfada64230ad17badb739f15a492b4c2c065
tree6d670167e5d6101747b5a1758c5ca5118f3eb99b
parentc9e02d3e69f909a6eb215286c6109f2b3f1e68a2

std.Build.WriteFileStep: integrate with cache system

And additionally support writing files to source files. This means a custom build step in zig's own build.zig is no longer needed for copying zig.h because it is handled by WriteFileStep.

2 files changed, 175 insertions(+), 100 deletions(-)

build.zig+2-29
...@@ -509,35 +509,8 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {...@@ -509,35 +509,8 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
509 run_opt.addArg("-o");509 run_opt.addArg("-o");
510 run_opt.addFileSourceArg(.{ .path = "stage1/zig1.wasm" });510 run_opt.addFileSourceArg(.{ .path = "stage1/zig1.wasm" });
511511
512 const CopyFileStep = struct {512 const copy_zig_h = b.addWriteFiles();
513 const Step = std.Build.Step;513 copy_zig_h.addCopyFileToSource(.{ .path = "lib/zig.h" }, "stage1/zig.h");
514 const FileSource = std.Build.FileSource;
515 const CopyFileStep = @This();
516
517 step: Step,
518 builder: *std.Build,
519 source: FileSource,
520 dest_rel_path: []const u8,
521
522 pub fn init(builder: *std.Build, source: FileSource, dest_rel_path: []const u8) CopyFileStep {
523 return CopyFileStep{
524 .builder = builder,
525 .step = Step.init(.custom, builder.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), builder.allocator, make),
526 .source = source.dupe(builder),
527 .dest_rel_path = builder.dupePath(dest_rel_path),
528 };
529 }
530
531 fn make(step: *Step) !void {
532 const self = @fieldParentPtr(CopyFileStep, "step", step);
533 const full_src_path = self.source.getPath(self.builder);
534 const full_dest_path = self.builder.pathFromRoot(self.dest_rel_path);
535 try self.builder.updateFile(full_src_path, full_dest_path);
536 }
537 };
538
539 const copy_zig_h = try b.allocator.create(CopyFileStep);
540 copy_zig_h.* = CopyFileStep.init(b, .{ .path = "lib/zig.h" }, "stage1/zig.h");
541514
542 const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm");515 const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm");
543 update_zig1_step.dependOn(&run_opt.step);516 update_zig1_step.dependOn(&run_opt.step);
lib/std/Build/WriteFileStep.zig+173-71
...@@ -1,55 +1,117 @@...@@ -1,55 +1,117 @@
1const std = @import("../std.zig");1//! WriteFileStep is primarily used to create a directory in an appropriate
2const Step = std.Build.Step;2//! location inside the local cache which has a set of files that have either
3const fs = std.fs;3//! been generated during the build, or are copied from the source package.
4const ArrayList = std.ArrayList;4//!
55//! However, this step has an additional capability of writing data to paths
6const WriteFileStep = @This();6//! relative to the package root, effectively mutating the package's source
77//! files. Be careful with the latter functionality; it should not be used
8pub const base_id = .write_file;8//! during the normal build process, but as a utility run by a developer with
9//! intention to update source files, which will then be committed to version
10//! control.
911
10step: Step,12step: Step,
11builder: *std.Build,13builder: *std.Build,
12files: std.TailQueue(File),14/// The elements here are pointers because we need stable pointers for the
15/// GeneratedFile field.
16files: std.ArrayListUnmanaged(*File),
17output_source_files: std.ArrayListUnmanaged(OutputSourceFile),
18
19pub const base_id = .write_file;
1320
14pub const File = struct {21pub const File = struct {
15 source: std.Build.GeneratedFile,22 generated_file: std.Build.GeneratedFile,
16 basename: []const u8,23 sub_path: []const u8,
24 contents: Contents,
25};
26
27pub const OutputSourceFile = struct {
28 contents: Contents,
29 sub_path: []const u8,
30};
31
32pub const Contents = union(enum) {
17 bytes: []const u8,33 bytes: []const u8,
34 copy: std.Build.FileSource,
18};35};
1936
20pub fn init(builder: *std.Build) WriteFileStep {37pub fn init(builder: *std.Build) WriteFileStep {
21 return WriteFileStep{38 return .{
22 .builder = builder,39 .builder = builder,
23 .step = Step.init(.write_file, "writefile", builder.allocator, make),40 .step = Step.init(.write_file, "writefile", builder.allocator, make),
24 .files = .{},41 .files = .{},
42 .output_source_files = .{},
25 };43 };
26}44}
2745
28pub fn add(self: *WriteFileStep, basename: []const u8, bytes: []const u8) void {46pub fn add(wf: *WriteFileStep, sub_path: []const u8, bytes: []const u8) void {
29 const node = self.builder.allocator.create(std.TailQueue(File).Node) catch @panic("unhandled error");47 const gpa = wf.builder.allocator;
30 node.* = .{48 const file = gpa.create(File) catch @panic("OOM");
31 .data = .{49 file.* = .{
32 .source = std.Build.GeneratedFile{ .step = &self.step },50 .generated_file = .{ .step = &wf.step },
33 .basename = self.builder.dupePath(basename),51 .sub_path = wf.builder.dupePath(sub_path),
34 .bytes = self.builder.dupe(bytes),52 .contents = .{ .bytes = wf.builder.dupe(bytes) },
35 },53 };
54 wf.files.append(gpa, file) catch @panic("OOM");
55}
56
57/// Place the file into the generated directory within the local cache,
58/// along with all the rest of the files added to this step. The parameter
59/// here is the destination path relative to the local cache directory
60/// associated with this WriteFileStep. It may be a basename, or it may
61/// include sub-directories, in which case this step will ensure the
62/// required sub-path exists.
63/// This is the option expected to be used most commonly with `addCopyFile`.
64pub fn addCopyFile(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void {
65 const gpa = wf.builder.allocator;
66 const file = gpa.create(File) catch @panic("OOM");
67 file.* = .{
68 .generated_file = .{ .step = &wf.step },
69 .sub_path = wf.builder.dupePath(sub_path),
70 .contents = .{ .copy = source },
36 };71 };
72 wf.files.append(gpa, file) catch @panic("OOM");
73}
3774
38 self.files.append(node);75/// A path relative to the package root.
76/// Be careful with this because it updates source files. This should not be
77/// used as part of the normal build process, but as a utility occasionally
78/// run by a developer with intent to modify source files and then commit
79/// those changes to version control.
80/// A file added this way is not available with `getFileSource`.
81pub fn addCopyFileToSource(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void {
82 wf.output_source_files.append(wf.builder.allocator, .{
83 .contents = .{ .copy = source },
84 .sub_path = sub_path,
85 }) catch @panic("OOM");
39}86}
4087
41/// Gets a file source for the given basename. If the file does not exist, returns `null`.88/// Gets a file source for the given sub_path. If the file does not exist, returns `null`.
42pub fn getFileSource(step: *WriteFileStep, basename: []const u8) ?std.Build.FileSource {89pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSource {
43 var it = step.files.first;90 for (wf.files.items) |file| {
44 while (it) |node| : (it = node.next) {91 if (std.mem.eql(u8, file.sub_path, sub_path)) {
45 if (std.mem.eql(u8, node.data.basename, basename))92 return .{ .generated = &file.generated_file };
46 return std.Build.FileSource{ .generated = &node.data.source };93 }
47 }94 }
48 return null;95 return null;
49}96}
5097
51fn make(step: *Step) !void {98fn make(step: *Step) !void {
52 const self = @fieldParentPtr(WriteFileStep, "step", step);99 const wf = @fieldParentPtr(WriteFileStep, "step", step);
100
101 // Writing to source files is kind of an extra capability of this
102 // WriteFileStep - arguably it should be a different step. But anyway here
103 // it is, it happens unconditionally and does not interact with the other
104 // files here.
105 for (wf.output_source_files.items) |output_source_file| {
106 const basename = fs.path.basename(output_source_file.sub_path);
107 if (fs.path.dirname(output_source_file.sub_path)) |dirname| {
108 var dir = try wf.builder.build_root.handle.makeOpenPath(dirname, .{});
109 defer dir.close();
110 try writeFile(wf, dir, output_source_file.contents, basename);
111 } else {
112 try writeFile(wf, wf.builder.build_root.handle, output_source_file.contents, basename);
113 }
114 }
53115
54 // The cache is used here not really as a way to speed things up - because writing116 // The cache is used here not really as a way to speed things up - because writing
55 // the data to a file would probably be very fast - but as a way to find a canonical117 // the data to a file would probably be very fast - but as a way to find a canonical
...@@ -58,56 +120,96 @@ fn make(step: *Step) !void {...@@ -58,56 +120,96 @@ fn make(step: *Step) !void {
58 // If, for example, a hard-coded path was used as the location to put WriteFileStep120 // If, for example, a hard-coded path was used as the location to put WriteFileStep
59 // files, then two WriteFileSteps executing in parallel might clobber each other.121 // files, then two WriteFileSteps executing in parallel might clobber each other.
60122
61 // TODO port the cache system from the compiler to zig std lib. Until then123 var man = wf.builder.cache.obtain();
62 // we directly construct the path, and no "cache hit" detection happens;124 defer man.deinit();
63 // the files are always written.125
64 // Note there is similar code over in ConfigHeaderStep.
65 const Hasher = std.crypto.auth.siphash.SipHash128(1, 3);
66 // Random bytes to make WriteFileStep unique. Refresh this with126 // Random bytes to make WriteFileStep unique. Refresh this with
67 // new random bytes when WriteFileStep implementation is modified127 // new random bytes when WriteFileStep implementation is modified
68 // in a non-backwards-compatible way.128 // in a non-backwards-compatible way.
69 var hash = Hasher.init("eagVR1dYXoE7ARDP");129 man.hash.add(@as(u32, 0xd767ee59));
70130
71 {131 for (wf.files.items) |file| {
72 var it = self.files.first;132 man.hash.addBytes(file.sub_path);
73 while (it) |node| : (it = node.next) {133 switch (file.contents) {
74 hash.update(node.data.basename);134 .bytes => |bytes| {
75 hash.update(node.data.bytes);135 man.hash.addBytes(bytes);
76 hash.update("|");136 },
137 .copy => |file_source| {
138 _ = try man.addFile(file_source.getPath(wf.builder), null);
139 },
77 }140 }
78 }141 }
79 var digest: [16]u8 = undefined;142
80 hash.final(&digest);143 if (man.hit() catch |err| failWithCacheError(man, err)) {
81 var hash_basename: [digest.len * 2]u8 = undefined;144 // Cache hit, skip writing file data.
82 _ = std.fmt.bufPrint(145 const digest = man.final();
83 &hash_basename,146 for (wf.files.items) |file| {
84 "{s}",147 file.generated_file.path = try wf.builder.cache_root.join(
85 .{std.fmt.fmtSliceHexLower(&digest)},148 wf.builder.allocator,
86 ) catch unreachable;149 &.{ "o", &digest, file.sub_path },
87150 );
88 const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{151 }
89 "o", &hash_basename,152 return;
90 });153 }
91 var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {154
92 std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });155 const digest = man.final();
156 const cache_path = "o" ++ fs.path.sep_str ++ digest;
157
158 var cache_dir = wf.builder.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| {
159 std.debug.print("unable to make path {s}: {s}\n", .{ cache_path, @errorName(err) });
93 return err;160 return err;
94 };161 };
95 defer dir.close();162 defer cache_dir.close();
96 {163
97 var it = self.files.first;164 for (wf.files.items) |file| {
98 while (it) |node| : (it = node.next) {165 const basename = fs.path.basename(file.sub_path);
99 dir.writeFile(node.data.basename, node.data.bytes) catch |err| {166 if (fs.path.dirname(file.sub_path)) |dirname| {
100 std.debug.print("unable to write {s} into {s}: {s}\n", .{167 var dir = try wf.builder.cache_root.handle.makeOpenPath(dirname, .{});
101 node.data.basename,168 defer dir.close();
102 output_dir,169 try writeFile(wf, dir, file.contents, basename);
103 @errorName(err),170 } else {
104 });171 try writeFile(wf, cache_dir, file.contents, basename);
105 return err;
106 };
107 node.data.source.path = try fs.path.join(
108 self.builder.allocator,
109 &[_][]const u8{ output_dir, node.data.basename },
110 );
111 }172 }
173
174 file.generated_file.path = try wf.builder.cache_root.join(
175 wf.builder.allocator,
176 &.{ cache_path, file.sub_path },
177 );
112 }178 }
179
180 try man.writeManifest();
113}181}
182
183fn writeFile(wf: *WriteFileStep, dir: fs.Dir, contents: Contents, basename: []const u8) !void {
184 // TODO after landing concurrency PR, improve error reporting here
185 switch (contents) {
186 .bytes => |bytes| return dir.writeFile(basename, bytes),
187 .copy => |file_source| {
188 const source_path = file_source.getPath(wf.builder);
189 const prev_status = try fs.Dir.updateFile(fs.cwd(), source_path, dir, basename, .{});
190 _ = prev_status; // TODO logging (affected by open PR regarding concurrency)
191 },
192 }
193}
194
195/// TODO consolidate this with the same function in RunStep?
196/// Also properly deal with concurrency (see open PR)
197fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
198 const i = man.failed_file_index orelse failWithSimpleError(err);
199 const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
200 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
201 std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
202 std.process.exit(1);
203}
204
205fn failWithSimpleError(err: anyerror) noreturn {
206 std.debug.print("{s}\n", .{@errorName(err)});
207 std.process.exit(1);
208}
209
210const std = @import("../std.zig");
211const Step = std.Build.Step;
212const fs = std.fs;
213const ArrayList = std.ArrayList;
214
215const WriteFileStep = @This();