| ... | ... | @@ -1,205 +1,294 @@ |
| 1 | | fn make(step: *Step, options: Step.MakeOptions) !void { |
| 2 | | _ = options; |
| 3 | | const b = step.owner; |
| 4 | | const graph = b.graph; |
| 1 | const WriteFile = @This(); |
| 2 | |
| 3 | const std = @import("std"); |
| 4 | const Io = std.Io; |
| 5 | const assert = std.debug.assert; |
| 6 | const Path = std.Build.Cache.Path; |
| 7 | const allocPrint = std.fmt.allocPrint; |
| 8 | const Configuration = std.Build.Configuration; |
| 9 | |
| 10 | const Step = @import("../Step.zig"); |
| 11 | const Maker = @import("../../Maker.zig"); |
| 12 | |
| 13 | pub fn make( |
| 14 | wf: *WriteFile, |
| 15 | step_index: Configuration.Step.Index, |
| 16 | maker: *Maker, |
| 17 | progress_node: std.Progress.Node, |
| 18 | ) Step.ExtendedMakeError!void { |
| 19 | _ = wf; |
| 20 | const graph = maker.graph; |
| 21 | const gpa = maker.gpa; |
| 22 | const arena = maker.graph.arena; // TODO don't leak into process arena |
| 5 | 23 | const io = graph.io; |
| 6 | | const arena = b.allocator; |
| 7 | | const gpa = graph.cache.gpa; |
| 8 | | const write_file: *WriteFile = @fieldParentPtr("step", step); |
| 24 | const step = maker.stepByIndex(step_index); |
| 25 | const conf = &maker.scanned_config.configuration; |
| 26 | const conf_step = step_index.ptr(conf); |
| 27 | const conf_wf = conf_step.extended.get(conf.extra).write_file; |
| 28 | const cache_root = graph.local_cache_root; |
| 29 | const directories = conf_wf.directories.slice; |
| 9 | 30 | |
| 10 | | const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); |
| 11 | | var open_dirs_count: usize = 0; |
| 31 | const open_dir_cache = try arena.alloc(Io.Dir, directories.len); |
| 32 | var open_dirs_count: u32 = 0; |
| 12 | 33 | defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); |
| 13 | 34 | |
| 14 | | switch (write_file.mode) { |
| 15 | | .whole_cached => { |
| 16 | | step.clearWatchInputs(); |
| 35 | // Doesn't yet include contents of directories. |
| 36 | var total_items: usize = conf_wf.embeds.slice.len + conf_wf.copies.slice.len + conf_wf.directories.slice.len; |
| 37 | progress_node.setEstimatedTotalItems(total_items); |
| 17 | 38 | |
| 18 | | // The cache is used here not really as a way to speed things up - because writing |
| 19 | | // the data to a file would probably be very fast - but as a way to find a canonical |
| 20 | | // location to put build artifacts. |
| 39 | switch (conf_wf.flags.mode) { |
| 40 | .whole_cached => { |
| 41 | step.clearWatchInputs(maker); |
| 21 | 42 | |
| 22 | | // If, for example, a hard-coded path was used as the location to put WriteFile |
| 23 | | // files, then two WriteFiles executing in parallel might clobber each other. |
| 43 | // The cache is used here primarily as a way to find a canonical |
| 44 | // location to put build artifacts without parallel step execution |
| 45 | // clobbering each other. |
| 24 | 46 | |
| 25 | | var man = b.graph.cache.obtain(); |
| 47 | var man = graph.cache.obtain(); |
| 26 | 48 | defer man.deinit(); |
| 27 | 49 | |
| 28 | | for (write_file.files.items) |file| { |
| 29 | | man.hash.addBytes(file.sub_path); |
| 30 | | |
| 31 | | switch (file.contents) { |
| 32 | | .bytes => |bytes| { |
| 33 | | man.hash.addBytes(bytes); |
| 34 | | }, |
| 35 | | .copy => |lazy_path| { |
| 36 | | const path = lazy_path.getPath3(b, step); |
| 37 | | _ = try man.addFilePath(path, null); |
| 38 | | try step.addWatchInput(lazy_path); |
| 39 | | }, |
| 40 | | } |
| 50 | for (conf_wf.embeds.slice) |*embed| { |
| 51 | man.hash.addBytes(embed.sub_path.slice(conf)); |
| 52 | man.hash.addBytes(embed.contents.slice(conf)); |
| 53 | } |
| 54 | |
| 55 | for (conf_wf.copies.slice) |*copy| { |
| 56 | man.hash.addBytes(copy.sub_path.slice(conf)); |
| 57 | const src_lazy_path = copy.src_file.get(conf); |
| 58 | const source_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); |
| 59 | _ = try man.addFilePath(source_path, null); |
| 60 | try step.addWatchInput(maker, arena, src_lazy_path); |
| 41 | 61 | } |
| 42 | 62 | |
| 43 | | for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { |
| 44 | | man.hash.addBytes(dir.sub_path); |
| 45 | | for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); |
| 46 | | if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); |
| 63 | for (directories, open_dir_cache) |conf_dir, *opened_dir| { |
| 64 | const exclude_extensions = conf_dir.exclude_extensions.slice(conf) orelse &.{}; |
| 65 | const include_extensions = conf_dir.include_extensions.slice(conf); |
| 66 | |
| 67 | man.hash.addBytes(conf_dir.sub_path.slice(conf)); |
| 68 | for (exclude_extensions) |ext| man.hash.addBytes(ext.slice(conf)); |
| 69 | if (include_extensions) |includes| for (includes) |inc| { |
| 70 | man.hash.addBytes(inc.slice(conf)); |
| 71 | }; |
| 47 | 72 | |
| 48 | | const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); |
| 49 | | const src_dir_path = dir.source.getPath3(b, step); |
| 73 | const src_lazy_path = conf_dir.src_path.get(conf); |
| 74 | const need_derived_inputs = try step.addDirectoryWatchInput(maker, src_lazy_path); |
| 75 | const src_dir_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); |
| 50 | 76 | |
| 51 | 77 | var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { |
| 52 | | return step.fail("unable to open source directory '{f}': {s}", .{ |
| 53 | | src_dir_path, @errorName(err), |
| 54 | | }); |
| 78 | return step.fail(maker, "failed opening source directory {f}: {t}", .{ src_dir_path, err }); |
| 55 | 79 | }; |
| 56 | | open_dir_cache_elem.* = src_dir; |
| 80 | opened_dir.* = src_dir; |
| 57 | 81 | open_dirs_count += 1; |
| 58 | 82 | |
| 59 | 83 | var it = try src_dir.walk(gpa); |
| 60 | 84 | defer it.deinit(); |
| 61 | | while (try it.next(io)) |entry| { |
| 62 | | if (!dir.options.pathIncluded(entry.path)) continue; |
| 85 | while (it.next(io) catch |err| switch (err) { |
| 86 | error.Canceled, error.OutOfMemory => |e| return e, |
| 87 | else => |e| return step.fail(maker, "failed iterating dir {f}: {t}", .{ src_dir_path, e }), |
| 88 | }) |entry| { |
| 89 | if (!pathIncluded(conf, exclude_extensions, include_extensions, entry.path)) continue; |
| 63 | 90 | |
| 64 | 91 | switch (entry.kind) { |
| 65 | 92 | .directory => { |
| 66 | 93 | if (need_derived_inputs) { |
| 67 | 94 | const entry_path = try src_dir_path.join(arena, entry.path); |
| 68 | | try step.addDirectoryWatchInputFromPath(entry_path); |
| 95 | try step.addDirectoryWatchInputFromPath(maker, entry_path); |
| 69 | 96 | } |
| 70 | 97 | }, |
| 71 | 98 | .file => { |
| 72 | 99 | const entry_path = try src_dir_path.join(arena, entry.path); |
| 73 | 100 | _ = try man.addFilePath(entry_path, null); |
| 101 | total_items += 1; |
| 74 | 102 | }, |
| 75 | 103 | else => continue, |
| 76 | 104 | } |
| 77 | 105 | } |
| 78 | 106 | } |
| 79 | 107 | |
| 80 | | if (try step.cacheHit(&man)) { |
| 108 | if (try step.cacheHit(maker, &man)) { |
| 81 | 109 | const digest = man.final(); |
| 82 | | write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); |
| 110 | maker.generatedPath(conf_wf.generated_directory).* = .{ |
| 111 | .root_dir = cache_root, |
| 112 | .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), |
| 113 | }; |
| 83 | 114 | assert(step.result_cached); |
| 84 | 115 | return; |
| 85 | 116 | } |
| 86 | 117 | |
| 87 | 118 | const digest = man.final(); |
| 88 | | const cache_path = "o" ++ Dir.path.sep_str ++ digest; |
| 89 | | |
| 90 | | write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); |
| 119 | const out_path: Path = .{ |
| 120 | .root_dir = cache_root, |
| 121 | .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), |
| 122 | }; |
| 91 | 123 | |
| 92 | | try operate(write_file, open_dir_cache, .{ |
| 93 | | .root_dir = b.cache_root, |
| 94 | | .sub_path = cache_path, |
| 95 | | }); |
| 124 | progress_node.setEstimatedTotalItems(total_items); |
| 125 | try operate(maker, step_index, open_dir_cache, out_path, progress_node); |
| 126 | try step.writeManifest(maker, &man); |
| 96 | 127 | |
| 97 | | try step.writeManifest(&man); |
| 128 | maker.generatedPath(conf_wf.generated_directory).* = out_path; |
| 98 | 129 | }, |
| 99 | 130 | .tmp => { |
| 100 | 131 | step.result_cached = false; |
| 101 | 132 | |
| 102 | 133 | var rand_int: u64 = undefined; |
| 103 | 134 | io.random(@ptrCast(&rand_int)); |
| 104 | | const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); |
| 135 | const hex_digest = std.fmt.hex(rand_int); |
| 105 | 136 | |
| 106 | | write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); |
| 137 | const out_path: Path = .{ |
| 138 | .root_dir = cache_root, |
| 139 | .sub_path = try Io.Dir.path.join(arena, &.{ "tmp", &hex_digest }), |
| 140 | }; |
| 107 | 141 | |
| 108 | | try operate(write_file, open_dir_cache, .{ |
| 109 | | .root_dir = b.cache_root, |
| 110 | | .sub_path = tmp_dir_sub_path, |
| 111 | | }); |
| 142 | try operate(maker, step_index, open_dir_cache, out_path, progress_node); |
| 143 | |
| 144 | maker.generatedPath(conf_wf.generated_directory).* = out_path; |
| 112 | 145 | }, |
| 113 | | .mutate => |lp| { |
| 146 | .mutate => { |
| 114 | 147 | step.result_cached = false; |
| 115 | | const root_path = try lp.getPath4(b, step); |
| 116 | | write_file.generated_directory.path = try root_path.toString(arena); |
| 117 | | try operate(write_file, open_dir_cache, root_path); |
| 148 | const root_path = try maker.resolveLazyPathIndex(arena, conf_wf.mutate_path.value.?, step_index); |
| 149 | try operate(maker, step_index, open_dir_cache, root_path, progress_node); |
| 150 | maker.generatedPath(conf_wf.generated_directory).* = root_path; |
| 118 | 151 | }, |
| 119 | 152 | } |
| 120 | 153 | } |
| 121 | 154 | |
| 122 | | fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { |
| 123 | | const step = &write_file.step; |
| 124 | | const b = step.owner; |
| 125 | | const io = b.graph.io; |
| 126 | | const gpa = b.graph.cache.gpa; |
| 127 | | const arena = b.allocator; |
| 128 | | |
| 129 | | var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| |
| 130 | | return step.fail("unable to make path {f}: {t}", .{ root_path, err }); |
| 131 | | defer cache_dir.close(io); |
| 132 | | |
| 133 | | for (write_file.files.items) |file| { |
| 134 | | if (Dir.path.dirname(file.sub_path)) |dirname| { |
| 135 | | cache_dir.createDirPath(io, dirname) catch |err| { |
| 136 | | return step.fail("unable to make path '{f}{c}{s}': {t}", .{ |
| 137 | | root_path, Dir.path.sep, dirname, err, |
| 138 | | }); |
| 155 | fn operate( |
| 156 | maker: *Maker, |
| 157 | step_index: Configuration.Step.Index, |
| 158 | open_dir_cache: []const Io.Dir, |
| 159 | root_path: std.Build.Cache.Path, |
| 160 | progress_node: std.Progress.Node, |
| 161 | ) !void { |
| 162 | const graph = maker.graph; |
| 163 | const gpa = maker.gpa; |
| 164 | const arena = maker.graph.arena; // TODO don't leak into process arena |
| 165 | const io = graph.io; |
| 166 | const step = maker.stepByIndex(step_index); |
| 167 | const conf = &maker.scanned_config.configuration; |
| 168 | const conf_step = step_index.ptr(conf); |
| 169 | const conf_wf = conf_step.extended.get(conf.extra).write_file; |
| 170 | |
| 171 | const root_directory: std.Build.Cache.Directory = .{ |
| 172 | .handle = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| |
| 173 | return step.fail(maker, "failed creating path {f}: {t}", .{ root_path, err }), |
| 174 | .path = try root_path.toString(arena), |
| 175 | }; |
| 176 | defer root_directory.handle.close(io); |
| 177 | |
| 178 | for (conf_wf.embeds.slice) |*embed| { |
| 179 | const dest_path: Path = .{ |
| 180 | .root_dir = root_directory, |
| 181 | .sub_path = embed.sub_path.slice(conf), |
| 182 | }; |
| 183 | if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| { |
| 184 | const dirname_path: Path = .{ |
| 185 | .root_dir = root_directory, |
| 186 | .sub_path = dirname, |
| 139 | 187 | }; |
| 188 | dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| |
| 189 | return step.fail(maker, "failed creating path {f}: {t}", .{ dirname_path, err }); |
| 140 | 190 | } |
| 141 | | switch (file.contents) { |
| 142 | | .bytes => |bytes| { |
| 143 | | cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { |
| 144 | | return step.fail("unable to write file '{f}{c}{s}': {t}", .{ |
| 145 | | root_path, Dir.path.sep, file.sub_path, err, |
| 146 | | }); |
| 147 | | }; |
| 148 | | }, |
| 149 | | .copy => |file_source| { |
| 150 | | const source_path = file_source.getPath2(b, step); |
| 151 | | const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { |
| 152 | | return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ |
| 153 | | source_path, root_path, Dir.path.sep, file.sub_path, err, |
| 154 | | }); |
| 155 | | }; |
| 156 | | // At this point we already will mark the step as a cache miss. |
| 157 | | // But this is kind of a partial cache hit since individual |
| 158 | | // file copies may be avoided. Oh well, this information is |
| 159 | | // discarded. |
| 160 | | _ = prev_status; |
| 161 | | }, |
| 191 | dest_path.root_dir.handle.writeFile(io, .{ |
| 192 | .sub_path = dest_path.sub_path, |
| 193 | .data = embed.contents.slice(conf), |
| 194 | }) catch |err| return step.fail(maker, "failed writing contents to file {f}: {t}", .{ dest_path, err }); |
| 195 | progress_node.completeOne(); |
| 196 | } |
| 197 | |
| 198 | for (conf_wf.copies.slice) |*copy| { |
| 199 | const dest_path: Path = .{ |
| 200 | .root_dir = root_directory, |
| 201 | .sub_path = copy.sub_path.slice(conf), |
| 202 | }; |
| 203 | // Rather than passing make_path = true below, this optimizes for the |
| 204 | // more common case where the directory does not exist. |
| 205 | if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| { |
| 206 | const dirname_path: Path = .{ |
| 207 | .root_dir = root_directory, |
| 208 | .sub_path = dirname, |
| 209 | }; |
| 210 | dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| |
| 211 | return step.fail(maker, "failed creating path {f}: {t}", .{ dirname_path, err }); |
| 162 | 212 | } |
| 213 | const source_path = try maker.resolveLazyPathIndex(arena, copy.src_file, step_index); |
| 214 | Io.Dir.copyFile( |
| 215 | source_path.root_dir.handle, |
| 216 | source_path.sub_path, |
| 217 | dest_path.root_dir.handle, |
| 218 | dest_path.sub_path, |
| 219 | io, |
| 220 | .{}, |
| 221 | ) catch |err| return step.fail(maker, "failed copying file from {f} to {f}: {t}", .{ |
| 222 | source_path, dest_path, err, |
| 223 | }); |
| 224 | progress_node.completeOne(); |
| 163 | 225 | } |
| 164 | 226 | |
| 165 | | for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { |
| 166 | | const src_dir_path = dir.source.getPath3(b, step); |
| 167 | | const dest_dirname = dir.sub_path; |
| 227 | for (conf_wf.directories.slice, open_dir_cache) |conf_dir, already_open_dir| { |
| 228 | const exclude_extensions = conf_dir.exclude_extensions.slice(conf) orelse &.{}; |
| 229 | const include_extensions = conf_dir.include_extensions.slice(conf); |
| 168 | 230 | |
| 169 | | if (dest_dirname.len != 0) { |
| 170 | | cache_dir.createDirPath(io, dest_dirname) catch |err| { |
| 171 | | return step.fail("unable to make path '{f}{c}{s}': {t}", .{ |
| 172 | | root_path, Dir.path.sep, dest_dirname, err, |
| 173 | | }); |
| 174 | | }; |
| 231 | const src_dir_path = try maker.resolveLazyPathIndex(arena, conf_dir.src_path, step_index); |
| 232 | const dest_dir_path: Path = .{ |
| 233 | .root_dir = root_directory, |
| 234 | .sub_path = conf_dir.sub_path.slice(conf), |
| 235 | }; |
| 236 | |
| 237 | if (dest_dir_path.sub_path.len != 0) { |
| 238 | dest_dir_path.root_dir.handle.createDirPath(io, dest_dir_path.sub_path) catch |err| |
| 239 | return step.fail(maker, "failed creating path {f}: {t}", .{ dest_dir_path, err }); |
| 175 | 240 | } |
| 176 | 241 | |
| 177 | 242 | var it = try already_open_dir.walk(gpa); |
| 178 | 243 | defer it.deinit(); |
| 179 | | while (try it.next(io)) |entry| { |
| 180 | | if (!dir.options.pathIncluded(entry.path)) continue; |
| 244 | while (it.next(io) catch |err| switch (err) { |
| 245 | error.Canceled, error.OutOfMemory => |e| return e, |
| 246 | else => |e| return step.fail(maker, "failed iterating dir {f}: {t}", .{ src_dir_path, e }), |
| 247 | }) |entry| { |
| 248 | if (!pathIncluded(conf, exclude_extensions, include_extensions, entry.path)) continue; |
| 181 | 249 | |
| 182 | 250 | const src_entry_path = try src_dir_path.join(arena, entry.path); |
| 183 | | const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); |
| 251 | const dest_path = try dest_dir_path.join(arena, entry.path); |
| 184 | 252 | switch (entry.kind) { |
| 185 | | .directory => try cache_dir.createDirPath(io, dest_path), |
| 253 | .directory => dest_path.root_dir.handle.createDirPath(io, dest_path.sub_path) catch |err| { |
| 254 | return step.fail(maker, "failed creating path {f}: {t}", .{ dest_path, err }); |
| 255 | }, |
| 186 | 256 | .file => { |
| 187 | | const prev_status = Io.Dir.updateFile( |
| 257 | Io.Dir.copyFile( |
| 188 | 258 | src_entry_path.root_dir.handle, |
| 189 | | io, |
| 190 | 259 | src_entry_path.sub_path, |
| 191 | | cache_dir, |
| 192 | | dest_path, |
| 260 | dest_path.root_dir.handle, |
| 261 | dest_path.sub_path, |
| 262 | io, |
| 193 | 263 | .{}, |
| 194 | | ) catch |err| { |
| 195 | | return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ |
| 196 | | src_entry_path, root_path, Dir.path.sep, dest_path, err, |
| 197 | | }); |
| 198 | | }; |
| 199 | | _ = prev_status; |
| 264 | ) catch |err| return step.fail(maker, "failed copying file from {f} to {f}: {t}", .{ |
| 265 | src_entry_path, dest_path, err, |
| 266 | }); |
| 267 | progress_node.completeOne(); |
| 200 | 268 | }, |
| 201 | 269 | else => continue, |
| 202 | 270 | } |
| 203 | 271 | } |
| 204 | 272 | } |
| 205 | 273 | } |
| 274 | |
| 275 | fn pathIncluded( |
| 276 | conf: *const Configuration, |
| 277 | exclude_extensions: []const Configuration.String, |
| 278 | include_extensions: ?[]const Configuration.String, |
| 279 | path: []const u8, |
| 280 | ) bool { |
| 281 | for (exclude_extensions) |ext| { |
| 282 | if (std.mem.endsWith(u8, path, ext.slice(conf))) |
| 283 | return false; |
| 284 | } |
| 285 | if (include_extensions) |incs| { |
| 286 | for (incs) |inc| { |
| 287 | if (std.mem.endsWith(u8, path, inc.slice(conf))) |
| 288 | return true; |
| 289 | } else { |
| 290 | return false; |
| 291 | } |
| 292 | } |
| 293 | return true; |
| 294 | } |