authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-04 22:44:21-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-05 06:26:30-07:00
logb29e3fa2cd667cc967b4c7dfb5023e5ac0224d96
tree7b436f99dc6063e380813a0a357351f301fdbaad
parentb04e48566c58ed22fdb0dbe7ac866877ad53133c

std.Build: enhancements to ConfigHeaderStep

Breaking API change to std.Build.addConfigHeader. It now uses an options struct. Introduce std.Build.CompileStep.installConfigHeader which also accepts an options struct. This is used to add a generated config file into the set of installed header files for a particular compilation artifact. std.Build.ConfigHeaderStep now additionally supports a "blank" style where a header is generated from scratch. It no longer exposes `output_dir`. Instead it exposes a FileSource via `output_file`. It now additionally accepts an `include_path` option which affects the include path of CompileStep when using the `#include` directive, as well as affecting the default installation subdirectory for header installation purposes. The hash used for the directory to store the generated config file now includes the contents of the generated file. This fixes possible race conditions when generating multiple header files simultaneously. The values hash table is now an array hash map, to preserve order for the "blank" use case. I also took the opportunity to remove output_dir from TranslateCStep and WriteFileStep. This is technically a breaking change, but it was always naughty to access these fields.

5 files changed, 181 insertions(+), 158 deletions(-)

lib/std/Build.zig+7-3
......@@ -598,13 +598,17 @@ pub fn addSystemCommand(self: *Build, argv: []const []const u8) *RunStep {
598598 return run_step;
599599}
600600
601/// Using the `values` provided, produces a C header file, possibly based on a
602/// template input file (e.g. config.h.in).
603/// When an input template file is provided, this function will fail the build
604/// when an option not found in the input file is provided in `values`, and
605/// when an option found in the input file is missing from `values`.
601606pub fn addConfigHeader(
602607 b: *Build,
603 source: FileSource,
604 style: ConfigHeaderStep.Style,
608 options: ConfigHeaderStep.Options,
605609 values: anytype,
606610) *ConfigHeaderStep {
607 const config_header_step = ConfigHeaderStep.create(b, source, style);
611 const config_header_step = ConfigHeaderStep.create(b, options);
608612 config_header_step.addValues(values);
609613 return config_header_step;
610614}
lib/std/Build/CompileStep.zig+21-6
......@@ -442,10 +442,24 @@ pub fn installHeader(a: *CompileStep, src_path: []const u8, dest_rel_path: []con
442442 a.installed_headers.append(&install_file.step) catch @panic("OOM");
443443}
444444
445pub fn installConfigHeader(a: *CompileStep, config_header: *ConfigHeaderStep) void {
446 const install_file = a.builder.addInstallFileWithDir(config_header.getOutputSource(), .header, config_header.output_path);
447 a.builder.getInstallStep().dependOn(&install_file.step);
448 a.installed_headers.append(&install_file.step) catch unreachable;
445pub const InstallConfigHeaderOptions = struct {
446 install_dir: InstallDir = .header,
447 dest_rel_path: ?[]const u8 = null,
448};
449
450pub fn installConfigHeader(
451 cs: *CompileStep,
452 config_header: *ConfigHeaderStep,
453 options: InstallConfigHeaderOptions,
454) void {
455 const dest_rel_path = options.dest_rel_path orelse config_header.include_path;
456 const install_file = cs.builder.addInstallFileWithDir(
457 .{ .generated = &config_header.output_file },
458 options.install_dir,
459 dest_rel_path,
460 );
461 cs.builder.getInstallStep().dependOn(&install_file.step);
462 cs.installed_headers.append(&install_file.step) catch @panic("OOM");
449463}
450464
451465pub fn installHeadersDirectory(
......@@ -1628,8 +1642,9 @@ fn make(step: *Step) !void {
16281642 }
16291643 },
16301644 .config_header_step => |config_header| {
1631 try zig_args.append("-I");
1632 try zig_args.append(config_header.output_dir);
1645 const full_file_path = config_header.output_file.path.?;
1646 const header_dir_path = full_file_path[0 .. full_file_path.len - config_header.include_path.len];
1647 try zig_args.appendSlice(&.{ "-I", header_dir_path });
16331648 },
16341649 }
16351650 }
lib/std/Build/ConfigHeaderStep.zig+146-133
......@@ -4,15 +4,22 @@ const Step = std.Build.Step;
44
55pub const base_id: Step.Id = .config_header;
66
7pub const Style = enum {
7pub const Style = union(enum) {
88 /// The configure format supported by autotools. It uses `#undef foo` to
99 /// mark lines that can be substituted with different values.
10 autoconf,
10 autoconf: std.Build.FileSource,
1111 /// The configure format supported by CMake. It uses `@@FOO@@` and
1212 /// `#cmakedefine` for template substitution.
13 cmake,
14 /// Generate a c header from scratch with the values passed.
15 generated,
13 cmake: std.Build.FileSource,
14 /// Instead of starting with an input file, start with nothing.
15 blank,
16
17 pub fn getFileSource(style: Style) ?std.Build.FileSource {
18 switch (style) {
19 .autoconf, .cmake => |s| return s,
20 .blank => return null,
21 }
22 }
1623};
1724
1825pub const Value = union(enum) {
......@@ -26,91 +33,96 @@ pub const Value = union(enum) {
2633
2734step: Step,
2835builder: *std.Build,
29source: std.Build.FileSource,
36values: std.StringArrayHashMap(Value),
37output_file: std.Build.GeneratedFile,
38
3039style: Style,
31values: std.StringHashMap(Value),
32gen_keys: std.ArrayList([]const u8),
33gen_values: std.ArrayList(Value),
34max_bytes: usize = 2 * 1024 * 1024,
35output_dir: []const u8,
36output_path: []const u8,
37output_gen: std.build.GeneratedFile,
38
39pub fn create(builder: *std.Build, source: std.Build.FileSource, style: Style) *ConfigHeaderStep {
40max_bytes: usize,
41include_path: []const u8,
42
43pub const Options = struct {
44 style: Style = .blank,
45 max_bytes: usize = 2 * 1024 * 1024,
46 include_path: ?[]const u8 = null,
47};
48
49pub fn create(builder: *std.Build, options: Options) *ConfigHeaderStep {
4050 const self = builder.allocator.create(ConfigHeaderStep) catch @panic("OOM");
41 const name = builder.fmt("configure header {s}", .{source.getDisplayName()});
51 const name = if (options.style.getFileSource()) |s|
52 builder.fmt("configure {s} header {s}", .{ @tagName(options.style), s.getDisplayName() })
53 else
54 builder.fmt("configure {s} header", .{@tagName(options.style)});
4255 self.* = .{
4356 .builder = builder,
4457 .step = Step.init(base_id, name, builder.allocator, make),
45 .source = source,
46 .style = style,
47 .values = std.StringHashMap(Value).init(builder.allocator),
48 .gen_keys = std.ArrayList([]const u8).init(builder.allocator),
49 .gen_values = std.ArrayList(Value).init(builder.allocator),
50 .output_dir = undefined,
51 .output_path = "config.h",
52 .output_gen = std.build.GeneratedFile{ .step = &self.step },
58 .style = options.style,
59 .values = std.StringArrayHashMap(Value).init(builder.allocator),
60
61 .max_bytes = options.max_bytes,
62 .include_path = "config.h",
63 .output_file = .{ .step = &self.step },
5364 };
5465
55 switch (source) {
66 if (options.style.getFileSource()) |s| switch (s) {
5667 .path => |p| {
57 self.output_path = p;
58
59 switch (style) {
60 .autoconf, .cmake => {
61 if (std.mem.endsWith(u8, p, ".h.in")) {
62 self.output_path = p[0 .. p.len - 3];
63 }
64 },
65 else => {},
68 const basename = std.fs.path.basename(p);
69 if (std.mem.endsWith(u8, basename, ".h.in")) {
70 self.include_path = basename[0 .. basename.len - 3];
6671 }
6772 },
6873 else => {},
74 };
75
76 if (options.include_path) |include_path| {
77 self.include_path = include_path;
6978 }
7079
7180 return self;
7281}
7382
74pub fn getOutputSource(self: *ConfigHeaderStep) std.build.FileSource {
75 return std.build.FileSource{ .generated = &self.output_gen };
76}
77
7883pub fn addValues(self: *ConfigHeaderStep, values: anytype) void {
7984 return addValuesInner(self, values) catch @panic("OOM");
8085}
8186
8287fn addValuesInner(self: *ConfigHeaderStep, values: anytype) !void {
8388 inline for (@typeInfo(@TypeOf(values)).Struct.fields) |field| {
84 const val = try getValue(self, field.type, @field(values, field.name));
85 switch (self.style) {
86 .generated => {
87 try self.gen_keys.append(field.name);
88 try self.gen_values.append(val);
89 },
90 else => try self.values.put(field.name, val),
91 }
89 try putValue(self, field.name, field.type, @field(values, field.name));
9290 }
9391}
9492
95fn getValue(self: *ConfigHeaderStep, comptime T: type, v: T) !Value {
93fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v: T) !void {
9694 switch (@typeInfo(T)) {
97 .Null => return .undef,
98 .Void => return .defined,
99 .Bool => return .{ .boolean = v },
100 .Int, .ComptimeInt => return .{ .int = v },
101 .EnumLiteral => return .{ .ident = @tagName(v) },
95 .Null => {
96 try self.values.put(field_name, .undef);
97 },
98 .Void => {
99 try self.values.put(field_name, .defined);
100 },
101 .Bool => {
102 try self.values.put(field_name, .{ .boolean = v });
103 },
104 .Int => {
105 try self.values.put(field_name, .{ .int = v });
106 },
107 .ComptimeInt => {
108 try self.values.put(field_name, .{ .int = v });
109 },
110 .EnumLiteral => {
111 try self.values.put(field_name, .{ .ident = @tagName(v) });
112 },
102113 .Optional => {
103114 if (v) |x| {
104 return getValue(self, @TypeOf(x), x);
115 return putValue(self, field_name, @TypeOf(x), x);
105116 } else {
106 return .undef;
117 try self.values.put(field_name, .undef);
107118 }
108119 },
109120 .Pointer => |ptr| {
110121 switch (@typeInfo(ptr.child)) {
111122 .Array => |array| {
112123 if (ptr.size == .One and array.child == u8) {
113 return .{ .string = v };
124 try self.values.put(field_name, .{ .string = v });
125 return;
114126 }
115127 },
116128 else => {},
......@@ -125,11 +137,6 @@ fn getValue(self: *ConfigHeaderStep, comptime T: type, v: T) !Value {
125137fn make(step: *Step) !void {
126138 const self = @fieldParentPtr(ConfigHeaderStep, "step", step);
127139 const gpa = self.builder.allocator;
128 const src_path = self.source.getPath(self.builder);
129 const contents = switch (self.style) {
130 .generated => src_path,
131 else => try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes),
132 };
133140
134141 // The cache is used here not really as a way to speed things up - because writing
135142 // the data to a file would probably be very fast - but as a way to find a canonical
......@@ -146,9 +153,30 @@ fn make(step: *Step) !void {
146153 // Random bytes to make ConfigHeaderStep unique. Refresh this with new
147154 // random bytes when ConfigHeaderStep implementation is modified in a
148155 // non-backwards-compatible way.
149 var hash = Hasher.init("X1pQzdDt91Zlh7Eh");
150 hash.update(self.source.getDisplayName());
151 hash.update(contents);
156 var hash = Hasher.init("PGuDTpidxyMqnkGM");
157
158 var output = std.ArrayList(u8).init(gpa);
159 defer output.deinit();
160
161 try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n");
162
163 switch (self.style) {
164 .autoconf => |file_source| {
165 const src_path = file_source.getPath(self.builder);
166 const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
167 try render_autoconf(contents, &output, self.values, src_path);
168 },
169 .cmake => |file_source| {
170 const src_path = file_source.getPath(self.builder);
171 const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
172 try render_cmake(contents, &output, self.values, src_path);
173 },
174 .blank => {
175 try render_blank(&output, self.values, self.include_path);
176 },
177 }
178
179 hash.update(output.items);
152180
153181 var digest: [16]u8 = undefined;
154182 hash.final(&digest);
......@@ -159,7 +187,7 @@ fn make(step: *Step) !void {
159187 .{std.fmt.fmtSliceHexLower(&digest)},
160188 ) catch unreachable;
161189
162 self.output_dir = try std.fs.path.join(gpa, &[_][]const u8{
190 const output_dir = try std.fs.path.join(gpa, &[_][]const u8{
163191 self.builder.cache_root, "o", &hash_basename,
164192 });
165193
......@@ -168,45 +196,33 @@ fn make(step: *Step) !void {
168196 // output_path is libavutil/avconfig.h
169197 // We want to open directory zig-cache/o/HASH/libavutil/
170198 // but keep output_dir as zig-cache/o/HASH for -I include
171 var outdir = self.output_dir;
172 var outpath = self.output_path;
173 if (std.fs.path.dirname(self.output_path)) |d| {
174 outdir = try std.fs.path.join(gpa, &[_][]const u8{ self.output_dir, d });
175 outpath = std.fs.path.basename(self.output_path);
176 }
199 const sub_dir_path = if (std.fs.path.dirname(self.include_path)) |d|
200 try std.fs.path.join(gpa, &.{ output_dir, d })
201 else
202 output_dir;
177203
178 var dir = std.fs.cwd().makeOpenPath(outdir, .{}) catch |err| {
179 std.debug.print("unable to make path {s}: {s}\n", .{ outdir, @errorName(err) });
204 var dir = std.fs.cwd().makeOpenPath(sub_dir_path, .{}) catch |err| {
205 std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
180206 return err;
181207 };
182208 defer dir.close();
183209
184 var values_copy = try self.values.clone();
185 defer values_copy.deinit();
186
187 var output = std.ArrayList(u8).init(gpa);
188 defer output.deinit();
189 try output.ensureTotalCapacity(contents.len);
190
191 try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n");
192
193 switch (self.style) {
194 .autoconf => try render_autoconf(contents, &output, &values_copy, src_path),
195 .cmake => try render_cmake(contents, &output, &values_copy, src_path),
196 .generated => try render_generated(gpa, &output, &self.gen_keys, &self.gen_values, self.source.getDisplayName()),
197 }
210 try dir.writeFile(std.fs.path.basename(self.include_path), output.items);
198211
199 try dir.writeFile(outpath, output.items);
200
201 self.output_gen.path = try std.fs.path.join(gpa, &[_][]const u8{ self.output_dir, self.output_path });
212 self.output_file.path = try std.fs.path.join(self.builder.allocator, &.{
213 output_dir, self.include_path,
214 });
202215}
203216
204217fn render_autoconf(
205218 contents: []const u8,
206219 output: *std.ArrayList(u8),
207 values_copy: *std.StringHashMap(Value),
220 values: std.StringArrayHashMap(Value),
208221 src_path: []const u8,
209222) !void {
223 var values_copy = try values.clone();
224 defer values_copy.deinit();
225
210226 var any_errors = false;
211227 var line_index: u32 = 0;
212228 var line_it = std.mem.split(u8, contents, "\n");
......@@ -224,7 +240,7 @@ fn render_autoconf(
224240 continue;
225241 }
226242 const name = it.rest();
227 const kv = values_copy.fetchRemove(name) orelse {
243 const kv = values_copy.fetchSwapRemove(name) orelse {
228244 std.debug.print("{s}:{d}: error: unspecified config header value: '{s}'\n", .{
229245 src_path, line_index + 1, name,
230246 });
......@@ -234,12 +250,8 @@ fn render_autoconf(
234250 try renderValue(output, name, kv.value);
235251 }
236252
237 {
238 var it = values_copy.iterator();
239 while (it.next()) |entry| {
240 const name = entry.key_ptr.*;
241 std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
242 }
253 for (values_copy.keys()) |name| {
254 std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
243255 }
244256
245257 if (any_errors) {
......@@ -250,9 +262,12 @@ fn render_autoconf(
250262fn render_cmake(
251263 contents: []const u8,
252264 output: *std.ArrayList(u8),
253 values_copy: *std.StringHashMap(Value),
265 values: std.StringArrayHashMap(Value),
254266 src_path: []const u8,
255267) !void {
268 var values_copy = try values.clone();
269 defer values_copy.deinit();
270
256271 var any_errors = false;
257272 var line_index: u32 = 0;
258273 var line_it = std.mem.split(u8, contents, "\n");
......@@ -276,7 +291,7 @@ fn render_cmake(
276291 any_errors = true;
277292 continue;
278293 };
279 const kv = values_copy.fetchRemove(name) orelse {
294 const kv = values_copy.fetchSwapRemove(name) orelse {
280295 std.debug.print("{s}:{d}: error: unspecified config header value: '{s}'\n", .{
281296 src_path, line_index + 1, name,
282297 });
......@@ -286,12 +301,8 @@ fn render_cmake(
286301 try renderValue(output, name, kv.value);
287302 }
288303
289 {
290 var it = values_copy.iterator();
291 while (it.next()) |entry| {
292 const name = entry.key_ptr.*;
293 std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
294 }
304 for (values_copy.keys()) |name| {
305 std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
295306 }
296307
297308 if (any_errors) {
......@@ -299,6 +310,36 @@ fn render_cmake(
299310 }
300311}
301312
313fn render_blank(
314 output: *std.ArrayList(u8),
315 defines: std.StringArrayHashMap(Value),
316 include_path: []const u8,
317) !void {
318 const include_guard_name = try output.allocator.dupe(u8, include_path);
319 for (include_guard_name) |*byte| {
320 switch (byte.*) {
321 'a'...'z' => byte.* = byte.* - 'a' + 'A',
322 'A'...'Z', '0'...'9' => continue,
323 else => byte.* = '_',
324 }
325 }
326
327 try output.appendSlice("#ifndef ");
328 try output.appendSlice(include_guard_name);
329 try output.appendSlice("\n#define ");
330 try output.appendSlice(include_guard_name);
331 try output.appendSlice("\n");
332
333 const values = defines.values();
334 for (defines.keys()) |name, i| {
335 try renderValue(output, name, values[i]);
336 }
337
338 try output.appendSlice("#endif /* ");
339 try output.appendSlice(include_guard_name);
340 try output.appendSlice(" */\n");
341}
342
302343fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
303344 switch (value) {
304345 .undef => {
......@@ -329,31 +370,3 @@ fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void
329370 },
330371 }
331372}
332
333fn render_generated(
334 gpa: std.mem.Allocator,
335 output: *std.ArrayList(u8),
336 keys: *std.ArrayList([]const u8),
337 values: *std.ArrayList(Value),
338 src_path: []const u8,
339) !void {
340 var include_guard = try gpa.dupe(u8, src_path);
341 defer gpa.free(include_guard);
342
343 for (include_guard) |*ch| {
344 if (ch.* == '.' or std.fs.path.isSep(ch.*)) {
345 ch.* = '_';
346 } else {
347 ch.* = std.ascii.toUpper(ch.*);
348 }
349 }
350
351 try output.writer().print("#ifndef {s}\n", .{include_guard});
352 try output.writer().print("#define {s}\n", .{include_guard});
353
354 for (keys.items) |k, i| {
355 try renderValue(output, k, values.items[i]);
356 }
357
358 try output.writer().print("#endif /* {s} */\n", .{include_guard});
359}
lib/std/Build/TranslateCStep.zig+2-9
......@@ -15,7 +15,6 @@ builder: *std.Build,
1515source: std.Build.FileSource,
1616include_dirs: std.ArrayList([]const u8),
1717c_macros: std.ArrayList([]const u8),
18output_dir: ?[]const u8,
1918out_basename: []const u8,
2019target: CrossTarget,
2120optimize: std.builtin.OptimizeMode,
......@@ -36,7 +35,6 @@ pub fn create(builder: *std.Build, options: Options) *TranslateCStep {
3635 .source = source,
3736 .include_dirs = std.ArrayList([]const u8).init(builder.allocator),
3837 .c_macros = std.ArrayList([]const u8).init(builder.allocator),
39 .output_dir = null,
4038 .out_basename = undefined,
4139 .target = options.target,
4240 .optimize = options.optimize,
......@@ -122,15 +120,10 @@ fn make(step: *Step) !void {
122120 const output_path = mem.trimRight(u8, output_path_nl, "\r\n");
123121
124122 self.out_basename = fs.path.basename(output_path);
125 if (self.output_dir) |output_dir| {
126 const full_dest = try fs.path.join(self.builder.allocator, &[_][]const u8{ output_dir, self.out_basename });
127 try self.builder.updateFile(output_path, full_dest);
128 } else {
129 self.output_dir = fs.path.dirname(output_path).?;
130 }
123 const output_dir = fs.path.dirname(output_path).?;
131124
132125 self.output_file.path = try fs.path.join(
133126 self.builder.allocator,
134 &[_][]const u8{ self.output_dir.?, self.out_basename },
127 &[_][]const u8{ output_dir, self.out_basename },
135128 );
136129}
lib/std/Build/WriteFileStep.zig+5-7
......@@ -9,7 +9,6 @@ pub const base_id = .write_file;
99
1010step: Step,
1111builder: *std.Build,
12output_dir: []const u8,
1312files: std.TailQueue(File),
1413
1514pub const File = struct {
......@@ -23,7 +22,6 @@ pub fn init(builder: *std.Build) WriteFileStep {
2322 .builder = builder,
2423 .step = Step.init(.write_file, "writefile", builder.allocator, make),
2524 .files = .{},
26 .output_dir = undefined,
2725 };
2826}
2927
......@@ -87,11 +85,11 @@ fn make(step: *Step) !void {
8785 .{std.fmt.fmtSliceHexLower(&digest)},
8886 ) catch unreachable;
8987
90 self.output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
88 const output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
9189 self.builder.cache_root, "o", &hash_basename,
9290 });
93 var dir = fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| {
94 std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) });
91 var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {
92 std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
9593 return err;
9694 };
9795 defer dir.close();
......@@ -101,14 +99,14 @@ fn make(step: *Step) !void {
10199 dir.writeFile(node.data.basename, node.data.bytes) catch |err| {
102100 std.debug.print("unable to write {s} into {s}: {s}\n", .{
103101 node.data.basename,
104 self.output_dir,
102 output_dir,
105103 @errorName(err),
106104 });
107105 return err;
108106 };
109107 node.data.source.path = try fs.path.join(
110108 self.builder.allocator,
111 &[_][]const u8{ self.output_dir, node.data.basename },
109 &[_][]const u8{ output_dir, node.data.basename },
112110 );
113111 }
114112 }