authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-08 14:10:55-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-01-08 14:10:55-05:00
logfcee1bf9930ad48300863d1b85b1f0a7ee924fbc
tree437d8d096baf6bb15b5c247e450a5aed5290e176
parent18f6ef613c7ae18433f1e7998b787f7a6b840215
parentc53a556a61fabbfa664d0bc433d0edf8092ee0db
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14241 from ziglang/zig-build-config-header-step

add std.build.ConfigHeaderStep

6 files changed, 309 insertions(+), 16 deletions(-)

lib/std/build.zig+13
......@@ -21,6 +21,7 @@ const ThisModule = @This();
2121
2222pub const CheckFileStep = @import("build/CheckFileStep.zig");
2323pub const CheckObjectStep = @import("build/CheckObjectStep.zig");
24pub const ConfigHeaderStep = @import("build/ConfigHeaderStep.zig");
2425pub const EmulatableRunStep = @import("build/EmulatableRunStep.zig");
2526pub const FmtStep = @import("build/FmtStep.zig");
2627pub const InstallArtifactStep = @import("build/InstallArtifactStep.zig");
......@@ -364,6 +365,17 @@ pub const Builder = struct {
364365 return run_step;
365366 }
366367
368 pub fn addConfigHeader(
369 b: *Builder,
370 source: FileSource,
371 style: ConfigHeaderStep.Style,
372 values: anytype,
373 ) *ConfigHeaderStep {
374 const config_header_step = ConfigHeaderStep.create(b, source, style);
375 config_header_step.addValues(values);
376 return config_header_step;
377 }
378
367379 /// Allocator.dupe without the need to handle out of memory.
368380 pub fn dupe(self: *Builder, bytes: []const u8) []u8 {
369381 return self.allocator.dupe(u8, bytes) catch unreachable;
......@@ -1427,6 +1439,7 @@ pub const Step = struct {
14271439 emulatable_run,
14281440 check_file,
14291441 check_object,
1442 config_header,
14301443 install_raw,
14311444 options,
14321445 custom,
lib/std/build/ConfigHeaderStep.zig created+245
......@@ -0,0 +1,245 @@
1const std = @import("../std.zig");
2const ConfigHeaderStep = @This();
3const Step = std.build.Step;
4const Builder = std.build.Builder;
5
6pub const base_id: Step.Id = .config_header;
7
8pub const Style = enum {
9 /// The configure format supported by autotools. It uses `#undef foo` to
10 /// mark lines that can be substituted with different values.
11 autoconf,
12 /// The configure format supported by CMake. It uses `@@FOO@@` and
13 /// `#cmakedefine` for template substitution.
14 cmake,
15};
16
17pub const Value = union(enum) {
18 undef,
19 defined,
20 boolean: bool,
21 int: i64,
22 ident: []const u8,
23 string: []const u8,
24};
25
26step: Step,
27builder: *Builder,
28source: std.build.FileSource,
29style: Style,
30values: std.StringHashMap(Value),
31max_bytes: usize = 2 * 1024 * 1024,
32output_dir: []const u8,
33output_basename: []const u8,
34
35pub fn create(builder: *Builder, source: std.build.FileSource, style: Style) *ConfigHeaderStep {
36 const self = builder.allocator.create(ConfigHeaderStep) catch @panic("OOM");
37 const name = builder.fmt("configure header {s}", .{source.getDisplayName()});
38 self.* = .{
39 .builder = builder,
40 .step = Step.init(base_id, name, builder.allocator, make),
41 .source = source,
42 .style = style,
43 .values = std.StringHashMap(Value).init(builder.allocator),
44 .output_dir = undefined,
45 .output_basename = "config.h",
46 };
47 switch (source) {
48 .path => |p| {
49 const basename = std.fs.path.basename(p);
50 if (std.mem.endsWith(u8, basename, ".h.in")) {
51 self.output_basename = basename[0 .. basename.len - 3];
52 }
53 },
54 else => {},
55 }
56 return self;
57}
58
59pub fn addValues(self: *ConfigHeaderStep, values: anytype) void {
60 return addValuesInner(self, values) catch @panic("OOM");
61}
62
63fn addValuesInner(self: *ConfigHeaderStep, values: anytype) !void {
64 inline for (@typeInfo(@TypeOf(values)).Struct.fields) |field| {
65 switch (@typeInfo(field.type)) {
66 .Null => {
67 try self.values.put(field.name, .undef);
68 },
69 .Void => {
70 try self.values.put(field.name, .defined);
71 },
72 .Bool => {
73 try self.values.put(field.name, .{ .boolean = @field(values, field.name) });
74 },
75 .ComptimeInt => {
76 try self.values.put(field.name, .{ .int = @field(values, field.name) });
77 },
78 .EnumLiteral => {
79 try self.values.put(field.name, .{ .ident = @tagName(@field(values, field.name)) });
80 },
81 .Pointer => |ptr| {
82 switch (@typeInfo(ptr.child)) {
83 .Array => |array| {
84 if (ptr.size == .One and array.child == u8) {
85 try self.values.put(field.name, .{ .string = @field(values, field.name) });
86 continue;
87 }
88 },
89 else => {},
90 }
91
92 @compileError("unsupported ConfigHeaderStep value type: " ++
93 @typeName(field.type));
94 },
95 else => @compileError("unsupported ConfigHeaderStep value type: " ++
96 @typeName(field.type)),
97 }
98 }
99}
100
101fn make(step: *Step) !void {
102 const self = @fieldParentPtr(ConfigHeaderStep, "step", step);
103 const gpa = self.builder.allocator;
104 const src_path = self.source.getPath(self.builder);
105 const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
106
107 // The cache is used here not really as a way to speed things up - because writing
108 // the data to a file would probably be very fast - but as a way to find a canonical
109 // location to put build artifacts.
110
111 // If, for example, a hard-coded path was used as the location to put ConfigHeaderStep
112 // files, then two ConfigHeaderStep executing in parallel might clobber each other.
113
114 // TODO port the cache system from the compiler to zig std lib. Until then
115 // we construct the path directly, and no "cache hit" detection happens;
116 // the files are always written.
117 // Note there is very similar code over in WriteFileStep
118 const Hasher = std.crypto.auth.siphash.SipHash128(1, 3);
119 // Random bytes to make ConfigHeaderStep unique. Refresh this with new
120 // random bytes when ConfigHeaderStep implementation is modified in a
121 // non-backwards-compatible way.
122 var hash = Hasher.init("X1pQzdDt91Zlh7Eh");
123 hash.update(self.source.getDisplayName());
124 hash.update(contents);
125
126 var digest: [16]u8 = undefined;
127 hash.final(&digest);
128 var hash_basename: [digest.len * 2]u8 = undefined;
129 _ = std.fmt.bufPrint(
130 &hash_basename,
131 "{s}",
132 .{std.fmt.fmtSliceHexLower(&digest)},
133 ) catch unreachable;
134
135 self.output_dir = try std.fs.path.join(gpa, &[_][]const u8{
136 self.builder.cache_root, "o", &hash_basename,
137 });
138 var dir = std.fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| {
139 std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) });
140 return err;
141 };
142 defer dir.close();
143
144 var values_copy = try self.values.clone();
145 defer values_copy.deinit();
146
147 var output = std.ArrayList(u8).init(gpa);
148 defer output.deinit();
149 try output.ensureTotalCapacity(contents.len);
150
151 try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n");
152
153 switch (self.style) {
154 .autoconf => try render_autoconf(contents, &output, &values_copy, src_path),
155 .cmake => try render_cmake(contents, &output, &values_copy, src_path),
156 }
157
158 try dir.writeFile(self.output_basename, output.items);
159}
160
161fn render_autoconf(
162 contents: []const u8,
163 output: *std.ArrayList(u8),
164 values_copy: *std.StringHashMap(Value),
165 src_path: []const u8,
166) !void {
167 var any_errors = false;
168 var line_index: u32 = 0;
169 var line_it = std.mem.split(u8, contents, "\n");
170 while (line_it.next()) |line| : (line_index += 1) {
171 if (!std.mem.startsWith(u8, line, "#")) {
172 try output.appendSlice(line);
173 try output.appendSlice("\n");
174 continue;
175 }
176 var it = std.mem.tokenize(u8, line[1..], " \t\r");
177 const undef = it.next().?;
178 if (!std.mem.eql(u8, undef, "undef")) {
179 try output.appendSlice(line);
180 try output.appendSlice("\n");
181 continue;
182 }
183 const name = it.rest();
184 const kv = values_copy.fetchRemove(name) orelse {
185 std.debug.print("{s}:{d}: error: unspecified config header value: '{s}'\n", .{
186 src_path, line_index + 1, name,
187 });
188 any_errors = true;
189 continue;
190 };
191 switch (kv.value) {
192 .undef => {
193 try output.appendSlice("/* #undef ");
194 try output.appendSlice(name);
195 try output.appendSlice(" */\n");
196 },
197 .defined => {
198 try output.appendSlice("#define ");
199 try output.appendSlice(name);
200 try output.appendSlice("\n");
201 },
202 .boolean => |b| {
203 try output.appendSlice("#define ");
204 try output.appendSlice(name);
205 try output.appendSlice(" ");
206 try output.appendSlice(if (b) "true\n" else "false\n");
207 },
208 .int => |i| {
209 try output.writer().print("#define {s} {d}\n", .{ name, i });
210 },
211 .ident => |ident| {
212 try output.writer().print("#define {s} {s}\n", .{ name, ident });
213 },
214 .string => |string| {
215 // TODO: use C-specific escaping instead of zig string literals
216 try output.writer().print("#define {s} \"{}\"\n", .{ name, std.zig.fmtEscapes(string) });
217 },
218 }
219 }
220
221 {
222 var it = values_copy.iterator();
223 while (it.next()) |entry| {
224 const name = entry.key_ptr.*;
225 std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
226 }
227 }
228
229 if (any_errors) {
230 return error.HeaderConfigFailed;
231 }
232}
233
234fn render_cmake(
235 contents: []const u8,
236 output: *std.ArrayList(u8),
237 values_copy: *std.StringHashMap(Value),
238 src_path: []const u8,
239) !void {
240 _ = contents;
241 _ = output;
242 _ = values_copy;
243 _ = src_path;
244 @panic("TODO: render_cmake is not implemented yet");
245}
lib/std/build/LibExeObjStep.zig+11
......@@ -28,6 +28,7 @@ const EmulatableRunStep = std.build.EmulatableRunStep;
2828const CheckObjectStep = std.build.CheckObjectStep;
2929const RunStep = std.build.RunStep;
3030const OptionsStep = std.build.OptionsStep;
31const ConfigHeaderStep = std.build.ConfigHeaderStep;
3132const LibExeObjStep = @This();
3233
3334pub const base_id = .lib_exe_obj;
......@@ -266,6 +267,7 @@ pub const IncludeDir = union(enum) {
266267 raw_path: []const u8,
267268 raw_path_system: []const u8,
268269 other_step: *LibExeObjStep,
270 config_header_step: *ConfigHeaderStep,
269271};
270272
271273pub const Kind = enum {
......@@ -932,6 +934,11 @@ pub fn addIncludePath(self: *LibExeObjStep, path: []const u8) void {
932934 self.include_dirs.append(IncludeDir{ .raw_path = self.builder.dupe(path) }) catch unreachable;
933935}
934936
937pub fn addConfigHeader(self: *LibExeObjStep, config_header: *ConfigHeaderStep) void {
938 self.step.dependOn(&config_header.step);
939 self.include_dirs.append(.{ .config_header_step = config_header }) catch @panic("OOM");
940}
941
935942pub fn addLibraryPath(self: *LibExeObjStep, path: []const u8) void {
936943 self.lib_paths.append(self.builder.dupe(path)) catch unreachable;
937944}
......@@ -1684,6 +1691,10 @@ fn make(step: *Step) !void {
16841691 try zig_args.append("-isystem");
16851692 try zig_args.append(fs.path.dirname(h_path).?);
16861693 },
1694 .config_header_step => |config_header| {
1695 try zig_args.append("-I");
1696 try zig_args.append(config_header.output_dir);
1697 },
16871698 }
16881699 }
16891700
lib/std/build/RunStep.zig+2-2
......@@ -17,7 +17,7 @@ const max_stdout_size = 1 * 1024 * 1024; // 1 MiB
1717
1818const RunStep = @This();
1919
20pub const base_id = .run;
20pub const base_id: Step.Id = .run;
2121
2222step: Step,
2323builder: *Builder,
......@@ -59,7 +59,7 @@ pub fn create(builder: *Builder, name: []const u8) *RunStep {
5959 const self = builder.allocator.create(RunStep) catch unreachable;
6060 self.* = RunStep{
6161 .builder = builder,
62 .step = Step.init(.run, name, builder.allocator, make),
62 .step = Step.init(base_id, name, builder.allocator, make),
6363 .argv = ArrayList(Arg).init(builder.allocator),
6464 .cwd = null,
6565 .env_map = null,
lib/std/build/WriteFileStep.zig+16-14
......@@ -63,14 +63,15 @@ fn make(step: *Step) !void {
6363 // files, then two WriteFileSteps executing in parallel might clobber each other.
6464
6565 // TODO port the cache system from the compiler to zig std lib. Until then
66 // we use blake2b directly and construct the path, and no "cache hit"
67 // detection happens; the files are always written.
68 var hash = std.crypto.hash.blake2.Blake2b384.init(.{});
69
66 // we directly construct the path, and no "cache hit" detection happens;
67 // the files are always written.
68 // Note there is similar code over in ConfigHeaderStep.
69 const Hasher = std.crypto.auth.siphash.SipHash128(1, 3);
7070 // Random bytes to make WriteFileStep unique. Refresh this with
7171 // new random bytes when WriteFileStep implementation is modified
7272 // in a non-backwards-compatible way.
73 hash.update("eagVR1dYXoE7ARDP");
73 var hash = Hasher.init("eagVR1dYXoE7ARDP");
74
7475 {
7576 var it = self.files.first;
7677 while (it) |node| : (it = node.next) {
......@@ -79,21 +80,22 @@ fn make(step: *Step) !void {
7980 hash.update("|");
8081 }
8182 }
82 var digest: [48]u8 = undefined;
83 var digest: [16]u8 = undefined;
8384 hash.final(&digest);
84 var hash_basename: [64]u8 = undefined;
85 _ = fs.base64_encoder.encode(&hash_basename, &digest);
86 self.output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
87 self.builder.cache_root,
88 "o",
85 var hash_basename: [digest.len * 2]u8 = undefined;
86 _ = std.fmt.bufPrint(
8987 &hash_basename,
88 "{s}",
89 .{std.fmt.fmtSliceHexLower(&digest)},
90 ) catch unreachable;
91
92 self.output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
93 self.builder.cache_root, "o", &hash_basename,
9094 });
91 // TODO replace with something like fs.makePathAndOpenDir
92 fs.cwd().makePath(self.output_dir) catch |err| {
95 var dir = fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| {
9396 std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) });
9497 return err;
9598 };
96 var dir = try fs.cwd().openDir(self.output_dir, .{});
9799 defer dir.close();
98100 {
99101 var it = self.files.first;
lib/std/crypto/siphash.zig+22
......@@ -87,6 +87,11 @@ fn SipHashStateless(comptime T: type, comptime c_rounds: usize, comptime d_round
8787 self.msg_len +%= @truncate(u8, b.len);
8888 }
8989
90 pub fn peek(self: Self) [digest_length]u8 {
91 var copy = self;
92 return copy.finalResult();
93 }
94
9095 pub fn final(self: *Self, b: []const u8) T {
9196 std.debug.assert(b.len < 8);
9297
......@@ -124,6 +129,12 @@ fn SipHashStateless(comptime T: type, comptime c_rounds: usize, comptime d_round
124129 return (@as(u128, b2) << 64) | b1;
125130 }
126131
132 pub fn finalResult(self: *Self) [digest_length]u8 {
133 var result: [digest_length]u8 = undefined;
134 self.final(&result);
135 return result;
136 }
137
127138 fn round(self: *Self, b: [8]u8) void {
128139 const m = mem.readIntLittle(u64, b[0..8]);
129140 self.v3 ^= m;
......@@ -205,12 +216,23 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)
205216 self.buf_len += @intCast(u8, b[off + aligned_len ..].len);
206217 }
207218
219 pub fn peek(self: Self) [mac_length]u8 {
220 var copy = self;
221 return copy.finalResult();
222 }
223
208224 /// Return an authentication tag for the current state
209225 /// Assumes `out` is less than or equal to `mac_length`.
210226 pub fn final(self: *Self, out: *[mac_length]u8) void {
211227 mem.writeIntLittle(T, out, self.state.final(self.buf[0..self.buf_len]));
212228 }
213229
230 pub fn finalResult(self: *Self) [mac_length]u8 {
231 var result: [mac_length]u8 = undefined;
232 self.final(&result);
233 return result;
234 }
235
214236 /// Return an authentication tag for a message and a key
215237 pub fn create(out: *[mac_length]u8, msg: []const u8, key: *const [key_length]u8) void {
216238 var ctx = Self.init(key);