authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-24 19:15:30-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-05-25 18:54:34-07:00
log92803903f3e96995cc705087615bd67749757862
treef0c4353850516fa3e6fc5fd0ac370d2c23153b15
parente8e7fbf8432899d12888bcabeba17ea8e51dc62d

Configuration: implement FlagLengthPrefixedList

3 files changed, 209 insertions(+), 67 deletions(-)

lib/compiler/Maker/ScannedConfig.zig+16-5
......@@ -55,17 +55,24 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi
5555 try s.value(field_value.slice(c), .{});
5656 },
5757 Configuration.Deps => {
58 var deps_field = try s.beginTuple(.{});
59 for (field_value.slice(c)) |dep| {
60 try deps_field.field(@intFromEnum(dep), .{});
61 }
62 try deps_field.end();
58 try printValue(sc, s, []Configuration.Step.Index, field_value.slice(c));
6359 },
6460 Configuration.MaxRss => {
6561 try s.value(field_value.toBytes(), .{});
6662 },
6763 else => switch (@typeInfo(Field)) {
6864 .int => try s.int(field_value),
65 .pointer => |info| switch (info.size) {
66 .slice => {
67 var slice_field = try s.beginTuple(.{});
68 for (field_value) |elem| {
69 try slice_field.fieldPrefix();
70 try printValue(sc, s, info.child, elem);
71 }
72 try slice_field.end();
73 },
74 else => comptime unreachable,
75 },
6976 .@"enum" => {
7077 if (@hasDecl(Field, "storage")) switch (Field.storage) {
7178 .extended => {
......@@ -74,6 +81,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi
7481 try sub_struct.end();
7582 },
7683 .flag_optional => comptime unreachable,
84 .flag_length_prefixed_list => comptime unreachable,
7785 .enum_optional => comptime unreachable,
7886 } else if (std.enums.tagName(Field, field_value)) |name| {
7987 try s.ident(name);
......@@ -93,6 +101,9 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi
93101 try s.value(null, .{});
94102 }
95103 },
104 .flag_length_prefixed_list => {
105 try printValue(sc, s, @TypeOf(field_value.slice), field_value.slice);
106 },
96107 .extended => @compileError("TODO"),
97108 },
98109 else => @compileError("not implemented: " ++ @typeName(Field)),
lib/compiler/configure_runner.zig+49-3
......@@ -279,6 +279,10 @@ const Serialize = struct {
279279 return (try addOptionalLazyPathEnum(s, lp)).unwrap();
280280 }
281281
282 fn addLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.LazyPath {
283 return @enumFromInt(@intFromEnum(try addOptionalLazyPathEnum(s, lp)));
284 }
285
282286 fn addOptionalSemVer(s: *Serialize, sem_ver: ?std.SemanticVersion) !?Configuration.String {
283287 return if (sem_ver) |sv| try s.wc.addSemVer(sv) else null;
284288 }
......@@ -286,6 +290,20 @@ const Serialize = struct {
286290 fn addOptionalString(s: *Serialize, opt_slice: ?[]const u8) !?Configuration.String {
287291 return if (opt_slice) |slice| try s.wc.addString(slice) else null;
288292 }
293
294 fn initStringList(s: *Serialize, list: []const []const u8) ![]const Configuration.String {
295 const wc = s.wc;
296 const result = try s.arena.alloc(Configuration.String, list.len);
297 for (result, list) |*dest, src| dest.* = try wc.addString(src);
298 return result;
299 }
300
301 fn initOptionalStringList(s: *Serialize, list: []const ?[]const u8) ![]const Configuration.OptionalString {
302 const wc = s.wc;
303 const result = try s.arena.alloc(Configuration.OptionalString, list.len);
304 for (result, list) |*dest, src| dest.* = try wc.addOptionalString(src);
305 return result;
306 }
289307};
290308
291309fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
......@@ -320,7 +338,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
320338 // Add and then de-duplicate dependencies.
321339 const deps = d: {
322340 const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len);
323 for (try wc.prepareDeps(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step|
341 for (try wc.reserveLengthPrefixed(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step|
324342 dep.* = @intCast(step_map.getIndex(dep_step).?);
325343 break :d try wc.dedupeDeps(deps);
326344 };
......@@ -340,11 +358,35 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
340358 },
341359 .compile => e: {
342360 const c: *Step.Compile = @fieldParentPtr("step", step);
361 const exec_cmd_args: []const ?[]const u8 = c.exec_cmd_args orelse &.{};
362 const installed_headers: []u32 = try arena.alloc(u32, c.installed_headers.items.len);
363 for (installed_headers, c.installed_headers.items) |*dst, src| switch (src) {
364 .file => |file| {
365 dst.* = try wc.addExtra(@as(Configuration.Step.Compile.InstalledHeader.File, .{
366 .source = try s.addLazyPath(file.source),
367 .dest_sub_path = try wc.addString(file.dest_rel_path),
368 }));
369 },
370 .directory => |directory| {
371 const include_extensions = directory.options.include_extensions orelse &.{};
372 dst.* = try wc.addExtra(@as(Configuration.Step.Compile.InstalledHeader.Directory, .{
373 .flags = .{
374 .include_extensions = include_extensions.len != 0,
375 .exclude_extensions = directory.options.exclude_extensions.len != 0,
376 },
377 .source = try s.addLazyPath(directory.source),
378 .dest_sub_path = try wc.addString(directory.dest_rel_path),
379 .exclude_extensions = .{ .slice = try s.initStringList(directory.options.exclude_extensions) },
380 .include_extensions = .{ .slice = try s.initStringList(include_extensions) },
381 }));
382 },
383 };
384
343385 const extra_index = try wc.addExtra(@as(Configuration.Step.Compile, .{
344386 .flags = .{
345387 .filters_len = c.filters.len != 0,
346 .exec_cmd_args_len = if (c.exec_cmd_args) |a| a.len != 0 else false,
347 .installed_headers_len = c.installed_headers.items.len != 0,
388 .exec_cmd_args_len = exec_cmd_args.len != 0,
389 .installed_headers_len = installed_headers.len != 0,
348390 .force_undefined_symbols_len = c.force_undefined_symbols.entries.len != 0,
349391
350392 .verbose_link = c.verbose_link,
......@@ -466,6 +508,10 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
466508 .hexstring => |*hexstring| try wc.addString(hexstring.toSlice()),
467509 .none, .fast, .uuid, .sha1, .md5 => null,
468510 } else null },
511 .filters = .{ .slice = try s.initStringList(c.filters) },
512 .exec_cmd_args = .{ .slice = try s.initOptionalStringList(exec_cmd_args) },
513 .installed_headers = .initErased(installed_headers),
514 .force_undefined_symbols = .{ .slice = try s.initStringList(c.force_undefined_symbols.keys()) },
469515 }));
470516
471517 log.err("TODO serialize the trailing Compile step data", .{});
lib/std/zig/Configuration.zig+144-59
......@@ -33,7 +33,9 @@ pub const Header = extern struct {
3333pub const Wip = struct {
3434 gpa: Allocator,
3535 string_table: StringTable = .empty,
36 deps_table: DepsTable = .empty,
36 /// De-duplicates an array inside `extra` that has first element length
37 /// followed by length elements.
38 length_prefixed_table: LengthPrefixedTable = .empty,
3739 targets_table: TargetsTable = .empty,
3840
3941 string_bytes: std.ArrayList(u8) = .empty,
......@@ -44,23 +46,23 @@ pub const Wip = struct {
4446 path_deps: std.MultiArrayList(Path) = .empty,
4547 extra: std.ArrayList(u32) = .empty,
4648
47 const DepsTable = std.HashMapUnmanaged(Deps, void, DepsTableContext, std.hash_map.default_max_load_percentage);
49 const LengthPrefixedTable = std.HashMapUnmanaged(u32, void, LengthPrefixedContext, std.hash_map.default_max_load_percentage);
4850 const TargetsTable = std.HashMapUnmanaged(TargetQuery.Index, void, TargetsTableContext, std.hash_map.default_max_load_percentage);
4951
50 const DepsTableContext = struct {
52 const LengthPrefixedContext = struct {
5153 extra: []const u32,
5254
53 pub fn eql(ctx: @This(), a: Deps, b: Deps) bool {
54 const len_a = ctx.extra[@intFromEnum(a)];
55 const len_b = ctx.extra[@intFromEnum(b)];
56 const slice_a = ctx.extra[@intFromEnum(a) + 1 ..][0..len_a];
57 const slice_b = ctx.extra[@intFromEnum(b) + 1 ..][0..len_b];
55 pub fn eql(ctx: @This(), a: u32, b: u32) bool {
56 const len_a = ctx.extra[a];
57 const len_b = ctx.extra[b];
58 const slice_a = ctx.extra[a + 1 ..][0..len_a];
59 const slice_b = ctx.extra[b + 1 ..][0..len_b];
5860 return std.mem.eql(u32, slice_a, slice_b);
5961 }
6062
61 pub fn hash(ctx: @This(), key: Deps) u64 {
62 const len = ctx.extra[@intFromEnum(key)];
63 const slice = ctx.extra[@intFromEnum(key) + 1 ..][0..len];
63 pub fn hash(ctx: @This(), key: u32) u64 {
64 const len = ctx.extra[key];
65 const slice = ctx.extra[key + 1 ..][0..len];
6466 return std.hash_map.hashString(@ptrCast(slice));
6567 }
6668 };
......@@ -174,6 +176,10 @@ pub const Wip = struct {
174176 return new_off;
175177 }
176178
179 pub fn addOptionalString(wip: *Wip, bytes: ?[]const u8) Allocator.Error!OptionalString {
180 return .init(try addString(wip, bytes orelse return .none));
181 }
182
177183 pub fn addSemVer(wip: *Wip, sv: std.SemanticVersion) Allocator.Error!String {
178184 var buffer: [256]u8 = undefined;
179185 var writer: std.Io.Writer = .fixed(&buffer);
......@@ -324,27 +330,32 @@ pub const Wip = struct {
324330 }
325331 }
326332
327 pub fn prepareDeps(wip: *Wip, n: usize) Allocator.Error![]u32 {
333 pub fn reserveLengthPrefixed(wip: *Wip, n: usize) Allocator.Error![]u32 {
328334 const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1);
329335 slice[0] = @intCast(n);
330336 return slice[1..];
331337 }
332338
333 pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps {
339 pub fn dedupeLengthPrefixed(wip: *Wip, index: u32) Allocator.Error!u32 {
340 assert(wip.extra.items.len == index + wip.extra.items[index] + 1);
334341 const gpa = wip.gpa;
335 const gop = try wip.deps_table.getOrPutContext(gpa, deps, @as(DepsTableContext, .{
342 const gop = try wip.length_prefixed_table.getOrPutContext(gpa, index, @as(LengthPrefixedContext, .{
336343 .extra = wip.extra.items,
337344 }));
338345 if (gop.found_existing) {
339 wip.extra.items.len = @intFromEnum(deps);
346 wip.extra.items.len = index;
340347 return gop.key_ptr.*;
341348 } else {
342 return deps;
349 return index;
343350 }
344351 }
345352
353 pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps {
354 return @enumFromInt(try dedupeLengthPrefixed(wip, @intFromEnum(deps)));
355 }
356
346357 pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 {
347 const extra_len = Storage.calculateExtraLenUpperBound(@TypeOf(extra));
358 const extra_len = Storage.extraLen(extra);
348359 try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len);
349360 return addExtraAssumeCapacity(wip, extra);
350361 }
......@@ -397,7 +408,7 @@ pub const Step = extern struct {
397408 owner: Package.Index,
398409 deps: Deps,
399410 max_rss: MaxRss,
400 extended: Storage.ExtendedIndex(Flags, union(Tag) {
411 extended: Storage.Extended(Flags, union(Tag) {
401412 check_file: CheckFile,
402413 check_object: CheckObject,
403414 compile: Compile,
......@@ -585,10 +596,10 @@ pub const Step = extern struct {
585596 root_module: Module.Index,
586597 root_name: String,
587598
588 //filters: FlagLengthPrefixedList(.flags, .filters_len, String),
589 //exec_cmd_args: FlagLengthPrefixedList(.flags, .exec_cmd_args_len, u32),
590 //installed_headers: FlagLengthPrefixedList(.flags, .installed_headers_len, InstalledHeader),
591 //force_undefined_symbols: FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String),
599 filters: Storage.FlagLengthPrefixedList(.flags, .filters_len, String),
600 exec_cmd_args: Storage.FlagLengthPrefixedList(.flags, .exec_cmd_args_len, OptionalString),
601 installed_headers: Storage.FlagLengthPrefixedList(.flags, .installed_headers_len, Storage.Extended(InstalledHeader.Flags, InstalledHeader)),
602 force_undefined_symbols: Storage.FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String),
592603 //exacts: EnumConditionalPrefixedList(.flags4, .expect_errors, .exact, String),
593604 linker_script: Storage.FlagOptional(.flags4, .linker_script, LazyPath),
594605 version_script: Storage.FlagOptional(.flags4, .version_script, LazyPath),
......@@ -612,6 +623,46 @@ pub const Step = extern struct {
612623 error_limit: Storage.FlagOptional(.flags4, .error_limit, u32),
613624 build_id: Storage.EnumOptional(.flags3, .build_id, .hexstring, String),
614625
626 pub const InstalledHeader = union(@This().Tag) {
627 file: File,
628 directory: Directory,
629
630 pub const Flags = packed struct(u32) {
631 tag: InstalledHeader.Tag,
632 _: u24 = 0,
633 };
634
635 pub const Tag = enum(u8) {
636 file,
637 directory,
638 };
639
640 pub const File = struct {
641 flags: @This().Flags = .{},
642 source: LazyPath,
643 dest_sub_path: String,
644
645 pub const Flags = packed struct(u32) {
646 tag: InstalledHeader.Tag = .file,
647 _: u24 = 0,
648 };
649 };
650
651 pub const Directory = struct {
652 flags: @This().Flags,
653 source: LazyPath,
654 dest_sub_path: String,
655 exclude_extensions: Storage.FlagLengthPrefixedList(.flags, .exclude_extensions, String),
656 include_extensions: Storage.FlagLengthPrefixedList(.flags, .include_extensions, String),
657
658 pub const Flags = packed struct(u32) {
659 tag: InstalledHeader.Tag = .directory,
660 exclude_extensions: bool,
661 include_extensions: bool,
662 _: u22 = 0,
663 };
664 };
665 };
615666 pub const ExpectErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none };
616667 pub const TestRunnerMode = enum(u2) { default, simple, server };
617668 pub const Entry = enum(u2) { default, disabled, enabled, symbol_name };
......@@ -1636,6 +1687,7 @@ pub const Storage = enum {
16361687 flag_optional,
16371688 enum_optional,
16381689 extended,
1690 flag_length_prefixed_list,
16391691
16401692 /// The presence of the field is determined by a boolean within a packed
16411693 /// struct.
......@@ -1674,16 +1726,7 @@ pub const Storage = enum {
16741726
16751727 /// The field indexes into an auxilary buffer, with the first element being
16761728 /// a packed struct that contains the tag.
1677 pub fn Extended(comptime U: type) type {
1678 return struct {
1679 value: U,
1680
1681 pub const storage: Storage = .extended;
1682 };
1683 }
1684
1685 /// Equivalent to `Extended` but works in an `extern struct`.
1686 pub fn ExtendedIndex(comptime BaseFlags: type, comptime U: type) type {
1729 pub fn Extended(comptime BaseFlags: type, comptime U: type) type {
16871730 return enum(u32) {
16881731 _,
16891732
......@@ -1699,6 +1742,30 @@ pub const Storage = enum {
16991742 };
17001743 }
17011744
1745 /// A field in flags determines whether the length is zero or nonzero. If the length is
1746 /// nonzero, then there is a length field followed by the list.
1747 ///
1748 /// When deserializing, the slice field is set. When serializing, the index
1749 /// field must be set.
1750 pub fn FlagLengthPrefixedList(
1751 comptime flags_arg: @EnumLiteral(),
1752 comptime flag_arg: @EnumLiteral(),
1753 comptime ValueArg: type,
1754 ) type {
1755 return struct {
1756 slice: []const Value,
1757
1758 pub const storage: Storage = .flag_length_prefixed_list;
1759 pub const flags = flags_arg;
1760 pub const flag = flag_arg;
1761 pub const Value = ValueArg;
1762
1763 pub fn initErased(s: []const u32) @This() {
1764 return .{ .slice = @ptrCast(s) };
1765 }
1766 };
1767 }
1768
17021769 pub fn dataLength(buffer: []const u32, i: usize, comptime S: type) usize {
17031770 var end = i;
17041771 _ = data(buffer, &end, S);
......@@ -1769,6 +1836,15 @@ pub const Storage = enum {
17691836 };
17701837 },
17711838 .extended => @compileError("TODO"),
1839 .flag_length_prefixed_list => {
1840 const flags = @field(container, @tagName(Field.flags));
1841 const flag = @field(flags, @tagName(Field.flag));
1842 if (!flag) return .{ .slice = &.{} };
1843 const data_start = i.* + 1;
1844 const len = buffer[data_start - 1];
1845 defer i.* = data_start + len;
1846 return .{ .slice = @ptrCast(buffer[data_start..][0..len]) };
1847 },
17721848 },
17731849 },
17741850 .@"extern" => comptime unreachable,
......@@ -1787,11 +1863,36 @@ pub const Storage = enum {
17871863 return i;
17881864 }
17891865
1790 fn calculateExtraLenUpperBound(comptime Extra: type) comptime_int {
1791 var i = 0;
1792 const fields = @typeInfo(Extra).@"struct".fields;
1866 fn extraFieldLen(field: anytype) usize {
1867 const Field = @TypeOf(field);
1868 return switch (@typeInfo(Field)) {
1869 .int => |info| switch (info.bits) {
1870 32 => 1,
1871 64 => 2,
1872 else => comptime unreachable,
1873 },
1874 .@"enum" => 1,
1875 .@"struct" => |info| switch (info.layout) {
1876 .@"packed" => switch (info.backing_integer.?) {
1877 u32 => 1,
1878 u64 => 2,
1879 else => comptime unreachable,
1880 },
1881 .auto => switch (Field.storage) {
1882 .flag_optional, .enum_optional, .extended => 1,
1883 .flag_length_prefixed_list => field.slice.len + 1,
1884 },
1885 .@"extern" => comptime unreachable,
1886 },
1887 else => @compileError("bad type: " ++ @typeName(Field)),
1888 };
1889 }
1890
1891 fn extraLen(extra: anytype) usize {
1892 const fields = @typeInfo(@TypeOf(extra)).@"struct".fields;
1893 var i: usize = 0;
17931894 inline for (fields) |field| {
1794 i += calculateExtraFieldLenUpperBound(field.type);
1895 i += Storage.extraFieldLen(@field(extra, field.name));
17951896 }
17961897 return i;
17971898 }
......@@ -1836,6 +1937,13 @@ pub const Storage = enum {
18361937 return if (value.value) |v| setExtraField(buffer, i, Field.Value, v) else 0;
18371938 },
18381939 .extended => @compileError("TODO"),
1940 .flag_length_prefixed_list => {
1941 const len: u32 = @intCast(value.slice.len);
1942 if (len == 0) return 0;
1943 buffer[i] = len;
1944 @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice)));
1945 return len + 1;
1946 },
18391947 },
18401948 },
18411949 .@"extern" => comptime unreachable,
......@@ -1843,29 +1951,6 @@ pub const Storage = enum {
18431951 else => @compileError("bad field type: " ++ @typeName(Field)),
18441952 }
18451953 }
1846
1847 fn calculateExtraFieldLenUpperBound(comptime Field: type) comptime_int {
1848 return switch (@typeInfo(Field)) {
1849 .int => |info| switch (info.bits) {
1850 32 => 1,
1851 64 => 2,
1852 else => comptime unreachable,
1853 },
1854 .@"enum" => 1,
1855 .@"struct" => |info| switch (info.layout) {
1856 .@"packed" => switch (info.backing_integer.?) {
1857 u32 => 1,
1858 u64 => 2,
1859 else => comptime unreachable,
1860 },
1861 .auto => switch (Field.storage) {
1862 .flag_optional, .enum_optional, .extended => 1,
1863 },
1864 .@"extern" => comptime unreachable,
1865 },
1866 else => comptime unreachable,
1867 };
1868 }
18691954};
18701955
18711956pub fn extraData(c: *const Configuration, comptime T: type, index: usize) T {