1//! To get started, run this tool with no args and read the help message.
2//!
3//! The build systems of glibc, musl, FreeBSD, and NetBSD require specifying a single target
4//! architecture. Meanwhile, Zig supports out-of-the-box cross compilation for
5//! every target. So the process to create libc headers that Zig ships is to use
6//! this tool.
7//!
8//! First, use the glibc, musl, FreeBSD, and NetBSD build systems to create installations of all the
9//! targets in the `glibc_targets`, `musl_targets`, `freebsd_targets`, `netbsd_targets`, and
10//! `openbsd_targets` variables. Next, run this tool to create a new directory which puts .h files into
11//! <arch> subdirectories, with `generic` being files that apply to all architectures.
12//! You'll then have to manually update Zig source repo with these new files.
13
14const std = @import("std");
15const Io = std.Io;
16const Dir = std.Io.Dir;
17const Arch = std.Target.Cpu.Arch;
18const Abi = std.Target.Abi;
19const OsTag = std.Target.Os.Tag;
20const assert = std.debug.assert;
21const Blake3 = std.crypto.hash.Blake3;
22
23const LibCTarget = struct {
24 arch: Arch,
25 abi: Abi,
26 dest: ?[]const u8 = null,
27};
28
29const glibc_targets = [_]LibCTarget{
30 .{ .arch = .arc, .abi = .gnu },
31 .{ .arch = .arm, .abi = .gnueabi, .dest = "arm-linux-gnu" },
32 .{ .arch = .arm, .abi = .gnueabihf, .dest = "arm-linux-gnu" },
33 .{ .arch = .armeb, .abi = .gnueabi, .dest = "arm-linux-gnu" },
34 .{ .arch = .armeb, .abi = .gnueabihf, .dest = "arm-linux-gnu" },
35 .{ .arch = .aarch64, .abi = .gnu, .dest = "aarch64-linux-gnu" },
36 .{ .arch = .aarch64_be, .abi = .gnu, .dest = "aarch64-linux-gnu" },
37 .{ .arch = .csky, .abi = .gnueabi, .dest = "csky-linux-gnu" },
38 .{ .arch = .csky, .abi = .gnueabihf, .dest = "csky-linux-gnu" },
39 .{ .arch = .loongarch32, .abi = .gnu, .dest = "loongarch-linux-gnu" },
40 .{ .arch = .loongarch32, .abi = .gnusf, .dest = "loongarch-linux-gnu" },
41 .{ .arch = .loongarch64, .abi = .gnu, .dest = "loongarch-linux-gnu" },
42 .{ .arch = .loongarch64, .abi = .gnusf, .dest = "loongarch-linux-gnu" },
43 .{ .arch = .m68k, .abi = .gnu },
44 .{ .arch = .mips, .abi = .gnueabi, .dest = "mips-linux-gnu" },
45 .{ .arch = .mips, .abi = .gnueabihf, .dest = "mips-linux-gnu" },
46 .{ .arch = .mipsel, .abi = .gnueabi, .dest = "mips-linux-gnu" },
47 .{ .arch = .mipsel, .abi = .gnueabihf, .dest = "mips-linux-gnu" },
48 .{ .arch = .mips64, .abi = .gnuabi64, .dest = "mips-linux-gnu" },
49 .{ .arch = .mips64, .abi = .gnuabin32, .dest = "mips-linux-gnu" },
50 .{ .arch = .mips64el, .abi = .gnuabi64, .dest = "mips-linux-gnu" },
51 .{ .arch = .mips64el, .abi = .gnuabin32, .dest = "mips-linux-gnu" },
52 .{ .arch = .powerpc64le, .abi = .gnu, .dest = "powerpc-linux-gnu" },
53 .{ .arch = .riscv32, .abi = .gnu, .dest = "riscv-linux-gnu" },
54 .{ .arch = .riscv64, .abi = .gnu, .dest = "riscv-linux-gnu" },
55 .{ .arch = .s390x, .abi = .gnu },
56 .{ .arch = .sparc, .abi = .gnu, .dest = "sparc-linux-gnu" },
57 .{ .arch = .sparc64, .abi = .gnu, .dest = "sparc-linux-gnu" },
58 .{ .arch = .x86, .abi = .gnu, .dest = "x86-linux-gnu" },
59 .{ .arch = .x86_64, .abi = .gnu, .dest = "x86-linux-gnu" },
60 .{ .arch = .x86_64, .abi = .gnux32, .dest = "x86-linux-gnu" },
61};
62
63const musl_targets = [_]LibCTarget{
64 .{ .arch = .arm, .abi = .musl },
65 .{ .arch = .aarch64, .abi = .musl },
66 .{ .arch = .hexagon, .abi = .musl },
67 .{ .arch = .loongarch64, .abi = .musl },
68 .{ .arch = .m68k, .abi = .musl },
69 .{ .arch = .mips, .abi = .musl },
70 .{ .arch = .mips64, .abi = .musl },
71 .{ .arch = .mips64, .abi = .muslabin32 },
72 .{ .arch = .powerpc, .abi = .musl },
73 .{ .arch = .powerpc64, .abi = .musl },
74 .{ .arch = .riscv32, .abi = .musl },
75 .{ .arch = .riscv64, .abi = .musl },
76 .{ .arch = .s390x, .abi = .musl },
77 .{ .arch = .x86, .abi = .musl },
78 .{ .arch = .x86_64, .abi = .musl },
79 .{ .arch = .x86_64, .abi = .muslx32 },
80};
81
82const freebsd_targets = [_]LibCTarget{
83 .{ .arch = .arm, .abi = .eabihf },
84 .{ .arch = .aarch64, .abi = .none },
85 .{ .arch = .powerpc64, .abi = .none },
86 .{ .arch = .riscv64, .abi = .none },
87 .{ .arch = .x86, .abi = .none },
88 .{ .arch = .x86_64, .abi = .none },
89};
90
91const netbsd_targets = [_]LibCTarget{
92 .{ .arch = .arm, .abi = .eabi, .dest = "arm-netbsd-eabi" },
93 .{ .arch = .arm, .abi = .eabihf, .dest = "arm-netbsd-eabi" },
94 .{ .arch = .aarch64, .abi = .none },
95 .{ .arch = .m68k, .abi = .none },
96 .{ .arch = .mips, .abi = .eabi, .dest = "mips-netbsd-eabi" },
97 .{ .arch = .mips, .abi = .eabihf, .dest = "mips-netbsd-eabi" },
98 .{ .arch = .powerpc, .abi = .eabihf, .dest = "powerpc-netbsd-eabi" },
99 .{ .arch = .riscv32, .abi = .none },
100 .{ .arch = .riscv64, .abi = .none },
101 .{ .arch = .sparc, .abi = .none },
102 .{ .arch = .sparc64, .abi = .none },
103 .{ .arch = .x86, .abi = .none },
104 .{ .arch = .x86_64, .abi = .none },
105};
106
107const openbsd_targets = [_]LibCTarget{
108 .{ .arch = .arm, .abi = .eabi },
109 .{ .arch = .aarch64, .abi = .none },
110 .{ .arch = .mips64, .abi = .none },
111 .{ .arch = .mips64el, .abi = .none },
112 .{ .arch = .powerpc, .abi = .eabihf },
113 .{ .arch = .powerpc64, .abi = .none },
114 .{ .arch = .riscv64, .abi = .none },
115 .{ .arch = .sparc64, .abi = .none },
116 .{ .arch = .x86, .abi = .none },
117 .{ .arch = .x86_64, .abi = .none },
118};
119
120const Contents = struct {
121 bytes: []const u8,
122 hit_count: usize,
123 hash: []const u8,
124 is_generic: bool,
125
126 fn hitCountLessThan(context: void, lhs: *const Contents, rhs: *const Contents) bool {
127 _ = context;
128 return lhs.hit_count < rhs.hit_count;
129 }
130};
131
132const HashToContents = std.StringHashMap(Contents);
133const TargetToHash = std.array_hash_map.String([]const u8);
134const PathTable = std.StringHashMap(*TargetToHash);
135
136const LibCVendor = enum {
137 musl,
138 glibc,
139 freebsd,
140 netbsd,
141 openbsd,
142};
143
144pub fn main(init: std.process.Init) !void {
145 const arena = init.arena.allocator();
146 const io = init.io;
147 const args = try init.minimal.args.toSlice(arena);
148 const cwd_path = try std.process.currentPathAlloc(io, arena);
149 const environ_map = init.environ_map;
150
151 var search_paths = std.array_list.Managed([]const u8).init(arena);
152 var opt_out_dir: ?[]const u8 = null;
153 var opt_abi: ?[]const u8 = null;
154
155 var arg_i: usize = 1;
156 while (arg_i < args.len) : (arg_i += 1) {
157 if (std.mem.eql(u8, args[arg_i], "--help"))
158 usageAndExit(args[0]);
159 if (arg_i + 1 >= args.len) {
160 std.debug.print("expected argument after '{s}'\n", .{args[arg_i]});
161 usageAndExit(args[0]);
162 }
163
164 if (std.mem.eql(u8, args[arg_i], "--search-path")) {
165 try search_paths.append(args[arg_i + 1]);
166 } else if (std.mem.eql(u8, args[arg_i], "--out")) {
167 assert(opt_out_dir == null);
168 opt_out_dir = args[arg_i + 1];
169 } else if (std.mem.eql(u8, args[arg_i], "--abi")) {
170 assert(opt_abi == null);
171 opt_abi = args[arg_i + 1];
172 } else {
173 std.debug.print("unrecognized argument: {s}\n", .{args[arg_i]});
174 usageAndExit(args[0]);
175 }
176
177 arg_i += 1;
178 }
179
180 const out_dir = opt_out_dir orelse usageAndExit(args[0]);
181 const abi_name = opt_abi orelse usageAndExit(args[0]);
182 const vendor = std.meta.stringToEnum(LibCVendor, abi_name) orelse {
183 std.debug.print("unrecognized C ABI: {s}\n", .{abi_name});
184 usageAndExit(args[0]);
185 };
186
187 const generic_name = try std.fmt.allocPrint(arena, "generic-{s}", .{abi_name});
188 const libc_targets = switch (vendor) {
189 .glibc => &glibc_targets,
190 .musl => &musl_targets,
191 .freebsd => &freebsd_targets,
192 .netbsd => &netbsd_targets,
193 .openbsd => &openbsd_targets,
194 };
195
196 var path_table = PathTable.init(arena);
197 var hash_to_contents = HashToContents.init(arena);
198 var max_bytes_saved: usize = 0;
199 var total_bytes: usize = 0;
200
201 var hasher = Blake3.init(.{});
202
203 for (libc_targets) |libc_target| {
204 const libc_dir = switch (vendor) {
205 .glibc => try std.zig.target.glibcRuntimeTriple(arena, libc_target.arch, .linux, libc_target.abi),
206 .musl => std.zig.target.muslArchName(libc_target.arch, libc_target.abi),
207 .freebsd => switch (libc_target.arch) {
208 .arm => "armv7",
209 .x86 => "i386",
210 .x86_64 => "amd64",
211
212 .aarch64,
213 .powerpc,
214 .powerpc64,
215 .riscv64,
216 => |a| @tagName(a),
217
218 else => unreachable,
219 },
220 .netbsd => switch (libc_target.arch) {
221 .arm => if (libc_target.abi == .eabihf) "evbarmv7hf" else "evbarmv7",
222 .aarch64 => "evbarm64",
223 .m68k => "mac68k",
224 .mips => if (libc_target.abi == .eabihf) "evbmips" else "evbmipssf",
225 .powerpc => if (libc_target.abi == .eabihf) "evbppc" else "evbppcsf",
226 .x86 => "i386",
227 .x86_64 => "amd64",
228
229 .riscv32,
230 .riscv64,
231 .sparc,
232 .sparc64,
233 => |a| @tagName(a),
234
235 else => unreachable,
236 },
237 .openbsd => switch (libc_target.arch) {
238 .arm => "armv7",
239 .aarch64 => "arm64",
240 .mips64 => "octeon",
241 .mips64el => "loongson",
242 .powerpc => "macppc",
243 .x86 => "i386",
244 .x86_64 => "amd64",
245
246 .powerpc64,
247 .riscv64,
248 .sparc64,
249 => |a| @tagName(a),
250
251 else => unreachable,
252 },
253 };
254
255 const dest_target = if (libc_target.dest) |dest| dest else try std.fmt.allocPrint(arena, "{s}-{s}-{s}", .{
256 @tagName(libc_target.arch),
257 switch (vendor) {
258 .musl, .glibc => "linux",
259 .freebsd => "freebsd",
260 .netbsd => "netbsd",
261 .openbsd => "openbsd",
262 },
263 @tagName(libc_target.abi),
264 });
265
266 search: for (search_paths.items) |search_path| {
267 const sub_path = switch (vendor) {
268 .glibc,
269 .freebsd,
270 .netbsd,
271 .openbsd,
272 => &[_][]const u8{ search_path, libc_dir, "usr", "include" },
273 .musl => &[_][]const u8{ search_path, libc_dir, "usr", "local", "musl", "include" },
274 };
275 const target_include_dir = try Dir.path.join(arena, sub_path);
276 var dir_stack = std.array_list.Managed([]const u8).init(arena);
277 try dir_stack.append(target_include_dir);
278
279 while (dir_stack.pop()) |full_dir_name| {
280 var dir = Dir.cwd().openDir(io, full_dir_name, .{ .iterate = true }) catch |err| switch (err) {
281 error.FileNotFound => continue :search,
282 error.AccessDenied => continue :search,
283 else => return err,
284 };
285 defer dir.close(io);
286
287 var dir_it = dir.iterate();
288
289 while (try dir_it.next(io)) |entry| {
290 const full_path = try Dir.path.join(arena, &[_][]const u8{ full_dir_name, entry.name });
291 switch (entry.kind) {
292 .directory => try dir_stack.append(full_path),
293 .file, .sym_link => {
294 const rel_path = try Dir.path.relative(arena, cwd_path, environ_map, target_include_dir, full_path);
295 const max_size = 2 * 1024 * 1024 * 1024;
296 const raw_bytes = try Dir.cwd().readFileAlloc(io, full_path, arena, .limited(max_size));
297 const trimmed = std.mem.trim(u8, raw_bytes, " \r\n\t");
298 total_bytes += raw_bytes.len;
299 const hash = try arena.alloc(u8, 32);
300 hasher = Blake3.init(.{});
301 hasher.update(rel_path);
302 hasher.update(trimmed);
303 hasher.final(hash);
304 const gop = try hash_to_contents.getOrPut(hash);
305 if (gop.found_existing) {
306 max_bytes_saved += raw_bytes.len;
307 gop.value_ptr.hit_count += 1;
308 std.debug.print("duplicate: {s} {s} ({B})\n", .{
309 libc_dir, rel_path, raw_bytes.len,
310 });
311 } else {
312 gop.value_ptr.* = Contents{
313 .bytes = trimmed,
314 .hit_count = 1,
315 .hash = hash,
316 .is_generic = false,
317 };
318 }
319 const path_gop = try path_table.getOrPut(rel_path);
320 const target_to_hash = if (path_gop.found_existing) path_gop.value_ptr.* else blk: {
321 const ptr = try arena.create(TargetToHash);
322 ptr.* = .empty;
323 path_gop.value_ptr.* = ptr;
324 break :blk ptr;
325 };
326 // When `dest` is set, there are a few rare cases where we expect to overwrite a header. For
327 // example, `bits/long-double.h` differs very slightly between `powerpc64le-linux-gnu` and
328 // other `powerpc*-linux-gnu` targets, and we unify those targets as `powerpc-linux-gnu`. In
329 // such cases, we manually patch the affected header after processing, so it's fine that
330 // only one header wins here.
331 if (libc_target.dest != null) {
332 const hash_gop = try target_to_hash.getOrPut(arena, dest_target);
333 if (hash_gop.found_existing) std.debug.print("overwrote: {s} {s} {s}\n", .{
334 libc_dir,
335 rel_path,
336 dest_target,
337 }) else hash_gop.value_ptr.* = hash;
338 } else {
339 try target_to_hash.putNoClobber(arena, dest_target, hash);
340 }
341 },
342 else => std.debug.print("warning: weird file: {s}\n", .{full_path}),
343 }
344 }
345 }
346 break;
347 } else {
348 std.debug.print("warning: libc target not found: {s}\n", .{libc_dir});
349 }
350 }
351 std.debug.print("summary: {B} could be reduced to {B}\n", .{
352 total_bytes,
353 total_bytes - max_bytes_saved,
354 });
355 try Dir.cwd().createDirPath(io, out_dir);
356
357 var missed_opportunity_bytes: usize = 0;
358 // iterate path_table. for each path, put all the hashes into a list. sort by hit_count.
359 // the hash with the highest hit_count gets to be the "generic" one. everybody else
360 // gets their header in a separate arch directory.
361 var path_it = path_table.iterator();
362 while (path_it.next()) |path_kv| {
363 var contents_list = std.array_list.Managed(*Contents).init(arena);
364 {
365 var hash_it = path_kv.value_ptr.*.iterator();
366 while (hash_it.next()) |hash_kv| {
367 const contents = hash_to_contents.getPtr(hash_kv.value_ptr.*).?;
368 try contents_list.append(contents);
369 }
370 }
371 std.mem.sort(*Contents, contents_list.items, {}, Contents.hitCountLessThan);
372 const best_contents = contents_list.pop().?;
373 if (best_contents.hit_count > 1) {
374 // worth it to make it generic
375 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, generic_name, path_kv.key_ptr.* });
376 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?);
377 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = best_contents.bytes });
378 best_contents.is_generic = true;
379 while (contents_list.pop()) |contender| {
380 if (contender.hit_count > 1) {
381 const this_missed_bytes = contender.hit_count * contender.bytes.len;
382 missed_opportunity_bytes += this_missed_bytes;
383 std.debug.print("Missed opportunity ({B}): {s}\n", .{
384 this_missed_bytes,
385 path_kv.key_ptr.*,
386 });
387 } else break;
388 }
389 }
390 var hash_it = path_kv.value_ptr.*.iterator();
391 while (hash_it.next()) |hash_kv| {
392 const contents = hash_to_contents.get(hash_kv.value_ptr.*).?;
393 if (contents.is_generic) continue;
394
395 const dest_target = hash_kv.key_ptr.*;
396 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, dest_target, path_kv.key_ptr.* });
397 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?);
398 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = contents.bytes });
399 }
400 }
401}
402
403fn usageAndExit(arg0: []const u8) noreturn {
404 std.debug.print("Usage: {s} [--search-path <dir>] --out <dir> --abi <name>\n", .{arg0});
405 std.debug.print("--search-path can be used any number of times.\n", .{});
406 std.debug.print(" subdirectories of search paths look like, e.g. x86_64-linux-gnu\n", .{});
407 std.debug.print("--out is a dir that will be created, and populated with the results\n", .{});
408 std.debug.print("--abi is either glibc, musl, freebsd, netbsd, or openbsd\n", .{});
409 std.process.exit(1);
410}