authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-14 15:25:30-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-14 15:28:09-07:00
log26798018b780846eb0613b2061835985bf0c58ea
tree1c7620cb2aecda0537ee6adbce4c3110d9394cc2
parent40cb712d13aff4bfe83256858ad6b18d82e70211

stage2: implement writing archive files

9 files changed, 246 insertions(+), 11 deletions(-)

src-self-hosted/Compilation.zig+2
...@@ -535,6 +535,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -535,6 +535,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
535 var hash = cache.hash;535 var hash = cache.hash;
536 if (options.c_source_files.len >= 1) {536 if (options.c_source_files.len >= 1) {
537 hash.addBytes(options.c_source_files[0].src_path);537 hash.addBytes(options.c_source_files[0].src_path);
538 } else if (options.link_objects.len >= 1) {
539 hash.addBytes(options.link_objects[0]);
538 }540 }
539541
540 const digest = hash.final();542 const digest = hash.final();
src-self-hosted/link.zig+119-4
...@@ -1,4 +1,5 @@...@@ -1,4 +1,5 @@
1const std = @import("std");1const std = @import("std");
2const mem = std.mem;
2const Allocator = std.mem.Allocator;3const Allocator = std.mem.Allocator;
3const Compilation = @import("Compilation.zig");4const Compilation = @import("Compilation.zig");
4const Module = @import("Module.zig");5const Module = @import("Module.zig");
...@@ -9,6 +10,7 @@ const Type = @import("type.zig").Type;...@@ -9,6 +10,7 @@ const Type = @import("type.zig").Type;
9const Cache = @import("Cache.zig");10const Cache = @import("Cache.zig");
10const build_options = @import("build_options");11const build_options = @import("build_options");
11const LibCInstallation = @import("libc_installation.zig").LibCInstallation;12const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
13const log = std.log.scoped(.link);
1214
13pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ build_options.version;15pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ build_options.version;
1416
...@@ -279,9 +281,11 @@ pub const File = struct {...@@ -279,9 +281,11 @@ pub const File = struct {
279 }281 }
280 }282 }
281283
284 /// Commit pending changes and write headers. Takes into account final output mode
285 /// and `use_lld`, not only `effectiveOutputMode`.
282 pub fn flush(base: *File, comp: *Compilation) !void {286 pub fn flush(base: *File, comp: *Compilation) !void {
283 const use_lld = build_options.have_llvm and base.options.use_lld;287 const use_lld = build_options.have_llvm and base.options.use_lld;
284 if (base.options.output_mode == .Lib and base.options.link_mode == .Static and288 if (use_lld and base.options.output_mode == .Lib and base.options.link_mode == .Static and
285 !base.options.target.isWasm())289 !base.options.target.isWasm())
286 {290 {
287 return base.linkAsArchive(comp);291 return base.linkAsArchive(comp);
...@@ -295,6 +299,18 @@ pub const File = struct {...@@ -295,6 +299,18 @@ pub const File = struct {
295 }299 }
296 }300 }
297301
302 /// Commit pending changes and write headers. Works based on `effectiveOutputMode`
303 /// rather than final output mode.
304 pub fn flushModule(base: *File, comp: *Compilation) !void {
305 switch (base.tag) {
306 .coff => return @fieldParentPtr(Coff, "base", base).flushModule(comp),
307 .elf => return @fieldParentPtr(Elf, "base", base).flushModule(comp),
308 .macho => return @fieldParentPtr(MachO, "base", base).flushModule(comp),
309 .c => return @fieldParentPtr(C, "base", base).flushModule(comp),
310 .wasm => return @fieldParentPtr(Wasm, "base", base).flushModule(comp),
311 }
312 }
313
298 pub fn freeDecl(base: *File, decl: *Module.Decl) void {314 pub fn freeDecl(base: *File, decl: *Module.Decl) void {
299 switch (base.tag) {315 switch (base.tag) {
300 .coff => @fieldParentPtr(Coff, "base", base).freeDecl(decl),316 .coff => @fieldParentPtr(Coff, "base", base).freeDecl(decl),
...@@ -343,9 +359,108 @@ pub const File = struct {...@@ -343,9 +359,108 @@ pub const File = struct {
343 }359 }
344360
345 fn linkAsArchive(base: *File, comp: *Compilation) !void {361 fn linkAsArchive(base: *File, comp: *Compilation) !void {
346 // TODO follow pattern from ELF linkWithLLD362 const tracy = trace(@src());
347 // ZigLLVMWriteArchive363 defer tracy.end();
348 return error.TODOMakeArchive;364
365 var arena_allocator = std.heap.ArenaAllocator.init(base.allocator);
366 defer arena_allocator.deinit();
367 const arena = &arena_allocator.allocator;
368
369 const directory = base.options.directory; // Just an alias to make it shorter to type.
370
371 // If there is no Zig code to compile, then we should skip flushing the output file because it
372 // will not be part of the linker line anyway.
373 const module_obj_path: ?[]const u8 = if (base.options.module) |module| blk: {
374 try base.flushModule(comp);
375
376 const obj_basename = base.intermediary_basename.?;
377 const full_obj_path = if (directory.path) |dir_path|
378 try std.fs.path.join(arena, &[_][]const u8{ dir_path, obj_basename })
379 else
380 obj_basename;
381 break :blk full_obj_path;
382 } else null;
383
384 // This function follows the same pattern as link.Elf.linkWithLLD so if you want some
385 // insight as to what's going on here you can read that function body which is more
386 // well-commented.
387
388 const id_symlink_basename = "llvm-ar.id";
389
390 base.releaseLock();
391
392 var ch = comp.cache_parent.obtain();
393 defer ch.deinit();
394
395 try ch.addListOfFiles(base.options.objects);
396 for (comp.c_object_table.items()) |entry| {
397 _ = try ch.addFile(entry.key.status.success.object_path, null);
398 }
399 try ch.addOptionalFile(module_obj_path);
400
401 // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
402 _ = try ch.hit();
403 const digest = ch.final();
404
405 var prev_digest_buf: [digest.len]u8 = undefined;
406 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| b: {
407 log.debug("archive new_digest={} readlink error: {}", .{ digest, @errorName(err) });
408 break :b prev_digest_buf[0..0];
409 };
410 if (mem.eql(u8, prev_digest, &digest)) {
411 log.debug("archive digest={} match - skipping invocation", .{digest});
412 base.lock = ch.toOwnedLock();
413 return;
414 }
415
416 // We are about to change the output file to be different, so we invalidate the build hash now.
417 directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
418 error.FileNotFound => {},
419 else => |e| return e,
420 };
421
422 var object_files = std.ArrayList([*:0]const u8).init(base.allocator);
423 defer object_files.deinit();
424
425 try object_files.ensureCapacity(base.options.objects.len + comp.c_object_table.items().len + 1);
426 for (base.options.objects) |obj_path| {
427 object_files.appendAssumeCapacity(try arena.dupeZ(u8, obj_path));
428 }
429 for (comp.c_object_table.items()) |entry| {
430 object_files.appendAssumeCapacity(try arena.dupeZ(u8, entry.key.status.success.object_path));
431 }
432 if (module_obj_path) |p| {
433 object_files.appendAssumeCapacity(try arena.dupeZ(u8, p));
434 }
435
436 const full_out_path = if (directory.path) |dir_path|
437 try std.fs.path.join(arena, &[_][]const u8{ dir_path, base.options.sub_path })
438 else
439 base.options.sub_path;
440 const full_out_path_z = try arena.dupeZ(u8, full_out_path);
441
442 if (base.options.debug_link) {
443 std.debug.print("ar rcs {}", .{full_out_path_z});
444 for (object_files.items) |arg| {
445 std.debug.print(" {}", .{arg});
446 }
447 std.debug.print("\n", .{});
448 }
449
450 const llvm = @import("llvm.zig");
451 const os_type = @import("target.zig").osToLLVM(base.options.target.os.tag);
452 const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type);
453 if (bad) return error.UnableToWriteArchive;
454
455 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
456 std.log.warn("failed to save archive hash digest symlink: {}", .{@errorName(err)});
457 };
458
459 ch.writeManifest() catch |err| {
460 std.log.warn("failed to write cache manifest when archiving: {}", .{@errorName(err)});
461 };
462
463 base.lock = ch.toOwnedLock();
349 }464 }
350465
351 pub const Tag = enum {466 pub const Tag = enum {
src-self-hosted/link/C.zig+4
...@@ -74,6 +74,10 @@ pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void {...@@ -74,6 +74,10 @@ pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void {
74}74}
7575
76pub fn flush(self: *C, comp: *Compilation) !void {76pub fn flush(self: *C, comp: *Compilation) !void {
77 return self.flushModule(comp);
78}
79
80pub fn flushModule(self: *C, comp: *Compilation) !void {
77 const tracy = trace(@src());81 const tracy = trace(@src());
78 defer tracy.end();82 defer tracy.end();
7983
src-self-hosted/link/Coff.zig+9
...@@ -11,6 +11,7 @@ const Module = @import("../Module.zig");...@@ -11,6 +11,7 @@ const Module = @import("../Module.zig");
11const Compilation = @import("../Compilation.zig");11const Compilation = @import("../Compilation.zig");
12const codegen = @import("../codegen.zig");12const codegen = @import("../codegen.zig");
13const link = @import("../link.zig");13const link = @import("../link.zig");
14const build_options = @import("build_options");
1415
15const allocation_padding = 4 / 3;16const allocation_padding = 4 / 3;
16const minimum_text_block_size = 64 * allocation_padding;17const minimum_text_block_size = 64 * allocation_padding;
...@@ -724,6 +725,14 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl,...@@ -724,6 +725,14 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl,
724}725}
725726
726pub fn flush(self: *Coff, comp: *Compilation) !void {727pub fn flush(self: *Coff, comp: *Compilation) !void {
728 if (build_options.have_llvm and self.base.options.use_lld) {
729 return error.CoffLinkingWithLLDUnimplemented;
730 } else {
731 return self.flushModule(comp);
732 }
733}
734
735pub fn flushModule(self: *Coff, comp: *Compilation) !void {
727 const tracy = trace(@src());736 const tracy = trace(@src());
728 defer tracy.end();737 defer tracy.end();
729738
src-self-hosted/link/Elf.zig+4-5
...@@ -719,12 +719,11 @@ pub fn flush(self: *Elf, comp: *Compilation) !void {...@@ -719,12 +719,11 @@ pub fn flush(self: *Elf, comp: *Compilation) !void {
719 .Exe, .Obj => {},719 .Exe, .Obj => {},
720 .Lib => return error.TODOImplementWritingLibFiles,720 .Lib => return error.TODOImplementWritingLibFiles,
721 }721 }
722 return self.flushInner(comp);722 return self.flushModule(comp);
723 }723 }
724}724}
725725
726/// Commit pending changes and write headers.726pub fn flushModule(self: *Elf, comp: *Compilation) !void {
727fn flushInner(self: *Elf, comp: *Compilation) !void {
728 const tracy = trace(@src());727 const tracy = trace(@src());
729 defer tracy.end();728 defer tracy.end();
730729
...@@ -1221,7 +1220,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {...@@ -1221,7 +1220,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
1221 // If there is no Zig code to compile, then we should skip flushing the output file because it1220 // If there is no Zig code to compile, then we should skip flushing the output file because it
1222 // will not be part of the linker line anyway.1221 // will not be part of the linker line anyway.
1223 const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: {1222 const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: {
1224 try self.flushInner(comp);1223 try self.flushModule(comp);
12251224
1226 const obj_basename = self.base.intermediary_basename.?;1225 const obj_basename = self.base.intermediary_basename.?;
1227 const full_obj_path = if (directory.path) |dir_path|1226 const full_obj_path = if (directory.path) |dir_path|
...@@ -1239,7 +1238,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {...@@ -1239,7 +1238,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
1239 // After a successful link, we store the id in the metadata of a symlink named "id.txt" in1238 // After a successful link, we store the id in the metadata of a symlink named "id.txt" in
1240 // the artifact directory. So, now, we check if this symlink exists, and if it matches1239 // the artifact directory. So, now, we check if this symlink exists, and if it matches
1241 // our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD.1240 // our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD.
1242 const id_symlink_basename = "id.txt";1241 const id_symlink_basename = "lld.id";
12431242
1244 // We are about to obtain this lock, so here we give other processes a chance first.1243 // We are about to obtain this lock, so here we give other processes a chance first.
1245 self.base.releaseLock();1244 self.base.releaseLock();
src-self-hosted/link/MachO.zig+10-1
...@@ -9,9 +9,10 @@ const macho = std.macho;...@@ -9,9 +9,10 @@ const macho = std.macho;
9const codegen = @import("../codegen.zig");9const codegen = @import("../codegen.zig");
10const math = std.math;10const math = std.math;
11const mem = std.mem;11const mem = std.mem;
12
12const trace = @import("../tracy.zig").trace;13const trace = @import("../tracy.zig").trace;
13const Type = @import("../type.zig").Type;14const Type = @import("../type.zig").Type;
1415const build_options = @import("build_options");
15const Module = @import("../Module.zig");16const Module = @import("../Module.zig");
16const Compilation = @import("../Compilation.zig");17const Compilation = @import("../Compilation.zig");
17const link = @import("../link.zig");18const link = @import("../link.zig");
...@@ -178,6 +179,14 @@ pub fn createEmpty(gpa: *Allocator, options: link.Options) !*MachO {...@@ -178,6 +179,14 @@ pub fn createEmpty(gpa: *Allocator, options: link.Options) !*MachO {
178}179}
179180
180pub fn flush(self: *MachO, comp: *Compilation) !void {181pub fn flush(self: *MachO, comp: *Compilation) !void {
182 if (build_options.have_llvm and self.base.options.use_lld) {
183 return error.MachOLLDLinkingUnimplemented;
184 } else {
185 return self.flushModule(comp);
186 }
187}
188
189pub fn flushModule(self: *MachO, comp: *Compilation) !void {
181 const tracy = trace(@src());190 const tracy = trace(@src());
182 defer tracy.end();191 defer tracy.end();
183192
src-self-hosted/link/Wasm.zig+9
...@@ -11,6 +11,7 @@ const Compilation = @import("../Compilation.zig");...@@ -11,6 +11,7 @@ const Compilation = @import("../Compilation.zig");
11const codegen = @import("../codegen/wasm.zig");11const codegen = @import("../codegen/wasm.zig");
12const link = @import("../link.zig");12const link = @import("../link.zig");
13const trace = @import("../tracy.zig").trace;13const trace = @import("../tracy.zig").trace;
14const build_options = @import("build_options");
1415
15/// Various magic numbers defined by the wasm spec16/// Various magic numbers defined by the wasm spec
16const spec = struct {17const spec = struct {
...@@ -135,6 +136,14 @@ pub fn freeDecl(self: *Wasm, decl: *Module.Decl) void {...@@ -135,6 +136,14 @@ pub fn freeDecl(self: *Wasm, decl: *Module.Decl) void {
135}136}
136137
137pub fn flush(self: *Wasm, comp: *Compilation) !void {138pub fn flush(self: *Wasm, comp: *Compilation) !void {
139 if (build_options.have_llvm and self.base.options.use_lld) {
140 return error.WasmLinkingWithLLDUnimplemented;
141 } else {
142 return self.flushModule(comp);
143 }
144}
145
146pub fn flushModule(self: *Wasm, comp: *Compilation) !void {
138 const tracy = trace(@src());147 const tracy = trace(@src());
139 defer tracy.end();148 defer tracy.end();
140149
src-self-hosted/llvm.zig+48-1
...@@ -2,7 +2,7 @@...@@ -2,7 +2,7 @@
2//! to bootstrap if it does not depend on translate-c.2//! to bootstrap if it does not depend on translate-c.
33
4pub const Link = ZigLLDLink;4pub const Link = ZigLLDLink;
5pub extern fn ZigLLDLink(5extern fn ZigLLDLink(
6 oformat: ObjectFormatType,6 oformat: ObjectFormatType,
7 args: [*:null]const ?[*:0]const u8,7 args: [*:null]const ?[*:0]const u8,
8 arg_count: usize,8 arg_count: usize,
...@@ -25,3 +25,50 @@ extern fn LLVMGetHostCPUName() ?[*:0]u8;...@@ -25,3 +25,50 @@ extern fn LLVMGetHostCPUName() ?[*:0]u8;
2525
26pub const GetNativeFeatures = ZigLLVMGetNativeFeatures;26pub const GetNativeFeatures = ZigLLVMGetNativeFeatures;
27extern fn ZigLLVMGetNativeFeatures() ?[*:0]u8;27extern fn ZigLLVMGetNativeFeatures() ?[*:0]u8;
28
29pub const WriteArchive = ZigLLVMWriteArchive;
30extern fn ZigLLVMWriteArchive(
31 archive_name: [*:0]const u8,
32 file_names_ptr: [*]const [*:0]const u8,
33 file_names_len: usize,
34 os_type: OSType,
35) bool;
36
37pub const OSType = extern enum(c_int) {
38 UnknownOS = 0,
39 Ananas = 1,
40 CloudABI = 2,
41 Darwin = 3,
42 DragonFly = 4,
43 FreeBSD = 5,
44 Fuchsia = 6,
45 IOS = 7,
46 KFreeBSD = 8,
47 Linux = 9,
48 Lv2 = 10,
49 MacOSX = 11,
50 NetBSD = 12,
51 OpenBSD = 13,
52 Solaris = 14,
53 Win32 = 15,
54 Haiku = 16,
55 Minix = 17,
56 RTEMS = 18,
57 NaCl = 19,
58 CNK = 20,
59 AIX = 21,
60 CUDA = 22,
61 NVCL = 23,
62 AMDHSA = 24,
63 PS4 = 25,
64 ELFIAMCU = 26,
65 TvOS = 27,
66 WatchOS = 28,
67 Mesa3D = 29,
68 Contiki = 30,
69 AMDPAL = 31,
70 HermitCore = 32,
71 Hurd = 33,
72 WASI = 34,
73 Emscripten = 35,
74};
src-self-hosted/target.zig+41
...@@ -1,4 +1,5 @@...@@ -1,4 +1,5 @@
1const std = @import("std");1const std = @import("std");
2const llvm = @import("llvm.zig");
23
3pub const ArchOsAbi = struct {4pub const ArchOsAbi = struct {
4 arch: std.Target.Cpu.Arch,5 arch: std.Target.Cpu.Arch,
...@@ -168,3 +169,43 @@ pub fn supportsStackProbing(target: std.Target) bool {...@@ -168,3 +169,43 @@ pub fn supportsStackProbing(target: std.Target) bool {
168 return target.os.tag != .windows and target.os.tag != .uefi and169 return target.os.tag != .windows and target.os.tag != .uefi and
169 (target.cpu.arch == .i386 or target.cpu.arch == .x86_64);170 (target.cpu.arch == .i386 or target.cpu.arch == .x86_64);
170}171}
172
173pub fn osToLLVM(os_tag: std.Target.Os.Tag) llvm.OSType {
174 return switch (os_tag) {
175 .freestanding, .other => .UnknownOS,
176 .windows, .uefi => .Win32,
177 .ananas => .Ananas,
178 .cloudabi => .CloudABI,
179 .dragonfly => .DragonFly,
180 .freebsd => .FreeBSD,
181 .fuchsia => .Fuchsia,
182 .ios => .IOS,
183 .kfreebsd => .KFreeBSD,
184 .linux => .Linux,
185 .lv2 => .Lv2,
186 .macosx => .MacOSX,
187 .netbsd => .NetBSD,
188 .openbsd => .OpenBSD,
189 .solaris => .Solaris,
190 .haiku => .Haiku,
191 .minix => .Minix,
192 .rtems => .RTEMS,
193 .nacl => .NaCl,
194 .cnk => .CNK,
195 .aix => .AIX,
196 .cuda => .CUDA,
197 .nvcl => .NVCL,
198 .amdhsa => .AMDHSA,
199 .ps4 => .PS4,
200 .elfiamcu => .ELFIAMCU,
201 .tvos => .TvOS,
202 .watchos => .WatchOS,
203 .mesa3d => .Mesa3D,
204 .contiki => .Contiki,
205 .amdpal => .AMDPAL,
206 .hermit => .HermitCore,
207 .hurd => .Hurd,
208 .wasi => .WASI,
209 .emscripten => .Emscripten,
210 };
211}