authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-08-30 12:30:17+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-08-30 12:30:17+02:00
logf21245f5e773c61a8d1f5fb91309faadf0d2f103
tree25a2134dd570cee064f16fa0bb1144619e62cce0
parentea9f2513a3b57aa4fda7825ca407672ce8d9da31

macho: refactor resolving and parsing dependent dylibs

3 files changed, 93 insertions(+), 69 deletions(-)

src/link/MachO.zig+81-61
...@@ -401,26 +401,25 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No...@@ -401,26 +401,25 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
401 parent: u16,401 parent: u16,
402 }, .Dynamic).init(arena);402 }, .Dynamic).init(arena);
403403
404 var parse_ctx = ParseErrorCtx.init(arena);
405
406 for (libs.keys(), libs.values()) |path, lib| {404 for (libs.keys(), libs.values()) |path, lib| {
407 const in_file = try std.fs.cwd().openFile(path, .{});405 const in_file = try std.fs.cwd().openFile(path, .{});
408 defer in_file.close();406 defer in_file.close();
409 defer parse_ctx.detected_targets.clearRetainingCapacity();407
408 var parse_ctx = ParseErrorCtx.init(self.base.allocator);
409 defer parse_ctx.deinit();
410
410 self.parseLibrary(411 self.parseLibrary(
411 in_file,412 in_file,
412 path,413 path,
413 lib,414 lib,
414 false,415 false,
416 false,
415 &dependent_libs,417 &dependent_libs,
416 &parse_ctx,418 &parse_ctx,
417 ) catch |err| try self.handleAndReportParseError(path, err, &parse_ctx);419 ) catch |err| try self.handleAndReportParseError(path, err, &parse_ctx);
418 }420 }
419421
420 self.parseDependentLibs(&dependent_libs, &parse_ctx) catch |err| {422 try self.parseDependentLibs(&dependent_libs);
421 // TODO convert to error
422 log.err("parsing dependent libraries failed with err {s}", .{@errorName(err)});
423 };
424 }423 }
425424
426 var actions = std.ArrayList(ResolveAction).init(self.base.allocator);425 var actions = std.ArrayList(ResolveAction).init(self.base.allocator);
...@@ -674,7 +673,7 @@ fn resolveLibSystemInDirs(arena: Allocator, dirs: []const []const u8, out_libs:...@@ -674,7 +673,7 @@ fn resolveLibSystemInDirs(arena: Allocator, dirs: []const []const u8, out_libs:
674 // Try stub file first. If we hit it, then we're done as the stub file673 // Try stub file first. If we hit it, then we're done as the stub file
675 // re-exports every single symbol definition.674 // re-exports every single symbol definition.
676 for (dirs) |dir| {675 for (dirs) |dir| {
677 if (try resolveLib(arena, dir, "System", ".tbd")) |full_path| {676 if (try resolveLib(arena, dir, "libSystem", ".tbd")) |full_path| {
678 try out_libs.put(full_path, .{ .needed = true, .weak = false, .path = full_path });677 try out_libs.put(full_path, .{ .needed = true, .weak = false, .path = full_path });
679 return true;678 return true;
680 }679 }
...@@ -682,8 +681,8 @@ fn resolveLibSystemInDirs(arena: Allocator, dirs: []const []const u8, out_libs:...@@ -682,8 +681,8 @@ fn resolveLibSystemInDirs(arena: Allocator, dirs: []const []const u8, out_libs:
682 // If we didn't hit the stub file, try .dylib next. However, libSystem.dylib681 // If we didn't hit the stub file, try .dylib next. However, libSystem.dylib
683 // doesn't export libc.dylib which we'll need to resolve subsequently also.682 // doesn't export libc.dylib which we'll need to resolve subsequently also.
684 for (dirs) |dir| {683 for (dirs) |dir| {
685 if (try resolveLib(arena, dir, "System", ".dylib")) |libsystem_path| {684 if (try resolveLib(arena, dir, "libSystem", ".dylib")) |libsystem_path| {
686 if (try resolveLib(arena, dir, "c", ".dylib")) |libc_path| {685 if (try resolveLib(arena, dir, "libc", ".dylib")) |libc_path| {
687 try out_libs.put(libsystem_path, .{ .needed = true, .weak = false, .path = libsystem_path });686 try out_libs.put(libsystem_path, .{ .needed = true, .weak = false, .path = libsystem_path });
688 try out_libs.put(libc_path, .{ .needed = true, .weak = false, .path = libc_path });687 try out_libs.put(libc_path, .{ .needed = true, .weak = false, .path = libc_path });
689 return true;688 return true;
...@@ -700,7 +699,7 @@ fn resolveLib(...@@ -700,7 +699,7 @@ fn resolveLib(
700 name: []const u8,699 name: []const u8,
701 ext: []const u8,700 ext: []const u8,
702) !?[]const u8 {701) !?[]const u8 {
703 const search_name = try std.fmt.allocPrint(arena, "lib{s}{s}", .{ name, ext });702 const search_name = try std.fmt.allocPrint(arena, "{s}{s}", .{ name, ext });
704 const full_path = try fs.path.join(arena, &[_][]const u8{ search_dir, search_name });703 const full_path = try fs.path.join(arena, &[_][]const u8{ search_dir, search_name });
705704
706 // Check if the file exists.705 // Check if the file exists.
...@@ -747,7 +746,7 @@ pub fn parsePositional(...@@ -747,7 +746,7 @@ pub fn parsePositional(
747 .path = null,746 .path = null,
748 .needed = false,747 .needed = false,
749 .weak = false,748 .weak = false,
750 }, must_link, dependent_libs, ctx);749 }, must_link, false, dependent_libs, ctx);
751 }750 }
752}751}
753752
...@@ -790,7 +789,7 @@ fn parseObject(...@@ -790,7 +789,7 @@ fn parseObject(
790 (detected_platform != null and !detected_platform.?.eqlTarget(this_platform)))789 (detected_platform != null and !detected_platform.?.eqlTarget(this_platform)))
791 {790 {
792 const platform = detected_platform orelse this_platform;791 const platform = detected_platform orelse this_platform;
793 try ctx.detected_targets.append(try platform.allocPrintTarget(ctx.arena, detected_cpu_arch));792 try ctx.detected_targets.append(try platform.allocPrintTarget(ctx.arena(), detected_cpu_arch));
794 return error.InvalidTarget;793 return error.InvalidTarget;
795 }794 }
796795
...@@ -803,6 +802,7 @@ pub fn parseLibrary(...@@ -803,6 +802,7 @@ pub fn parseLibrary(
803 path: []const u8,802 path: []const u8,
804 lib: link.SystemLib,803 lib: link.SystemLib,
805 must_link: bool,804 must_link: bool,
805 is_dependent: bool,
806 dependent_libs: anytype,806 dependent_libs: anytype,
807 ctx: *ParseErrorCtx,807 ctx: *ParseErrorCtx,
808) ParseError!void {808) ParseError!void {
...@@ -819,6 +819,7 @@ pub fn parseLibrary(...@@ -819,6 +819,7 @@ pub fn parseLibrary(
819 try self.parseDylib(file, path, offset, dependent_libs, .{819 try self.parseDylib(file, path, offset, dependent_libs, .{
820 .needed = lib.needed,820 .needed = lib.needed,
821 .weak = lib.weak,821 .weak = lib.weak,
822 .dependent = is_dependent,
822 }, ctx);823 }, ctx);
823 } else return error.UnknownFileType;824 } else return error.UnknownFileType;
824 } else if (Archive.isArchive(file, 0)) {825 } else if (Archive.isArchive(file, 0)) {
...@@ -827,11 +828,13 @@ pub fn parseLibrary(...@@ -827,11 +828,13 @@ pub fn parseLibrary(
827 try self.parseDylib(file, path, 0, dependent_libs, .{828 try self.parseDylib(file, path, 0, dependent_libs, .{
828 .needed = lib.needed,829 .needed = lib.needed,
829 .weak = lib.weak,830 .weak = lib.weak,
831 .dependent = is_dependent,
830 }, ctx);832 }, ctx);
831 } else {833 } else {
832 self.parseLibStub(file, path, dependent_libs, .{834 self.parseLibStub(file, path, dependent_libs, .{
833 .needed = lib.needed,835 .needed = lib.needed,
834 .weak = lib.weak,836 .weak = lib.weak,
837 .dependent = is_dependent,
835 }, ctx) catch |err| switch (err) {838 }, ctx) catch |err| switch (err) {
836 error.NotLibStub, error.UnexpectedToken => return error.UnknownFileType,839 error.NotLibStub, error.UnexpectedToken => return error.UnknownFileType,
837 else => |e| return e,840 else => |e| return e,
...@@ -853,9 +856,9 @@ pub fn parseFatLibrary(...@@ -853,9 +856,9 @@ pub fn parseFatLibrary(
853 const offset = for (fat_archs) |arch| {856 const offset = for (fat_archs) |arch| {
854 if (arch.tag == cpu_arch) break arch.offset;857 if (arch.tag == cpu_arch) break arch.offset;
855 } else {858 } else {
856 try ctx.detected_targets.ensureTotalCapacityPrecise(fat_archs.len);859 try ctx.detected_targets.ensureUnusedCapacity(fat_archs.len);
857 for (fat_archs) |arch| {860 for (fat_archs) |arch| {
858 ctx.detected_targets.appendAssumeCapacity(try ctx.arena.dupe(u8, @tagName(arch.tag)));861 ctx.detected_targets.appendAssumeCapacity(try ctx.arena().dupe(u8, @tagName(arch.tag)));
859 }862 }
860 return error.InvalidTargetFatLibrary;863 return error.InvalidTargetFatLibrary;
861 };864 };
...@@ -952,7 +955,7 @@ fn parseDylib(...@@ -952,7 +955,7 @@ fn parseDylib(
952 const contents = try file.readToEndAllocOptions(gpa, file_size, file_size, @alignOf(u64), null);955 const contents = try file.readToEndAllocOptions(gpa, file_size, file_size, @alignOf(u64), null);
953 defer gpa.free(contents);956 defer gpa.free(contents);
954957
955 var dylib = Dylib{ .weak = dylib_options.weak };958 var dylib = Dylib{ .path = try gpa.dupe(u8, path), .weak = dylib_options.weak };
956 errdefer dylib.deinit(gpa);959 errdefer dylib.deinit(gpa);
957960
958 try dylib.parseFromBinary(961 try dylib.parseFromBinary(
...@@ -976,7 +979,7 @@ fn parseDylib(...@@ -976,7 +979,7 @@ fn parseDylib(
976 (detected_platform != null and !detected_platform.?.eqlTarget(this_platform)))979 (detected_platform != null and !detected_platform.?.eqlTarget(this_platform)))
977 {980 {
978 const platform = detected_platform orelse this_platform;981 const platform = detected_platform orelse this_platform;
979 try ctx.detected_targets.append(try platform.allocPrintTarget(ctx.arena, detected_cpu_arch));982 try ctx.detected_targets.append(try platform.allocPrintTarget(ctx.arena(), detected_cpu_arch));
980 return error.InvalidTarget;983 return error.InvalidTarget;
981 }984 }
982985
...@@ -1014,13 +1017,13 @@ fn parseLibStub(...@@ -1014,13 +1017,13 @@ fn parseLibStub(
1014 if (!matcher.matchesTarget(targets)) {1017 if (!matcher.matchesTarget(targets)) {
1015 try ctx.detected_targets.ensureUnusedCapacity(targets.len);1018 try ctx.detected_targets.ensureUnusedCapacity(targets.len);
1016 for (targets) |t| {1019 for (targets) |t| {
1017 ctx.detected_targets.appendAssumeCapacity(try ctx.arena.dupe(u8, t));1020 ctx.detected_targets.appendAssumeCapacity(try ctx.arena().dupe(u8, t));
1018 }1021 }
1019 return error.InvalidTarget;1022 return error.InvalidTarget;
1020 }1023 }
1021 }1024 }
10221025
1023 var dylib = Dylib{ .weak = dylib_options.weak };1026 var dylib = Dylib{ .path = try gpa.dupe(u8, path), .weak = dylib_options.weak };
1024 errdefer dylib.deinit(gpa);1027 errdefer dylib.deinit(gpa);
10251028
1026 try dylib.parseFromStub(1029 try dylib.parseFromStub(
...@@ -1067,7 +1070,7 @@ fn addDylib(self: *MachO, dylib: Dylib, dylib_options: DylibOpts) ParseError!voi...@@ -1067,7 +1070,7 @@ fn addDylib(self: *MachO, dylib: Dylib, dylib_options: DylibOpts) ParseError!voi
1067 }1070 }
1068}1071}
10691072
1070pub fn parseDependentLibs(self: *MachO, dependent_libs: anytype, ctx: *ParseErrorCtx) ParseError!void {1073pub fn parseDependentLibs(self: *MachO, dependent_libs: anytype) !void {
1071 const tracy = trace(@src());1074 const tracy = trace(@src());
1072 defer tracy.end();1075 defer tracy.end();
10731076
...@@ -1081,12 +1084,13 @@ pub fn parseDependentLibs(self: *MachO, dependent_libs: anytype, ctx: *ParseErro...@@ -1081,12 +1084,13 @@ pub fn parseDependentLibs(self: *MachO, dependent_libs: anytype, ctx: *ParseErro
1081 const arena = arena_alloc.allocator();1084 const arena = arena_alloc.allocator();
1082 defer arena_alloc.deinit();1085 defer arena_alloc.deinit();
10831086
1084 outer: while (dependent_libs.readItem()) |dep_id| {1087 while (dependent_libs.readItem()) |dep_id| {
1085 defer dep_id.id.deinit(gpa);1088 defer dep_id.id.deinit(gpa);
10861089
1087 if (self.dylibs_map.contains(dep_id.id.name)) continue;1090 if (self.dylibs_map.contains(dep_id.id.name)) continue;
10881091
1089 const weak = self.dylibs.items[dep_id.parent].weak;1092 const parent = &self.dylibs.items[dep_id.parent];
1093 const weak = parent.weak;
1090 const has_ext = blk: {1094 const has_ext = blk: {
1091 const basename = fs.path.basename(dep_id.id.name);1095 const basename = fs.path.basename(dep_id.id.name);
1092 break :blk mem.lastIndexOfScalar(u8, basename, '.') != null;1096 break :blk mem.lastIndexOfScalar(u8, basename, '.') != null;
...@@ -1097,46 +1101,50 @@ pub fn parseDependentLibs(self: *MachO, dependent_libs: anytype, ctx: *ParseErro...@@ -1097,46 +1101,50 @@ pub fn parseDependentLibs(self: *MachO, dependent_libs: anytype, ctx: *ParseErro
1097 break :blk dep_id.id.name[0..index];1101 break :blk dep_id.id.name[0..index];
1098 } else dep_id.id.name;1102 } else dep_id.id.name;
10991103
1100 for (&[_][]const u8{ extension, ".tbd" }) |ext| {1104 const maybe_full_path = full_path: {
1101 const with_ext = try std.fmt.allocPrint(arena, "{s}{s}", .{ without_ext, ext });1105 if (self.base.options.sysroot) |root| {
1102 const full_path = if (self.base.options.sysroot) |root|1106 for (&[_][]const u8{ extension, ".tbd" }) |ext| {
1103 try fs.path.join(arena, &.{ root, with_ext })1107 if (try resolveLib(arena, root, without_ext, ext)) |full_path| break :full_path full_path;
1104 else1108 }
1105 with_ext;1109 }
11061110
1107 const file = std.fs.cwd().openFile(full_path, .{}) catch |err| switch (err) {1111 for (&[_][]const u8{ extension, ".tbd" }) |ext| {
1108 error.FileNotFound => continue,1112 if (try resolveLib(arena, "", without_ext, ext)) |full_path| break :full_path full_path;
1109 else => |e| return e,1113 }
1110 };1114
1111 defer file.close();1115 break :full_path null;
1116 };
1117
1118 const full_path = maybe_full_path orelse {
1119 try self.misc_errors.ensureUnusedCapacity(gpa, 1);
1120 var notes = try gpa.alloc(File.ErrorMsg, 1);
1121 errdefer gpa.free(notes);
1122 const parent_name = if (parent.id) |id| id.name else parent.path;
1123 notes[0] = .{ .msg = try std.fmt.allocPrint(gpa, "a dependency of {s}", .{parent_name}) };
1124 self.misc_errors.appendAssumeCapacity(.{
1125 .msg = try std.fmt.allocPrint(gpa, "missing dynamic library dependency: '{s}'", .{dep_id.id.name}),
1126 .notes = notes,
1127 });
1128 continue;
1129 };
11121130
1113 log.debug("trying dependency at fully resolved path {s}", .{full_path});1131 const file = try std.fs.cwd().openFile(full_path, .{});
1132 defer file.close();
11141133
1115 const offset: u64 = if (fat.isFatLibrary(file)) blk: {1134 log.debug("parsing dependency {s} at fully resolved path {s}", .{ dep_id.id.name, full_path });
1116 const offset = try self.parseFatLibrary(file, self.base.options.target.cpu.arch, ctx);
1117 try file.seekTo(offset);
1118 break :blk offset;
1119 } else 0;
11201135
1121 if (Dylib.isDylib(file, offset)) {1136 var parse_ctx = ParseErrorCtx.init(gpa);
1122 try self.parseDylib(file, full_path, offset, dependent_libs, .{1137 defer parse_ctx.deinit();
1123 .dependent = true,1138
1124 .weak = weak,1139 self.parseLibrary(file, full_path, .{
1125 }, ctx);1140 .path = null,
1126 } else {1141 .needed = false,
1127 self.parseLibStub(file, full_path, dependent_libs, .{1142 .weak = weak,
1128 .dependent = true,1143 }, false, true, dependent_libs, &parse_ctx) catch |err|
1129 .weak = weak,1144 try self.handleAndReportParseError(full_path, err, &parse_ctx);
1130 }, ctx) catch |err| switch (err) {
1131 error.NotLibStub, error.UnexpectedToken => continue,
1132 else => |e| return e,
1133 };
1134 }
1135 continue :outer;
1136 }
11371145
1138 // TODO convert into an error1146 // TODO I think that it would be nice to rewrite this error to include metadata for failed dependency
1139 log.err("{s}: unable to resolve dependency", .{dep_id.id.name});1147 // in addition to parsing error
1140 }1148 }
1141}1149}
11421150
...@@ -4854,11 +4862,23 @@ pub fn getSectionPrecedence(header: macho.section_64) u8 {...@@ -4854,11 +4862,23 @@ pub fn getSectionPrecedence(header: macho.section_64) u8 {
4854}4862}
48554863
4856pub const ParseErrorCtx = struct {4864pub const ParseErrorCtx = struct {
4857 arena: Allocator,4865 arena_allocator: std.heap.ArenaAllocator,
4858 detected_targets: std.ArrayList([]const u8),4866 detected_targets: std.ArrayList([]const u8),
48594867
4860 pub fn init(arena: Allocator) ParseErrorCtx {4868 pub fn init(gpa: Allocator) ParseErrorCtx {
4861 return .{ .arena = arena, .detected_targets = std.ArrayList([]const u8).init(arena) };4869 return .{
4870 .arena_allocator = std.heap.ArenaAllocator.init(gpa),
4871 .detected_targets = std.ArrayList([]const u8).init(gpa),
4872 };
4873 }
4874
4875 pub fn deinit(ctx: *ParseErrorCtx) void {
4876 ctx.arena_allocator.deinit();
4877 ctx.detected_targets.deinit();
4878 }
4879
4880 pub fn arena(ctx: *ParseErrorCtx) Allocator {
4881 return ctx.arena_allocator.allocator();
4862 }4882 }
4863};4883};
48644884
...@@ -4890,7 +4910,7 @@ pub fn handleAndReportParseError(...@@ -4890,7 +4910,7 @@ pub fn handleAndReportParseError(
4890 ),4910 ),
4891 error.InvalidTargetFatLibrary => try self.reportParseError(4911 error.InvalidTargetFatLibrary => try self.reportParseError(
4892 path,4912 path,
4893 "invalid architecture in univeral library: expected '{s}', but found '{s}'",4913 "invalid architecture in universal library: expected '{s}', but found '{s}'",
4894 .{ @tagName(cpu_arch), targets_string.items },4914 .{ @tagName(cpu_arch), targets_string.items },
4895 ),4915 ),
4896 else => unreachable,4916 else => unreachable,
src/link/MachO/Dylib.zig+2
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1path: []const u8,
1id: ?Id = null,2id: ?Id = null,
2weak: bool = false,3weak: bool = false,
3/// Header is only set if Dylib is parsed directly from a binary and not a stub file.4/// Header is only set if Dylib is parsed directly from a binary and not a stub file.
...@@ -106,6 +107,7 @@ pub fn isDylib(file: std.fs.File, fat_offset: u64) bool {...@@ -106,6 +107,7 @@ pub fn isDylib(file: std.fs.File, fat_offset: u64) bool {
106}107}
107108
108pub fn deinit(self: *Dylib, allocator: Allocator) void {109pub fn deinit(self: *Dylib, allocator: Allocator) void {
110 allocator.free(self.path);
109 for (self.symbols.keys()) |key| {111 for (self.symbols.keys()) |key| {
110 allocator.free(key);112 allocator.free(key);
111 }113 }
src/link/MachO/zld.zig+10-8
...@@ -345,12 +345,13 @@ pub fn linkWithZld(...@@ -345,12 +345,13 @@ pub fn linkWithZld(
345 parent: u16,345 parent: u16,
346 }, .Dynamic).init(arena);346 }, .Dynamic).init(arena);
347347
348 var parse_ctx = MachO.ParseErrorCtx.init(arena);
349
350 for (positionals.items) |obj| {348 for (positionals.items) |obj| {
351 const in_file = try std.fs.cwd().openFile(obj.path, .{});349 const in_file = try std.fs.cwd().openFile(obj.path, .{});
352 defer in_file.close();350 defer in_file.close();
353 defer parse_ctx.detected_targets.clearRetainingCapacity();351
352 var parse_ctx = MachO.ParseErrorCtx.init(gpa);
353 defer parse_ctx.deinit();
354
354 macho_file.parsePositional(355 macho_file.parsePositional(
355 in_file,356 in_file,
356 obj.path,357 obj.path,
...@@ -363,21 +364,22 @@ pub fn linkWithZld(...@@ -363,21 +364,22 @@ pub fn linkWithZld(
363 for (libs.keys(), libs.values()) |path, lib| {364 for (libs.keys(), libs.values()) |path, lib| {
364 const in_file = try std.fs.cwd().openFile(path, .{});365 const in_file = try std.fs.cwd().openFile(path, .{});
365 defer in_file.close();366 defer in_file.close();
366 defer parse_ctx.detected_targets.clearRetainingCapacity();367
368 var parse_ctx = MachO.ParseErrorCtx.init(gpa);
369 defer parse_ctx.deinit();
370
367 macho_file.parseLibrary(371 macho_file.parseLibrary(
368 in_file,372 in_file,
369 path,373 path,
370 lib,374 lib,
371 false,375 false,
376 false,
372 &dependent_libs,377 &dependent_libs,
373 &parse_ctx,378 &parse_ctx,
374 ) catch |err| try macho_file.handleAndReportParseError(path, err, &parse_ctx);379 ) catch |err| try macho_file.handleAndReportParseError(path, err, &parse_ctx);
375 }380 }
376381
377 macho_file.parseDependentLibs(&dependent_libs, &parse_ctx) catch |err| {382 try macho_file.parseDependentLibs(&dependent_libs);
378 // TODO convert to error
379 log.err("parsing dependent libraries failed with err {s}", .{@errorName(err)});
380 };
381383
382 var actions = std.ArrayList(MachO.ResolveAction).init(gpa);384 var actions = std.ArrayList(MachO.ResolveAction).init(gpa);
383 defer actions.deinit();385 defer actions.deinit();