authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-10-24 12:12:46+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-10-24 19:03:00+02:00
log8087ec8e8c9e3abf8cf2f3952127aa97749610a5
tree2d9302cc45c2dc8dde46fd5241902c3f72e4ab57
parent2f3add4f301fb0fd6f72fb6d00a39342a2255c29

elf: improve parsing of ld scripts and actually test linking against them

6 files changed, 78 insertions(+), 167 deletions(-)

src/link/Elf.zig+33-56
......@@ -1353,10 +1353,8 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
13531353 }
13541354
13551355 for (positionals.items) |obj| {
1356 const in_file = try std.fs.cwd().openFile(obj.path, .{});
1357 defer in_file.close();
13581356 var parse_ctx: ParseErrorCtx = .{ .detected_cpu_arch = undefined };
1359 self.parsePositional(in_file, obj.path, obj.must_link, &parse_ctx) catch |err|
1357 self.parsePositional(obj.path, obj.must_link, &parse_ctx) catch |err|
13601358 try self.handleAndReportParseError(obj.path, err, &parse_ctx);
13611359 }
13621360
......@@ -1437,9 +1435,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
14371435
14381436 for (system_libs.items) |lib| {
14391437 var parse_ctx: ParseErrorCtx = .{ .detected_cpu_arch = undefined };
1440 const in_file = try std.fs.cwd().openFile(lib.path, .{});
1441 defer in_file.close();
1442 self.parseLibrary(in_file, lib, false, &parse_ctx) catch |err|
1438 self.parseLibrary(lib, false, &parse_ctx) catch |err|
14431439 try self.handleAndReportParseError(lib.path, err, &parse_ctx);
14441440 }
14451441
......@@ -1456,10 +1452,8 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
14561452 if (csu.crtn) |v| try positionals.append(.{ .path = v });
14571453
14581454 for (positionals.items) |obj| {
1459 const in_file = try std.fs.cwd().openFile(obj.path, .{});
1460 defer in_file.close();
14611455 var parse_ctx: ParseErrorCtx = .{ .detected_cpu_arch = undefined };
1462 self.parsePositional(in_file, obj.path, obj.must_link, &parse_ctx) catch |err|
1456 self.parsePositional(obj.path, obj.must_link, &parse_ctx) catch |err|
14631457 try self.handleAndReportParseError(obj.path, err, &parse_ctx);
14641458 }
14651459
......@@ -1679,51 +1673,40 @@ const ParseError = error{
16791673 InvalidCharacter,
16801674} || LdScript.Error || std.os.AccessError || std.os.SeekError || std.fs.File.OpenError || std.fs.File.ReadError;
16811675
1682fn parsePositional(
1683 self: *Elf,
1684 in_file: std.fs.File,
1685 path: []const u8,
1686 must_link: bool,
1687 ctx: *ParseErrorCtx,
1688) ParseError!void {
1676fn parsePositional(self: *Elf, path: []const u8, must_link: bool, ctx: *ParseErrorCtx) ParseError!void {
16891677 const tracy = trace(@src());
16901678 defer tracy.end();
1691
1692 if (Object.isObject(in_file)) {
1693 try self.parseObject(in_file, path, ctx);
1679 if (try Object.isObject(path)) {
1680 try self.parseObject(path, ctx);
16941681 } else {
1695 try self.parseLibrary(in_file, .{ .path = path }, must_link, ctx);
1682 try self.parseLibrary(.{ .path = path }, must_link, ctx);
16961683 }
16971684}
16981685
1699fn parseLibrary(
1700 self: *Elf,
1701 in_file: std.fs.File,
1702 lib: SystemLib,
1703 must_link: bool,
1704 ctx: *ParseErrorCtx,
1705) ParseError!void {
1686fn parseLibrary(self: *Elf, lib: SystemLib, must_link: bool, ctx: *ParseErrorCtx) ParseError!void {
17061687 const tracy = trace(@src());
17071688 defer tracy.end();
17081689
1709 if (Archive.isArchive(in_file)) {
1710 try self.parseArchive(in_file, lib.path, must_link, ctx);
1711 } else if (SharedObject.isSharedObject(in_file)) {
1712 try self.parseSharedObject(in_file, lib, ctx);
1690 if (try Archive.isArchive(lib.path)) {
1691 try self.parseArchive(lib.path, must_link, ctx);
1692 } else if (try SharedObject.isSharedObject(lib.path)) {
1693 try self.parseSharedObject(lib, ctx);
17131694 } else {
17141695 // TODO if the script has a top-level comment identifying it as GNU ld script,
17151696 // then report parse errors. Otherwise return UnknownFileType.
1716 self.parseLdScript(in_file, lib, ctx) catch |err| switch (err) {
1697 self.parseLdScript(lib, ctx) catch |err| switch (err) {
17171698 else => return error.UnknownFileType,
17181699 };
17191700 }
17201701}
17211702
1722fn parseObject(self: *Elf, in_file: std.fs.File, path: []const u8, ctx: *ParseErrorCtx) ParseError!void {
1703fn parseObject(self: *Elf, path: []const u8, ctx: *ParseErrorCtx) ParseError!void {
17231704 const tracy = trace(@src());
17241705 defer tracy.end();
17251706
17261707 const gpa = self.base.allocator;
1708 const in_file = try std.fs.cwd().openFile(path, .{});
1709 defer in_file.close();
17271710 const data = try in_file.readToEndAlloc(gpa, std.math.maxInt(u32));
17281711 const index = @as(File.Index, @intCast(try self.files.addOne(gpa)));
17291712 self.files.set(index, .{ .object = .{
......@@ -1740,17 +1723,13 @@ fn parseObject(self: *Elf, in_file: std.fs.File, path: []const u8, ctx: *ParseEr
17401723 if (ctx.detected_cpu_arch != self.base.options.target.cpu.arch) return error.InvalidCpuArch;
17411724}
17421725
1743fn parseArchive(
1744 self: *Elf,
1745 in_file: std.fs.File,
1746 path: []const u8,
1747 must_link: bool,
1748 ctx: *ParseErrorCtx,
1749) ParseError!void {
1726fn parseArchive(self: *Elf, path: []const u8, must_link: bool, ctx: *ParseErrorCtx) ParseError!void {
17501727 const tracy = trace(@src());
17511728 defer tracy.end();
17521729
17531730 const gpa = self.base.allocator;
1731 const in_file = try std.fs.cwd().openFile(path, .{});
1732 defer in_file.close();
17541733 const data = try in_file.readToEndAlloc(gpa, std.math.maxInt(u32));
17551734 var archive = Archive{ .path = try gpa.dupe(u8, path), .data = data };
17561735 defer archive.deinit(gpa);
......@@ -1773,16 +1752,13 @@ fn parseArchive(
17731752 }
17741753}
17751754
1776fn parseSharedObject(
1777 self: *Elf,
1778 in_file: std.fs.File,
1779 lib: SystemLib,
1780 ctx: *ParseErrorCtx,
1781) ParseError!void {
1755fn parseSharedObject(self: *Elf, lib: SystemLib, ctx: *ParseErrorCtx) ParseError!void {
17821756 const tracy = trace(@src());
17831757 defer tracy.end();
17841758
17851759 const gpa = self.base.allocator;
1760 const in_file = try std.fs.cwd().openFile(lib.path, .{});
1761 defer in_file.close();
17861762 const data = try in_file.readToEndAlloc(gpa, std.math.maxInt(u32));
17871763 const index = @as(File.Index, @intCast(try self.files.addOne(gpa)));
17881764 self.files.set(index, .{ .shared_object = .{
......@@ -1801,11 +1777,13 @@ fn parseSharedObject(
18011777 if (ctx.detected_cpu_arch != self.base.options.target.cpu.arch) return error.InvalidCpuArch;
18021778}
18031779
1804fn parseLdScript(self: *Elf, in_file: std.fs.File, lib: SystemLib, ctx: *ParseErrorCtx) ParseError!void {
1780fn parseLdScript(self: *Elf, lib: SystemLib, ctx: *ParseErrorCtx) ParseError!void {
18051781 const tracy = trace(@src());
18061782 defer tracy.end();
18071783
18081784 const gpa = self.base.allocator;
1785 const in_file = try std.fs.cwd().openFile(lib.path, .{});
1786 defer in_file.close();
18091787 const data = try in_file.readToEndAlloc(gpa, std.math.maxInt(u32));
18101788 defer gpa.free(data);
18111789
......@@ -1871,11 +1849,8 @@ fn parseLdScript(self: *Elf, in_file: std.fs.File, lib: SystemLib, ctx: *ParseEr
18711849 }
18721850
18731851 const full_path = test_path.items;
1874 const scr_file = try std.fs.cwd().openFile(full_path, .{});
1875 defer scr_file.close();
1876
18771852 var scr_ctx: ParseErrorCtx = .{ .detected_cpu_arch = undefined };
1878 self.parseLibrary(scr_file, .{
1853 self.parseLibrary(.{
18791854 .needed = scr_obj.needed,
18801855 .path = full_path,
18811856 }, false, &scr_ctx) catch |err| try self.handleAndReportParseError(full_path, err, &scr_ctx);
......@@ -1893,14 +1868,16 @@ fn accessLibPath(
18931868 const sep = fs.path.sep_str;
18941869 const target = self.base.options.target;
18951870 test_path.clearRetainingCapacity();
1871 const prefix = if (link_mode != null) "lib" else "";
1872 const suffix = if (link_mode) |mode| switch (mode) {
1873 .Static => target.staticLibSuffix(),
1874 .Dynamic => target.dynamicLibSuffix(),
1875 } else "";
18961876 try test_path.writer().print("{s}" ++ sep ++ "{s}{s}{s}", .{
18971877 lib_dir_path,
1898 target.libPrefix(),
1878 prefix,
18991879 lib_name,
1900 if (link_mode) |mode| switch (mode) {
1901 .Static => target.staticLibSuffix(),
1902 .Dynamic => target.dynamicLibSuffix(),
1903 } else "",
1880 suffix,
19041881 });
19051882 if (checked_paths) |cpaths| {
19061883 try cpaths.append(try self.base.allocator.dupe(u8, test_path.items));
src/link/Elf/Archive.zig+3-2
......@@ -62,10 +62,11 @@ const ar_hdr = extern struct {
6262 }
6363};
6464
65pub fn isArchive(file: std.fs.File) bool {
65pub fn isArchive(path: []const u8) !bool {
66 const file = try std.fs.cwd().openFile(path, .{});
67 defer file.close();
6668 const reader = file.reader();
6769 const magic = reader.readBytesNoEof(Archive.SARMAG) catch return false;
68 defer file.seekTo(0) catch {};
6970 if (!mem.eql(u8, &magic, ARMAG)) return false;
7071 return true;
7172}
src/link/Elf/LdScript.zig+3-105
......@@ -83,7 +83,8 @@ fn doParse(scr: *LdScript, ctx: struct {
8383 const cmd = ctx.parser.getCommand(cmd_id);
8484 switch (cmd) {
8585 .output_format => scr.cpu_arch = try ctx.parser.outputFormat(),
86 .group => try ctx.parser.group(ctx.args),
86 // TODO we should verify that group only contains libraries
87 .input, .group => try ctx.parser.group(ctx.args),
8788 else => return error.UnexpectedToken,
8889 }
8990 } else break;
......@@ -102,6 +103,7 @@ const LineColumn = struct {
102103
103104const Command = enum {
104105 output_format,
106 input,
105107 group,
106108 as_needed,
107109
......@@ -420,110 +422,6 @@ const TokenIterator = struct {
420422 }
421423};
422424
423const testing = std.testing;
424
425fn testExpectedTokens(input: []const u8, expected: []const Token.Id) !void {
426 var given = std.ArrayList(Token.Id).init(testing.allocator);
427 defer given.deinit();
428
429 var tokenizer = Tokenizer{ .source = input };
430 while (true) {
431 const tok = tokenizer.next();
432 if (tok.id == .invalid) {
433 std.debug.print(" {s} => '{s}'\n", .{ @tagName(tok.id), tok.get(input) });
434 }
435 try given.append(tok.id);
436 if (tok.id == .eof) break;
437 }
438
439 try testing.expectEqualSlices(Token.Id, expected, given.items);
440}
441
442test "Tokenizer - just comments" {
443 try testExpectedTokens(
444 \\/* GNU ld script
445 \\ Use the shared library, but some functions are only in
446 \\ the static library, so try that secondarily. */
447 , &.{ .comment, .eof });
448}
449
450test "Tokenizer - comments with a simple command" {
451 try testExpectedTokens(
452 \\/* GNU ld script
453 \\ Use the shared library, but some functions are only in
454 \\ the static library, so try that secondarily. */
455 \\OUTPUT_FORMAT(elf64-x86-64)
456 , &.{ .comment, .new_line, .command, .lparen, .literal, .rparen, .eof });
457}
458
459test "Tokenizer - libc.so" {
460 try testExpectedTokens(
461 \\/* GNU ld script
462 \\ Use the shared library, but some functions are only in
463 \\ the static library, so try that secondarily. */
464 \\OUTPUT_FORMAT(elf64-x86-64)
465 \\GROUP ( /a/b/c.so.6 /a/d/e.a AS_NEEDED ( /f/g/h.so.2 ) )
466 , &.{
467 .comment, .new_line, // GNU comment
468 .command, .lparen, .literal, .rparen, .new_line, // output format
469 .command, .lparen, .literal, .literal, // group start
470 .command, .lparen, .literal, .rparen, // as needed
471 .rparen, // group end
472 .eof,
473 });
474}
475
476test "Parser - output format" {
477 const source =
478 \\OUTPUT_FORMAT(elf64-x86-64)
479 ;
480 var tokenizer = Tokenizer{ .source = source };
481 var tokens = std.ArrayList(Token).init(testing.allocator);
482 defer tokens.deinit();
483 while (true) {
484 const tok = tokenizer.next();
485 try testing.expect(tok.id != .invalid);
486 try tokens.append(tok);
487 if (tok.id == .eof) break;
488 }
489 var it = TokenIterator{ .tokens = tokens.items };
490 var parser = Parser{ .source = source, .it = &it };
491 const tok_id = try parser.require(.command);
492 try testing.expectEqual(parser.getCommand(tok_id), .output_format);
493 const cpu_arch = try parser.outputFormat();
494 try testing.expectEqual(cpu_arch, .x86_64);
495}
496
497test "Parser - group with as-needed" {
498 const source =
499 \\GROUP ( /a/b/c.so.6 /a/d/e.a AS_NEEDED ( /f/g/h.so.2 ) )
500 ;
501 var tokenizer = Tokenizer{ .source = source };
502 var tokens = std.ArrayList(Token).init(testing.allocator);
503 defer tokens.deinit();
504 while (true) {
505 const tok = tokenizer.next();
506 try testing.expect(tok.id != .invalid);
507 try tokens.append(tok);
508 if (tok.id == .eof) break;
509 }
510 var it = TokenIterator{ .tokens = tokens.items };
511 var parser = Parser{ .source = source, .it = &it };
512
513 var args = std.ArrayList(Elf.LinkObject).init(testing.allocator);
514 defer args.deinit();
515 const tok_id = try parser.require(.command);
516 try testing.expectEqual(parser.getCommand(tok_id), .group);
517 try parser.group(&args);
518
519 try testing.expectEqualStrings("/a/b/c.so.6", args.items[0].path);
520 try testing.expect(args.items[0].needed);
521 try testing.expectEqualStrings("/a/d/e.a", args.items[1].path);
522 try testing.expect(args.items[1].needed);
523 try testing.expectEqualStrings("/f/g/h.so.2", args.items[2].path);
524 try testing.expect(!args.items[2].needed);
525}
526
527425const LdScript = @This();
528426
529427const std = @import("std");
src/link/Elf/Object.zig+3-2
......@@ -22,10 +22,11 @@ num_dynrelocs: u32 = 0,
2222
2323output_symtab_size: Elf.SymtabSize = .{},
2424
25pub fn isObject(file: std.fs.File) bool {
25pub fn isObject(path: []const u8) !bool {
26 const file = try std.fs.cwd().openFile(path, .{});
27 defer file.close();
2628 const reader = file.reader();
2729 const header = reader.readStruct(elf.Elf64_Ehdr) catch return false;
28 defer file.seekTo(0) catch {};
2930 if (!mem.eql(u8, header.e_ident[0..4], "\x7fELF")) return false;
3031 if (header.e_ident[elf.EI_VERSION] != 1) return false;
3132 if (header.e_type != elf.ET.REL) return false;
src/link/Elf/SharedObject.zig+3-2
......@@ -22,10 +22,11 @@ alive: bool,
2222
2323output_symtab_size: Elf.SymtabSize = .{},
2424
25pub fn isSharedObject(file: std.fs.File) bool {
25pub fn isSharedObject(path: []const u8) !bool {
26 const file = try std.fs.cwd().openFile(path, .{});
27 defer file.close();
2628 const reader = file.reader();
2729 const header = reader.readStruct(elf.Elf64_Ehdr) catch return false;
28 defer file.seekTo(0) catch {};
2930 if (!mem.eql(u8, header.e_ident[0..4], "\x7fELF")) return false;
3031 if (header.e_ident[elf.EI_VERSION] != 1) return false;
3132 if (header.e_type != elf.ET.DYN) return false;
test/link/elf.zig+33
......@@ -75,6 +75,7 @@ pub fn build(b: *Build) void {
7575 elf_step.dependOn(testLargeAlignmentExe(b, .{ .target = glibc_target }));
7676 elf_step.dependOn(testLargeBss(b, .{ .target = glibc_target }));
7777 elf_step.dependOn(testLinkOrder(b, .{ .target = glibc_target }));
78 elf_step.dependOn(testLdScript(b, .{ .target = glibc_target }));
7879 // https://github.com/ziglang/zig/issues/17451
7980 // elf_step.dependOn(testNoEhFrameHdr(b, .{ .target = glibc_target }));
8081 elf_step.dependOn(testPie(b, .{ .target = glibc_target }));
......@@ -1568,6 +1569,38 @@ fn testLinkOrder(b: *Build, opts: Options) *Step {
15681569 return test_step;
15691570}
15701571
1572fn testLdScript(b: *Build, opts: Options) *Step {
1573 const test_step = addTestStep(b, "ld-script", opts);
1574
1575 const dso = addSharedLibrary(b, "bar", opts);
1576 addCSourceBytes(dso, "int foo() { return 42; }", &.{});
1577
1578 const scripts = WriteFile.create(b);
1579 _ = scripts.add("liba.so", "INPUT(libfoo.so)");
1580 _ = scripts.add("libfoo.so", "GROUP(AS_NEEDED(-lbar))");
1581
1582 const exe = addExecutable(b, "main", opts);
1583 addCSourceBytes(exe,
1584 \\int foo();
1585 \\int main() {
1586 \\ return foo() - 42;
1587 \\}
1588 , &.{});
1589 exe.linkSystemLibrary2("a", .{});
1590 exe.addLibraryPath(scripts.getDirectory());
1591 exe.addLibraryPath(dso.getEmittedBinDirectory());
1592 exe.addRPath(dso.getEmittedBinDirectory());
1593 exe.linkLibC();
1594 // https://github.com/ziglang/zig/issues/17619
1595 exe.pie = true;
1596
1597 const run = addRunArtifact(exe);
1598 run.expectExitCode(0);
1599 test_step.dependOn(&run.step);
1600
1601 return test_step;
1602}
1603
15711604fn testLinkingC(b: *Build, opts: Options) *Step {
15721605 const test_step = addTestStep(b, "linking-c", opts);
15731606