authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-12-13 11:22:46+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-12-13 11:22:46+01:00
log2492488501ee04e132cb552fd11a3025990ea047
treef4f68b8b7a8a32e417ad5244633896544bf02ce7
parenta38af5f542199378483aba9f4598634ebf012b7b

lib/std/Build/CheckObject: introduce scoped checks; implement for MachO


1 files changed, 234 insertions(+), 122 deletions(-)

lib/std/Build/Step/CheckObject.zig+234-122
......@@ -246,10 +246,12 @@ const ComputeCompareExpected = struct {
246246};
247247
248248const Check = struct {
249 kind: Kind,
249250 actions: std.ArrayList(Action),
250251
251 fn create(allocator: Allocator) Check {
252 fn create(allocator: Allocator, kind: Kind) Check {
252253 return .{
254 .kind = kind,
253255 .actions = std.ArrayList(Action).init(allocator),
254256 };
255257 }
......@@ -289,15 +291,26 @@ const Check = struct {
289291 .expected = expected,
290292 }) catch @panic("OOM");
291293 }
294
295 const Kind = enum {
296 headers,
297 symtab,
298 indirect_symtab,
299 dynamic_symtab,
300 archive_symtab,
301 dynamic_section,
302 dyld_info,
303 compute_compare,
304 };
292305};
293306
294307/// Creates a new empty sequence of actions.
295pub fn checkStart(self: *CheckObject) void {
296 const new_check = Check.create(self.step.owner.allocator);
308fn checkStart(self: *CheckObject, kind: Check.Kind) void {
309 const new_check = Check.create(self.step.owner.allocator, kind);
297310 self.checks.append(new_check) catch @panic("OOM");
298311}
299312
300/// Adds an exact match phrase to the latest created Check with `CheckObject.checkStart()`.
313/// Adds an exact match phrase to the latest created Check.
301314pub fn checkExact(self: *CheckObject, phrase: []const u8) void {
302315 self.checkExactInner(phrase, null);
303316}
......@@ -314,7 +327,7 @@ fn checkExactInner(self: *CheckObject, phrase: []const u8, file_source: ?std.Bui
314327 last.exact(.{ .string = self.step.owner.dupe(phrase), .file_source = file_source });
315328}
316329
317/// Adds a fuzzy match phrase to the latest created Check with `CheckObject.checkStart()`.
330/// Adds a fuzzy match phrase to the latest created Check.
318331pub fn checkContains(self: *CheckObject, phrase: []const u8) void {
319332 self.checkContainsInner(phrase, null);
320333}
......@@ -331,8 +344,7 @@ fn checkContainsInner(self: *CheckObject, phrase: []const u8, file_source: ?std.
331344 last.contains(.{ .string = self.step.owner.dupe(phrase), .file_source = file_source });
332345}
333346
334/// Adds an exact match phrase with variable extractor to the latest created Check
335/// with `CheckObject.checkStart()`.
347/// Adds an exact match phrase with variable extractor to the latest created Check.
336348pub fn checkExtract(self: *CheckObject, phrase: []const u8) void {
337349 self.checkExtractInner(phrase, null);
338350}
......@@ -349,7 +361,7 @@ fn checkExtractInner(self: *CheckObject, phrase: []const u8, file_source: ?std.B
349361 last.extract(.{ .string = self.step.owner.dupe(phrase), .file_source = file_source });
350362}
351363
352/// Adds another searched phrase to the latest created Check with `CheckObject.checkStart(...)`
364/// Adds another searched phrase to the latest created Check
353365/// however ensures there is no matching phrase in the output.
354366pub fn checkNotPresent(self: *CheckObject, phrase: []const u8) void {
355367 self.checkNotPresentInner(phrase, null);
......@@ -367,6 +379,11 @@ fn checkNotPresentInner(self: *CheckObject, phrase: []const u8, file_source: ?st
367379 last.notPresent(.{ .string = self.step.owner.dupe(phrase), .file_source = file_source });
368380}
369381
382/// Creates a new check checking in the file headers (section, program headers, etc.).
383pub fn checkInHeaders(self: *CheckObject) void {
384 self.checkStart(.headers);
385}
386
370387/// Creates a new check checking specifically symbol table parsed and dumped from the object
371388/// file.
372389pub fn checkInSymtab(self: *CheckObject) void {
......@@ -377,7 +394,7 @@ pub fn checkInSymtab(self: *CheckObject) void {
377394 .coff => @panic("TODO symtab for coff"),
378395 else => @panic("TODO other file formats"),
379396 };
380 self.checkStart();
397 self.checkStart(.symtab);
381398 self.checkExact(label);
382399}
383400
......@@ -389,7 +406,19 @@ pub fn checkInDyldInfo(self: *CheckObject) void {
389406 .macho => MachODumper.dyld_info_label,
390407 else => @panic("Unsupported target platform"),
391408 };
392 self.checkStart();
409 self.checkStart(.dyld_info);
410 self.checkExact(label);
411}
412
413/// Creates a new check checking specifically indirect symbol table parsed and dumped
414/// from the object file.
415/// This check is target-dependent and applicable to MachO only.
416pub fn checkInIndirectSymtab(self: *CheckObject) void {
417 const label = switch (self.obj_format) {
418 .macho => MachODumper.indirect_symtab_label,
419 else => @panic("Unsupported target platform"),
420 };
421 self.checkStart(.indirect_symtab);
393422 self.checkExact(label);
394423}
395424
......@@ -401,7 +430,7 @@ pub fn checkInDynamicSymtab(self: *CheckObject) void {
401430 .elf => ElfDumper.dynamic_symtab_label,
402431 else => @panic("Unsupported target platform"),
403432 };
404 self.checkStart();
433 self.checkStart(.dynamic_symtab);
405434 self.checkExact(label);
406435}
407436
......@@ -413,7 +442,7 @@ pub fn checkInDynamicSection(self: *CheckObject) void {
413442 .elf => ElfDumper.dynamic_section_label,
414443 else => @panic("Unsupported target platform"),
415444 };
416 self.checkStart();
445 self.checkStart(.dynamic_section);
417446 self.checkExact(label);
418447}
419448
......@@ -424,7 +453,7 @@ pub fn checkInArchiveSymtab(self: *CheckObject) void {
424453 .elf => ElfDumper.archive_symtab_label,
425454 else => @panic("TODO other file formats"),
426455 };
427 self.checkStart();
456 self.checkStart(.archive_symtab);
428457 self.checkExact(label);
429458}
430459
......@@ -436,7 +465,7 @@ pub fn checkComputeCompare(
436465 program: []const u8,
437466 expected: ComputeCompareExpected,
438467) void {
439 var new_check = Check.create(self.step.owner.allocator);
468 var new_check = Check.create(self.step.owner.allocator, .compute_compare);
440469 new_check.computeCmp(.{ .string = self.step.owner.dupe(program) }, expected);
441470 self.checks.append(new_check) catch @panic("OOM");
442471}
......@@ -457,17 +486,35 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
457486 null,
458487 ) catch |err| return step.fail("unable to read '{s}': {s}", .{ src_path, @errorName(err) });
459488
460 const output = switch (self.obj_format) {
461 .macho => try MachODumper.parseAndDump(step, contents),
462 .elf => try ElfDumper.parseAndDump(step, contents),
463 .coff => @panic("TODO coff parser"),
464 .wasm => try WasmDumper.parseAndDump(step, contents),
465 else => unreachable,
466 };
467
468489 var vars = std.StringHashMap(u64).init(gpa);
469
470490 for (self.checks.items) |chk| {
491 if (chk.kind == .compute_compare) {
492 assert(chk.actions.items.len == 1);
493 const act = chk.actions.items[0];
494 assert(act.tag == .compute_cmp);
495 const res = act.computeCmp(b, step, vars) catch |err| switch (err) {
496 error.UnknownVariable => return step.fail("Unknown variable", .{}),
497 else => |e| return e,
498 };
499 if (!res) {
500 return step.fail(
501 \\
502 \\========= comparison failed for action: ===========
503 \\{s} {}
504 \\===================================================
505 , .{ act.phrase.resolve(b, step), act.expected.? });
506 }
507 continue;
508 }
509
510 const output = switch (self.obj_format) {
511 .macho => try MachODumper.parseAndDump(step, chk.kind, contents),
512 .elf => try ElfDumper.parseAndDump(step, chk.kind, contents),
513 .coff => return step.fail("TODO coff parser", .{}),
514 .wasm => try WasmDumper.parseAndDump(step, chk.kind, contents),
515 else => unreachable,
516 };
517
471518 var it = mem.tokenizeAny(u8, output, "\r\n");
472519 for (chk.actions.items) |act| {
473520 switch (act.tag) {
......@@ -485,6 +532,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
485532 , .{ act.phrase.resolve(b, step), output });
486533 }
487534 },
535
488536 .contains => {
489537 while (it.next()) |line| {
490538 if (act.contains(b, step, line)) break;
......@@ -499,6 +547,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
499547 , .{ act.phrase.resolve(b, step), output });
500548 }
501549 },
550
502551 .not_present => {
503552 while (it.next()) |line| {
504553 if (act.notPresent(b, step, line)) continue;
......@@ -512,6 +561,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
512561 , .{ act.phrase.resolve(b, step), output });
513562 }
514563 },
564
515565 .extract => {
516566 while (it.next()) |line| {
517567 if (try act.extract(b, step, line, &vars)) break;
......@@ -526,28 +576,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
526576 , .{ act.phrase.resolve(b, step), output });
527577 }
528578 },
529 .compute_cmp => {
530 const res = act.computeCmp(b, step, vars) catch |err| switch (err) {
531 error.UnknownVariable => {
532 return step.fail(
533 \\========= from parsed file: =====================
534 \\{s}
535 \\=================================================
536 , .{output});
537 },
538 else => |e| return e,
539 };
540 if (!res) {
541 return step.fail(
542 \\
543 \\========= comparison failed for action: ===========
544 \\{s} {}
545 \\========= from parsed file: =======================
546 \\{s}
547 \\===================================================
548 , .{ act.phrase.resolve(b, step), act.expected.?, output });
549 }
550 },
579
580 .compute_cmp => unreachable,
551581 }
552582 }
553583 }
......@@ -557,13 +587,20 @@ const MachODumper = struct {
557587 const LoadCommandIterator = macho.LoadCommandIterator;
558588 const dyld_info_label = "dyld info data";
559589 const symtab_label = "symbol table";
590 const indirect_symtab_label = "indirect symbol table";
560591
561592 const Symtab = struct {
562 symbols: []align(1) const macho.nlist_64,
563 strings: []const u8,
593 symbols: []align(1) const macho.nlist_64 = &[0]macho.nlist_64{},
594 strings: []const u8 = &[0]u8{},
595 indirect_symbols: []align(1) const u32 = &[0]u32{},
596
597 fn getString(symtab: Symtab, off: u32) []const u8 {
598 assert(off < symtab.strings.len);
599 return mem.sliceTo(@as([*:0]const u8, @ptrCast(symtab.strings.ptr + off)), 0);
600 }
564601 };
565602
566 fn parseAndDump(step: *Step, bytes: []const u8) ![]const u8 {
603 fn parseAndDump(step: *Step, kind: Check.Kind, bytes: []const u8) ![]const u8 {
567604 const gpa = step.owner.allocator;
568605 var stream = std.io.fixedBufferStream(bytes);
569606 const reader = stream.reader();
......@@ -576,7 +613,7 @@ const MachODumper = struct {
576613 var output = std.ArrayList(u8).init(gpa);
577614 const writer = output.writer();
578615
579 var symtab: ?Symtab = null;
616 var symtab: Symtab = .{};
580617 var segments = std.ArrayList(macho.segment_command_64).init(gpa);
581618 defer segments.deinit();
582619 var sections = std.ArrayList(macho.section_64).init(gpa);
......@@ -586,82 +623,109 @@ const MachODumper = struct {
586623 var text_seg: ?u8 = null;
587624 var dyld_info_lc: ?macho.dyld_info_command = null;
588625
589 try dumpHeader(hdr, writer);
626 {
627 var it: LoadCommandIterator = .{
628 .ncmds = hdr.ncmds,
629 .buffer = bytes[@sizeOf(macho.mach_header_64)..][0..hdr.sizeofcmds],
630 };
631 var i: usize = 0;
632 while (it.next()) |cmd| {
633 switch (cmd.cmd()) {
634 .SEGMENT_64 => {
635 const seg = cmd.cast(macho.segment_command_64).?;
636 try sections.ensureUnusedCapacity(seg.nsects);
637 for (cmd.getSections()) |sect| {
638 sections.appendAssumeCapacity(sect);
639 }
640 const seg_id: u8 = @intCast(segments.items.len);
641 try segments.append(seg);
642 if (mem.eql(u8, seg.segName(), "__TEXT")) {
643 text_seg = seg_id;
644 }
645 },
646 .SYMTAB => {
647 const lc = cmd.cast(macho.symtab_command).?;
648 const symbols = @as([*]align(1) const macho.nlist_64, @ptrCast(bytes.ptr + lc.symoff))[0..lc.nsyms];
649 const strings = bytes[lc.stroff..][0..lc.strsize];
650 symtab.symbols = symbols;
651 symtab.strings = strings;
652 },
653 .DYSYMTAB => {
654 const lc = cmd.cast(macho.dysymtab_command).?;
655 const indexes = @as([*]align(1) const u32, @ptrCast(bytes.ptr + lc.indirectsymoff))[0..lc.nindirectsyms];
656 symtab.indirect_symbols = indexes;
657 },
658 .LOAD_DYLIB,
659 .LOAD_WEAK_DYLIB,
660 .REEXPORT_DYLIB,
661 => {
662 try imports.append(cmd.getDylibPathName());
663 },
664 .DYLD_INFO_ONLY => {
665 dyld_info_lc = cmd.cast(macho.dyld_info_command).?;
666 },
667 else => {},
668 }
590669
591 var it: LoadCommandIterator = .{
592 .ncmds = hdr.ncmds,
593 .buffer = bytes[@sizeOf(macho.mach_header_64)..][0..hdr.sizeofcmds],
594 };
595 var i: usize = 0;
596 while (it.next()) |cmd| {
597 switch (cmd.cmd()) {
598 .SEGMENT_64 => {
599 const seg = cmd.cast(macho.segment_command_64).?;
600 try sections.ensureUnusedCapacity(seg.nsects);
601 for (cmd.getSections()) |sect| {
602 sections.appendAssumeCapacity(sect);
603 }
604 const seg_id: u8 = @intCast(segments.items.len);
605 try segments.append(seg);
606 if (mem.eql(u8, seg.segName(), "__TEXT")) {
607 text_seg = seg_id;
608 }
609 },
610 .SYMTAB => {
611 const lc = cmd.cast(macho.symtab_command).?;
612 const symbols = @as([*]align(1) const macho.nlist_64, @ptrCast(bytes.ptr + lc.symoff))[0..lc.nsyms];
613 const strings = bytes[lc.stroff..][0..lc.strsize];
614 symtab = .{ .symbols = symbols, .strings = strings };
615 },
616 .LOAD_DYLIB,
617 .LOAD_WEAK_DYLIB,
618 .REEXPORT_DYLIB,
619 => {
620 try imports.append(cmd.getDylibPathName());
621 },
622 .DYLD_INFO_ONLY => {
623 dyld_info_lc = cmd.cast(macho.dyld_info_command).?;
624 },
625 else => {},
670 i += 1;
626671 }
672 }
627673
628 try dumpLoadCommand(cmd, i, writer);
629 try writer.writeByte('\n');
674 switch (kind) {
675 .headers => {
676 try dumpHeader(hdr, writer);
630677
631 i += 1;
632 }
678 var it: LoadCommandIterator = .{
679 .ncmds = hdr.ncmds,
680 .buffer = bytes[@sizeOf(macho.mach_header_64)..][0..hdr.sizeofcmds],
681 };
682 var i: usize = 0;
683 while (it.next()) |cmd| {
684 try dumpLoadCommand(cmd, i, writer);
685 try writer.writeByte('\n');
633686
634 if (symtab) |stab| {
635 try dumpSymtab(sections.items, imports.items, stab, writer);
636 }
687 i += 1;
688 }
689 },
637690
638 if (dyld_info_lc) |lc| {
639 try writer.writeAll(dyld_info_label ++ "\n");
640 if (lc.rebase_size > 0) {
641 const data = bytes[lc.rebase_off..][0..lc.rebase_size];
642 try writer.writeAll("rebase info\n");
643 try dumpRebaseInfo(gpa, data, segments.items, writer);
644 }
645 if (lc.bind_size > 0) {
646 const data = bytes[lc.bind_off..][0..lc.bind_size];
647 try writer.writeAll("bind info\n");
648 try dumpBindInfo(gpa, data, segments.items, imports.items, writer);
649 }
650 if (lc.weak_bind_size > 0) {
651 const data = bytes[lc.weak_bind_off..][0..lc.weak_bind_size];
652 try writer.writeAll("weak bind info\n");
653 try dumpBindInfo(gpa, data, segments.items, imports.items, writer);
654 }
655 if (lc.lazy_bind_size > 0) {
656 const data = bytes[lc.lazy_bind_off..][0..lc.lazy_bind_size];
657 try writer.writeAll("lazy bind info\n");
658 try dumpBindInfo(gpa, data, segments.items, imports.items, writer);
659 }
660 if (lc.export_size > 0) {
661 const data = bytes[lc.export_off..][0..lc.export_size];
662 try writer.writeAll("exports\n");
663 try dumpExportsTrie(gpa, data, segments.items[text_seg.?], writer);
664 }
691 .symtab => if (symtab.symbols.len > 0) {
692 try dumpSymtab(sections.items, imports.items, symtab, writer);
693 } else return step.fail("no symbol table found", .{}),
694
695 .indirect_symtab => if (symtab.symbols.len > 0 and symtab.indirect_symbols.len > 0) {
696 try dumpIndirectSymtab(gpa, sections.items, symtab, writer);
697 } else return step.fail("no indirect symbol table found", .{}),
698
699 .dyld_info => if (dyld_info_lc) |lc| {
700 try writer.writeAll(dyld_info_label ++ "\n");
701 if (lc.rebase_size > 0) {
702 const data = bytes[lc.rebase_off..][0..lc.rebase_size];
703 try writer.writeAll("rebase info\n");
704 try dumpRebaseInfo(gpa, data, segments.items, writer);
705 }
706 if (lc.bind_size > 0) {
707 const data = bytes[lc.bind_off..][0..lc.bind_size];
708 try writer.writeAll("bind info\n");
709 try dumpBindInfo(gpa, data, segments.items, imports.items, writer);
710 }
711 if (lc.weak_bind_size > 0) {
712 const data = bytes[lc.weak_bind_off..][0..lc.weak_bind_size];
713 try writer.writeAll("weak bind info\n");
714 try dumpBindInfo(gpa, data, segments.items, imports.items, writer);
715 }
716 if (lc.lazy_bind_size > 0) {
717 const data = bytes[lc.lazy_bind_off..][0..lc.lazy_bind_size];
718 try writer.writeAll("lazy bind info\n");
719 try dumpBindInfo(gpa, data, segments.items, imports.items, writer);
720 }
721 if (lc.export_size > 0) {
722 const data = bytes[lc.export_off..][0..lc.export_size];
723 try writer.writeAll("exports\n");
724 try dumpExportsTrie(gpa, data, segments.items[text_seg.?], writer);
725 }
726 } else return step.fail("no dyld info found", .{}),
727
728 else => return step.fail("invalid check kind for MachO file format: {s}", .{@tagName(kind)}),
665729 }
666730
667731 return output.toOwnedSlice();
......@@ -971,7 +1035,7 @@ const MachODumper = struct {
9711035
9721036 for (symtab.symbols) |sym| {
9731037 if (sym.stab()) continue;
974 const sym_name = mem.sliceTo(@as([*:0]const u8, @ptrCast(symtab.strings.ptr + sym.n_strx)), 0);
1038 const sym_name = symtab.getString(sym.n_strx);
9751039 if (sym.sect()) {
9761040 const sect = sections[sym.n_sect - 1];
9771041 try writer.print("{x} ({s},{s})", .{
......@@ -1021,6 +1085,52 @@ const MachODumper = struct {
10211085 }
10221086 }
10231087
1088 fn dumpIndirectSymtab(
1089 gpa: Allocator,
1090 sections: []const macho.section_64,
1091 symtab: Symtab,
1092 writer: anytype,
1093 ) !void {
1094 try writer.writeAll(indirect_symtab_label ++ "\n");
1095
1096 var sects = std.ArrayList(macho.section_64).init(gpa);
1097 defer sects.deinit();
1098 try sects.ensureUnusedCapacity(3);
1099
1100 for (sections) |sect| {
1101 if (mem.eql(u8, sect.sectName(), "__stubs")) sects.appendAssumeCapacity(sect);
1102 if (mem.eql(u8, sect.sectName(), "__got")) sects.appendAssumeCapacity(sect);
1103 if (mem.eql(u8, sect.sectName(), "__la_symbol_ptr")) sects.appendAssumeCapacity(sect);
1104 }
1105
1106 const sortFn = struct {
1107 fn sortFn(ctx: void, lhs: macho.section_64, rhs: macho.section_64) bool {
1108 _ = ctx;
1109 return lhs.reserved1 < rhs.reserved1;
1110 }
1111 }.sortFn;
1112 mem.sort(macho.section_64, sects.items, {}, sortFn);
1113
1114 var i: usize = 0;
1115 while (i < sects.items.len) : (i += 1) {
1116 const sect = sects.items[i];
1117 const start = sect.reserved1;
1118 const end = if (i + 1 >= sects.items.len) symtab.indirect_symbols.len else sects.items[i + 1].reserved1;
1119 const entry_size = blk: {
1120 if (mem.eql(u8, sect.sectName(), "__stubs")) break :blk sect.reserved2;
1121 break :blk @sizeOf(u64);
1122 };
1123
1124 try writer.print("{s},{s}\n", .{ sect.segName(), sect.sectName() });
1125 try writer.print("nentries {d}\n", .{end - start});
1126 for (symtab.indirect_symbols[start..end], 0..) |index, j| {
1127 const sym = symtab.symbols[index];
1128 const addr = sect.addr + entry_size * j;
1129 try writer.print("0x{x} {d} {s}\n", .{ addr, index, symtab.getString(sym.n_strx) });
1130 }
1131 }
1132 }
1133
10241134 fn dumpRebaseInfo(
10251135 gpa: Allocator,
10261136 data: []const u8,
......@@ -1443,7 +1553,8 @@ const ElfDumper = struct {
14431553 const dynamic_section_label = "dynamic section";
14441554 const archive_symtab_label = "archive symbol table";
14451555
1446 fn parseAndDump(step: *Step, bytes: []const u8) ![]const u8 {
1556 fn parseAndDump(step: *Step, kind: Check.Kind, bytes: []const u8) ![]const u8 {
1557 _ = kind;
14471558 const gpa = step.owner.allocator;
14481559 return parseAndDumpArchive(gpa, bytes) catch |err| switch (err) {
14491560 error.InvalidArchiveMagicNumber => try parseAndDumpObject(gpa, bytes),
......@@ -2090,7 +2201,8 @@ const ElfDumper = struct {
20902201const WasmDumper = struct {
20912202 const symtab_label = "symbols";
20922203
2093 fn parseAndDump(step: *Step, bytes: []const u8) ![]const u8 {
2204 fn parseAndDump(step: *Step, kind: Check.Kind, bytes: []const u8) ![]const u8 {
2205 _ = kind;
20942206 const gpa = step.owner.allocator;
20952207 var fbs = std.io.fixedBufferStream(bytes);
20962208 const reader = fbs.reader();