Commit a3c74aca99c7eaf719c745ec6e9ee7366cad1910 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-07-24 17:41:44-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-07-25 18:52:21-07:00
log	a3c74aca99c7eaf719c745ec6e9ee7366cad1910
tree	b0396505cb9623d1a906501ed97905f7cc8bb6c6
parent	90dfd86ebee1639e5455ace4e34157ff9b68ac0f

add --debug-rt CLI arg to the compiler + bonus edits

The flag makes compiler_rt and libfuzzer be in debug mode.

Also:
* fuzzer: override debug logs and disable debug logs for frequently
  called functions
* std.Build.Fuzz: fix bug of rerunning the old unit test binary
* report errors from rebuilding the unit tests better
* link.Elf: additionally add tsan lib and fuzzer lib to the hash

9 files changed, 116 insertions(+), 29 deletions(-)

lib
- compiler
  - build_runner.zig
- fuzzer.zig
- std
  - Build.zig
  - Build
    Fuzz.zig
    Step
    Compile.zig
    Run.zig
src
- Compilation.zig
- link
  - Elf.zig
- main.zig

lib/compiler/build_runner.zig+6-4

 @@ -208,6 +208,8 @@ pub fn main() !void {
                 try debug_log_scopes.append(next_arg);
             } else if (mem.eql(u8, arg, "--debug-pkg-config")) {
                 builder.debug_pkg_config = true;
+            } else if (mem.eql(u8, arg, "--debug-rt")) {
+                graph.debug_compiler_runtime_libs = true;
             } else if (mem.eql(u8, arg, "--debug-compile-errors")) {
                 builder.debug_compile_errors = true;
             } else if (mem.eql(u8, arg, "--system")) {
 @@ -1072,7 +1074,8 @@ fn workerMakeOneStep(
         std.debug.lockStdErr();
         defer std.debug.unlockStdErr();
-        printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {};
+        const gpa = b.allocator;
+        printErrorMessages(gpa, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {};
     }
     handle_result: {
 @@ -1126,14 +1129,12 @@ fn workerMakeOneStep(
 }
 pub fn printErrorMessages(
-    b: *std.Build,
+    gpa: Allocator,
     failing_step: *Step,
     ttyconf: std.io.tty.Config,
     stderr: File,
     prominent_compile_errors: bool,
 ) !void {
-    const gpa = b.allocator;
     // Provide context for where these error messages are coming from by
     // printing the corresponding Step subtree.
 @@ -1313,6 +1314,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
         \\  --seed [integer]             For shuffling dependency traversal order (default: random)
         \\  --debug-log [scope]          Enable debugging the compiler
         \\  --debug-pkg-config           Fail if unknown pkg-config flags encountered
+        \\  --debug-rt                   Debug compiler runtime libraries
         \\  --verbose-link               Enable compiler debug output for linking
         \\  --verbose-air                Enable compiler debug output for Zig AIR
         \\  --verbose-llvm-ir[=file]     Enable compiler debug output for LLVM IR

lib/fuzzer.zig+62-10

 @@ -1,14 +1,40 @@
 const std = @import("std");
 const Allocator = std.mem.Allocator;
+pub const std_options = .{
+    .logFn = logOverride,
+};
+var log_file: ?std.fs.File = null;
+fn logOverride(
+    comptime level: std.log.Level,
+    comptime scope: @TypeOf(.EnumLiteral),
+    comptime format: []const u8,
+    args: anytype,
+) void {
+    const f = if (log_file) |f| f else f: {
+        const f = std.fs.cwd().createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file");
+        log_file = f;
+        break :f f;
+    };
+    const prefix1 = comptime level.asText();
+    const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): ";
+    f.writer().print(prefix1 ++ prefix2 ++ format ++ "\n", args) catch @panic("failed to write to fuzzer log");
+}
 export threadlocal var __sancov_lowest_stack: usize = 0;
 export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void {
     std.log.debug("__sanitizer_cov_8bit_counters_init start={*}, stop={*}", .{ start, stop });
 }
-export fn __sanitizer_cov_pcs_init(pcs_beg: [*]const usize, pcs_end: [*]const usize) void {
+export fn __sanitizer_cov_pcs_init(pc_start: [*]const usize, pc_end: [*]const usize) void {
-    std.log.debug("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}", .{ pcs_beg, pcs_end });
+    std.log.debug("__sanitizer_cov_pcs_init pc_start={*}, pc_end={*}", .{ pc_start, pc_end });
+    fuzzer.pc_range = .{
+        .start = @intFromPtr(pc_start),
+        .end = @intFromPtr(pc_start),
+    };
 }
 export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void {
 @@ -48,34 +74,45 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void {
     const len = cases_ptr[0];
     const val_size_in_bits = cases_ptr[1];
     const cases = cases_ptr[2..][0..len];
-    std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{
+    _ = val;
-        pc, val, val_size_in_bits, cases.len,
+    _ = pc;
-    });
+    _ = val_size_in_bits;
+    _ = cases;
+    //std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{
+    //    pc, val, val_size_in_bits, cases.len,
+    //});
 }
 export fn __sanitizer_cov_trace_pc_indir(callee: usize) void {
     const pc = @returnAddress();
-    std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee });
+    _ = callee;
+    _ = pc;
+    //std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee });
 }
 fn handleCmp(pc: usize, arg1: u64, arg2: u64) void {
-    std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 });
+    _ = pc;
+    _ = arg1;
+    _ = arg2;
+    //std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 });
 }
 const Fuzzer = struct {
     gpa: Allocator,
     rng: std.Random.DefaultPrng,
     input: std.ArrayListUnmanaged(u8),
+    pc_range: PcRange,
+    count: usize,
     const Slice = extern struct {
         ptr: [*]const u8,
         len: usize,
-        fn toSlice(s: Slice) []const u8 {
+        fn toZig(s: Slice) []const u8 {
             return s.ptr[0..s.len];
         }
-        fn fromSlice(s: []const u8) Slice {
+        fn fromZig(s: []const u8) Slice {
             return .{
                 .ptr = s.ptr,
                 .len = s.len,
 @@ -83,14 +120,27 @@ const Fuzzer = struct {
         }
     };
+    const PcRange = struct {
+        start: usize,
+        end: usize,
+    };
     fn next(f: *Fuzzer) ![]const u8 {
         const gpa = f.gpa;
+        // Prepare next input.
         const rng = fuzzer.rng.random();
         const len = rng.uintLessThan(usize, 64);
         try f.input.resize(gpa, len);
         rng.bytes(f.input.items);
+        f.resetCoverage();
+        f.count += 1;
         return f.input.items;
     }
+    fn resetCoverage(f: *Fuzzer) void {
+        _ = f;
+    }
 };
 var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{};
 @@ -99,10 +149,12 @@ var fuzzer: Fuzzer = .{
     .gpa = general_purpose_allocator.allocator(),
     .rng = std.Random.DefaultPrng.init(0),
     .input = .{},
+    .pc_range = .{ .start = 0, .end = 0 },
+    .count = 0,
 };
 export fn fuzzer_next() Fuzzer.Slice {
-    return Fuzzer.Slice.fromSlice(fuzzer.next() catch |err| switch (err) {
+    return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) {
         error.OutOfMemory => @panic("out of memory"),
     });
 }

lib/std/Build.zig+1

...	@@ -113,6 +113,7 @@ pub const Graph = struct {	...	@@ -113,6 +113,7 @@ pub const Graph = struct {
113	arena: Allocator,	113	arena: Allocator,
114	system_library_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .{},	114	system_library_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .{},
115	system_package_mode: bool = false,	115	system_package_mode: bool = false,
		116	debug_compiler_runtime_libs: bool = false,
116	cache: Cache,	117	cache: Cache,
117	zig_exe: [:0]const u8,	118	zig_exe: [:0]const u8,
118	env_map: EnvMap,	119	env_map: EnvMap,

lib/std/Build/Fuzz.zig+23-13

...	@@ -55,22 +55,32 @@ pub fn start(	...	@@ -55,22 +55,32 @@ pub fn start(
55	}	55	}
56		56
57	fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {	57	fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
58	const compile_step = run.producer.?;	58	const gpa = run.step.owner.allocator;
59	const prog_node = parent_prog_node.start(compile_step.step.name, 0);	59	const stderr = std.io.getStdErr();
		60
		61	const compile = run.producer.?;
		62	const prog_node = parent_prog_node.start(compile.step.name, 0);
60	defer prog_node.end();	63	defer prog_node.end();
61	if (compile_step.rebuildInFuzzMode(prog_node)) \|rebuilt_bin_path\| {	64
		65	const result = compile.rebuildInFuzzMode(prog_node);
		66
		67	const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0;
		68	const show_error_msgs = compile.step.result_error_msgs.items.len > 0;
		69	const show_stderr = compile.step.result_stderr.len > 0;
		70
		71	if (show_error_msgs or show_compile_errors or show_stderr) {
		72	std.debug.lockStdErr();
		73	defer std.debug.unlockStdErr();
		74	build_runner.printErrorMessages(gpa, &compile.step, ttyconf, stderr, false) catch {};
		75	}
		76
		77	if (result) \|rebuilt_bin_path\| {
62	run.rebuilt_executable = rebuilt_bin_path;	78	run.rebuilt_executable = rebuilt_bin_path;
63	} else \|err\| switch (err) {	79	} else \|err\| switch (err) {
64	error.MakeFailed => {	80	error.MakeFailed => {},
65	const b = run.step.owner;
66	const stderr = std.io.getStdErr();
67	std.debug.lockStdErr();
68	defer std.debug.unlockStdErr();
69	build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {};
70	},
71	else => {	81	else => {
72	std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{	82	std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{
73	compile_step.step.name, @errorName(err),	83	compile.step.name, @errorName(err),
74	});	84	});
75	},	85	},
76	}	86	}
...	@@ -82,6 +92,7 @@ fn fuzzWorkerRun(	...	@@ -82,6 +92,7 @@ fn fuzzWorkerRun(
82	ttyconf: std.io.tty.Config,	92	ttyconf: std.io.tty.Config,
83	parent_prog_node: std.Progress.Node,	93	parent_prog_node: std.Progress.Node,
84	) void {	94	) void {
		95	const gpa = run.step.owner.allocator;
85	const test_name = run.cached_test_metadata.?.testName(unit_test_index);	96	const test_name = run.cached_test_metadata.?.testName(unit_test_index);
86		97
87	const prog_node = parent_prog_node.start(test_name, 0);	98	const prog_node = parent_prog_node.start(test_name, 0);
...	@@ -89,11 +100,10 @@ fn fuzzWorkerRun(	...	@@ -89,11 +100,10 @@ fn fuzzWorkerRun(
89		100
90	run.rerunInFuzzMode(unit_test_index, prog_node) catch \|err\| switch (err) {	101	run.rerunInFuzzMode(unit_test_index, prog_node) catch \|err\| switch (err) {
91	error.MakeFailed => {	102	error.MakeFailed => {
92	const b = run.step.owner;
93	const stderr = std.io.getStdErr();	103	const stderr = std.io.getStdErr();
94	std.debug.lockStdErr();	104	std.debug.lockStdErr();
95	defer std.debug.unlockStdErr();	105	defer std.debug.unlockStdErr();
96	build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {};	106	build_runner.printErrorMessages(gpa, &run.step, ttyconf, stderr, false) catch {};
97	},	107	},
98	else => {	108	else => {
99	std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{	109	std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{

lib/std/Build/Step/Compile.zig+10

 @@ -1483,6 +1483,8 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
     try zig_args.append("--global-cache-dir");
     try zig_args.append(b.graph.global_cache_root.path orelse ".");
+    if (b.graph.debug_compiler_runtime_libs) try zig_args.append("--debug-rt");
     try zig_args.append("--name");
     try zig_args.append(compile.name);
 @@ -1840,6 +1842,14 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
 }
 pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 {
+    const gpa = c.step.owner.allocator;
+    c.step.result_error_msgs.clearRetainingCapacity();
+    c.step.result_stderr = "";
+    c.step.result_error_bundle.deinit(gpa);
+    c.step.result_error_bundle = std.zig.ErrorBundle.empty;
     const zig_args = try getZigArgs(c, true);
     const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false);
     return maybe_output_bin_path.?;

lib/std/Build/Step/Run.zig+4-1

 @@ -865,7 +865,10 @@ pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.
             },
             .artifact => |pa| {
                 const artifact = pa.artifact;
-                const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?;
+                const file_path = if (artifact == run.producer.?)
+                    run.rebuilt_executable.?
+                else
+                    (artifact.installed_path orelse artifact.generated_bin.?.path.?);
                 try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path }));
             },
             .output_file, .output_directory => unreachable,

src/Compilation.zig+3-1

 @@ -2180,7 +2180,9 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
                 comp.bin_file = try link.File.createEmpty(arena, comp, emit, whole.lf_open_opts);
             }
         },
-        .incremental => {},
+        .incremental => {
+            log.debug("Compilation.update for {s}, CacheMode.incremental", .{comp.root_name});
+        },
     }
     // From this point we add a preliminary set of file system inputs that

src/link/Elf.zig+2

...	@@ -2286,6 +2286,8 @@ fn linkWithLLD(self: *Elf, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: s	...	@@ -2286,6 +2286,8 @@ fn linkWithLLD(self: *Elf, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: s
2286	}	2286	}
2287	try man.addOptionalFile(module_obj_path);	2287	try man.addOptionalFile(module_obj_path);
2288	try man.addOptionalFile(compiler_rt_path);	2288	try man.addOptionalFile(compiler_rt_path);
		2289	try man.addOptionalFile(if (comp.tsan_lib) \|l\| l.full_object_path else null);
		2290	try man.addOptionalFile(if (comp.fuzzer_lib) \|l\| l.full_object_path else null);
2289		2291
2290	// We can skip hashing libc and libc++ components that we are in charge of building from Zig	2292	// We can skip hashing libc and libc++ components that we are in charge of building from Zig
2291	// installation sources because they are always a product of the compiler version + target information.	2293	// installation sources because they are always a product of the compiler version + target information.

src/main.zig+5

...	@@ -655,6 +655,7 @@ const usage_build_generic =	...	@@ -655,6 +655,7 @@ const usage_build_generic =
655	\\ --debug-log [scope] Enable printing debug/info log messages for scope	655	\\ --debug-log [scope] Enable printing debug/info log messages for scope
656	\\ --debug-compile-errors Crash with helpful diagnostics at the first compile error	656	\\ --debug-compile-errors Crash with helpful diagnostics at the first compile error
657	\\ --debug-link-snapshot Enable dumping of the linker's state in JSON format	657	\\ --debug-link-snapshot Enable dumping of the linker's state in JSON format
		658	\\ --debug-rt Debug compiler runtime libraries
658	\\	659	\\
659	;	660	;
660		661
...	@@ -912,6 +913,7 @@ fn buildOutputType(	...	@@ -912,6 +913,7 @@ fn buildOutputType(
912	var minor_subsystem_version: ?u16 = null;	913	var minor_subsystem_version: ?u16 = null;
913	var mingw_unicode_entry_point: bool = false;	914	var mingw_unicode_entry_point: bool = false;
914	var enable_link_snapshots: bool = false;	915	var enable_link_snapshots: bool = false;
		916	var debug_compiler_runtime_libs = false;
915	var opt_incremental: ?bool = null;	917	var opt_incremental: ?bool = null;
916	var install_name: ?[]const u8 = null;	918	var install_name: ?[]const u8 = null;
917	var hash_style: link.File.Elf.HashStyle = .both;	919	var hash_style: link.File.Elf.HashStyle = .both;
...	@@ -1367,6 +1369,8 @@ fn buildOutputType(	...	@@ -1367,6 +1369,8 @@ fn buildOutputType(
1367	} else {	1369	} else {
1368	enable_link_snapshots = true;	1370	enable_link_snapshots = true;
1369	}	1371	}
		1372	} else if (mem.eql(u8, arg, "--debug-rt")) {
		1373	debug_compiler_runtime_libs = true;
1370	} else if (mem.eql(u8, arg, "-fincremental")) {	1374	} else if (mem.eql(u8, arg, "-fincremental")) {
1371	dev.check(.incremental);	1375	dev.check(.incremental);
1372	opt_incremental = true;	1376	opt_incremental = true;
...	@@ -3408,6 +3412,7 @@ fn buildOutputType(	...	@@ -3408,6 +3412,7 @@ fn buildOutputType(
3408	// noise when --search-prefix and --mod are combined.	3412	// noise when --search-prefix and --mod are combined.
3409	.global_cc_argv = try cc_argv.toOwnedSlice(arena),	3413	.global_cc_argv = try cc_argv.toOwnedSlice(arena),
3410	.file_system_inputs = &file_system_inputs,	3414	.file_system_inputs = &file_system_inputs,
		3415	.debug_compiler_runtime_libs = debug_compiler_runtime_libs,
3411	}) catch \|err\| switch (err) {	3416	}) catch \|err\| switch (err) {
3412	error.LibCUnavailable => {	3417	error.LibCUnavailable => {
3413	const triple_name = try target.zigTriple(arena);	3418	const triple_name = try target.zigTriple(arena);