Commit dec7e45f7c7e61a3778767bbc7f8e1e9a33b01fa | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-08-04 15:27:13-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-08-07 00:48:32-07:00
log	dec7e45f7c7e61a3778767bbc7f8e1e9a33b01fa
tree	8039451139cd95214974c5056538aca823f526d2
parent	22925636f7afc0f334f1d44257c007a1d2ccd63f

fuzzer web UI: receive coverage information

* libfuzzer: track unique runs instead of deduplicated runs
  - easier for consumers to notice when to recheck the covered bits.
* move common definitions to `std.Build.Fuzz.abi`.

build runner sends all the information needed to fuzzer web interface
client needed in order to display inline coverage information along with
source code.

7 files changed, 892 insertions(+), 663 deletions(-)

lib
- fuzzer.zig
- fuzzer
  - index.html
  - main.js
  - wasm
    main.zig
- std
  - Build
    Fuzz.zig
    Fuzz
    WebServer.zig
    abi.zig

lib/fuzzer.zig+5-10

...	...	@@ -3,6 +3,7 @@ const std = @import("std");
3	3	const Allocator = std.mem.Allocator;
4	4	const assert = std.debug.assert;
5	5	const fatal = std.process.fatal;
	6	const SeenPcsHeader = std.Build.Fuzz.abi.SeenPcsHeader;
6	7
7	8	pub const std_options = .{
8	9	.logFn = logOverride,
...	...	@@ -120,13 +121,6 @@ const Fuzzer = struct {
120	121	/// information, available to other processes.
121	122	coverage_id: u64,
122	123
123		const SeenPcsHeader = extern struct {
124		n_runs: usize,
125		deduplicated_runs: usize,
126		pcs_len: usize,
127		lowest_stack: usize,
128		};
129
130	124	const RunMap = std.ArrayHashMapUnmanaged(Run, void, Run.HashContext, false);
131	125
132	126	const Coverage = struct {
...	...	@@ -247,7 +241,7 @@ const Fuzzer = struct {
247	241	} else {
248	242	const header: SeenPcsHeader = .{
249	243	.n_runs = 0,
250		.deduplicated_runs = 0,
	244	.unique_runs = 0,
251	245	.pcs_len = flagged_pcs.len,
252	246	.lowest_stack = std.math.maxInt(usize),
253	247	};
...	...	@@ -292,8 +286,6 @@ const Fuzzer = struct {
292	286	});
293	287	if (gop.found_existing) {
294	288	//std.log.info("duplicate analysis: score={d} id={d}", .{ analysis.score, analysis.id });
295		const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
296		_ = @atomicRmw(usize, &header.deduplicated_runs, .Add, 1, .monotonic);
297	289	if (f.input.items.len < gop.key_ptr.input.len or gop.key_ptr.score == 0) {
298	290	gpa.free(gop.key_ptr.input);
299	291	gop.key_ptr.input = try gpa.dupe(u8, f.input.items);
...	...	@@ -325,6 +317,9 @@ const Fuzzer = struct {
325	317	_ = @atomicRmw(u8, elem, .Or, mask, .monotonic);
326	318	}
327	319	}
	320
	321	const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
	322	_ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
328	323	}
329	324
330	325	if (f.recent_cases.entries.len >= 100) {

lib/fuzzer/index.html+1

...	...	@@ -124,6 +124,7 @@
124	124	</style>
125	125	</head>
126	126	<body>
	127	<p id="status">Loading JavaScript...</p>
127	128	<div id="sectSource" class="hidden">
128	129	<h2>Source Code</h2>
129	130	<pre><code id="sourceText"></code></pre>

lib/fuzzer/main.js+143-90

...	...	@@ -1,95 +1,148 @@
1	1	(function() {
2		const domSectSource = document.getElementById("sectSource");
3		const domSourceText = document.getElementById("sourceText");
4
5		let wasm_promise = fetch("main.wasm");
6		let sources_promise = fetch("sources.tar").then(function(response) {
7		if (!response.ok) throw new Error("unable to download sources");
8		return response.arrayBuffer();
9		});
10		var wasm_exports = null;
11
12		const text_decoder = new TextDecoder();
13		const text_encoder = new TextEncoder();
14
15		const eventSource = new EventSource("events");
16		eventSource.addEventListener('message', onMessage, false);
17
18		WebAssembly.instantiateStreaming(wasm_promise, {
19		js: {
20		log: function(ptr, len) {
	2	const domStatus = document.getElementById("status");
	3	const domSectSource = document.getElementById("sectSource");
	4	const domSourceText = document.getElementById("sourceText");
	5
	6	let wasm_promise = fetch("main.wasm");
	7	let sources_promise = fetch("sources.tar").then(function(response) {
	8	if (!response.ok) throw new Error("unable to download sources");
	9	return response.arrayBuffer();
	10	});
	11	var wasm_exports = null;
	12
	13	const text_decoder = new TextDecoder();
	14	const text_encoder = new TextEncoder();
	15
	16	domStatus.textContent = "Loading WebAssembly...";
	17	WebAssembly.instantiateStreaming(wasm_promise, {
	18	js: {
	19	log: function(ptr, len) {
	20	const msg = decodeString(ptr, len);
	21	console.log(msg);
	22	},
	23	panic: function (ptr, len) {
21	24	const msg = decodeString(ptr, len);
22		console.log(msg);
23		},
24		panic: function (ptr, len) {
25		const msg = decodeString(ptr, len);
26		throw new Error("panic: " + msg);
27		},
	25	throw new Error("panic: " + msg);
28	26	},
29		}).then(function(obj) {
30		wasm_exports = obj.instance.exports;
31		window.wasm = obj; // for debugging
32
33		sources_promise.then(function(buffer) {
34		const js_array = new Uint8Array(buffer);
35		const ptr = wasm_exports.alloc(js_array.length);
36		const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);
37		wasm_array.set(js_array);
38		wasm_exports.unpack(ptr, js_array.length);
39
40		render();
41		});
	27	emitSourceIndexChange: onSourceIndexChange,
	28	emitCoverageUpdate: onCoverageUpdate,
	29	},
	30	}).then(function(obj) {
	31	wasm_exports = obj.instance.exports;
	32	window.wasm = obj; // for debugging
	33	domStatus.textContent = "Loading sources tarball...";
	34
	35	sources_promise.then(function(buffer) {
	36	domStatus.textContent = "Parsing sources...";
	37	const js_array = new Uint8Array(buffer);
	38	const ptr = wasm_exports.alloc(js_array.length);
	39	const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);
	40	wasm_array.set(js_array);
	41	wasm_exports.unpack(ptr, js_array.length);
	42
	43	domStatus.textContent = "Waiting for server to send source location metadata...";
	44	connectWebSocket();
42	45	});
43
44		function onMessage(e) {
45		console.log("Message", e.data);
46		}
47
48		function render() {
49		domSectSource.classList.add("hidden");
50
51		// TODO this is temporary debugging data
52		renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");
53		}
54
55		function renderSource(path) {
56		const decl_index = findFileRoot(path);
57		if (decl_index == null) throw new Error("file not found: " + path);
58
59		const h2 = domSectSource.children[0];
60		h2.innerText = path;
61		domSourceText.innerHTML = declSourceHtml(decl_index);
62
63		domSectSource.classList.remove("hidden");
64		}
65
66		function findFileRoot(path) {
67		setInputString(path);
68		const result = wasm_exports.find_file_root();
69		if (result === -1) return null;
70		return result;
71		}
72
73		function decodeString(ptr, len) {
74		if (len === 0) return "";
75		return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
76		}
77
78		function setInputString(s) {
79		const jsArray = text_encoder.encode(s);
80		const len = jsArray.length;
81		const ptr = wasm_exports.set_input_string(len);
82		const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len);
83		wasmArray.set(jsArray);
84		}
85
86		function declSourceHtml(decl_index) {
87		return unwrapString(wasm_exports.decl_source_html(decl_index));
88		}
89
90		function unwrapString(bigint) {
91		const ptr = Number(bigint & 0xffffffffn);
92		const len = Number(bigint >> 32n);
93		return decodeString(ptr, len);
94		}
	46	});
	47
	48	function connectWebSocket() {
	49	const host = window.document.location.host;
	50	const pathname = window.document.location.pathname;
	51	const isHttps = window.document.location.protocol === 'https:';
	52	const match = host.match(/^(.+):(\d+)$/);
	53	const defaultPort = isHttps ? 443 : 80;
	54	const port = match ? parseInt(match[2], 10) : defaultPort;
	55	const hostName = match ? match[1] : host;
	56	const wsProto = isHttps ? "wss:" : "ws:";
	57	const wsUrl = wsProto + '//' + hostName + ':' + port + pathname;
	58	ws = new WebSocket(wsUrl);
	59	ws.binaryType = "arraybuffer";
	60	ws.addEventListener('message', onWebSocketMessage, false);
	61	ws.addEventListener('error', timeoutThenCreateNew, false);
	62	ws.addEventListener('close', timeoutThenCreateNew, false);
	63	ws.addEventListener('open', onWebSocketOpen, false);
	64	}
	65
	66	function onWebSocketOpen() {
	67	console.log("web socket opened");
	68	}
	69
	70	function onWebSocketMessage(ev) {
	71	wasmOnMessage(ev.data);
	72	}
	73
	74	function timeoutThenCreateNew() {
	75	ws.removeEventListener('message', onWebSocketMessage, false);
	76	ws.removeEventListener('error', timeoutThenCreateNew, false);
	77	ws.removeEventListener('close', timeoutThenCreateNew, false);
	78	ws.removeEventListener('open', onWebSocketOpen, false);
	79	ws = null;
	80	setTimeout(connectWebSocket, 1000);
	81	}
	82
	83	function wasmOnMessage(data) {
	84	const jsArray = new Uint8Array(data);
	85	const ptr = wasm_exports.message_begin(jsArray.length);
	86	const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, jsArray.length);
	87	wasmArray.set(jsArray);
	88	wasm_exports.message_end();
	89	}
	90
	91	function onSourceIndexChange() {
	92	console.log("source location index metadata updated");
	93	render();
	94	}
	95
	96	function onCoverageUpdate() {
	97	console.log("coverage update");
	98	}
	99
	100	function render() {
	101	domStatus.classList.add("hidden");
	102	domSectSource.classList.add("hidden");
	103
	104	// TODO this is temporary debugging data
	105	renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");
	106	}
	107
	108	function renderSource(path) {
	109	const decl_index = findFileRoot(path);
	110	if (decl_index == null) throw new Error("file not found: " + path);
	111
	112	const h2 = domSectSource.children[0];
	113	h2.innerText = path;
	114	domSourceText.innerHTML = declSourceHtml(decl_index);
	115
	116	domSectSource.classList.remove("hidden");
	117	}
	118
	119	function findFileRoot(path) {
	120	setInputString(path);
	121	const result = wasm_exports.find_file_root();
	122	if (result === -1) return null;
	123	return result;
	124	}
	125
	126	function decodeString(ptr, len) {
	127	if (len === 0) return "";
	128	return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
	129	}
	130
	131	function setInputString(s) {
	132	const jsArray = text_encoder.encode(s);
	133	const len = jsArray.length;
	134	const ptr = wasm_exports.set_input_string(len);
	135	const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len);
	136	wasmArray.set(jsArray);
	137	}
	138
	139	function declSourceHtml(decl_index) {
	140	return unwrapString(wasm_exports.decl_source_html(decl_index));
	141	}
	142
	143	function unwrapString(bigint) {
	144	const ptr = Number(bigint & 0xffffffffn);
	145	const len = Number(bigint >> 32n);
	146	return decodeString(ptr, len);
	147	}
95	148	})();

lib/fuzzer/wasm/main.zig+80-3

...	...	@@ -1,16 +1,19 @@
1	1	const std = @import("std");
2	2	const assert = std.debug.assert;
	3	const abi = std.Build.Fuzz.abi;
	4	const gpa = std.heap.wasm_allocator;
	5	const log = std.log;
	6	const Coverage = std.debug.Coverage;
3	7
4	8	const Walk = @import("Walk");
5	9	const Decl = Walk.Decl;
6	10	const html_render = @import("html_render");
7	11
8		const gpa = std.heap.wasm_allocator;
9		const log = std.log;
10
11	12	const js = struct {
12	13	extern "js" fn log(ptr: [*]const u8, len: usize) void;
13	14	extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn;
	15	extern "js" fn emitSourceIndexChange() void;
	16	extern "js" fn emitCoverageUpdate() void;
14	17	};
15	18
16	19	pub const std_options: std.Options = .{
...	...	@@ -45,6 +48,26 @@ export fn alloc(n: usize) [*]u8 {
45	48	return slice.ptr;
46	49	}
47	50
	51	var message_buffer: std.ArrayListAlignedUnmanaged(u8, @alignOf(u64)) = .{};
	52
	53	/// Resizes the message buffer to be the correct length; returns the pointer to
	54	/// the query string.
	55	export fn message_begin(len: usize) [*]u8 {
	56	message_buffer.resize(gpa, len) catch @panic("OOM");
	57	return message_buffer.items.ptr;
	58	}
	59
	60	export fn message_end() void {
	61	const msg_bytes = message_buffer.items;
	62
	63	const tag: abi.ToClientTag = @enumFromInt(msg_bytes[0]);
	64	switch (tag) {
	65	.source_index => return sourceIndexMessage(msg_bytes) catch @panic("OOM"),
	66	.coverage_update => return coverageUpdateMessage(msg_bytes) catch @panic("OOM"),
	67	_ => unreachable,
	68	}
	69	}
	70
48	71	export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
49	72	const tar_bytes = tar_ptr[0..tar_len];
50	73	log.debug("received {d} bytes of tar file", .{tar_bytes.len});
...	...	@@ -141,3 +164,57 @@ fn fatal(comptime format: []const u8, args: anytype) noreturn {
141	164	};
142	165	js.panic(line.ptr, line.len);
143	166	}
	167
	168	fn sourceIndexMessage(msg_bytes: []u8) error{OutOfMemory}!void {
	169	const Header = abi.SourceIndexHeader;
	170	const header: Header = @bitCast(msg_bytes[0..@sizeOf(Header)].*);
	171
	172	const directories_start = @sizeOf(Header);
	173	const directories_end = directories_start + header.directories_len * @sizeOf(Coverage.String);
	174	const files_start = directories_end;
	175	const files_end = files_start + header.files_len * @sizeOf(Coverage.File);
	176	const source_locations_start = files_end;
	177	const source_locations_end = source_locations_start + header.source_locations_len * @sizeOf(Coverage.SourceLocation);
	178	const string_bytes = msg_bytes[source_locations_end..][0..header.string_bytes_len];
	179
	180	const directories: []const Coverage.String = @alignCast(std.mem.bytesAsSlice(Coverage.String, msg_bytes[directories_start..directories_end]));
	181	const files: []const Coverage.File = @alignCast(std.mem.bytesAsSlice(Coverage.File, msg_bytes[files_start..files_end]));
	182	const source_locations: []const Coverage.SourceLocation = @alignCast(std.mem.bytesAsSlice(Coverage.SourceLocation, msg_bytes[source_locations_start..source_locations_end]));
	183
	184	try updateCoverage(directories, files, source_locations, string_bytes);
	185	js.emitSourceIndexChange();
	186	}
	187
	188	fn coverageUpdateMessage(msg_bytes: []u8) error{OutOfMemory}!void {
	189	recent_coverage_update.clearRetainingCapacity();
	190	recent_coverage_update.appendSlice(gpa, msg_bytes) catch @panic("OOM");
	191	js.emitCoverageUpdate();
	192	}
	193
	194	var coverage = Coverage.init;
	195	var coverage_source_locations: std.ArrayListUnmanaged(Coverage.SourceLocation) = .{};
	196	/// Contains the most recent coverage update message, unmodified.
	197	var recent_coverage_update: std.ArrayListUnmanaged(u8) = .{};
	198
	199	fn updateCoverage(
	200	directories: []const Coverage.String,
	201	files: []const Coverage.File,
	202	source_locations: []const Coverage.SourceLocation,
	203	string_bytes: []const u8,
	204	) !void {
	205	coverage.directories.clearRetainingCapacity();
	206	coverage.files.clearRetainingCapacity();
	207	coverage.string_bytes.clearRetainingCapacity();
	208	coverage_source_locations.clearRetainingCapacity();
	209
	210	try coverage_source_locations.appendSlice(gpa, source_locations);
	211	try coverage.string_bytes.appendSlice(gpa, string_bytes);
	212
	213	try coverage.files.entries.resize(gpa, files.len);
	214	@memcpy(coverage.files.entries.items(.key), files);
	215	try coverage.files.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items });
	216
	217	try coverage.directories.entries.resize(gpa, directories.len);
	218	@memcpy(coverage.directories.entries.items(.key), directories);
	219	try coverage.directories.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items });
	220	}

lib/std/Build/Fuzz.zig+3-560

...	...	@@ -6,11 +6,13 @@ const assert = std.debug.assert;
6	6	const fatal = std.process.fatal;
7	7	const Allocator = std.mem.Allocator;
8	8	const log = std.log;
9		const Coverage = std.debug.Coverage;
10	9
11	10	const Fuzz = @This();
12	11	const build_runner = @import("root");
13	12
	13	pub const WebServer = @import("Fuzz/WebServer.zig");
	14	pub const abi = @import("Fuzz/abi.zig");
	15
14	16	pub fn start(
15	17	gpa: Allocator,
16	18	arena: Allocator,
...	...	@@ -97,565 +99,6 @@ pub fn start(
97	99	log.err("all fuzz workers crashed", .{});
98	100	}
99	101
100		pub const WebServer = struct {
101		gpa: Allocator,
102		global_cache_directory: Build.Cache.Directory,
103		zig_lib_directory: Build.Cache.Directory,
104		zig_exe_path: []const u8,
105		listen_address: std.net.Address,
106		fuzz_run_steps: []const *Step.Run,
107
108		/// Messages from fuzz workers. Protected by mutex.
109		msg_queue: std.ArrayListUnmanaged(Msg),
110		/// Protects `msg_queue` only.
111		mutex: std.Thread.Mutex,
112		/// Signaled when there is a message in `msg_queue`.
113		condition: std.Thread.Condition,
114
115		coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
116		/// Protects `coverage_files` only.
117		coverage_mutex: std.Thread.Mutex,
118		/// Signaled when `coverage_files` changes.
119		coverage_condition: std.Thread.Condition,
120
121		const CoverageMap = struct {
122		mapped_memory: []align(std.mem.page_size) const u8,
123		coverage: Coverage,
124
125		fn deinit(cm: *CoverageMap, gpa: Allocator) void {
126		std.posix.munmap(cm.mapped_memory);
127		cm.coverage.deinit(gpa);
128		cm.* = undefined;
129		}
130		};
131
132		const Msg = union(enum) {
133		coverage: struct {
134		id: u64,
135		run: *Step.Run,
136		},
137		};
138
139		fn run(ws: *WebServer) void {
140		var http_server = ws.listen_address.listen(.{
141		.reuse_address = true,
142		}) catch \|err\| {
143		log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.in.getPort(), @errorName(err) });
144		return;
145		};
146		const port = http_server.listen_address.in.getPort();
147		log.info("web interface listening at http://127.0.0.1:{d}/", .{port});
148
149		while (true) {
150		const connection = http_server.accept() catch \|err\| {
151		log.err("failed to accept connection: {s}", .{@errorName(err)});
152		return;
153		};
154		_ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch \|err\| {
155		log.err("unable to spawn connection thread: {s}", .{@errorName(err)});
156		connection.stream.close();
157		continue;
158		};
159		}
160		}
161
162		fn accept(ws: *WebServer, connection: std.net.Server.Connection) void {
163		defer connection.stream.close();
164
165		var read_buffer: [8000]u8 = undefined;
166		var server = std.http.Server.init(connection, &read_buffer);
167		while (server.state == .ready) {
168		var request = server.receiveHead() catch \|err\| switch (err) {
169		error.HttpConnectionClosing => return,
170		else => {
171		log.err("closing http connection: {s}", .{@errorName(err)});
172		return;
173		},
174		};
175		serveRequest(ws, &request) catch \|err\| switch (err) {
176		error.AlreadyReported => return,
177		else => \|e\| {
178		log.err("unable to serve {s}: {s}", .{ request.head.target, @errorName(e) });
179		return;
180		},
181		};
182		}
183		}
184
185		fn serveRequest(ws: WebServer, request: std.http.Server.Request) !void {
186		if (std.mem.eql(u8, request.head.target, "/") or
187		std.mem.eql(u8, request.head.target, "/debug") or
188		std.mem.eql(u8, request.head.target, "/debug/"))
189		{
190		try serveFile(ws, request, "fuzzer/index.html", "text/html");
191		} else if (std.mem.eql(u8, request.head.target, "/main.js") or
192		std.mem.eql(u8, request.head.target, "/debug/main.js"))
193		{
194		try serveFile(ws, request, "fuzzer/main.js", "application/javascript");
195		} else if (std.mem.eql(u8, request.head.target, "/main.wasm")) {
196		try serveWasm(ws, request, .ReleaseFast);
197		} else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) {
198		try serveWasm(ws, request, .Debug);
199		} else if (std.mem.eql(u8, request.head.target, "/sources.tar") or
200		std.mem.eql(u8, request.head.target, "/debug/sources.tar"))
201		{
202		try serveSourcesTar(ws, request);
203		} else if (std.mem.eql(u8, request.head.target, "/events") or
204		std.mem.eql(u8, request.head.target, "/debug/events"))
205		{
206		try serveEvents(ws, request);
207		} else {
208		try request.respond("not found", .{
209		.status = .not_found,
210		.extra_headers = &.{
211		.{ .name = "content-type", .value = "text/plain" },
212		},
213		});
214		}
215		}
216
217		fn serveFile(
218		ws: *WebServer,
219		request: *std.http.Server.Request,
220		name: []const u8,
221		content_type: []const u8,
222		) !void {
223		const gpa = ws.gpa;
224		// The desired API is actually sendfile, which will require enhancing std.http.Server.
225		// We load the file with every request so that the user can make changes to the file
226		// and refresh the HTML page without restarting this server.
227		const file_contents = ws.zig_lib_directory.handle.readFileAlloc(gpa, name, 10 * 1024 * 1024) catch \|err\| {
228		log.err("failed to read '{}{s}': {s}", .{ ws.zig_lib_directory, name, @errorName(err) });
229		return error.AlreadyReported;
230		};
231		defer gpa.free(file_contents);
232		try request.respond(file_contents, .{
233		.extra_headers = &.{
234		.{ .name = "content-type", .value = content_type },
235		cache_control_header,
236		},
237		});
238		}
239
240		fn serveWasm(
241		ws: *WebServer,
242		request: *std.http.Server.Request,
243		optimize_mode: std.builtin.OptimizeMode,
244		) !void {
245		const gpa = ws.gpa;
246
247		var arena_instance = std.heap.ArenaAllocator.init(gpa);
248		defer arena_instance.deinit();
249		const arena = arena_instance.allocator();
250
251		// Do the compilation every request, so that the user can edit the files
252		// and see the changes without restarting the server.
253		const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode);
254		// std.http.Server does not have a sendfile API yet.
255		const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);
256		defer gpa.free(file_contents);
257		try request.respond(file_contents, .{
258		.extra_headers = &.{
259		.{ .name = "content-type", .value = "application/wasm" },
260		cache_control_header,
261		},
262		});
263		}
264
265		fn buildWasmBinary(
266		ws: *WebServer,
267		arena: Allocator,
268		optimize_mode: std.builtin.OptimizeMode,
269		) ![]const u8 {
270		const gpa = ws.gpa;
271
272		const main_src_path: Build.Cache.Path = .{
273		.root_dir = ws.zig_lib_directory,
274		.sub_path = "fuzzer/wasm/main.zig",
275		};
276		const walk_src_path: Build.Cache.Path = .{
277		.root_dir = ws.zig_lib_directory,
278		.sub_path = "docs/wasm/Walk.zig",
279		};
280		const html_render_src_path: Build.Cache.Path = .{
281		.root_dir = ws.zig_lib_directory,
282		.sub_path = "docs/wasm/html_render.zig",
283		};
284
285		var argv: std.ArrayListUnmanaged([]const u8) = .{};
286
287		try argv.appendSlice(arena, &.{
288		ws.zig_exe_path, "build-exe", //
289		"-fno-entry", //
290		"-O", @tagName(optimize_mode), //
291		"-target", "wasm32-freestanding", //
292		"-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
293		"--cache-dir", ws.global_cache_directory.path orelse ".", //
294		"--global-cache-dir", ws.global_cache_directory.path orelse ".", //
295		"--name", "fuzzer", //
296		"-rdynamic", //
297		"--dep", "Walk", //
298		"--dep", "html_render", //
299		try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), //
300		try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), //
301		"--dep", "Walk", //
302		try std.fmt.allocPrint(arena, "-Mhtml_render={}", .{html_render_src_path}), //
303		"--listen=-",
304		});
305
306		var child = std.process.Child.init(argv.items, gpa);
307		child.stdin_behavior = .Pipe;
308		child.stdout_behavior = .Pipe;
309		child.stderr_behavior = .Pipe;
310		try child.spawn();
311
312		var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
313		.stdout = child.stdout.?,
314		.stderr = child.stderr.?,
315		});
316		defer poller.deinit();
317
318		try sendMessage(child.stdin.?, .update);
319		try sendMessage(child.stdin.?, .exit);
320
321		const Header = std.zig.Server.Message.Header;
322		var result: ?[]const u8 = null;
323		var result_error_bundle = std.zig.ErrorBundle.empty;
324
325		const stdout = poller.fifo(.stdout);
326
327		poll: while (true) {
328		while (stdout.readableLength() < @sizeOf(Header)) {
329		if (!(try poller.poll())) break :poll;
330		}
331		const header = stdout.reader().readStruct(Header) catch unreachable;
332		while (stdout.readableLength() < header.bytes_len) {
333		if (!(try poller.poll())) break :poll;
334		}
335		const body = stdout.readableSliceOfLen(header.bytes_len);
336
337		switch (header.tag) {
338		.zig_version => {
339		if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
340		return error.ZigProtocolVersionMismatch;
341		}
342		},
343		.error_bundle => {
344		const EbHdr = std.zig.Server.Message.ErrorBundle;
345		const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body));
346		const extra_bytes =
347		body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len];
348		const string_bytes =
349		body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len];
350		// TODO: use @ptrCast when the compiler supports it
351		const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes);
352		const extra_array = try arena.alloc(u32, unaligned_extra.len);
353		@memcpy(extra_array, unaligned_extra);
354		result_error_bundle = .{
355		.string_bytes = try arena.dupe(u8, string_bytes),
356		.extra = extra_array,
357		};
358		},
359		.emit_bin_path => {
360		const EbpHdr = std.zig.Server.Message.EmitBinPath;
361		const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
362		if (!ebp_hdr.flags.cache_hit) {
363		log.info("source changes detected; rebuilt wasm component", .{});
364		}
365		result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
366		},
367		else => {}, // ignore other messages
368		}
369
370		stdout.discard(body.len);
371		}
372
373		const stderr = poller.fifo(.stderr);
374		if (stderr.readableLength() > 0) {
375		const owned_stderr = try stderr.toOwnedSlice();
376		defer gpa.free(owned_stderr);
377		std.debug.print("{s}", .{owned_stderr});
378		}
379
380		// Send EOF to stdin.
381		child.stdin.?.close();
382		child.stdin = null;
383
384		switch (try child.wait()) {
385		.Exited => \|code\| {
386		if (code != 0) {
387		log.err(
388		"the following command exited with error code {d}:\n{s}",
389		.{ code, try Build.Step.allocPrintCmd(arena, null, argv.items) },
390		);
391		return error.WasmCompilationFailed;
392		}
393		},
394		.Signal, .Stopped, .Unknown => {
395		log.err(
396		"the following command terminated unexpectedly:\n{s}",
397		.{try Build.Step.allocPrintCmd(arena, null, argv.items)},
398		);
399		return error.WasmCompilationFailed;
400		},
401		}
402
403		if (result_error_bundle.errorMessageCount() > 0) {
404		const color = std.zig.Color.auto;
405		result_error_bundle.renderToStdErr(color.renderOptions());
406		log.err("the following command failed with {d} compilation errors:\n{s}", .{
407		result_error_bundle.errorMessageCount(),
408		try Build.Step.allocPrintCmd(arena, null, argv.items),
409		});
410		return error.WasmCompilationFailed;
411		}
412
413		return result orelse {
414		log.err("child process failed to report result\n{s}", .{
415		try Build.Step.allocPrintCmd(arena, null, argv.items),
416		});
417		return error.WasmCompilationFailed;
418		};
419		}
420
421		fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
422		const header: std.zig.Client.Message.Header = .{
423		.tag = tag,
424		.bytes_len = 0,
425		};
426		try file.writeAll(std.mem.asBytes(&header));
427		}
428
429		fn serveEvents(ws: WebServer, request: std.http.Server.Request) !void {
430		var send_buffer: [0x4000]u8 = undefined;
431		var response = request.respondStreaming(.{
432		.send_buffer = &send_buffer,
433		.respond_options = .{
434		.extra_headers = &.{
435		.{ .name = "content-type", .value = "text/event-stream" },
436		},
437		.transfer_encoding = .none,
438		},
439		});
440
441		ws.coverage_mutex.lock();
442		defer ws.coverage_mutex.unlock();
443
444		if (getStats(ws)) \|stats\| {
445		try response.writer().print("data: {d}\n\n", .{stats.n_runs});
446		} else {
447		try response.writeAll("data: loading debug information\n\n");
448		}
449		try response.flush();
450
451		while (true) {
452		ws.coverage_condition.timedWait(&ws.coverage_mutex, std.time.ns_per_ms * 500) catch {};
453		if (getStats(ws)) \|stats\| {
454		try response.writer().print("data: {d}\n\n", .{stats.n_runs});
455		try response.flush();
456		}
457		}
458		}
459
460		const Stats = struct {
461		n_runs: u64,
462		};
463
464		fn getStats(ws: *WebServer) ?Stats {
465		const coverage_maps = ws.coverage_files.values();
466		if (coverage_maps.len == 0) return null;
467		// TODO: make each events URL correspond to one coverage map
468		const ptr = coverage_maps[0].mapped_memory;
469		const SeenPcsHeader = extern struct {
470		n_runs: usize,
471		deduplicated_runs: usize,
472		pcs_len: usize,
473		lowest_stack: usize,
474		};
475		const header: *const SeenPcsHeader = @ptrCast(ptr[0..@sizeOf(SeenPcsHeader)]);
476		return .{
477		.n_runs = @atomicLoad(usize, &header.n_runs, .monotonic),
478		};
479		}
480
481		fn serveSourcesTar(ws: WebServer, request: std.http.Server.Request) !void {
482		const gpa = ws.gpa;
483
484		var arena_instance = std.heap.ArenaAllocator.init(gpa);
485		defer arena_instance.deinit();
486		const arena = arena_instance.allocator();
487
488		var send_buffer: [0x4000]u8 = undefined;
489		var response = request.respondStreaming(.{
490		.send_buffer = &send_buffer,
491		.respond_options = .{
492		.extra_headers = &.{
493		.{ .name = "content-type", .value = "application/x-tar" },
494		cache_control_header,
495		},
496		},
497		});
498		const w = response.writer();
499
500		const DedupeTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
501		var dedupe_table: DedupeTable = .{};
502		defer dedupe_table.deinit(gpa);
503
504		for (ws.fuzz_run_steps) \|run_step\| {
505		const compile_step_inputs = run_step.producer.?.step.inputs.table;
506		for (compile_step_inputs.keys(), compile_step_inputs.values()) \|dir_path, *file_list\| {
507		try dedupe_table.ensureUnusedCapacity(gpa, file_list.items.len);
508		for (file_list.items) \|sub_path\| {
509		// Special file "." means the entire directory.
510		if (std.mem.eql(u8, sub_path, ".")) continue;
511		const joined_path = try dir_path.join(arena, sub_path);
512		_ = dedupe_table.getOrPutAssumeCapacity(joined_path);
513		}
514		}
515		}
516
517		const deduped_paths = dedupe_table.keys();
518		const SortContext = struct {
519		pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool {
520		_ = this;
521		return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) {
522		.lt => true,
523		.gt => false,
524		.eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path),
525		};
526		}
527		};
528		std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
529
530		for (deduped_paths) \|joined_path\| {
531		var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch \|err\| {
532		log.err("failed to open {}: {s}", .{ joined_path, @errorName(err) });
533		continue;
534		};
535		defer file.close();
536
537		const stat = file.stat() catch \|err\| {
538		log.err("failed to stat {}: {s}", .{ joined_path, @errorName(err) });
539		continue;
540		};
541		if (stat.kind != .file)
542		continue;
543
544		const padding = p: {
545		const remainder = stat.size % 512;
546		break :p if (remainder > 0) 512 - remainder else 0;
547		};
548
549		var file_header = std.tar.output.Header.init();
550		file_header.typeflag = .regular;
551		try file_header.setPath(joined_path.root_dir.path orelse ".", joined_path.sub_path);
552		try file_header.setSize(stat.size);
553		try file_header.updateChecksum();
554		try w.writeAll(std.mem.asBytes(&file_header));
555		try w.writeFile(file);
556		try w.writeByteNTimes(0, padding);
557		}
558
559		// intentionally omitting the pointless trailer
560		//try w.writeByteNTimes(0, 512 * 2);
561		try response.end();
562		}
563
564		const cache_control_header: std.http.Header = .{
565		.name = "cache-control",
566		.value = "max-age=0, must-revalidate",
567		};
568
569		fn coverageRun(ws: *WebServer) void {
570		ws.mutex.lock();
571		defer ws.mutex.unlock();
572
573		while (true) {
574		ws.condition.wait(&ws.mutex);
575		for (ws.msg_queue.items) \|msg\| switch (msg) {
576		.coverage => \|coverage\| prepareTables(ws, coverage.run, coverage.id) catch \|err\| switch (err) {
577		error.AlreadyReported => continue,
578		else => \|e\| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
579		},
580		};
581		ws.msg_queue.clearRetainingCapacity();
582		}
583		}
584
585		fn prepareTables(
586		ws: *WebServer,
587		run_step: *Step.Run,
588		coverage_id: u64,
589		) error{ OutOfMemory, AlreadyReported }!void {
590		const gpa = ws.gpa;
591
592		ws.coverage_mutex.lock();
593		defer ws.coverage_mutex.unlock();
594
595		const gop = try ws.coverage_files.getOrPut(gpa, coverage_id);
596		if (gop.found_existing) {
597		// We are fuzzing the same executable with multiple threads.
598		// Perhaps the same unit test; perhaps a different one. In any
599		// case, since the coverage file is the same, we only have to
600		// notice changes to that one file in order to learn coverage for
601		// this particular executable.
602		return;
603		}
604		errdefer _ = ws.coverage_files.pop();
605
606		gop.value_ptr.* = .{
607		.coverage = std.debug.Coverage.init,
608		.mapped_memory = undefined, // populated below
609		};
610		errdefer gop.value_ptr.coverage.deinit(gpa);
611
612		const rebuilt_exe_path: Build.Cache.Path = .{
613		.root_dir = Build.Cache.Directory.cwd(),
614		.sub_path = run_step.rebuilt_executable.?,
615		};
616		var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch \|err\| {
617		log.err("step '{s}': failed to load debug information for '{}': {s}", .{
618		run_step.step.name, rebuilt_exe_path, @errorName(err),
619		});
620		return error.AlreadyReported;
621		};
622		defer debug_info.deinit(gpa);
623
624		const coverage_file_path: Build.Cache.Path = .{
625		.root_dir = run_step.step.owner.cache_root,
626		.sub_path = "v/" ++ std.fmt.hex(coverage_id),
627		};
628		var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch \|err\| {
629		log.err("step '{s}': failed to load coverage file '{}': {s}", .{
630		run_step.step.name, coverage_file_path, @errorName(err),
631		});
632		return error.AlreadyReported;
633		};
634		defer coverage_file.close();
635
636		const file_size = coverage_file.getEndPos() catch \|err\| {
637		log.err("unable to check len of coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
638		return error.AlreadyReported;
639		};
640
641		const mapped_memory = std.posix.mmap(
642		null,
643		file_size,
644		std.posix.PROT.READ,
645		.{ .TYPE = .SHARED },
646		coverage_file.handle,
647		0,
648		) catch \|err\| {
649		log.err("failed to map coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
650		return error.AlreadyReported;
651		};
652
653		gop.value_ptr.mapped_memory = mapped_memory;
654
655		ws.coverage_condition.broadcast();
656		}
657		};
658
659	102	fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
660	103	const gpa = run.step.owner.allocator;
661	104	const stderr = std.io.getStdErr();

lib/std/Build/Fuzz/WebServer.zig created+605

...	...	@@ -0,0 +1,605 @@
	1	const builtin = @import("builtin");
	2
	3	const std = @import("../../std.zig");
	4	const Allocator = std.mem.Allocator;
	5	const Build = std.Build;
	6	const Step = std.Build.Step;
	7	const Coverage = std.debug.Coverage;
	8	const abi = std.Build.Fuzz.abi;
	9	const log = std.log;
	10
	11	const WebServer = @This();
	12
	13	gpa: Allocator,
	14	global_cache_directory: Build.Cache.Directory,
	15	zig_lib_directory: Build.Cache.Directory,
	16	zig_exe_path: []const u8,
	17	listen_address: std.net.Address,
	18	fuzz_run_steps: []const *Step.Run,
	19
	20	/// Messages from fuzz workers. Protected by mutex.
	21	msg_queue: std.ArrayListUnmanaged(Msg),
	22	/// Protects `msg_queue` only.
	23	mutex: std.Thread.Mutex,
	24	/// Signaled when there is a message in `msg_queue`.
	25	condition: std.Thread.Condition,
	26
	27	coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
	28	/// Protects `coverage_files` only.
	29	coverage_mutex: std.Thread.Mutex,
	30	/// Signaled when `coverage_files` changes.
	31	coverage_condition: std.Thread.Condition,
	32
	33	const CoverageMap = struct {
	34	mapped_memory: []align(std.mem.page_size) const u8,
	35	coverage: Coverage,
	36	source_locations: []Coverage.SourceLocation,
	37
	38	fn deinit(cm: *CoverageMap, gpa: Allocator) void {
	39	std.posix.munmap(cm.mapped_memory);
	40	cm.coverage.deinit(gpa);
	41	cm.* = undefined;
	42	}
	43	};
	44
	45	const Msg = union(enum) {
	46	coverage: struct {
	47	id: u64,
	48	run: *Step.Run,
	49	},
	50	};
	51
	52	pub fn run(ws: *WebServer) void {
	53	var http_server = ws.listen_address.listen(.{
	54	.reuse_address = true,
	55	}) catch \|err\| {
	56	log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.in.getPort(), @errorName(err) });
	57	return;
	58	};
	59	const port = http_server.listen_address.in.getPort();
	60	log.info("web interface listening at http://127.0.0.1:{d}/", .{port});
	61
	62	while (true) {
	63	const connection = http_server.accept() catch \|err\| {
	64	log.err("failed to accept connection: {s}", .{@errorName(err)});
	65	return;
	66	};
	67	_ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch \|err\| {
	68	log.err("unable to spawn connection thread: {s}", .{@errorName(err)});
	69	connection.stream.close();
	70	continue;
	71	};
	72	}
	73	}
	74
	75	fn accept(ws: *WebServer, connection: std.net.Server.Connection) void {
	76	defer connection.stream.close();
	77
	78	var read_buffer: [0x4000]u8 = undefined;
	79	var server = std.http.Server.init(connection, &read_buffer);
	80	var web_socket: std.http.WebSocket = undefined;
	81	var send_buffer: [0x4000]u8 = undefined;
	82	var ws_recv_buffer: [0x4000]u8 align(4) = undefined;
	83	while (server.state == .ready) {
	84	var request = server.receiveHead() catch \|err\| switch (err) {
	85	error.HttpConnectionClosing => return,
	86	else => {
	87	log.err("closing http connection: {s}", .{@errorName(err)});
	88	return;
	89	},
	90	};
	91	if (web_socket.init(&request, &send_buffer, &ws_recv_buffer) catch \|err\| {
	92	log.err("initializing web socket: {s}", .{@errorName(err)});
	93	return;
	94	}) {
	95	serveWebSocket(ws, &web_socket) catch \|err\| {
	96	log.err("unable to serve web socket connection: {s}", .{@errorName(err)});
	97	return;
	98	};
	99	} else {
	100	serveRequest(ws, &request) catch \|err\| switch (err) {
	101	error.AlreadyReported => return,
	102	else => \|e\| {
	103	log.err("unable to serve {s}: {s}", .{ request.head.target, @errorName(e) });
	104	return;
	105	},
	106	};
	107	}
	108	}
	109	}
	110
	111	fn serveRequest(ws: WebServer, request: std.http.Server.Request) !void {
	112	if (std.mem.eql(u8, request.head.target, "/") or
	113	std.mem.eql(u8, request.head.target, "/debug") or
	114	std.mem.eql(u8, request.head.target, "/debug/"))
	115	{
	116	try serveFile(ws, request, "fuzzer/index.html", "text/html");
	117	} else if (std.mem.eql(u8, request.head.target, "/main.js") or
	118	std.mem.eql(u8, request.head.target, "/debug/main.js"))
	119	{
	120	try serveFile(ws, request, "fuzzer/main.js", "application/javascript");
	121	} else if (std.mem.eql(u8, request.head.target, "/main.wasm")) {
	122	try serveWasm(ws, request, .ReleaseFast);
	123	} else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) {
	124	try serveWasm(ws, request, .Debug);
	125	} else if (std.mem.eql(u8, request.head.target, "/sources.tar") or
	126	std.mem.eql(u8, request.head.target, "/debug/sources.tar"))
	127	{
	128	try serveSourcesTar(ws, request);
	129	} else {
	130	try request.respond("not found", .{
	131	.status = .not_found,
	132	.extra_headers = &.{
	133	.{ .name = "content-type", .value = "text/plain" },
	134	},
	135	});
	136	}
	137	}
	138
	139	fn serveFile(
	140	ws: *WebServer,
	141	request: *std.http.Server.Request,
	142	name: []const u8,
	143	content_type: []const u8,
	144	) !void {
	145	const gpa = ws.gpa;
	146	// The desired API is actually sendfile, which will require enhancing std.http.Server.
	147	// We load the file with every request so that the user can make changes to the file
	148	// and refresh the HTML page without restarting this server.
	149	const file_contents = ws.zig_lib_directory.handle.readFileAlloc(gpa, name, 10 * 1024 * 1024) catch \|err\| {
	150	log.err("failed to read '{}{s}': {s}", .{ ws.zig_lib_directory, name, @errorName(err) });
	151	return error.AlreadyReported;
	152	};
	153	defer gpa.free(file_contents);
	154	try request.respond(file_contents, .{
	155	.extra_headers = &.{
	156	.{ .name = "content-type", .value = content_type },
	157	cache_control_header,
	158	},
	159	});
	160	}
	161
	162	fn serveWasm(
	163	ws: *WebServer,
	164	request: *std.http.Server.Request,
	165	optimize_mode: std.builtin.OptimizeMode,
	166	) !void {
	167	const gpa = ws.gpa;
	168
	169	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	170	defer arena_instance.deinit();
	171	const arena = arena_instance.allocator();
	172
	173	// Do the compilation every request, so that the user can edit the files
	174	// and see the changes without restarting the server.
	175	const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode);
	176	// std.http.Server does not have a sendfile API yet.
	177	const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);
	178	defer gpa.free(file_contents);
	179	try request.respond(file_contents, .{
	180	.extra_headers = &.{
	181	.{ .name = "content-type", .value = "application/wasm" },
	182	cache_control_header,
	183	},
	184	});
	185	}
	186
	187	fn buildWasmBinary(
	188	ws: *WebServer,
	189	arena: Allocator,
	190	optimize_mode: std.builtin.OptimizeMode,
	191	) ![]const u8 {
	192	const gpa = ws.gpa;
	193
	194	const main_src_path: Build.Cache.Path = .{
	195	.root_dir = ws.zig_lib_directory,
	196	.sub_path = "fuzzer/wasm/main.zig",
	197	};
	198	const walk_src_path: Build.Cache.Path = .{
	199	.root_dir = ws.zig_lib_directory,
	200	.sub_path = "docs/wasm/Walk.zig",
	201	};
	202	const html_render_src_path: Build.Cache.Path = .{
	203	.root_dir = ws.zig_lib_directory,
	204	.sub_path = "docs/wasm/html_render.zig",
	205	};
	206
	207	var argv: std.ArrayListUnmanaged([]const u8) = .{};
	208
	209	try argv.appendSlice(arena, &.{
	210	ws.zig_exe_path, "build-exe", //
	211	"-fno-entry", //
	212	"-O", @tagName(optimize_mode), //
	213	"-target", "wasm32-freestanding", //
	214	"-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
	215	"--cache-dir", ws.global_cache_directory.path orelse ".", //
	216	"--global-cache-dir", ws.global_cache_directory.path orelse ".", //
	217	"--name", "fuzzer", //
	218	"-rdynamic", //
	219	"-fsingle-threaded", //
	220	"--dep", "Walk", //
	221	"--dep", "html_render", //
	222	try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), //
	223	try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), //
	224	"--dep", "Walk", //
	225	try std.fmt.allocPrint(arena, "-Mhtml_render={}", .{html_render_src_path}), //
	226	"--listen=-",
	227	});
	228
	229	var child = std.process.Child.init(argv.items, gpa);
	230	child.stdin_behavior = .Pipe;
	231	child.stdout_behavior = .Pipe;
	232	child.stderr_behavior = .Pipe;
	233	try child.spawn();
	234
	235	var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
	236	.stdout = child.stdout.?,
	237	.stderr = child.stderr.?,
	238	});
	239	defer poller.deinit();
	240
	241	try sendMessage(child.stdin.?, .update);
	242	try sendMessage(child.stdin.?, .exit);
	243
	244	const Header = std.zig.Server.Message.Header;
	245	var result: ?[]const u8 = null;
	246	var result_error_bundle = std.zig.ErrorBundle.empty;
	247
	248	const stdout = poller.fifo(.stdout);
	249
	250	poll: while (true) {
	251	while (stdout.readableLength() < @sizeOf(Header)) {
	252	if (!(try poller.poll())) break :poll;
	253	}
	254	const header = stdout.reader().readStruct(Header) catch unreachable;
	255	while (stdout.readableLength() < header.bytes_len) {
	256	if (!(try poller.poll())) break :poll;
	257	}
	258	const body = stdout.readableSliceOfLen(header.bytes_len);
	259
	260	switch (header.tag) {
	261	.zig_version => {
	262	if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
	263	return error.ZigProtocolVersionMismatch;
	264	}
	265	},
	266	.error_bundle => {
	267	const EbHdr = std.zig.Server.Message.ErrorBundle;
	268	const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body));
	269	const extra_bytes =
	270	body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len];
	271	const string_bytes =
	272	body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len];
	273	// TODO: use @ptrCast when the compiler supports it
	274	const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes);
	275	const extra_array = try arena.alloc(u32, unaligned_extra.len);
	276	@memcpy(extra_array, unaligned_extra);
	277	result_error_bundle = .{
	278	.string_bytes = try arena.dupe(u8, string_bytes),
	279	.extra = extra_array,
	280	};
	281	},
	282	.emit_bin_path => {
	283	const EbpHdr = std.zig.Server.Message.EmitBinPath;
	284	const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
	285	if (!ebp_hdr.flags.cache_hit) {
	286	log.info("source changes detected; rebuilt wasm component", .{});
	287	}
	288	result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
	289	},
	290	else => {}, // ignore other messages
	291	}
	292
	293	stdout.discard(body.len);
	294	}
	295
	296	const stderr = poller.fifo(.stderr);
	297	if (stderr.readableLength() > 0) {
	298	const owned_stderr = try stderr.toOwnedSlice();
	299	defer gpa.free(owned_stderr);
	300	std.debug.print("{s}", .{owned_stderr});
	301	}
	302
	303	// Send EOF to stdin.
	304	child.stdin.?.close();
	305	child.stdin = null;
	306
	307	switch (try child.wait()) {
	308	.Exited => \|code\| {
	309	if (code != 0) {
	310	log.err(
	311	"the following command exited with error code {d}:\n{s}",
	312	.{ code, try Build.Step.allocPrintCmd(arena, null, argv.items) },
	313	);
	314	return error.WasmCompilationFailed;
	315	}
	316	},
	317	.Signal, .Stopped, .Unknown => {
	318	log.err(
	319	"the following command terminated unexpectedly:\n{s}",
	320	.{try Build.Step.allocPrintCmd(arena, null, argv.items)},
	321	);
	322	return error.WasmCompilationFailed;
	323	},
	324	}
	325
	326	if (result_error_bundle.errorMessageCount() > 0) {
	327	const color = std.zig.Color.auto;
	328	result_error_bundle.renderToStdErr(color.renderOptions());
	329	log.err("the following command failed with {d} compilation errors:\n{s}", .{
	330	result_error_bundle.errorMessageCount(),
	331	try Build.Step.allocPrintCmd(arena, null, argv.items),
	332	});
	333	return error.WasmCompilationFailed;
	334	}
	335
	336	return result orelse {
	337	log.err("child process failed to report result\n{s}", .{
	338	try Build.Step.allocPrintCmd(arena, null, argv.items),
	339	});
	340	return error.WasmCompilationFailed;
	341	};
	342	}
	343
	344	fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
	345	const header: std.zig.Client.Message.Header = .{
	346	.tag = tag,
	347	.bytes_len = 0,
	348	};
	349	try file.writeAll(std.mem.asBytes(&header));
	350	}
	351
	352	fn serveWebSocket(ws: WebServer, web_socket: std.http.WebSocket) !void {
	353	ws.coverage_mutex.lock();
	354	defer ws.coverage_mutex.unlock();
	355
	356	// On first connection, the client needs all the coverage information
	357	// so that subsequent updates can contain only the updated bits.
	358	var prev_unique_runs: usize = 0;
	359	try sendCoverageContext(ws, web_socket, &prev_unique_runs);
	360	while (true) {
	361	ws.coverage_condition.timedWait(&ws.coverage_mutex, std.time.ns_per_ms * 500) catch {};
	362	try sendCoverageContext(ws, web_socket, &prev_unique_runs);
	363	}
	364	}
	365
	366	fn sendCoverageContext(ws: WebServer, web_socket: std.http.WebSocket, prev_unique_runs: *usize) !void {
	367	const coverage_maps = ws.coverage_files.values();
	368	if (coverage_maps.len == 0) return;
	369	// TODO: make each events URL correspond to one coverage map
	370	const coverage_map = &coverage_maps[0];
	371	const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
	372	const seen_pcs = coverage_map.mapped_memory[@sizeOf(abi.SeenPcsHeader) + coverage_map.source_locations.len * @sizeOf(usize) ..];
	373	const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
	374	const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
	375	const lowest_stack = @atomicLoad(usize, &cov_header.lowest_stack, .monotonic);
	376	if (prev_unique_runs.* != unique_runs) {
	377	// There has been an update.
	378	if (prev_unique_runs.* == 0) {
	379	// We need to send initial context.
	380	const header: abi.SourceIndexHeader = .{
	381	.flags = .{},
	382	.directories_len = @intCast(coverage_map.coverage.directories.entries.len),
	383	.files_len = @intCast(coverage_map.coverage.files.entries.len),
	384	.source_locations_len = @intCast(coverage_map.source_locations.len),
	385	.string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len),
	386	};
	387	const iovecs: [5]std.posix.iovec_const = .{
	388	makeIov(std.mem.asBytes(&header)),
	389	makeIov(std.mem.sliceAsBytes(coverage_map.coverage.directories.keys())),
	390	makeIov(std.mem.sliceAsBytes(coverage_map.coverage.files.keys())),
	391	makeIov(std.mem.sliceAsBytes(coverage_map.source_locations)),
	392	makeIov(coverage_map.coverage.string_bytes.items),
	393	};
	394	try web_socket.writeMessagev(&iovecs, .binary);
	395	}
	396
	397	const header: abi.CoverageUpdateHeader = .{
	398	.n_runs = n_runs,
	399	.unique_runs = unique_runs,
	400	.lowest_stack = lowest_stack,
	401	};
	402	const iovecs: [2]std.posix.iovec_const = .{
	403	makeIov(std.mem.asBytes(&header)),
	404	makeIov(seen_pcs),
	405	};
	406	try web_socket.writeMessagev(&iovecs, .binary);
	407
	408	prev_unique_runs.* = unique_runs;
	409	}
	410	}
	411
	412	fn serveSourcesTar(ws: WebServer, request: std.http.Server.Request) !void {
	413	const gpa = ws.gpa;
	414
	415	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	416	defer arena_instance.deinit();
	417	const arena = arena_instance.allocator();
	418
	419	var send_buffer: [0x4000]u8 = undefined;
	420	var response = request.respondStreaming(.{
	421	.send_buffer = &send_buffer,
	422	.respond_options = .{
	423	.extra_headers = &.{
	424	.{ .name = "content-type", .value = "application/x-tar" },
	425	cache_control_header,
	426	},
	427	},
	428	});
	429	const w = response.writer();
	430
	431	const DedupeTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
	432	var dedupe_table: DedupeTable = .{};
	433	defer dedupe_table.deinit(gpa);
	434
	435	for (ws.fuzz_run_steps) \|run_step\| {
	436	const compile_step_inputs = run_step.producer.?.step.inputs.table;
	437	for (compile_step_inputs.keys(), compile_step_inputs.values()) \|dir_path, *file_list\| {
	438	try dedupe_table.ensureUnusedCapacity(gpa, file_list.items.len);
	439	for (file_list.items) \|sub_path\| {
	440	// Special file "." means the entire directory.
	441	if (std.mem.eql(u8, sub_path, ".")) continue;
	442	const joined_path = try dir_path.join(arena, sub_path);
	443	_ = dedupe_table.getOrPutAssumeCapacity(joined_path);
	444	}
	445	}
	446	}
	447
	448	const deduped_paths = dedupe_table.keys();
	449	const SortContext = struct {
	450	pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool {
	451	_ = this;
	452	return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) {
	453	.lt => true,
	454	.gt => false,
	455	.eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path),
	456	};
	457	}
	458	};
	459	std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
	460
	461	for (deduped_paths) \|joined_path\| {
	462	var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch \|err\| {
	463	log.err("failed to open {}: {s}", .{ joined_path, @errorName(err) });
	464	continue;
	465	};
	466	defer file.close();
	467
	468	const stat = file.stat() catch \|err\| {
	469	log.err("failed to stat {}: {s}", .{ joined_path, @errorName(err) });
	470	continue;
	471	};
	472	if (stat.kind != .file)
	473	continue;
	474
	475	const padding = p: {
	476	const remainder = stat.size % 512;
	477	break :p if (remainder > 0) 512 - remainder else 0;
	478	};
	479
	480	var file_header = std.tar.output.Header.init();
	481	file_header.typeflag = .regular;
	482	try file_header.setPath(joined_path.root_dir.path orelse ".", joined_path.sub_path);
	483	try file_header.setSize(stat.size);
	484	try file_header.updateChecksum();
	485	try w.writeAll(std.mem.asBytes(&file_header));
	486	try w.writeFile(file);
	487	try w.writeByteNTimes(0, padding);
	488	}
	489
	490	// intentionally omitting the pointless trailer
	491	//try w.writeByteNTimes(0, 512 * 2);
	492	try response.end();
	493	}
	494
	495	const cache_control_header: std.http.Header = .{
	496	.name = "cache-control",
	497	.value = "max-age=0, must-revalidate",
	498	};
	499
	500	pub fn coverageRun(ws: *WebServer) void {
	501	ws.mutex.lock();
	502	defer ws.mutex.unlock();
	503
	504	while (true) {
	505	ws.condition.wait(&ws.mutex);
	506	for (ws.msg_queue.items) \|msg\| switch (msg) {
	507	.coverage => \|coverage\| prepareTables(ws, coverage.run, coverage.id) catch \|err\| switch (err) {
	508	error.AlreadyReported => continue,
	509	else => \|e\| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
	510	},
	511	};
	512	ws.msg_queue.clearRetainingCapacity();
	513	}
	514	}
	515
	516	fn prepareTables(
	517	ws: *WebServer,
	518	run_step: *Step.Run,
	519	coverage_id: u64,
	520	) error{ OutOfMemory, AlreadyReported }!void {
	521	const gpa = ws.gpa;
	522
	523	ws.coverage_mutex.lock();
	524	defer ws.coverage_mutex.unlock();
	525
	526	const gop = try ws.coverage_files.getOrPut(gpa, coverage_id);
	527	if (gop.found_existing) {
	528	// We are fuzzing the same executable with multiple threads.
	529	// Perhaps the same unit test; perhaps a different one. In any
	530	// case, since the coverage file is the same, we only have to
	531	// notice changes to that one file in order to learn coverage for
	532	// this particular executable.
	533	return;
	534	}
	535	errdefer _ = ws.coverage_files.pop();
	536
	537	gop.value_ptr.* = .{
	538	.coverage = std.debug.Coverage.init,
	539	.mapped_memory = undefined, // populated below
	540	.source_locations = undefined, // populated below
	541	};
	542	errdefer gop.value_ptr.coverage.deinit(gpa);
	543
	544	const rebuilt_exe_path: Build.Cache.Path = .{
	545	.root_dir = Build.Cache.Directory.cwd(),
	546	.sub_path = run_step.rebuilt_executable.?,
	547	};
	548	var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch \|err\| {
	549	log.err("step '{s}': failed to load debug information for '{}': {s}", .{
	550	run_step.step.name, rebuilt_exe_path, @errorName(err),
	551	});
	552	return error.AlreadyReported;
	553	};
	554	defer debug_info.deinit(gpa);
	555
	556	const coverage_file_path: Build.Cache.Path = .{
	557	.root_dir = run_step.step.owner.cache_root,
	558	.sub_path = "v/" ++ std.fmt.hex(coverage_id),
	559	};
	560	var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch \|err\| {
	561	log.err("step '{s}': failed to load coverage file '{}': {s}", .{
	562	run_step.step.name, coverage_file_path, @errorName(err),
	563	});
	564	return error.AlreadyReported;
	565	};
	566	defer coverage_file.close();
	567
	568	const file_size = coverage_file.getEndPos() catch \|err\| {
	569	log.err("unable to check len of coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
	570	return error.AlreadyReported;
	571	};
	572
	573	const mapped_memory = std.posix.mmap(
	574	null,
	575	file_size,
	576	std.posix.PROT.READ,
	577	.{ .TYPE = .SHARED },
	578	coverage_file.handle,
	579	0,
	580	) catch \|err\| {
	581	log.err("failed to map coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
	582	return error.AlreadyReported;
	583	};
	584	gop.value_ptr.mapped_memory = mapped_memory;
	585
	586	const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
	587	const pcs_bytes = mapped_memory[@sizeOf(abi.SeenPcsHeader)..][0 .. header.pcs_len * @sizeOf(usize)];
	588	const pcs = std.mem.bytesAsSlice(usize, pcs_bytes);
	589	const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len);
	590	errdefer gpa.free(source_locations);
	591	debug_info.resolveAddresses(gpa, pcs, source_locations) catch \|err\| {
	592	log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)});
	593	return error.AlreadyReported;
	594	};
	595	gop.value_ptr.source_locations = source_locations;
	596
	597	ws.coverage_condition.broadcast();
	598	}
	599
	600	fn makeIov(s: []const u8) std.posix.iovec_const {
	601	return .{
	602	.base = s.ptr,
	603	.len = s.len,
	604	};
	605	}

lib/std/Build/Fuzz/abi.zig created+55

...	...	@@ -0,0 +1,55 @@
	1	//! This file is shared among Zig code running in wildly different contexts:
	2	//! libfuzzer, compiled alongside unit tests, the build runner, running on the
	3	//! host computer, and the fuzzing web interface webassembly code running in
	4	//! the browser. All of these components interface to some degree via an ABI.
	5
	6	/// libfuzzer uses this and its usize is the one that counts. To match the ABI,
	7	/// make the ints be the size of the target used with libfuzzer.
	8	///
	9	/// Trailing:
	10	/// * pc_addr: usize for each pcs_len
	11	/// * 1 bit per pc_addr, usize elements
	12	pub const SeenPcsHeader = extern struct {
	13	n_runs: usize,
	14	unique_runs: usize,
	15	pcs_len: usize,
	16	lowest_stack: usize,
	17	};
	18
	19	pub const ToClientTag = enum(u8) {
	20	source_index,
	21	coverage_update,
	22	_,
	23	};
	24
	25	/// Sent to the fuzzer web client on first connection to the websocket URL.
	26	///
	27	/// Trailing:
	28	/// * std.debug.Coverage.String for each directories_len
	29	/// * std.debug.Coverage.File for each files_len
	30	/// * std.debug.Coverage.SourceLocation for each source_locations_len
	31	/// * u8 for each string_bytes_len
	32	pub const SourceIndexHeader = extern struct {
	33	flags: Flags,
	34	directories_len: u32,
	35	files_len: u32,
	36	source_locations_len: u32,
	37	string_bytes_len: u32,
	38
	39	pub const Flags = packed struct(u32) {
	40	tag: ToClientTag = .source_index,
	41	_: u24 = 0,
	42	};
	43	};
	44
	45	/// Sent to the fuzzer web client whenever the set of covered source locations
	46	/// changes.
	47	///
	48	/// Trailing:
	49	/// * one bit per source_locations_len, contained in u8 elements
	50	pub const CoverageUpdateHeader = extern struct {
	51	tag: ToClientTag = .coverage_update,
	52	n_runs: u64 align(1),
	53	unique_runs: u64 align(1),
	54	lowest_stack: u64 align(1),
	55	};