Commit dec7e45f7c7e61a3778767bbc7f8e1e9a33b01fa | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-08-04 15:27:13-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-08-07 00:48:32-07:00
log	dec7e45f7c7e61a3778767bbc7f8e1e9a33b01fa
tree	8039451139cd95214974c5056538aca823f526d2
parent	22925636f7afc0f334f1d44257c007a1d2ccd63f

fuzzer web UI: receive coverage information

* libfuzzer: track unique runs instead of deduplicated runs
  - easier for consumers to notice when to recheck the covered bits.
* move common definitions to `std.Build.Fuzz.abi`.

build runner sends all the information needed to fuzzer web interface
client needed in order to display inline coverage information along with
source code.

7 files changed, 892 insertions(+), 663 deletions(-)

lib
- fuzzer.zig
- fuzzer
  - index.html
  - main.js
  - wasm
    main.zig
- std
  - Build
    Fuzz.zig
    Fuzz
    WebServer.zig
    abi.zig

lib/fuzzer.zig+5-10

...	@@ -3,6 +3,7 @@ const std = @import("std");	...	@@ -3,6 +3,7 @@ const std = @import("std");
3	const Allocator = std.mem.Allocator;	3	const Allocator = std.mem.Allocator;
4	const assert = std.debug.assert;	4	const assert = std.debug.assert;
5	const fatal = std.process.fatal;	5	const fatal = std.process.fatal;
		6	const SeenPcsHeader = std.Build.Fuzz.abi.SeenPcsHeader;
6		7
7	pub const std_options = .{	8	pub const std_options = .{
8	.logFn = logOverride,	9	.logFn = logOverride,
...	@@ -120,13 +121,6 @@ const Fuzzer = struct {	...	@@ -120,13 +121,6 @@ const Fuzzer = struct {
120	/// information, available to other processes.	121	/// information, available to other processes.
121	coverage_id: u64,	122	coverage_id: u64,
122		123
123	const SeenPcsHeader = extern struct {
124	n_runs: usize,
125	deduplicated_runs: usize,
126	pcs_len: usize,
127	lowest_stack: usize,
128	};
129
130	const RunMap = std.ArrayHashMapUnmanaged(Run, void, Run.HashContext, false);	124	const RunMap = std.ArrayHashMapUnmanaged(Run, void, Run.HashContext, false);
131		125
132	const Coverage = struct {	126	const Coverage = struct {
...	@@ -247,7 +241,7 @@ const Fuzzer = struct {	...	@@ -247,7 +241,7 @@ const Fuzzer = struct {
247	} else {	241	} else {
248	const header: SeenPcsHeader = .{	242	const header: SeenPcsHeader = .{
249	.n_runs = 0,	243	.n_runs = 0,
250	.deduplicated_runs = 0,	244	.unique_runs = 0,
251	.pcs_len = flagged_pcs.len,	245	.pcs_len = flagged_pcs.len,
252	.lowest_stack = std.math.maxInt(usize),	246	.lowest_stack = std.math.maxInt(usize),
253	};	247	};
...	@@ -292,8 +286,6 @@ const Fuzzer = struct {	...	@@ -292,8 +286,6 @@ const Fuzzer = struct {
292	});	286	});
293	if (gop.found_existing) {	287	if (gop.found_existing) {
294	//std.log.info("duplicate analysis: score={d} id={d}", .{ analysis.score, analysis.id });	288	//std.log.info("duplicate analysis: score={d} id={d}", .{ analysis.score, analysis.id });
295	const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
296	_ = @atomicRmw(usize, &header.deduplicated_runs, .Add, 1, .monotonic);
297	if (f.input.items.len < gop.key_ptr.input.len or gop.key_ptr.score == 0) {	289	if (f.input.items.len < gop.key_ptr.input.len or gop.key_ptr.score == 0) {
298	gpa.free(gop.key_ptr.input);	290	gpa.free(gop.key_ptr.input);
299	gop.key_ptr.input = try gpa.dupe(u8, f.input.items);	291	gop.key_ptr.input = try gpa.dupe(u8, f.input.items);
...	@@ -325,6 +317,9 @@ const Fuzzer = struct {	...	@@ -325,6 +317,9 @@ const Fuzzer = struct {
325	_ = @atomicRmw(u8, elem, .Or, mask, .monotonic);	317	_ = @atomicRmw(u8, elem, .Or, mask, .monotonic);
326	}	318	}
327	}	319	}
		320
		321	const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
		322	_ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
328	}	323	}
329		324
330	if (f.recent_cases.entries.len >= 100) {	325	if (f.recent_cases.entries.len >= 100) {

lib/fuzzer/index.html+1

...	@@ -124,6 +124,7 @@	...	@@ -124,6 +124,7 @@
124	</style>	124	</style>
125	</head>	125	</head>
126	<body>	126	<body>
		127	<p id="status">Loading JavaScript...</p>
127	<div id="sectSource" class="hidden">	128	<div id="sectSource" class="hidden">
128	<h2>Source Code</h2>	129	<h2>Source Code</h2>
129	<pre><code id="sourceText"></code></pre>	130	<pre><code id="sourceText"></code></pre>

lib/fuzzer/main.js+143-90

...	@@ -1,95 +1,148 @@	...	@@ -1,95 +1,148 @@
1	(function() {	1	(function() {
2	const domSectSource = document.getElementById("sectSource");	2	const domStatus = document.getElementById("status");
3	const domSourceText = document.getElementById("sourceText");	3	const domSectSource = document.getElementById("sectSource");
4		4	const domSourceText = document.getElementById("sourceText");
5	let wasm_promise = fetch("main.wasm");	5
6	let sources_promise = fetch("sources.tar").then(function(response) {	6	let wasm_promise = fetch("main.wasm");
7	if (!response.ok) throw new Error("unable to download sources");	7	let sources_promise = fetch("sources.tar").then(function(response) {
8	return response.arrayBuffer();	8	if (!response.ok) throw new Error("unable to download sources");
9	});	9	return response.arrayBuffer();
10	var wasm_exports = null;	10	});
11		11	var wasm_exports = null;
12	const text_decoder = new TextDecoder();	12
13	const text_encoder = new TextEncoder();	13	const text_decoder = new TextDecoder();
14		14	const text_encoder = new TextEncoder();
15	const eventSource = new EventSource("events");	15
16	eventSource.addEventListener('message', onMessage, false);	16	domStatus.textContent = "Loading WebAssembly...";
17		17	WebAssembly.instantiateStreaming(wasm_promise, {
18	WebAssembly.instantiateStreaming(wasm_promise, {	18	js: {
19	js: {	19	log: function(ptr, len) {
20	log: function(ptr, len) {	20	const msg = decodeString(ptr, len);
		21	console.log(msg);
		22	},
		23	panic: function (ptr, len) {
21	const msg = decodeString(ptr, len);	24	const msg = decodeString(ptr, len);
22	console.log(msg);	25	throw new Error("panic: " + msg);
23	},
24	panic: function (ptr, len) {
25	const msg = decodeString(ptr, len);
26	throw new Error("panic: " + msg);
27	},
28	},	26	},
29	}).then(function(obj) {	27	emitSourceIndexChange: onSourceIndexChange,
30	wasm_exports = obj.instance.exports;	28	emitCoverageUpdate: onCoverageUpdate,
31	window.wasm = obj; // for debugging	29	},
32		30	}).then(function(obj) {
33	sources_promise.then(function(buffer) {	31	wasm_exports = obj.instance.exports;
34	const js_array = new Uint8Array(buffer);	32	window.wasm = obj; // for debugging
35	const ptr = wasm_exports.alloc(js_array.length);	33	domStatus.textContent = "Loading sources tarball...";
36	const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);	34
37	wasm_array.set(js_array);	35	sources_promise.then(function(buffer) {
38	wasm_exports.unpack(ptr, js_array.length);	36	domStatus.textContent = "Parsing sources...";
39		37	const js_array = new Uint8Array(buffer);
40	render();	38	const ptr = wasm_exports.alloc(js_array.length);
41	});	39	const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);
		40	wasm_array.set(js_array);
		41	wasm_exports.unpack(ptr, js_array.length);
		42
		43	domStatus.textContent = "Waiting for server to send source location metadata...";
		44	connectWebSocket();
42	});	45	});
43		46	});
44	function onMessage(e) {	47
45	console.log("Message", e.data);	48	function connectWebSocket() {
46	}	49	const host = window.document.location.host;
47		50	const pathname = window.document.location.pathname;
48	function render() {	51	const isHttps = window.document.location.protocol === 'https:';
49	domSectSource.classList.add("hidden");	52	const match = host.match(/^(.+):(\d+)$/);
50		53	const defaultPort = isHttps ? 443 : 80;
51	// TODO this is temporary debugging data	54	const port = match ? parseInt(match[2], 10) : defaultPort;
52	renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");	55	const hostName = match ? match[1] : host;
53	}	56	const wsProto = isHttps ? "wss:" : "ws:";
54		57	const wsUrl = wsProto + '//' + hostName + ':' + port + pathname;
55	function renderSource(path) {	58	ws = new WebSocket(wsUrl);
56	const decl_index = findFileRoot(path);	59	ws.binaryType = "arraybuffer";
57	if (decl_index == null) throw new Error("file not found: " + path);	60	ws.addEventListener('message', onWebSocketMessage, false);
58		61	ws.addEventListener('error', timeoutThenCreateNew, false);
59	const h2 = domSectSource.children[0];	62	ws.addEventListener('close', timeoutThenCreateNew, false);
60	h2.innerText = path;	63	ws.addEventListener('open', onWebSocketOpen, false);
61	domSourceText.innerHTML = declSourceHtml(decl_index);	64	}
62		65
63	domSectSource.classList.remove("hidden");	66	function onWebSocketOpen() {
64	}	67	console.log("web socket opened");
65		68	}
66	function findFileRoot(path) {	69
67	setInputString(path);	70	function onWebSocketMessage(ev) {
68	const result = wasm_exports.find_file_root();	71	wasmOnMessage(ev.data);
69	if (result === -1) return null;	72	}
70	return result;	73
71	}	74	function timeoutThenCreateNew() {
72		75	ws.removeEventListener('message', onWebSocketMessage, false);
73	function decodeString(ptr, len) {	76	ws.removeEventListener('error', timeoutThenCreateNew, false);
74	if (len === 0) return "";	77	ws.removeEventListener('close', timeoutThenCreateNew, false);
75	return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));	78	ws.removeEventListener('open', onWebSocketOpen, false);
76	}	79	ws = null;
77		80	setTimeout(connectWebSocket, 1000);
78	function setInputString(s) {	81	}
79	const jsArray = text_encoder.encode(s);	82
80	const len = jsArray.length;	83	function wasmOnMessage(data) {
81	const ptr = wasm_exports.set_input_string(len);	84	const jsArray = new Uint8Array(data);
82	const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len);	85	const ptr = wasm_exports.message_begin(jsArray.length);
83	wasmArray.set(jsArray);	86	const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, jsArray.length);
84	}	87	wasmArray.set(jsArray);
85		88	wasm_exports.message_end();
86	function declSourceHtml(decl_index) {	89	}
87	return unwrapString(wasm_exports.decl_source_html(decl_index));	90
88	}	91	function onSourceIndexChange() {
89		92	console.log("source location index metadata updated");
90	function unwrapString(bigint) {	93	render();
91	const ptr = Number(bigint & 0xffffffffn);	94	}
92	const len = Number(bigint >> 32n);	95
93	return decodeString(ptr, len);	96	function onCoverageUpdate() {
94	}	97	console.log("coverage update");
		98	}
		99
		100	function render() {
		101	domStatus.classList.add("hidden");
		102	domSectSource.classList.add("hidden");
		103
		104	// TODO this is temporary debugging data
		105	renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");
		106	}
		107
		108	function renderSource(path) {
		109	const decl_index = findFileRoot(path);
		110	if (decl_index == null) throw new Error("file not found: " + path);
		111
		112	const h2 = domSectSource.children[0];
		113	h2.innerText = path;
		114	domSourceText.innerHTML = declSourceHtml(decl_index);
		115
		116	domSectSource.classList.remove("hidden");
		117	}
		118
		119	function findFileRoot(path) {
		120	setInputString(path);
		121	const result = wasm_exports.find_file_root();
		122	if (result === -1) return null;
		123	return result;
		124	}
		125
		126	function decodeString(ptr, len) {
		127	if (len === 0) return "";
		128	return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
		129	}
		130
		131	function setInputString(s) {
		132	const jsArray = text_encoder.encode(s);
		133	const len = jsArray.length;
		134	const ptr = wasm_exports.set_input_string(len);
		135	const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len);
		136	wasmArray.set(jsArray);
		137	}
		138
		139	function declSourceHtml(decl_index) {
		140	return unwrapString(wasm_exports.decl_source_html(decl_index));
		141	}
		142
		143	function unwrapString(bigint) {
		144	const ptr = Number(bigint & 0xffffffffn);
		145	const len = Number(bigint >> 32n);
		146	return decodeString(ptr, len);
		147	}
95	})();	148	})();

lib/fuzzer/wasm/main.zig+80-3

...	@@ -1,16 +1,19 @@	...	@@ -1,16 +1,19 @@
1	const std = @import("std");	1	const std = @import("std");
2	const assert = std.debug.assert;	2	const assert = std.debug.assert;
		3	const abi = std.Build.Fuzz.abi;
		4	const gpa = std.heap.wasm_allocator;
		5	const log = std.log;
		6	const Coverage = std.debug.Coverage;
3		7
4	const Walk = @import("Walk");	8	const Walk = @import("Walk");
5	const Decl = Walk.Decl;	9	const Decl = Walk.Decl;
6	const html_render = @import("html_render");	10	const html_render = @import("html_render");
7		11
8	const gpa = std.heap.wasm_allocator;
9	const log = std.log;
10
11	const js = struct {	12	const js = struct {
12	extern "js" fn log(ptr: [*]const u8, len: usize) void;	13	extern "js" fn log(ptr: [*]const u8, len: usize) void;
13	extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn;	14	extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn;
		15	extern "js" fn emitSourceIndexChange() void;
		16	extern "js" fn emitCoverageUpdate() void;
14	};	17	};
15		18
16	pub const std_options: std.Options = .{	19	pub const std_options: std.Options = .{
...	@@ -45,6 +48,26 @@ export fn alloc(n: usize) [*]u8 {	...	@@ -45,6 +48,26 @@ export fn alloc(n: usize) [*]u8 {
45	return slice.ptr;	48	return slice.ptr;
46	}	49	}
47		50
		51	var message_buffer: std.ArrayListAlignedUnmanaged(u8, @alignOf(u64)) = .{};
		52
		53	/// Resizes the message buffer to be the correct length; returns the pointer to
		54	/// the query string.
		55	export fn message_begin(len: usize) [*]u8 {
		56	message_buffer.resize(gpa, len) catch @panic("OOM");
		57	return message_buffer.items.ptr;
		58	}
		59
		60	export fn message_end() void {
		61	const msg_bytes = message_buffer.items;
		62
		63	const tag: abi.ToClientTag = @enumFromInt(msg_bytes[0]);
		64	switch (tag) {
		65	.source_index => return sourceIndexMessage(msg_bytes) catch @panic("OOM"),
		66	.coverage_update => return coverageUpdateMessage(msg_bytes) catch @panic("OOM"),
		67	_ => unreachable,
		68	}
		69	}
		70
48	export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {	71	export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
49	const tar_bytes = tar_ptr[0..tar_len];	72	const tar_bytes = tar_ptr[0..tar_len];
50	log.debug("received {d} bytes of tar file", .{tar_bytes.len});	73	log.debug("received {d} bytes of tar file", .{tar_bytes.len});
...	@@ -141,3 +164,57 @@ fn fatal(comptime format: []const u8, args: anytype) noreturn {	...	@@ -141,3 +164,57 @@ fn fatal(comptime format: []const u8, args: anytype) noreturn {
141	};	164	};
142	js.panic(line.ptr, line.len);	165	js.panic(line.ptr, line.len);
143	}	166	}
		167
		168	fn sourceIndexMessage(msg_bytes: []u8) error{OutOfMemory}!void {
		169	const Header = abi.SourceIndexHeader;
		170	const header: Header = @bitCast(msg_bytes[0..@sizeOf(Header)].*);
		171
		172	const directories_start = @sizeOf(Header);
		173	const directories_end = directories_start + header.directories_len * @sizeOf(Coverage.String);
		174	const files_start = directories_end;
		175	const files_end = files_start + header.files_len * @sizeOf(Coverage.File);
		176	const source_locations_start = files_end;
		177	const source_locations_end = source_locations_start + header.source_locations_len * @sizeOf(Coverage.SourceLocation);
		178	const string_bytes = msg_bytes[source_locations_end..][0..header.string_bytes_len];
		179
		180	const directories: []const Coverage.String = @alignCast(std.mem.bytesAsSlice(Coverage.String, msg_bytes[directories_start..directories_end]));
		181	const files: []const Coverage.File = @alignCast(std.mem.bytesAsSlice(Coverage.File, msg_bytes[files_start..files_end]));
		182	const source_locations: []const Coverage.SourceLocation = @alignCast(std.mem.bytesAsSlice(Coverage.SourceLocation, msg_bytes[source_locations_start..source_locations_end]));
		183
		184	try updateCoverage(directories, files, source_locations, string_bytes);
		185	js.emitSourceIndexChange();
		186	}
		187
		188	fn coverageUpdateMessage(msg_bytes: []u8) error{OutOfMemory}!void {
		189	recent_coverage_update.clearRetainingCapacity();
		190	recent_coverage_update.appendSlice(gpa, msg_bytes) catch @panic("OOM");
		191	js.emitCoverageUpdate();
		192	}
		193
		194	var coverage = Coverage.init;
		195	var coverage_source_locations: std.ArrayListUnmanaged(Coverage.SourceLocation) = .{};
		196	/// Contains the most recent coverage update message, unmodified.
		197	var recent_coverage_update: std.ArrayListUnmanaged(u8) = .{};
		198
		199	fn updateCoverage(
		200	directories: []const Coverage.String,
		201	files: []const Coverage.File,
		202	source_locations: []const Coverage.SourceLocation,
		203	string_bytes: []const u8,
		204	) !void {
		205	coverage.directories.clearRetainingCapacity();
		206	coverage.files.clearRetainingCapacity();
		207	coverage.string_bytes.clearRetainingCapacity();
		208	coverage_source_locations.clearRetainingCapacity();
		209
		210	try coverage_source_locations.appendSlice(gpa, source_locations);
		211	try coverage.string_bytes.appendSlice(gpa, string_bytes);
		212
		213	try coverage.files.entries.resize(gpa, files.len);
		214	@memcpy(coverage.files.entries.items(.key), files);
		215	try coverage.files.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items });
		216
		217	try coverage.directories.entries.resize(gpa, directories.len);
		218	@memcpy(coverage.directories.entries.items(.key), directories);
		219	try coverage.directories.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items });
		220	}

lib/std/Build/Fuzz.zig+3-560

...	@@ -6,11 +6,13 @@ const assert = std.debug.assert;	...	@@ -6,11 +6,13 @@ const assert = std.debug.assert;
6	const fatal = std.process.fatal;	6	const fatal = std.process.fatal;
7	const Allocator = std.mem.Allocator;	7	const Allocator = std.mem.Allocator;
8	const log = std.log;	8	const log = std.log;
9	const Coverage = std.debug.Coverage;
10		9
11	const Fuzz = @This();	10	const Fuzz = @This();
12	const build_runner = @import("root");	11	const build_runner = @import("root");
13		12
		13	pub const WebServer = @import("Fuzz/WebServer.zig");
		14	pub const abi = @import("Fuzz/abi.zig");
		15
14	pub fn start(	16	pub fn start(
15	gpa: Allocator,	17	gpa: Allocator,
16	arena: Allocator,	18	arena: Allocator,
...	@@ -97,565 +99,6 @@ pub fn start(	...	@@ -97,565 +99,6 @@ pub fn start(
97	log.err("all fuzz workers crashed", .{});	99	log.err("all fuzz workers crashed", .{});
98	}	100	}
99		101
100	pub const WebServer = struct {
101	gpa: Allocator,
102	global_cache_directory: Build.Cache.Directory,
103	zig_lib_directory: Build.Cache.Directory,
104	zig_exe_path: []const u8,
105	listen_address: std.net.Address,
106	fuzz_run_steps: []const *Step.Run,
107
108	/// Messages from fuzz workers. Protected by mutex.
109	msg_queue: std.ArrayListUnmanaged(Msg),
110	/// Protects `msg_queue` only.
111	mutex: std.Thread.Mutex,
112	/// Signaled when there is a message in `msg_queue`.
113	condition: std.Thread.Condition,
114
115	coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
116	/// Protects `coverage_files` only.
117	coverage_mutex: std.Thread.Mutex,
118	/// Signaled when `coverage_files` changes.
119	coverage_condition: std.Thread.Condition,
120
121	const CoverageMap = struct {
122	mapped_memory: []align(std.mem.page_size) const u8,
123	coverage: Coverage,
124
125	fn deinit(cm: *CoverageMap, gpa: Allocator) void {
126	std.posix.munmap(cm.mapped_memory);
127	cm.coverage.deinit(gpa);
128	cm.* = undefined;
129	}
130	};
131
132	const Msg = union(enum) {
133	coverage: struct {
134	id: u64,
135	run: *Step.Run,
136	},
137	};
138
139	fn run(ws: *WebServer) void {
140	var http_server = ws.listen_address.listen(.{
141	.reuse_address = true,
142	}) catch \|err\| {
143	log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.in.getPort(), @errorName(err) });
144	return;
145	};
146	const port = http_server.listen_address.in.getPort();
147	log.info("web interface listening at http://127.0.0.1:{d}/", .{port});
148
149	while (true) {
150	const connection = http_server.accept() catch \|err\| {
151	log.err("failed to accept connection: {s}", .{@errorName(err)});
152	return;
153	};
154	_ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch \|err\| {
155	log.err("unable to spawn connection thread: {s}", .{@errorName(err)});
156	connection.stream.close();
157	continue;
158	};
159	}
160	}
161
162	fn accept(ws: *WebServer, connection: std.net.Server.Connection) void {
163	defer connection.stream.close();
164
165	var read_buffer: [8000]u8 = undefined;
166	var server = std.http.Server.init(connection, &read_buffer);
167	while (server.state == .ready) {
168	var request = server.receiveHead() catch \|err\| switch (err) {
169	error.HttpConnectionClosing => return,
170	else => {
171	log.err("closing http connection: {s}", .{@errorName(err)});
172	return;
173	},
174	};
175	serveRequest(ws, &request) catch \|err\| switch (err) {
176	error.AlreadyReported => return,
177	else => \|e\| {
178	log.err("unable to serve {s}: {s}", .{ request.head.target, @errorName(e) });
179	return;
180	},
181	};
182	}
183	}
184
185	fn serveRequest(ws: WebServer, request: std.http.Server.Request) !void {
186	if (std.mem.eql(u8, request.head.target, "/") or
187	std.mem.eql(u8, request.head.target, "/debug") or
188	std.mem.eql(u8, request.head.target, "/debug/"))
189	{
190	try serveFile(ws, request, "fuzzer/index.html", "text/html");
191	} else if (std.mem.eql(u8, request.head.target, "/main.js") or
192	std.mem.eql(u8, request.head.target, "/debug/main.js"))
193	{
194	try serveFile(ws, request, "fuzzer/main.js", "application/javascript");
195	} else if (std.mem.eql(u8, request.head.target, "/main.wasm")) {
196	try serveWasm(ws, request, .ReleaseFast);
197	} else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) {
198	try serveWasm(ws, request, .Debug);
199	} else if (std.mem.eql(u8, request.head.target, "/sources.tar") or
200	std.mem.eql(u8, request.head.target, "/debug/sources.tar"))
201	{
202	try serveSourcesTar(ws, request);
203	} else if (std.mem.eql(u8, request.head.target, "/events") or
204	std.mem.eql(u8, request.head.target, "/debug/events"))
205	{
206	try serveEvents(ws, request);
207	} else {
208	try request.respond("not found", .{
209	.status = .not_found,
210	.extra_headers = &.{
211	.{ .name = "content-type", .value = "text/plain" },
212	},
213	});
214	}
215	}
216
217	fn serveFile(
218	ws: *WebServer,
219	request: *std.http.Server.Request,
220	name: []const u8,
221	content_type: []const u8,
222	) !void {
223	const gpa = ws.gpa;
224	// The desired API is actually sendfile, which will require enhancing std.http.Server.
225	// We load the file with every request so that the user can make changes to the file
226	// and refresh the HTML page without restarting this server.
227	const file_contents = ws.zig_lib_directory.handle.readFileAlloc(gpa, name, 10 * 1024 * 1024) catch \|err\| {
228	log.err("failed to read '{}{s}': {s}", .{ ws.zig_lib_directory, name, @errorName(err) });
229	return error.AlreadyReported;
230	};
231	defer gpa.free(file_contents);
232	try request.respond(file_contents, .{
233	.extra_headers = &.{
234	.{ .name = "content-type", .value = content_type },
235	cache_control_header,
236	},
237	});
238	}
239
240	fn serveWasm(
241	ws: *WebServer,
242	request: *std.http.Server.Request,
243	optimize_mode: std.builtin.OptimizeMode,
244	) !void {
245	const gpa = ws.gpa;
246
247	var arena_instance = std.heap.ArenaAllocator.init(gpa);
248	defer arena_instance.deinit();
249	const arena = arena_instance.allocator();
250
251	// Do the compilation every request, so that the user can edit the files
252	// and see the changes without restarting the server.
253	const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode);
254	// std.http.Server does not have a sendfile API yet.
255	const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);
256	defer gpa.free(file_contents);
257	try request.respond(file_contents, .{
258	.extra_headers = &.{
259	.{ .name = "content-type", .value = "application/wasm" },
260	cache_control_header,
261	},
262	});
263	}
264
265	fn buildWasmBinary(
266	ws: *WebServer,
267	arena: Allocator,
268	optimize_mode: std.builtin.OptimizeMode,
269	) ![]const u8 {
270	const gpa = ws.gpa;
271
272	const main_src_path: Build.Cache.Path = .{
273	.root_dir = ws.zig_lib_directory,
274	.sub_path = "fuzzer/wasm/main.zig",
275	};
276	const walk_src_path: Build.Cache.Path = .{
277	.root_dir = ws.zig_lib_directory,
278	.sub_path = "docs/wasm/Walk.zig",
279	};
280	const html_render_src_path: Build.Cache.Path = .{
281	.root_dir = ws.zig_lib_directory,
282	.sub_path = "docs/wasm/html_render.zig",
283	};
284
285	var argv: std.ArrayListUnmanaged([]const u8) = .{};
286
287	try argv.appendSlice(arena, &.{
288	ws.zig_exe_path, "build-exe", //
289	"-fno-entry", //
290	"-O", @tagName(optimize_mode), //
291	"-target", "wasm32-freestanding", //
292	"-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
293	"--cache-dir", ws.global_cache_directory.path orelse ".", //
294	"--global-cache-dir", ws.global_cache_directory.path orelse ".", //
295	"--name", "fuzzer", //
296	"-rdynamic", //
297	"--dep", "Walk", //
298	"--dep", "html_render", //
299	try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), //
300	try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), //
301	"--dep", "Walk", //
302	try std.fmt.allocPrint(arena, "-Mhtml_render={}", .{html_render_src_path}), //
303	"--listen=-",
304	});
305
306	var child = std.process.Child.init(argv.items, gpa);
307	child.stdin_behavior = .Pipe;
308	child.stdout_behavior = .Pipe;
309	child.stderr_behavior = .Pipe;
310	try child.spawn();
311
312	var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
313	.stdout = child.stdout.?,
314	.stderr = child.stderr.?,
315	});
316	defer poller.deinit();
317
318	try sendMessage(child.stdin.?, .update);
319	try sendMessage(child.stdin.?, .exit);
320
321	const Header = std.zig.Server.Message.Header;
322	var result: ?[]const u8 = null;
323	var result_error_bundle = std.zig.ErrorBundle.empty;
324
325	const stdout = poller.fifo(.stdout);
326
327	poll: while (true) {
328	while (stdout.readableLength() < @sizeOf(Header)) {
329	if (!(try poller.poll())) break :poll;
330	}
331	const header = stdout.reader().readStruct(Header) catch unreachable;
332	while (stdout.readableLength() < header.bytes_len) {
333	if (!(try poller.poll())) break :poll;
334	}
335	const body = stdout.readableSliceOfLen(header.bytes_len);
336
337	switch (header.tag) {
338	.zig_version => {
339	if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
340	return error.ZigProtocolVersionMismatch;
341	}
342	},
343	.error_bundle => {
344	const EbHdr = std.zig.Server.Message.ErrorBundle;
345	const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body));
346	const extra_bytes =
347	body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len];
348	const string_bytes =
349	body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len];
350	// TODO: use @ptrCast when the compiler supports it
351	const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes);
352	const extra_array = try arena.alloc(u32, unaligned_extra.len);
353	@memcpy(extra_array, unaligned_extra);
354	result_error_bundle = .{
355	.string_bytes = try arena.dupe(u8, string_bytes),
356	.extra = extra_array,
357	};
358	},
359	.emit_bin_path => {
360	const EbpHdr = std.zig.Server.Message.EmitBinPath;
361	const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
362	if (!ebp_hdr.flags.cache_hit) {
363	log.info("source changes detected; rebuilt wasm component", .{});
364	}
365	result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
366	},
367	else => {}, // ignore other messages
368	}
369
370	stdout.discard(body.len);
371	}
372
373	const stderr = poller.fifo(.stderr);
374	if (stderr.readableLength() > 0) {
375	const owned_stderr = try stderr.toOwnedSlice();
376	defer gpa.free(owned_stderr);
377	std.debug.print("{s}", .{owned_stderr});
378	}
379
380	// Send EOF to stdin.
381	child.stdin.?.close();
382	child.stdin = null;
383
384	switch (try child.wait()) {
385	.Exited => \|code\| {
386	if (code != 0) {
387	log.err(
388	"the following command exited with error code {d}:\n{s}",
389	.{ code, try Build.Step.allocPrintCmd(arena, null, argv.items) },
390	);
391	return error.WasmCompilationFailed;
392	}
393	},
394	.Signal, .Stopped, .Unknown => {
395	log.err(
396	"the following command terminated unexpectedly:\n{s}",
397	.{try Build.Step.allocPrintCmd(arena, null, argv.items)},
398	);
399	return error.WasmCompilationFailed;
400	},
401	}
402
403	if (result_error_bundle.errorMessageCount() > 0) {
404	const color = std.zig.Color.auto;
405	result_error_bundle.renderToStdErr(color.renderOptions());
406	log.err("the following command failed with {d} compilation errors:\n{s}", .{
407	result_error_bundle.errorMessageCount(),
408	try Build.Step.allocPrintCmd(arena, null, argv.items),
409	});
410	return error.WasmCompilationFailed;
411	}
412
413	return result orelse {
414	log.err("child process failed to report result\n{s}", .{
415	try Build.Step.allocPrintCmd(arena, null, argv.items),
416	});
417	return error.WasmCompilationFailed;
418	};
419	}
420
421	fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
422	const header: std.zig.Client.Message.Header = .{
423	.tag = tag,
424	.bytes_len = 0,
425	};
426	try file.writeAll(std.mem.asBytes(&header));
427	}
428
429	fn serveEvents(ws: WebServer, request: std.http.Server.Request) !void {
430	var send_buffer: [0x4000]u8 = undefined;
431	var response = request.respondStreaming(.{
432	.send_buffer = &send_buffer,
433	.respond_options = .{
434	.extra_headers = &.{
435	.{ .name = "content-type", .value = "text/event-stream" },
436	},
437	.transfer_encoding = .none,
438	},
439	});
440
441	ws.coverage_mutex.lock();
442	defer ws.coverage_mutex.unlock();
443
444	if (getStats(ws)) \|stats\| {
445	try response.writer().print("data: {d}\n\n", .{stats.n_runs});
446	} else {
447	try response.writeAll("data: loading debug information\n\n");
448	}
449	try response.flush();
450
451	while (true) {
452	ws.coverage_condition.timedWait(&ws.coverage_mutex, std.time.ns_per_ms * 500) catch {};
453	if (getStats(ws)) \|stats\| {
454	try response.writer().print("data: {d}\n\n", .{stats.n_runs});
455	try response.flush();
456	}
457	}
458	}
459
460	const Stats = struct {
461	n_runs: u64,
462	};
463
464	fn getStats(ws: *WebServer) ?Stats {
465	const coverage_maps = ws.coverage_files.values();
466	if (coverage_maps.len == 0) return null;
467	// TODO: make each events URL correspond to one coverage map
468	const ptr = coverage_maps[0].mapped_memory;
469	const SeenPcsHeader = extern struct {
470	n_runs: usize,
471	deduplicated_runs: usize,
472	pcs_len: usize,
473	lowest_stack: usize,
474	};
475	const header: *const SeenPcsHeader = @ptrCast(ptr[0..@sizeOf(SeenPcsHeader)]);
476	return .{
477	.n_runs = @atomicLoad(usize, &header.n_runs, .monotonic),
478	};
479	}
480
481	fn serveSourcesTar(ws: WebServer, request: std.http.Server.Request) !void {
482	const gpa = ws.gpa;
483
484	var arena_instance = std.heap.ArenaAllocator.init(gpa);
485	defer arena_instance.deinit();
486	const arena = arena_instance.allocator();
487
488	var send_buffer: [0x4000]u8 = undefined;
489	var response = request.respondStreaming(.{
490	.send_buffer = &send_buffer,
491	.respond_options = .{
492	.extra_headers = &.{
493	.{ .name = "content-type", .value = "application/x-tar" },
494	cache_control_header,
495	},
496	},
497	});
498	const w = response.writer();
499
500	const DedupeTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
501	var dedupe_table: DedupeTable = .{};
502	defer dedupe_table.deinit(gpa);
503
504	for (ws.fuzz_run_steps) \|run_step\| {
505	const compile_step_inputs = run_step.producer.?.step.inputs.table;
506	for (compile_step_inputs.keys(), compile_step_inputs.values()) \|dir_path, *file_list\| {
507	try dedupe_table.ensureUnusedCapacity(gpa, file_list.items.len);
508	for (file_list.items) \|sub_path\| {
509	// Special file "." means the entire directory.
510	if (std.mem.eql(u8, sub_path, ".")) continue;
511	const joined_path = try dir_path.join(arena, sub_path);
512	_ = dedupe_table.getOrPutAssumeCapacity(joined_path);
513	}
514	}
515	}
516
517	const deduped_paths = dedupe_table.keys();
518	const SortContext = struct {
519	pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool {
520	_ = this;
521	return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) {
522	.lt => true,
523	.gt => false,
524	.eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path),
525	};
526	}
527	};
528	std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
529
530	for (deduped_paths) \|joined_path\| {
531	var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch \|err\| {
532	log.err("failed to open {}: {s}", .{ joined_path, @errorName(err) });
533	continue;
534	};
535	defer file.close();
536
537	const stat = file.stat() catch \|err\| {
538	log.err("failed to stat {}: {s}", .{ joined_path, @errorName(err) });
539	continue;
540	};
541	if (stat.kind != .file)
542	continue;
543
544	const padding = p: {
545	const remainder = stat.size % 512;
546	break :p if (remainder > 0) 512 - remainder else 0;
547	};
548
549	var file_header = std.tar.output.Header.init();
550	file_header.typeflag = .regular;
551	try file_header.setPath(joined_path.root_dir.path orelse ".", joined_path.sub_path);
552	try file_header.setSize(stat.size);
553	try file_header.updateChecksum();
554	try w.writeAll(std.mem.asBytes(&file_header));
555	try w.writeFile(file);
556	try w.writeByteNTimes(0, padding);
557	}
558
559	// intentionally omitting the pointless trailer
560	//try w.writeByteNTimes(0, 512 * 2);
561	try response.end();
562	}
563
564	const cache_control_header: std.http.Header = .{
565	.name = "cache-control",
566	.value = "max-age=0, must-revalidate",
567	};
568
569	fn coverageRun(ws: *WebServer) void {
570	ws.mutex.lock();
571	defer ws.mutex.unlock();
572
573	while (true) {
574	ws.condition.wait(&ws.mutex);
575	for (ws.msg_queue.items) \|msg\| switch (msg) {
576	.coverage => \|coverage\| prepareTables(ws, coverage.run, coverage.id) catch \|err\| switch (err) {
577	error.AlreadyReported => continue,
578	else => \|e\| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
579	},
580	};
581	ws.msg_queue.clearRetainingCapacity();
582	}
583	}
584
585	fn prepareTables(
586	ws: *WebServer,
587	run_step: *Step.Run,
588	coverage_id: u64,
589	) error{ OutOfMemory, AlreadyReported }!void {
590	const gpa = ws.gpa;
591
592	ws.coverage_mutex.lock();
593	defer ws.coverage_mutex.unlock();
594
595	const gop = try ws.coverage_files.getOrPut(gpa, coverage_id);
596	if (gop.found_existing) {
597	// We are fuzzing the same executable with multiple threads.
598	// Perhaps the same unit test; perhaps a different one. In any
599	// case, since the coverage file is the same, we only have to
600	// notice changes to that one file in order to learn coverage for
601	// this particular executable.
602	return;
603	}
604	errdefer _ = ws.coverage_files.pop();
605
606	gop.value_ptr.* = .{
607	.coverage = std.debug.Coverage.init,
608	.mapped_memory = undefined, // populated below
609	};
610	errdefer gop.value_ptr.coverage.deinit(gpa);
611
612	const rebuilt_exe_path: Build.Cache.Path = .{
613	.root_dir = Build.Cache.Directory.cwd(),
614	.sub_path = run_step.rebuilt_executable.?,
615	};
616	var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch \|err\| {
617	log.err("step '{s}': failed to load debug information for '{}': {s}", .{
618	run_step.step.name, rebuilt_exe_path, @errorName(err),
619	});
620	return error.AlreadyReported;
621	};
622	defer debug_info.deinit(gpa);
623
624	const coverage_file_path: Build.Cache.Path = .{
625	.root_dir = run_step.step.owner.cache_root,
626	.sub_path = "v/" ++ std.fmt.hex(coverage_id),
627	};
628	var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch \|err\| {
629	log.err("step '{s}': failed to load coverage file '{}': {s}", .{
630	run_step.step.name, coverage_file_path, @errorName(err),
631	});
632	return error.AlreadyReported;
633	};
634	defer coverage_file.close();
635
636	const file_size = coverage_file.getEndPos() catch \|err\| {
637	log.err("unable to check len of coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
638	return error.AlreadyReported;
639	};
640
641	const mapped_memory = std.posix.mmap(
642	null,
643	file_size,
644	std.posix.PROT.READ,
645	.{ .TYPE = .SHARED },
646	coverage_file.handle,
647	0,
648	) catch \|err\| {
649	log.err("failed to map coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
650	return error.AlreadyReported;
651	};
652
653	gop.value_ptr.mapped_memory = mapped_memory;
654
655	ws.coverage_condition.broadcast();
656	}
657	};
658
659	fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {	102	fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
660	const gpa = run.step.owner.allocator;	103	const gpa = run.step.owner.allocator;
661	const stderr = std.io.getStdErr();	104	const stderr = std.io.getStdErr();

lib/std/Build/Fuzz/WebServer.zig created+605

...	@@ -0,0 +1,605 @@
1	const builtin = @import("builtin");
2
3	const std = @import("../../std.zig");
4	const Allocator = std.mem.Allocator;
5	const Build = std.Build;
6	const Step = std.Build.Step;
7	const Coverage = std.debug.Coverage;
8	const abi = std.Build.Fuzz.abi;
9	const log = std.log;
10
11	const WebServer = @This();
12
13	gpa: Allocator,
14	global_cache_directory: Build.Cache.Directory,
15	zig_lib_directory: Build.Cache.Directory,
16	zig_exe_path: []const u8,
17	listen_address: std.net.Address,
18	fuzz_run_steps: []const *Step.Run,
19
20	/// Messages from fuzz workers. Protected by mutex.
21	msg_queue: std.ArrayListUnmanaged(Msg),
22	/// Protects `msg_queue` only.
23	mutex: std.Thread.Mutex,
24	/// Signaled when there is a message in `msg_queue`.
25	condition: std.Thread.Condition,
26
27	coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
28	/// Protects `coverage_files` only.
29	coverage_mutex: std.Thread.Mutex,
30	/// Signaled when `coverage_files` changes.
31	coverage_condition: std.Thread.Condition,
32
33	const CoverageMap = struct {
34	mapped_memory: []align(std.mem.page_size) const u8,
35	coverage: Coverage,
36	source_locations: []Coverage.SourceLocation,
37
38	fn deinit(cm: *CoverageMap, gpa: Allocator) void {
39	std.posix.munmap(cm.mapped_memory);
40	cm.coverage.deinit(gpa);
41	cm.* = undefined;
42	}
43	};
44
45	const Msg = union(enum) {
46	coverage: struct {
47	id: u64,
48	run: *Step.Run,
49	},
50	};
51
52	pub fn run(ws: *WebServer) void {
53	var http_server = ws.listen_address.listen(.{
54	.reuse_address = true,
55	}) catch \|err\| {
56	log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.in.getPort(), @errorName(err) });
57	return;
58	};
59	const port = http_server.listen_address.in.getPort();
60	log.info("web interface listening at http://127.0.0.1:{d}/", .{port});
61
62	while (true) {
63	const connection = http_server.accept() catch \|err\| {
64	log.err("failed to accept connection: {s}", .{@errorName(err)});
65	return;
66	};
67	_ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch \|err\| {
68	log.err("unable to spawn connection thread: {s}", .{@errorName(err)});
69	connection.stream.close();
70	continue;
71	};
72	}
73	}
74
75	fn accept(ws: *WebServer, connection: std.net.Server.Connection) void {
76	defer connection.stream.close();
77
78	var read_buffer: [0x4000]u8 = undefined;
79	var server = std.http.Server.init(connection, &read_buffer);
80	var web_socket: std.http.WebSocket = undefined;
81	var send_buffer: [0x4000]u8 = undefined;
82	var ws_recv_buffer: [0x4000]u8 align(4) = undefined;
83	while (server.state == .ready) {
84	var request = server.receiveHead() catch \|err\| switch (err) {
85	error.HttpConnectionClosing => return,
86	else => {
87	log.err("closing http connection: {s}", .{@errorName(err)});
88	return;
89	},
90	};
91	if (web_socket.init(&request, &send_buffer, &ws_recv_buffer) catch \|err\| {
92	log.err("initializing web socket: {s}", .{@errorName(err)});
93	return;
94	}) {
95	serveWebSocket(ws, &web_socket) catch \|err\| {
96	log.err("unable to serve web socket connection: {s}", .{@errorName(err)});
97	return;
98	};
99	} else {
100	serveRequest(ws, &request) catch \|err\| switch (err) {
101	error.AlreadyReported => return,
102	else => \|e\| {
103	log.err("unable to serve {s}: {s}", .{ request.head.target, @errorName(e) });
104	return;
105	},
106	};
107	}
108	}
109	}
110
111	fn serveRequest(ws: WebServer, request: std.http.Server.Request) !void {
112	if (std.mem.eql(u8, request.head.target, "/") or
113	std.mem.eql(u8, request.head.target, "/debug") or
114	std.mem.eql(u8, request.head.target, "/debug/"))
115	{
116	try serveFile(ws, request, "fuzzer/index.html", "text/html");
117	} else if (std.mem.eql(u8, request.head.target, "/main.js") or
118	std.mem.eql(u8, request.head.target, "/debug/main.js"))
119	{
120	try serveFile(ws, request, "fuzzer/main.js", "application/javascript");
121	} else if (std.mem.eql(u8, request.head.target, "/main.wasm")) {
122	try serveWasm(ws, request, .ReleaseFast);
123	} else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) {
124	try serveWasm(ws, request, .Debug);
125	} else if (std.mem.eql(u8, request.head.target, "/sources.tar") or
126	std.mem.eql(u8, request.head.target, "/debug/sources.tar"))
127	{
128	try serveSourcesTar(ws, request);
129	} else {
130	try request.respond("not found", .{
131	.status = .not_found,
132	.extra_headers = &.{
133	.{ .name = "content-type", .value = "text/plain" },
134	},
135	});
136	}
137	}
138
139	fn serveFile(
140	ws: *WebServer,
141	request: *std.http.Server.Request,
142	name: []const u8,
143	content_type: []const u8,
144	) !void {
145	const gpa = ws.gpa;
146	// The desired API is actually sendfile, which will require enhancing std.http.Server.
147	// We load the file with every request so that the user can make changes to the file
148	// and refresh the HTML page without restarting this server.
149	const file_contents = ws.zig_lib_directory.handle.readFileAlloc(gpa, name, 10 * 1024 * 1024) catch \|err\| {
150	log.err("failed to read '{}{s}': {s}", .{ ws.zig_lib_directory, name, @errorName(err) });
151	return error.AlreadyReported;
152	};
153	defer gpa.free(file_contents);
154	try request.respond(file_contents, .{
155	.extra_headers = &.{
156	.{ .name = "content-type", .value = content_type },
157	cache_control_header,
158	},
159	});
160	}
161
162	fn serveWasm(
163	ws: *WebServer,
164	request: *std.http.Server.Request,
165	optimize_mode: std.builtin.OptimizeMode,
166	) !void {
167	const gpa = ws.gpa;
168
169	var arena_instance = std.heap.ArenaAllocator.init(gpa);
170	defer arena_instance.deinit();
171	const arena = arena_instance.allocator();
172
173	// Do the compilation every request, so that the user can edit the files
174	// and see the changes without restarting the server.
175	const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode);
176	// std.http.Server does not have a sendfile API yet.
177	const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);
178	defer gpa.free(file_contents);
179	try request.respond(file_contents, .{
180	.extra_headers = &.{
181	.{ .name = "content-type", .value = "application/wasm" },
182	cache_control_header,
183	},
184	});
185	}
186
187	fn buildWasmBinary(
188	ws: *WebServer,
189	arena: Allocator,
190	optimize_mode: std.builtin.OptimizeMode,
191	) ![]const u8 {
192	const gpa = ws.gpa;
193
194	const main_src_path: Build.Cache.Path = .{
195	.root_dir = ws.zig_lib_directory,
196	.sub_path = "fuzzer/wasm/main.zig",
197	};
198	const walk_src_path: Build.Cache.Path = .{
199	.root_dir = ws.zig_lib_directory,
200	.sub_path = "docs/wasm/Walk.zig",
201	};
202	const html_render_src_path: Build.Cache.Path = .{
203	.root_dir = ws.zig_lib_directory,
204	.sub_path = "docs/wasm/html_render.zig",
205	};
206
207	var argv: std.ArrayListUnmanaged([]const u8) = .{};
208
209	try argv.appendSlice(arena, &.{
210	ws.zig_exe_path, "build-exe", //
211	"-fno-entry", //
212	"-O", @tagName(optimize_mode), //
213	"-target", "wasm32-freestanding", //
214	"-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
215	"--cache-dir", ws.global_cache_directory.path orelse ".", //
216	"--global-cache-dir", ws.global_cache_directory.path orelse ".", //
217	"--name", "fuzzer", //
218	"-rdynamic", //
219	"-fsingle-threaded", //
220	"--dep", "Walk", //
221	"--dep", "html_render", //
222	try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), //
223	try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), //
224	"--dep", "Walk", //
225	try std.fmt.allocPrint(arena, "-Mhtml_render={}", .{html_render_src_path}), //
226	"--listen=-",
227	});
228
229	var child = std.process.Child.init(argv.items, gpa);
230	child.stdin_behavior = .Pipe;
231	child.stdout_behavior = .Pipe;
232	child.stderr_behavior = .Pipe;
233	try child.spawn();
234
235	var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
236	.stdout = child.stdout.?,
237	.stderr = child.stderr.?,
238	});
239	defer poller.deinit();
240
241	try sendMessage(child.stdin.?, .update);
242	try sendMessage(child.stdin.?, .exit);
243
244	const Header = std.zig.Server.Message.Header;
245	var result: ?[]const u8 = null;
246	var result_error_bundle = std.zig.ErrorBundle.empty;
247
248	const stdout = poller.fifo(.stdout);
249
250	poll: while (true) {
251	while (stdout.readableLength() < @sizeOf(Header)) {
252	if (!(try poller.poll())) break :poll;
253	}
254	const header = stdout.reader().readStruct(Header) catch unreachable;
255	while (stdout.readableLength() < header.bytes_len) {
256	if (!(try poller.poll())) break :poll;
257	}
258	const body = stdout.readableSliceOfLen(header.bytes_len);
259
260	switch (header.tag) {
261	.zig_version => {
262	if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
263	return error.ZigProtocolVersionMismatch;
264	}
265	},
266	.error_bundle => {
267	const EbHdr = std.zig.Server.Message.ErrorBundle;
268	const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body));
269	const extra_bytes =
270	body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len];
271	const string_bytes =
272	body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len];
273	// TODO: use @ptrCast when the compiler supports it
274	const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes);
275	const extra_array = try arena.alloc(u32, unaligned_extra.len);
276	@memcpy(extra_array, unaligned_extra);
277	result_error_bundle = .{
278	.string_bytes = try arena.dupe(u8, string_bytes),
279	.extra = extra_array,
280	};
281	},
282	.emit_bin_path => {
283	const EbpHdr = std.zig.Server.Message.EmitBinPath;
284	const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
285	if (!ebp_hdr.flags.cache_hit) {
286	log.info("source changes detected; rebuilt wasm component", .{});
287	}
288	result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
289	},
290	else => {}, // ignore other messages
291	}
292
293	stdout.discard(body.len);
294	}
295
296	const stderr = poller.fifo(.stderr);
297	if (stderr.readableLength() > 0) {
298	const owned_stderr = try stderr.toOwnedSlice();
299	defer gpa.free(owned_stderr);
300	std.debug.print("{s}", .{owned_stderr});
301	}
302
303	// Send EOF to stdin.
304	child.stdin.?.close();
305	child.stdin = null;
306
307	switch (try child.wait()) {
308	.Exited => \|code\| {
309	if (code != 0) {
310	log.err(
311	"the following command exited with error code {d}:\n{s}",
312	.{ code, try Build.Step.allocPrintCmd(arena, null, argv.items) },
313	);
314	return error.WasmCompilationFailed;
315	}
316	},
317	.Signal, .Stopped, .Unknown => {
318	log.err(
319	"the following command terminated unexpectedly:\n{s}",
320	.{try Build.Step.allocPrintCmd(arena, null, argv.items)},
321	);
322	return error.WasmCompilationFailed;
323	},
324	}
325
326	if (result_error_bundle.errorMessageCount() > 0) {
327	const color = std.zig.Color.auto;
328	result_error_bundle.renderToStdErr(color.renderOptions());
329	log.err("the following command failed with {d} compilation errors:\n{s}", .{
330	result_error_bundle.errorMessageCount(),
331	try Build.Step.allocPrintCmd(arena, null, argv.items),
332	});
333	return error.WasmCompilationFailed;
334	}
335
336	return result orelse {
337	log.err("child process failed to report result\n{s}", .{
338	try Build.Step.allocPrintCmd(arena, null, argv.items),
339	});
340	return error.WasmCompilationFailed;
341	};
342	}
343
344	fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
345	const header: std.zig.Client.Message.Header = .{
346	.tag = tag,
347	.bytes_len = 0,
348	};
349	try file.writeAll(std.mem.asBytes(&header));
350	}
351
352	fn serveWebSocket(ws: WebServer, web_socket: std.http.WebSocket) !void {
353	ws.coverage_mutex.lock();
354	defer ws.coverage_mutex.unlock();
355
356	// On first connection, the client needs all the coverage information
357	// so that subsequent updates can contain only the updated bits.
358	var prev_unique_runs: usize = 0;
359	try sendCoverageContext(ws, web_socket, &prev_unique_runs);
360	while (true) {
361	ws.coverage_condition.timedWait(&ws.coverage_mutex, std.time.ns_per_ms * 500) catch {};
362	try sendCoverageContext(ws, web_socket, &prev_unique_runs);
363	}
364	}
365
366	fn sendCoverageContext(ws: WebServer, web_socket: std.http.WebSocket, prev_unique_runs: *usize) !void {
367	const coverage_maps = ws.coverage_files.values();
368	if (coverage_maps.len == 0) return;
369	// TODO: make each events URL correspond to one coverage map
370	const coverage_map = &coverage_maps[0];
371	const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
372	const seen_pcs = coverage_map.mapped_memory[@sizeOf(abi.SeenPcsHeader) + coverage_map.source_locations.len * @sizeOf(usize) ..];
373	const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
374	const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
375	const lowest_stack = @atomicLoad(usize, &cov_header.lowest_stack, .monotonic);
376	if (prev_unique_runs.* != unique_runs) {
377	// There has been an update.
378	if (prev_unique_runs.* == 0) {
379	// We need to send initial context.
380	const header: abi.SourceIndexHeader = .{
381	.flags = .{},
382	.directories_len = @intCast(coverage_map.coverage.directories.entries.len),
383	.files_len = @intCast(coverage_map.coverage.files.entries.len),
384	.source_locations_len = @intCast(coverage_map.source_locations.len),
385	.string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len),
386	};
387	const iovecs: [5]std.posix.iovec_const = .{
388	makeIov(std.mem.asBytes(&header)),
389	makeIov(std.mem.sliceAsBytes(coverage_map.coverage.directories.keys())),
390	makeIov(std.mem.sliceAsBytes(coverage_map.coverage.files.keys())),
391	makeIov(std.mem.sliceAsBytes(coverage_map.source_locations)),
392	makeIov(coverage_map.coverage.string_bytes.items),
393	};
394	try web_socket.writeMessagev(&iovecs, .binary);
395	}
396
397	const header: abi.CoverageUpdateHeader = .{
398	.n_runs = n_runs,
399	.unique_runs = unique_runs,
400	.lowest_stack = lowest_stack,
401	};
402	const iovecs: [2]std.posix.iovec_const = .{
403	makeIov(std.mem.asBytes(&header)),
404	makeIov(seen_pcs),
405	};
406	try web_socket.writeMessagev(&iovecs, .binary);
407
408	prev_unique_runs.* = unique_runs;
409	}
410	}
411
412	fn serveSourcesTar(ws: WebServer, request: std.http.Server.Request) !void {
413	const gpa = ws.gpa;
414
415	var arena_instance = std.heap.ArenaAllocator.init(gpa);
416	defer arena_instance.deinit();
417	const arena = arena_instance.allocator();
418
419	var send_buffer: [0x4000]u8 = undefined;
420	var response = request.respondStreaming(.{
421	.send_buffer = &send_buffer,
422	.respond_options = .{
423	.extra_headers = &.{
424	.{ .name = "content-type", .value = "application/x-tar" },
425	cache_control_header,
426	},
427	},
428	});
429	const w = response.writer();
430
431	const DedupeTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
432	var dedupe_table: DedupeTable = .{};
433	defer dedupe_table.deinit(gpa);
434
435	for (ws.fuzz_run_steps) \|run_step\| {
436	const compile_step_inputs = run_step.producer.?.step.inputs.table;
437	for (compile_step_inputs.keys(), compile_step_inputs.values()) \|dir_path, *file_list\| {
438	try dedupe_table.ensureUnusedCapacity(gpa, file_list.items.len);
439	for (file_list.items) \|sub_path\| {
440	// Special file "." means the entire directory.
441	if (std.mem.eql(u8, sub_path, ".")) continue;
442	const joined_path = try dir_path.join(arena, sub_path);
443	_ = dedupe_table.getOrPutAssumeCapacity(joined_path);
444	}
445	}
446	}
447
448	const deduped_paths = dedupe_table.keys();
449	const SortContext = struct {
450	pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool {
451	_ = this;
452	return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) {
453	.lt => true,
454	.gt => false,
455	.eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path),
456	};
457	}
458	};
459	std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
460
461	for (deduped_paths) \|joined_path\| {
462	var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch \|err\| {
463	log.err("failed to open {}: {s}", .{ joined_path, @errorName(err) });
464	continue;
465	};
466	defer file.close();
467
468	const stat = file.stat() catch \|err\| {
469	log.err("failed to stat {}: {s}", .{ joined_path, @errorName(err) });
470	continue;
471	};
472	if (stat.kind != .file)
473	continue;
474
475	const padding = p: {
476	const remainder = stat.size % 512;
477	break :p if (remainder > 0) 512 - remainder else 0;
478	};
479
480	var file_header = std.tar.output.Header.init();
481	file_header.typeflag = .regular;
482	try file_header.setPath(joined_path.root_dir.path orelse ".", joined_path.sub_path);
483	try file_header.setSize(stat.size);
484	try file_header.updateChecksum();
485	try w.writeAll(std.mem.asBytes(&file_header));
486	try w.writeFile(file);
487	try w.writeByteNTimes(0, padding);
488	}
489
490	// intentionally omitting the pointless trailer
491	//try w.writeByteNTimes(0, 512 * 2);
492	try response.end();
493	}
494
495	const cache_control_header: std.http.Header = .{
496	.name = "cache-control",
497	.value = "max-age=0, must-revalidate",
498	};
499
500	pub fn coverageRun(ws: *WebServer) void {
501	ws.mutex.lock();
502	defer ws.mutex.unlock();
503
504	while (true) {
505	ws.condition.wait(&ws.mutex);
506	for (ws.msg_queue.items) \|msg\| switch (msg) {
507	.coverage => \|coverage\| prepareTables(ws, coverage.run, coverage.id) catch \|err\| switch (err) {
508	error.AlreadyReported => continue,
509	else => \|e\| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
510	},
511	};
512	ws.msg_queue.clearRetainingCapacity();
513	}
514	}
515
516	fn prepareTables(
517	ws: *WebServer,
518	run_step: *Step.Run,
519	coverage_id: u64,
520	) error{ OutOfMemory, AlreadyReported }!void {
521	const gpa = ws.gpa;
522
523	ws.coverage_mutex.lock();
524	defer ws.coverage_mutex.unlock();
525
526	const gop = try ws.coverage_files.getOrPut(gpa, coverage_id);
527	if (gop.found_existing) {
528	// We are fuzzing the same executable with multiple threads.
529	// Perhaps the same unit test; perhaps a different one. In any
530	// case, since the coverage file is the same, we only have to
531	// notice changes to that one file in order to learn coverage for
532	// this particular executable.
533	return;
534	}
535	errdefer _ = ws.coverage_files.pop();
536
537	gop.value_ptr.* = .{
538	.coverage = std.debug.Coverage.init,
539	.mapped_memory = undefined, // populated below
540	.source_locations = undefined, // populated below
541	};
542	errdefer gop.value_ptr.coverage.deinit(gpa);
543
544	const rebuilt_exe_path: Build.Cache.Path = .{
545	.root_dir = Build.Cache.Directory.cwd(),
546	.sub_path = run_step.rebuilt_executable.?,
547	};
548	var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch \|err\| {
549	log.err("step '{s}': failed to load debug information for '{}': {s}", .{
550	run_step.step.name, rebuilt_exe_path, @errorName(err),
551	});
552	return error.AlreadyReported;
553	};
554	defer debug_info.deinit(gpa);
555
556	const coverage_file_path: Build.Cache.Path = .{
557	.root_dir = run_step.step.owner.cache_root,
558	.sub_path = "v/" ++ std.fmt.hex(coverage_id),
559	};
560	var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch \|err\| {
561	log.err("step '{s}': failed to load coverage file '{}': {s}", .{
562	run_step.step.name, coverage_file_path, @errorName(err),
563	});
564	return error.AlreadyReported;
565	};
566	defer coverage_file.close();
567
568	const file_size = coverage_file.getEndPos() catch \|err\| {
569	log.err("unable to check len of coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
570	return error.AlreadyReported;
571	};
572
573	const mapped_memory = std.posix.mmap(
574	null,
575	file_size,
576	std.posix.PROT.READ,
577	.{ .TYPE = .SHARED },
578	coverage_file.handle,
579	0,
580	) catch \|err\| {
581	log.err("failed to map coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
582	return error.AlreadyReported;
583	};
584	gop.value_ptr.mapped_memory = mapped_memory;
585
586	const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
587	const pcs_bytes = mapped_memory[@sizeOf(abi.SeenPcsHeader)..][0 .. header.pcs_len * @sizeOf(usize)];
588	const pcs = std.mem.bytesAsSlice(usize, pcs_bytes);
589	const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len);
590	errdefer gpa.free(source_locations);
591	debug_info.resolveAddresses(gpa, pcs, source_locations) catch \|err\| {
592	log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)});
593	return error.AlreadyReported;
594	};
595	gop.value_ptr.source_locations = source_locations;
596
597	ws.coverage_condition.broadcast();
598	}
599
600	fn makeIov(s: []const u8) std.posix.iovec_const {
601	return .{
602	.base = s.ptr,
603	.len = s.len,
604	};
605	}

lib/std/Build/Fuzz/abi.zig created+55

...	@@ -0,0 +1,55 @@
1	//! This file is shared among Zig code running in wildly different contexts:
2	//! libfuzzer, compiled alongside unit tests, the build runner, running on the
3	//! host computer, and the fuzzing web interface webassembly code running in
4	//! the browser. All of these components interface to some degree via an ABI.
5
6	/// libfuzzer uses this and its usize is the one that counts. To match the ABI,
7	/// make the ints be the size of the target used with libfuzzer.
8	///
9	/// Trailing:
10	/// * pc_addr: usize for each pcs_len
11	/// * 1 bit per pc_addr, usize elements
12	pub const SeenPcsHeader = extern struct {
13	n_runs: usize,
14	unique_runs: usize,
15	pcs_len: usize,
16	lowest_stack: usize,
17	};
18
19	pub const ToClientTag = enum(u8) {
20	source_index,
21	coverage_update,
22	_,
23	};
24
25	/// Sent to the fuzzer web client on first connection to the websocket URL.
26	///
27	/// Trailing:
28	/// * std.debug.Coverage.String for each directories_len
29	/// * std.debug.Coverage.File for each files_len
30	/// * std.debug.Coverage.SourceLocation for each source_locations_len
31	/// * u8 for each string_bytes_len
32	pub const SourceIndexHeader = extern struct {
33	flags: Flags,
34	directories_len: u32,
35	files_len: u32,
36	source_locations_len: u32,
37	string_bytes_len: u32,
38
39	pub const Flags = packed struct(u32) {
40	tag: ToClientTag = .source_index,
41	_: u24 = 0,
42	};
43	};
44
45	/// Sent to the fuzzer web client whenever the set of covered source locations
46	/// changes.
47	///
48	/// Trailing:
49	/// * one bit per source_locations_len, contained in u8 elements
50	pub const CoverageUpdateHeader = extern struct {
51	tag: ToClientTag = .coverage_update,
52	n_runs: u64 align(1),
53	unique_runs: u64 align(1),
54	lowest_stack: u64 align(1),
55	};