authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-31 23:45:43-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-07 00:48:32-07:00
log107b27276602d1935a90ae97c57f640b090088b5
tree4606427ffab08b476973028265ee9d6c967304c6
parente0ffac4e3c2271a617616760f3084f5f01fb0785

fuzzer: share zig to html rendering with autodocs

9 files changed, 598 insertions(+), 434 deletions(-)

lib/compiler/std-docs.zig+4-5
...@@ -275,10 +275,6 @@ fn buildWasmBinary(...@@ -275,10 +275,6 @@ fn buildWasmBinary(
275) ![]const u8 {275) ![]const u8 {
276 const gpa = context.gpa;276 const gpa = context.gpa;
277277
278 const main_src_path = try std.fs.path.join(arena, &.{
279 context.zig_lib_directory, "docs", "wasm", "main.zig",
280 });
281
282 var argv: std.ArrayListUnmanaged([]const u8) = .{};278 var argv: std.ArrayListUnmanaged([]const u8) = .{};
283279
284 try argv.appendSlice(arena, &.{280 try argv.appendSlice(arena, &.{
...@@ -298,7 +294,10 @@ fn buildWasmBinary(...@@ -298,7 +294,10 @@ fn buildWasmBinary(
298 "--name",294 "--name",
299 "autodoc",295 "autodoc",
300 "-rdynamic",296 "-rdynamic",
301 main_src_path,297 "--dep",
298 "Walk",
299 try std.fmt.allocPrint(arena, "-Mroot={s}/docs/wasm/main.zig", .{context.zig_lib_directory}),
300 try std.fmt.allocPrint(arena, "-MWalk={s}/docs/wasm/Walk.zig", .{context.zig_lib_directory}),
302 "--listen=-",301 "--listen=-",
303 });302 });
304303
lib/docs/wasm/Decl.zig+9-9
...@@ -1,3 +1,12 @@...@@ -1,3 +1,12 @@
1const Decl = @This();
2const std = @import("std");
3const Ast = std.zig.Ast;
4const Walk = @import("Walk.zig");
5const gpa = std.heap.wasm_allocator;
6const assert = std.debug.assert;
7const log = std.log;
8const Oom = error{OutOfMemory};
9
1ast_node: Ast.Node.Index,10ast_node: Ast.Node.Index,
2file: Walk.File.Index,11file: Walk.File.Index,
3/// The decl whose namespace this is in.12/// The decl whose namespace this is in.
...@@ -215,12 +224,3 @@ pub fn find(search_string: []const u8) Decl.Index {...@@ -215,12 +224,3 @@ pub fn find(search_string: []const u8) Decl.Index {
215 }224 }
216 return current_decl_index;225 return current_decl_index;
217}226}
218
219const Decl = @This();
220const std = @import("std");
221const Ast = std.zig.Ast;
222const Walk = @import("Walk.zig");
223const gpa = std.heap.wasm_allocator;
224const assert = std.debug.assert;
225const log = std.log;
226const Oom = error{OutOfMemory};
lib/docs/wasm/Walk.zig+11-9
...@@ -1,4 +1,15 @@...@@ -1,4 +1,15 @@
1//! Find and annotate identifiers with links to their declarations.1//! Find and annotate identifiers with links to their declarations.
2
3const Walk = @This();
4const std = @import("std");
5const Ast = std.zig.Ast;
6const assert = std.debug.assert;
7const log = std.log;
8const gpa = std.heap.wasm_allocator;
9const Oom = error{OutOfMemory};
10
11pub const Decl = @import("Decl.zig");
12
2pub var files: std.StringArrayHashMapUnmanaged(File) = .{};13pub var files: std.StringArrayHashMapUnmanaged(File) = .{};
3pub var decls: std.ArrayListUnmanaged(Decl) = .{};14pub var decls: std.ArrayListUnmanaged(Decl) = .{};
4pub var modules: std.StringArrayHashMapUnmanaged(File.Index) = .{};15pub var modules: std.StringArrayHashMapUnmanaged(File.Index) = .{};
...@@ -1120,15 +1131,6 @@ pub fn isPrimitiveNonType(name: []const u8) bool {...@@ -1120,15 +1131,6 @@ pub fn isPrimitiveNonType(name: []const u8) bool {
1120// try w.root();1131// try w.root();
1121//}1132//}
11221133
1123const Walk = @This();
1124const std = @import("std");
1125const Ast = std.zig.Ast;
1126const assert = std.debug.assert;
1127const Decl = @import("Decl.zig");
1128const log = std.log;
1129const gpa = std.heap.wasm_allocator;
1130const Oom = error{OutOfMemory};
1131
1132fn shrinkToFit(m: anytype) void {1134fn shrinkToFit(m: anytype) void {
1133 m.shrinkAndFree(gpa, m.entries.len);1135 m.shrinkAndFree(gpa, m.entries.len);
1134}1136}
lib/docs/wasm/html_render.zig created+388
...@@ -0,0 +1,388 @@
1const std = @import("std");
2const Ast = std.zig.Ast;
3const assert = std.debug.assert;
4
5const Walk = @import("Walk");
6const Decl = Walk.Decl;
7
8const gpa = std.heap.wasm_allocator;
9const Oom = error{OutOfMemory};
10
11/// Delete this to find out where URL escaping needs to be added.
12pub const missing_feature_url_escape = true;
13
14pub const RenderSourceOptions = struct {
15 skip_doc_comments: bool = false,
16 skip_comments: bool = false,
17 collapse_whitespace: bool = false,
18 fn_link: Decl.Index = .none,
19};
20
21pub fn fileSourceHtml(
22 file_index: Walk.File.Index,
23 out: *std.ArrayListUnmanaged(u8),
24 root_node: Ast.Node.Index,
25 options: RenderSourceOptions,
26) !void {
27 const ast = file_index.get_ast();
28 const file = file_index.get();
29
30 const g = struct {
31 var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
32 };
33
34 const token_tags = ast.tokens.items(.tag);
35 const token_starts = ast.tokens.items(.start);
36 const main_tokens = ast.nodes.items(.main_token);
37
38 const start_token = ast.firstToken(root_node);
39 const end_token = ast.lastToken(root_node) + 1;
40
41 var cursor: usize = token_starts[start_token];
42
43 var indent: usize = 0;
44 if (std.mem.lastIndexOf(u8, ast.source[0..cursor], "\n")) |newline_index| {
45 for (ast.source[newline_index + 1 .. cursor]) |c| {
46 if (c == ' ') {
47 indent += 1;
48 } else {
49 break;
50 }
51 }
52 }
53
54 for (
55 token_tags[start_token..end_token],
56 token_starts[start_token..end_token],
57 start_token..,
58 ) |tag, start, token_index| {
59 const between = ast.source[cursor..start];
60 if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
61 if (!options.skip_comments) {
62 try out.appendSlice(gpa, "<span class=\"tok-comment\">");
63 try appendUnindented(out, between, indent);
64 try out.appendSlice(gpa, "</span>");
65 }
66 } else if (between.len > 0) {
67 if (options.collapse_whitespace) {
68 if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
69 try out.append(gpa, ' ');
70 } else {
71 try appendUnindented(out, between, indent);
72 }
73 }
74 if (tag == .eof) break;
75 const slice = ast.tokenSlice(token_index);
76 cursor = start + slice.len;
77 switch (tag) {
78 .eof => unreachable,
79
80 .keyword_addrspace,
81 .keyword_align,
82 .keyword_and,
83 .keyword_asm,
84 .keyword_async,
85 .keyword_await,
86 .keyword_break,
87 .keyword_catch,
88 .keyword_comptime,
89 .keyword_const,
90 .keyword_continue,
91 .keyword_defer,
92 .keyword_else,
93 .keyword_enum,
94 .keyword_errdefer,
95 .keyword_error,
96 .keyword_export,
97 .keyword_extern,
98 .keyword_for,
99 .keyword_if,
100 .keyword_inline,
101 .keyword_noalias,
102 .keyword_noinline,
103 .keyword_nosuspend,
104 .keyword_opaque,
105 .keyword_or,
106 .keyword_orelse,
107 .keyword_packed,
108 .keyword_anyframe,
109 .keyword_pub,
110 .keyword_resume,
111 .keyword_return,
112 .keyword_linksection,
113 .keyword_callconv,
114 .keyword_struct,
115 .keyword_suspend,
116 .keyword_switch,
117 .keyword_test,
118 .keyword_threadlocal,
119 .keyword_try,
120 .keyword_union,
121 .keyword_unreachable,
122 .keyword_usingnamespace,
123 .keyword_var,
124 .keyword_volatile,
125 .keyword_allowzero,
126 .keyword_while,
127 .keyword_anytype,
128 .keyword_fn,
129 => {
130 try out.appendSlice(gpa, "<span class=\"tok-kw\">");
131 try appendEscaped(out, slice);
132 try out.appendSlice(gpa, "</span>");
133 },
134
135 .string_literal,
136 .char_literal,
137 .multiline_string_literal_line,
138 => {
139 try out.appendSlice(gpa, "<span class=\"tok-str\">");
140 try appendEscaped(out, slice);
141 try out.appendSlice(gpa, "</span>");
142 },
143
144 .builtin => {
145 try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
146 try appendEscaped(out, slice);
147 try out.appendSlice(gpa, "</span>");
148 },
149
150 .doc_comment,
151 .container_doc_comment,
152 => {
153 if (!options.skip_doc_comments) {
154 try out.appendSlice(gpa, "<span class=\"tok-comment\">");
155 try appendEscaped(out, slice);
156 try out.appendSlice(gpa, "</span>");
157 }
158 },
159
160 .identifier => i: {
161 if (options.fn_link != .none) {
162 const fn_link = options.fn_link.get();
163 const fn_token = main_tokens[fn_link.ast_node];
164 if (token_index == fn_token + 1) {
165 try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
166 _ = missing_feature_url_escape;
167 try fn_link.fqn(out);
168 try out.appendSlice(gpa, "\">");
169 try appendEscaped(out, slice);
170 try out.appendSlice(gpa, "</a>");
171 break :i;
172 }
173 }
174
175 if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
176 try out.appendSlice(gpa, "<span class=\"tok-fn\">");
177 try appendEscaped(out, slice);
178 try out.appendSlice(gpa, "</span>");
179 break :i;
180 }
181
182 if (Walk.isPrimitiveNonType(slice)) {
183 try out.appendSlice(gpa, "<span class=\"tok-null\">");
184 try appendEscaped(out, slice);
185 try out.appendSlice(gpa, "</span>");
186 break :i;
187 }
188
189 if (std.zig.primitives.isPrimitive(slice)) {
190 try out.appendSlice(gpa, "<span class=\"tok-type\">");
191 try appendEscaped(out, slice);
192 try out.appendSlice(gpa, "</span>");
193 break :i;
194 }
195
196 if (file.token_parents.get(token_index)) |field_access_node| {
197 g.field_access_buffer.clearRetainingCapacity();
198 try walkFieldAccesses(file_index, &g.field_access_buffer, field_access_node);
199 if (g.field_access_buffer.items.len > 0) {
200 try out.appendSlice(gpa, "<a href=\"#");
201 _ = missing_feature_url_escape;
202 try out.appendSlice(gpa, g.field_access_buffer.items);
203 try out.appendSlice(gpa, "\">");
204 try appendEscaped(out, slice);
205 try out.appendSlice(gpa, "</a>");
206 } else {
207 try appendEscaped(out, slice);
208 }
209 break :i;
210 }
211
212 {
213 g.field_access_buffer.clearRetainingCapacity();
214 try resolveIdentLink(file_index, &g.field_access_buffer, token_index);
215 if (g.field_access_buffer.items.len > 0) {
216 try out.appendSlice(gpa, "<a href=\"#");
217 _ = missing_feature_url_escape;
218 try out.appendSlice(gpa, g.field_access_buffer.items);
219 try out.appendSlice(gpa, "\">");
220 try appendEscaped(out, slice);
221 try out.appendSlice(gpa, "</a>");
222 break :i;
223 }
224 }
225
226 try appendEscaped(out, slice);
227 },
228
229 .number_literal => {
230 try out.appendSlice(gpa, "<span class=\"tok-number\">");
231 try appendEscaped(out, slice);
232 try out.appendSlice(gpa, "</span>");
233 },
234
235 .bang,
236 .pipe,
237 .pipe_pipe,
238 .pipe_equal,
239 .equal,
240 .equal_equal,
241 .equal_angle_bracket_right,
242 .bang_equal,
243 .l_paren,
244 .r_paren,
245 .semicolon,
246 .percent,
247 .percent_equal,
248 .l_brace,
249 .r_brace,
250 .l_bracket,
251 .r_bracket,
252 .period,
253 .period_asterisk,
254 .ellipsis2,
255 .ellipsis3,
256 .caret,
257 .caret_equal,
258 .plus,
259 .plus_plus,
260 .plus_equal,
261 .plus_percent,
262 .plus_percent_equal,
263 .plus_pipe,
264 .plus_pipe_equal,
265 .minus,
266 .minus_equal,
267 .minus_percent,
268 .minus_percent_equal,
269 .minus_pipe,
270 .minus_pipe_equal,
271 .asterisk,
272 .asterisk_equal,
273 .asterisk_asterisk,
274 .asterisk_percent,
275 .asterisk_percent_equal,
276 .asterisk_pipe,
277 .asterisk_pipe_equal,
278 .arrow,
279 .colon,
280 .slash,
281 .slash_equal,
282 .comma,
283 .ampersand,
284 .ampersand_equal,
285 .question_mark,
286 .angle_bracket_left,
287 .angle_bracket_left_equal,
288 .angle_bracket_angle_bracket_left,
289 .angle_bracket_angle_bracket_left_equal,
290 .angle_bracket_angle_bracket_left_pipe,
291 .angle_bracket_angle_bracket_left_pipe_equal,
292 .angle_bracket_right,
293 .angle_bracket_right_equal,
294 .angle_bracket_angle_bracket_right,
295 .angle_bracket_angle_bracket_right_equal,
296 .tilde,
297 => try appendEscaped(out, slice),
298
299 .invalid, .invalid_periodasterisks => return error.InvalidToken,
300 }
301 }
302}
303
304fn appendUnindented(out: *std.ArrayListUnmanaged(u8), s: []const u8, indent: usize) !void {
305 var it = std.mem.splitScalar(u8, s, '\n');
306 var is_first_line = true;
307 while (it.next()) |line| {
308 if (is_first_line) {
309 try appendEscaped(out, line);
310 is_first_line = false;
311 } else {
312 try out.appendSlice(gpa, "\n");
313 try appendEscaped(out, unindent(line, indent));
314 }
315 }
316}
317
318pub fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
319 for (s) |c| {
320 try out.ensureUnusedCapacity(gpa, 6);
321 switch (c) {
322 '&' => out.appendSliceAssumeCapacity("&amp;"),
323 '<' => out.appendSliceAssumeCapacity("&lt;"),
324 '>' => out.appendSliceAssumeCapacity("&gt;"),
325 '"' => out.appendSliceAssumeCapacity("&quot;"),
326 else => out.appendAssumeCapacity(c),
327 }
328 }
329}
330
331fn walkFieldAccesses(
332 file_index: Walk.File.Index,
333 out: *std.ArrayListUnmanaged(u8),
334 node: Ast.Node.Index,
335) Oom!void {
336 const ast = file_index.get_ast();
337 const node_tags = ast.nodes.items(.tag);
338 assert(node_tags[node] == .field_access);
339 const node_datas = ast.nodes.items(.data);
340 const main_tokens = ast.nodes.items(.main_token);
341 const object_node = node_datas[node].lhs;
342 const dot_token = main_tokens[node];
343 const field_ident = dot_token + 1;
344 switch (node_tags[object_node]) {
345 .identifier => {
346 const lhs_ident = main_tokens[object_node];
347 try resolveIdentLink(file_index, out, lhs_ident);
348 },
349 .field_access => {
350 try walkFieldAccesses(file_index, out, object_node);
351 },
352 else => {},
353 }
354 if (out.items.len > 0) {
355 try out.append(gpa, '.');
356 try out.appendSlice(gpa, ast.tokenSlice(field_ident));
357 }
358}
359
360fn resolveIdentLink(
361 file_index: Walk.File.Index,
362 out: *std.ArrayListUnmanaged(u8),
363 ident_token: Ast.TokenIndex,
364) Oom!void {
365 const decl_index = file_index.get().lookup_token(ident_token);
366 if (decl_index == .none) return;
367 try resolveDeclLink(decl_index, out);
368}
369
370fn unindent(s: []const u8, indent: usize) []const u8 {
371 var indent_idx: usize = 0;
372 for (s) |c| {
373 if (c == ' ' and indent_idx < indent) {
374 indent_idx += 1;
375 } else {
376 break;
377 }
378 }
379 return s[indent_idx..];
380}
381
382pub fn resolveDeclLink(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
383 const decl = decl_index.get();
384 switch (decl.categorize()) {
385 .alias => |alias_decl| try alias_decl.get().fqn(out),
386 else => try decl.fqn(out),
387 }
388}
lib/docs/wasm/main.zig+16-390
...@@ -1,15 +1,17 @@...@@ -1,15 +1,17 @@
1/// Delete this to find out where URL escaping needs to be added.
2const missing_feature_url_escape = true;
3
4const gpa = std.heap.wasm_allocator;
5
6const std = @import("std");1const std = @import("std");
7const log = std.log;2const log = std.log;
8const assert = std.debug.assert;3const assert = std.debug.assert;
9const Ast = std.zig.Ast;4const Ast = std.zig.Ast;
10const Walk = @import("Walk.zig");5const Walk = @import("Walk");
11const markdown = @import("markdown.zig");6const markdown = @import("markdown.zig");
12const Decl = @import("Decl.zig");7const Decl = Walk.Decl;
8
9const fileSourceHtml = @import("html_render.zig").fileSourceHtml;
10const appendEscaped = @import("html_render.zig").appendEscaped;
11const resolveDeclLink = @import("html_render.zig").resolveDeclLink;
12const missing_feature_url_escape = @import("html_render.zig").missing_feature_url_escape;
13
14const gpa = std.heap.wasm_allocator;
1315
14const js = struct {16const js = struct {
15 extern "js" fn log(ptr: [*]const u8, len: usize) void;17 extern "js" fn log(ptr: [*]const u8, len: usize) void;
...@@ -439,7 +441,7 @@ fn decl_field_html_fallible(...@@ -439,7 +441,7 @@ fn decl_field_html_fallible(
439 const decl = decl_index.get();441 const decl = decl_index.get();
440 const ast = decl.file.get_ast();442 const ast = decl.file.get_ast();
441 try out.appendSlice(gpa, "<pre><code>");443 try out.appendSlice(gpa, "<pre><code>");
442 try file_source_html(decl.file, out, field_node, .{});444 try fileSourceHtml(decl.file, out, field_node, .{});
443 try out.appendSlice(gpa, "</code></pre>");445 try out.appendSlice(gpa, "</code></pre>");
444446
445 const field = ast.fullContainerField(field_node).?;447 const field = ast.fullContainerField(field_node).?;
...@@ -478,7 +480,7 @@ fn decl_param_html_fallible(...@@ -478,7 +480,7 @@ fn decl_param_html_fallible(
478 try out.appendSlice(gpa, "<pre><code>");480 try out.appendSlice(gpa, "<pre><code>");
479 try appendEscaped(out, name);481 try appendEscaped(out, name);
480 try out.appendSlice(gpa, ": ");482 try out.appendSlice(gpa, ": ");
481 try file_source_html(decl.file, out, param_node, .{});483 try fileSourceHtml(decl.file, out, param_node, .{});
482 try out.appendSlice(gpa, "</code></pre>");484 try out.appendSlice(gpa, "</code></pre>");
483485
484 if (ast.tokens.items(.tag)[first_doc_comment] == .doc_comment) {486 if (ast.tokens.items(.tag)[first_doc_comment] == .doc_comment) {
...@@ -506,7 +508,7 @@ export fn decl_fn_proto_html(decl_index: Decl.Index, linkify_fn_name: bool) Stri...@@ -506,7 +508,7 @@ export fn decl_fn_proto_html(decl_index: Decl.Index, linkify_fn_name: bool) Stri
506 };508 };
507509
508 string_result.clearRetainingCapacity();510 string_result.clearRetainingCapacity();
509 file_source_html(decl.file, &string_result, proto_node, .{511 fileSourceHtml(decl.file, &string_result, proto_node, .{
510 .skip_doc_comments = true,512 .skip_doc_comments = true,
511 .skip_comments = true,513 .skip_comments = true,
512 .collapse_whitespace = true,514 .collapse_whitespace = true,
...@@ -521,7 +523,7 @@ export fn decl_source_html(decl_index: Decl.Index) String {...@@ -521,7 +523,7 @@ export fn decl_source_html(decl_index: Decl.Index) String {
521 const decl = decl_index.get();523 const decl = decl_index.get();
522524
523 string_result.clearRetainingCapacity();525 string_result.clearRetainingCapacity();
524 file_source_html(decl.file, &string_result, decl.ast_node, .{}) catch |err| {526 fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
525 fatal("unable to render source: {s}", .{@errorName(err)});527 fatal("unable to render source: {s}", .{@errorName(err)});
526 };528 };
527 return String.init(string_result.items);529 return String.init(string_result.items);
...@@ -533,7 +535,7 @@ export fn decl_doctest_html(decl_index: Decl.Index) String {...@@ -533,7 +535,7 @@ export fn decl_doctest_html(decl_index: Decl.Index) String {
533 return String.init("");535 return String.init("");
534536
535 string_result.clearRetainingCapacity();537 string_result.clearRetainingCapacity();
536 file_source_html(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {538 fileSourceHtml(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {
537 fatal("unable to render source: {s}", .{@errorName(err)});539 fatal("unable to render source: {s}", .{@errorName(err)});
538 };540 };
539 return String.init(string_result.items);541 return String.init(string_result.items);
...@@ -691,7 +693,7 @@ fn render_docs(...@@ -691,7 +693,7 @@ fn render_docs(
691 const content = doc.string(data.text.content);693 const content = doc.string(data.text.content);
692 if (resolve_decl_path(r.context, content)) |resolved_decl_index| {694 if (resolve_decl_path(r.context, content)) |resolved_decl_index| {
693 g.link_buffer.clearRetainingCapacity();695 g.link_buffer.clearRetainingCapacity();
694 try resolve_decl_link(resolved_decl_index, &g.link_buffer);696 try resolveDeclLink(resolved_decl_index, &g.link_buffer);
695697
696 try writer.writeAll("<a href=\"#");698 try writer.writeAll("<a href=\"#");
697 _ = missing_feature_url_escape;699 _ = missing_feature_url_escape;
...@@ -734,7 +736,7 @@ export fn decl_type_html(decl_index: Decl.Index) String {...@@ -734,7 +736,7 @@ export fn decl_type_html(decl_index: Decl.Index) String {
734 if (ast.fullVarDecl(decl.ast_node)) |var_decl| {736 if (ast.fullVarDecl(decl.ast_node)) |var_decl| {
735 if (var_decl.ast.type_node != 0) {737 if (var_decl.ast.type_node != 0) {
736 string_result.appendSlice(gpa, "<code>") catch @panic("OOM");738 string_result.appendSlice(gpa, "<code>") catch @panic("OOM");
737 file_source_html(decl.file, &string_result, var_decl.ast.type_node, .{739 fileSourceHtml(decl.file, &string_result, var_decl.ast.type_node, .{
738 .skip_comments = true,740 .skip_comments = true,
739 .collapse_whitespace = true,741 .collapse_whitespace = true,
740 }) catch |e| {742 }) catch |e| {
...@@ -902,382 +904,6 @@ export fn namespace_members(parent: Decl.Index, include_private: bool) Slice(Dec...@@ -902,382 +904,6 @@ export fn namespace_members(parent: Decl.Index, include_private: bool) Slice(Dec
902 return Slice(Decl.Index).init(g.members.items);904 return Slice(Decl.Index).init(g.members.items);
903}905}
904906
905const RenderSourceOptions = struct {
906 skip_doc_comments: bool = false,
907 skip_comments: bool = false,
908 collapse_whitespace: bool = false,
909 fn_link: Decl.Index = .none,
910};
911
912fn file_source_html(
913 file_index: Walk.File.Index,
914 out: *std.ArrayListUnmanaged(u8),
915 root_node: Ast.Node.Index,
916 options: RenderSourceOptions,
917) !void {
918 const ast = file_index.get_ast();
919 const file = file_index.get();
920
921 const g = struct {
922 var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
923 };
924
925 const token_tags = ast.tokens.items(.tag);
926 const token_starts = ast.tokens.items(.start);
927 const main_tokens = ast.nodes.items(.main_token);
928
929 const start_token = ast.firstToken(root_node);
930 const end_token = ast.lastToken(root_node) + 1;
931
932 var cursor: usize = token_starts[start_token];
933
934 var indent: usize = 0;
935 if (std.mem.lastIndexOf(u8, ast.source[0..cursor], "\n")) |newline_index| {
936 for (ast.source[newline_index + 1 .. cursor]) |c| {
937 if (c == ' ') {
938 indent += 1;
939 } else {
940 break;
941 }
942 }
943 }
944
945 for (
946 token_tags[start_token..end_token],
947 token_starts[start_token..end_token],
948 start_token..,
949 ) |tag, start, token_index| {
950 const between = ast.source[cursor..start];
951 if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
952 if (!options.skip_comments) {
953 try out.appendSlice(gpa, "<span class=\"tok-comment\">");
954 try appendUnindented(out, between, indent);
955 try out.appendSlice(gpa, "</span>");
956 }
957 } else if (between.len > 0) {
958 if (options.collapse_whitespace) {
959 if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
960 try out.append(gpa, ' ');
961 } else {
962 try appendUnindented(out, between, indent);
963 }
964 }
965 if (tag == .eof) break;
966 const slice = ast.tokenSlice(token_index);
967 cursor = start + slice.len;
968 switch (tag) {
969 .eof => unreachable,
970
971 .keyword_addrspace,
972 .keyword_align,
973 .keyword_and,
974 .keyword_asm,
975 .keyword_async,
976 .keyword_await,
977 .keyword_break,
978 .keyword_catch,
979 .keyword_comptime,
980 .keyword_const,
981 .keyword_continue,
982 .keyword_defer,
983 .keyword_else,
984 .keyword_enum,
985 .keyword_errdefer,
986 .keyword_error,
987 .keyword_export,
988 .keyword_extern,
989 .keyword_for,
990 .keyword_if,
991 .keyword_inline,
992 .keyword_noalias,
993 .keyword_noinline,
994 .keyword_nosuspend,
995 .keyword_opaque,
996 .keyword_or,
997 .keyword_orelse,
998 .keyword_packed,
999 .keyword_anyframe,
1000 .keyword_pub,
1001 .keyword_resume,
1002 .keyword_return,
1003 .keyword_linksection,
1004 .keyword_callconv,
1005 .keyword_struct,
1006 .keyword_suspend,
1007 .keyword_switch,
1008 .keyword_test,
1009 .keyword_threadlocal,
1010 .keyword_try,
1011 .keyword_union,
1012 .keyword_unreachable,
1013 .keyword_usingnamespace,
1014 .keyword_var,
1015 .keyword_volatile,
1016 .keyword_allowzero,
1017 .keyword_while,
1018 .keyword_anytype,
1019 .keyword_fn,
1020 => {
1021 try out.appendSlice(gpa, "<span class=\"tok-kw\">");
1022 try appendEscaped(out, slice);
1023 try out.appendSlice(gpa, "</span>");
1024 },
1025
1026 .string_literal,
1027 .char_literal,
1028 .multiline_string_literal_line,
1029 => {
1030 try out.appendSlice(gpa, "<span class=\"tok-str\">");
1031 try appendEscaped(out, slice);
1032 try out.appendSlice(gpa, "</span>");
1033 },
1034
1035 .builtin => {
1036 try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
1037 try appendEscaped(out, slice);
1038 try out.appendSlice(gpa, "</span>");
1039 },
1040
1041 .doc_comment,
1042 .container_doc_comment,
1043 => {
1044 if (!options.skip_doc_comments) {
1045 try out.appendSlice(gpa, "<span class=\"tok-comment\">");
1046 try appendEscaped(out, slice);
1047 try out.appendSlice(gpa, "</span>");
1048 }
1049 },
1050
1051 .identifier => i: {
1052 if (options.fn_link != .none) {
1053 const fn_link = options.fn_link.get();
1054 const fn_token = main_tokens[fn_link.ast_node];
1055 if (token_index == fn_token + 1) {
1056 try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
1057 _ = missing_feature_url_escape;
1058 try fn_link.fqn(out);
1059 try out.appendSlice(gpa, "\">");
1060 try appendEscaped(out, slice);
1061 try out.appendSlice(gpa, "</a>");
1062 break :i;
1063 }
1064 }
1065
1066 if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
1067 try out.appendSlice(gpa, "<span class=\"tok-fn\">");
1068 try appendEscaped(out, slice);
1069 try out.appendSlice(gpa, "</span>");
1070 break :i;
1071 }
1072
1073 if (Walk.isPrimitiveNonType(slice)) {
1074 try out.appendSlice(gpa, "<span class=\"tok-null\">");
1075 try appendEscaped(out, slice);
1076 try out.appendSlice(gpa, "</span>");
1077 break :i;
1078 }
1079
1080 if (std.zig.primitives.isPrimitive(slice)) {
1081 try out.appendSlice(gpa, "<span class=\"tok-type\">");
1082 try appendEscaped(out, slice);
1083 try out.appendSlice(gpa, "</span>");
1084 break :i;
1085 }
1086
1087 if (file.token_parents.get(token_index)) |field_access_node| {
1088 g.field_access_buffer.clearRetainingCapacity();
1089 try walk_field_accesses(file_index, &g.field_access_buffer, field_access_node);
1090 if (g.field_access_buffer.items.len > 0) {
1091 try out.appendSlice(gpa, "<a href=\"#");
1092 _ = missing_feature_url_escape;
1093 try out.appendSlice(gpa, g.field_access_buffer.items);
1094 try out.appendSlice(gpa, "\">");
1095 try appendEscaped(out, slice);
1096 try out.appendSlice(gpa, "</a>");
1097 } else {
1098 try appendEscaped(out, slice);
1099 }
1100 break :i;
1101 }
1102
1103 {
1104 g.field_access_buffer.clearRetainingCapacity();
1105 try resolve_ident_link(file_index, &g.field_access_buffer, token_index);
1106 if (g.field_access_buffer.items.len > 0) {
1107 try out.appendSlice(gpa, "<a href=\"#");
1108 _ = missing_feature_url_escape;
1109 try out.appendSlice(gpa, g.field_access_buffer.items);
1110 try out.appendSlice(gpa, "\">");
1111 try appendEscaped(out, slice);
1112 try out.appendSlice(gpa, "</a>");
1113 break :i;
1114 }
1115 }
1116
1117 try appendEscaped(out, slice);
1118 },
1119
1120 .number_literal => {
1121 try out.appendSlice(gpa, "<span class=\"tok-number\">");
1122 try appendEscaped(out, slice);
1123 try out.appendSlice(gpa, "</span>");
1124 },
1125
1126 .bang,
1127 .pipe,
1128 .pipe_pipe,
1129 .pipe_equal,
1130 .equal,
1131 .equal_equal,
1132 .equal_angle_bracket_right,
1133 .bang_equal,
1134 .l_paren,
1135 .r_paren,
1136 .semicolon,
1137 .percent,
1138 .percent_equal,
1139 .l_brace,
1140 .r_brace,
1141 .l_bracket,
1142 .r_bracket,
1143 .period,
1144 .period_asterisk,
1145 .ellipsis2,
1146 .ellipsis3,
1147 .caret,
1148 .caret_equal,
1149 .plus,
1150 .plus_plus,
1151 .plus_equal,
1152 .plus_percent,
1153 .plus_percent_equal,
1154 .plus_pipe,
1155 .plus_pipe_equal,
1156 .minus,
1157 .minus_equal,
1158 .minus_percent,
1159 .minus_percent_equal,
1160 .minus_pipe,
1161 .minus_pipe_equal,
1162 .asterisk,
1163 .asterisk_equal,
1164 .asterisk_asterisk,
1165 .asterisk_percent,
1166 .asterisk_percent_equal,
1167 .asterisk_pipe,
1168 .asterisk_pipe_equal,
1169 .arrow,
1170 .colon,
1171 .slash,
1172 .slash_equal,
1173 .comma,
1174 .ampersand,
1175 .ampersand_equal,
1176 .question_mark,
1177 .angle_bracket_left,
1178 .angle_bracket_left_equal,
1179 .angle_bracket_angle_bracket_left,
1180 .angle_bracket_angle_bracket_left_equal,
1181 .angle_bracket_angle_bracket_left_pipe,
1182 .angle_bracket_angle_bracket_left_pipe_equal,
1183 .angle_bracket_right,
1184 .angle_bracket_right_equal,
1185 .angle_bracket_angle_bracket_right,
1186 .angle_bracket_angle_bracket_right_equal,
1187 .tilde,
1188 => try appendEscaped(out, slice),
1189
1190 .invalid, .invalid_periodasterisks => return error.InvalidToken,
1191 }
1192 }
1193}
1194
1195fn unindent(s: []const u8, indent: usize) []const u8 {
1196 var indent_idx: usize = 0;
1197 for (s) |c| {
1198 if (c == ' ' and indent_idx < indent) {
1199 indent_idx += 1;
1200 } else {
1201 break;
1202 }
1203 }
1204 return s[indent_idx..];
1205}
1206
1207fn appendUnindented(out: *std.ArrayListUnmanaged(u8), s: []const u8, indent: usize) !void {
1208 var it = std.mem.splitScalar(u8, s, '\n');
1209 var is_first_line = true;
1210 while (it.next()) |line| {
1211 if (is_first_line) {
1212 try appendEscaped(out, line);
1213 is_first_line = false;
1214 } else {
1215 try out.appendSlice(gpa, "\n");
1216 try appendEscaped(out, unindent(line, indent));
1217 }
1218 }
1219}
1220
1221fn resolve_ident_link(
1222 file_index: Walk.File.Index,
1223 out: *std.ArrayListUnmanaged(u8),
1224 ident_token: Ast.TokenIndex,
1225) Oom!void {
1226 const decl_index = file_index.get().lookup_token(ident_token);
1227 if (decl_index == .none) return;
1228 try resolve_decl_link(decl_index, out);
1229}
1230
1231fn resolve_decl_link(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
1232 const decl = decl_index.get();
1233 switch (decl.categorize()) {
1234 .alias => |alias_decl| try alias_decl.get().fqn(out),
1235 else => try decl.fqn(out),
1236 }
1237}
1238
1239fn walk_field_accesses(
1240 file_index: Walk.File.Index,
1241 out: *std.ArrayListUnmanaged(u8),
1242 node: Ast.Node.Index,
1243) Oom!void {
1244 const ast = file_index.get_ast();
1245 const node_tags = ast.nodes.items(.tag);
1246 assert(node_tags[node] == .field_access);
1247 const node_datas = ast.nodes.items(.data);
1248 const main_tokens = ast.nodes.items(.main_token);
1249 const object_node = node_datas[node].lhs;
1250 const dot_token = main_tokens[node];
1251 const field_ident = dot_token + 1;
1252 switch (node_tags[object_node]) {
1253 .identifier => {
1254 const lhs_ident = main_tokens[object_node];
1255 try resolve_ident_link(file_index, out, lhs_ident);
1256 },
1257 .field_access => {
1258 try walk_field_accesses(file_index, out, object_node);
1259 },
1260 else => {},
1261 }
1262 if (out.items.len > 0) {
1263 try out.append(gpa, '.');
1264 try out.appendSlice(gpa, ast.tokenSlice(field_ident));
1265 }
1266}
1267
1268fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
1269 for (s) |c| {
1270 try out.ensureUnusedCapacity(gpa, 6);
1271 switch (c) {
1272 '&' => out.appendSliceAssumeCapacity("&amp;"),
1273 '<' => out.appendSliceAssumeCapacity("&lt;"),
1274 '>' => out.appendSliceAssumeCapacity("&gt;"),
1275 '"' => out.appendSliceAssumeCapacity("&quot;"),
1276 else => out.appendAssumeCapacity(c),
1277 }
1278 }
1279}
1280
1281fn count_scalar(haystack: []const u8, needle: u8) usize {907fn count_scalar(haystack: []const u8, needle: u8) usize {
1282 var total: usize = 0;908 var total: usize = 0;
1283 for (haystack) |elem| {909 for (haystack) |elem| {
lib/fuzzer/index.html+59-1
...@@ -2,12 +2,56 @@...@@ -2,12 +2,56 @@
2<html>2<html>
3 <head>3 <head>
4 <meta charset="utf-8">4 <meta charset="utf-8">
5 <title>Zig Documentation</title>5 <title>Zig Build System Interface</title>
6 <style type="text/css">6 <style type="text/css">
7 body {7 body {
8 font-family: system-ui, -apple-system, Roboto, "Segoe UI", sans-serif;8 font-family: system-ui, -apple-system, Roboto, "Segoe UI", sans-serif;
9 color: #000000;9 color: #000000;
10 }10 }
11 .hidden {
12 display: none;
13 }
14 table {
15 width: 100%;
16 }
17 a {
18 color: #2A6286;
19 }
20 pre{
21 font-family:"Source Code Pro",monospace;
22 font-size:1em;
23 background-color:#F5F5F5;
24 padding: 1em;
25 margin: 0;
26 overflow-x: auto;
27 }
28 :not(pre) > code {
29 white-space: break-spaces;
30 }
31 code {
32 font-family:"Source Code Pro",monospace;
33 font-size: 0.9em;
34 }
35 code a {
36 color: #000000;
37 }
38 kbd {
39 color: #000;
40 background-color: #fafbfc;
41 border-color: #d1d5da;
42 border-bottom-color: #c6cbd1;
43 box-shadow-color: #c6cbd1;
44 display: inline-block;
45 padding: 0.3em 0.2em;
46 font: 1.2em monospace;
47 line-height: 0.8em;
48 vertical-align: middle;
49 border: solid 1px;
50 border-radius: 3px;
51 box-shadow: inset 0 -1px 0;
52 cursor: default;
53 }
54
11 .tok-kw {55 .tok-kw {
12 color: #333;56 color: #333;
13 font-weight: bold;57 font-weight: bold;
...@@ -42,6 +86,16 @@...@@ -42,6 +86,16 @@
42 background-color: #111;86 background-color: #111;
43 color: #bbb;87 color: #bbb;
44 }88 }
89 pre {
90 background-color: #222;
91 color: #ccc;
92 }
93 a {
94 color: #88f;
95 }
96 code a {
97 color: #ccc;
98 }
45 .tok-kw {99 .tok-kw {
46 color: #eee;100 color: #eee;
47 }101 }
...@@ -70,6 +124,10 @@...@@ -70,6 +124,10 @@
70 </style>124 </style>
71 </head>125 </head>
72 <body>126 <body>
127 <div id="sectSource" class="hidden">
128 <h2>Source Code</h2>
129 <pre><code id="sourceText"></code></pre>
130 </div>
73 <script src="main.js"></script>131 <script src="main.js"></script>
74 </body>132 </body>
75</html>133</html>
lib/fuzzer/main.js+48
...@@ -1,4 +1,7 @@...@@ -1,4 +1,7 @@
1(function() {1(function() {
2 const domSectSource = document.getElementById("sectSource");
3 const domSourceText = document.getElementById("sourceText");
4
2 let wasm_promise = fetch("main.wasm");5 let wasm_promise = fetch("main.wasm");
3 let sources_promise = fetch("sources.tar").then(function(response) {6 let sources_promise = fetch("sources.tar").then(function(response) {
4 if (!response.ok) throw new Error("unable to download sources");7 if (!response.ok) throw new Error("unable to download sources");
...@@ -30,11 +33,56 @@...@@ -30,11 +33,56 @@
30 const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);33 const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);
31 wasm_array.set(js_array);34 wasm_array.set(js_array);
32 wasm_exports.unpack(ptr, js_array.length);35 wasm_exports.unpack(ptr, js_array.length);
36
37 render();
33 });38 });
34 });39 });
3540
41 function render() {
42 domSectSource.classList.add("hidden");
43
44 // TODO this is temporary debugging data
45 renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");
46 }
47
48 function renderSource(path) {
49 const decl_index = findFileRoot(path);
50 if (decl_index == null) throw new Error("file not found: " + path);
51
52 const h2 = domSectSource.children[0];
53 h2.innerText = path;
54 domSourceText.innerHTML = declSourceHtml(decl_index);
55
56 domSectSource.classList.remove("hidden");
57 }
58
59 function findFileRoot(path) {
60 setInputString(path);
61 const result = wasm_exports.find_file_root();
62 if (result === -1) return null;
63 return result;
64 }
65
36 function decodeString(ptr, len) {66 function decodeString(ptr, len) {
37 if (len === 0) return "";67 if (len === 0) return "";
38 return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));68 return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
39 }69 }
70
71 function setInputString(s) {
72 const jsArray = text_encoder.encode(s);
73 const len = jsArray.length;
74 const ptr = wasm_exports.set_input_string(len);
75 const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len);
76 wasmArray.set(jsArray);
77 }
78
79 function declSourceHtml(decl_index) {
80 return unwrapString(wasm_exports.decl_source_html(decl_index));
81 }
82
83 function unwrapString(bigint) {
84 const ptr = Number(bigint & 0xffffffffn);
85 const len = Number(bigint >> 32n);
86 return decodeString(ptr, len);
87 }
40})();88})();
lib/fuzzer/wasm/main.zig+44
...@@ -2,6 +2,8 @@ const std = @import("std");...@@ -2,6 +2,8 @@ const std = @import("std");
2const assert = std.debug.assert;2const assert = std.debug.assert;
33
4const Walk = @import("Walk");4const Walk = @import("Walk");
5const Decl = Walk.Decl;
6const html_render = @import("html_render");
57
6const gpa = std.heap.wasm_allocator;8const gpa = std.heap.wasm_allocator;
7const log = std.log;9const log = std.log;
...@@ -52,6 +54,48 @@ export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {...@@ -52,6 +54,48 @@ export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
52 };54 };
53}55}
5456
57/// Set by `set_input_string`.
58var input_string: std.ArrayListUnmanaged(u8) = .{};
59var string_result: std.ArrayListUnmanaged(u8) = .{};
60
61export fn set_input_string(len: usize) [*]u8 {
62 input_string.resize(gpa, len) catch @panic("OOM");
63 return input_string.items.ptr;
64}
65
66/// Looks up the root struct decl corresponding to a file by path.
67/// Uses `input_string`.
68export fn find_file_root() Decl.Index {
69 const file: Walk.File.Index = @enumFromInt(Walk.files.getIndex(input_string.items) orelse return .none);
70 return file.findRootDecl();
71}
72
73export fn decl_source_html(decl_index: Decl.Index) String {
74 const decl = decl_index.get();
75
76 string_result.clearRetainingCapacity();
77 html_render.fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
78 fatal("unable to render source: {s}", .{@errorName(err)});
79 };
80 return String.init(string_result.items);
81}
82
83const String = Slice(u8);
84
85fn Slice(T: type) type {
86 return packed struct(u64) {
87 ptr: u32,
88 len: u32,
89
90 fn init(s: []const T) @This() {
91 return .{
92 .ptr = @intFromPtr(s.ptr),
93 .len = s.len,
94 };
95 }
96 };
97}
98
55fn unpackInner(tar_bytes: []u8) !void {99fn unpackInner(tar_bytes: []u8) !void {
56 var fbs = std.io.fixedBufferStream(tar_bytes);100 var fbs = std.io.fixedBufferStream(tar_bytes);
57 var file_name_buffer: [1024]u8 = undefined;101 var file_name_buffer: [1024]u8 = undefined;
lib/std/Build/Fuzz.zig+19-20
...@@ -235,30 +235,29 @@ pub const WebServer = struct {...@@ -235,30 +235,29 @@ pub const WebServer = struct {
235 .root_dir = ws.zig_lib_directory,235 .root_dir = ws.zig_lib_directory,
236 .sub_path = "docs/wasm/Walk.zig",236 .sub_path = "docs/wasm/Walk.zig",
237 };237 };
238 const html_render_src_path: Build.Cache.Path = .{
239 .root_dir = ws.zig_lib_directory,
240 .sub_path = "docs/wasm/html_render.zig",
241 };
238242
239 var argv: std.ArrayListUnmanaged([]const u8) = .{};243 var argv: std.ArrayListUnmanaged([]const u8) = .{};
240244
241 try argv.appendSlice(arena, &.{245 try argv.appendSlice(arena, &.{
242 ws.zig_exe_path,246 ws.zig_exe_path, "build-exe", //
243 "build-exe",247 "-fno-entry", //
244 "-fno-entry",248 "-O", @tagName(optimize_mode), //
245 "-O",249 "-target", "wasm32-freestanding", //
246 @tagName(optimize_mode),250 "-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
247 "-target",251 "--cache-dir", ws.global_cache_directory.path orelse ".", //
248 "wasm32-freestanding",252 "--global-cache-dir", ws.global_cache_directory.path orelse ".", //
249 "-mcpu",253 "--name", "fuzzer", //
250 "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext",254 "-rdynamic", //
251 "--cache-dir",255 "--dep", "Walk", //
252 ws.global_cache_directory.path orelse ".",256 "--dep", "html_render", //
253 "--global-cache-dir",257 try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), //
254 ws.global_cache_directory.path orelse ".",258 try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), //
255 "--name",259 "--dep", "Walk", //
256 "fuzzer",260 try std.fmt.allocPrint(arena, "-Mhtml_render={}", .{html_render_src_path}), //
257 "-rdynamic",
258 "--dep",
259 "Walk",
260 try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}),
261 try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}),
262 "--listen=-",261 "--listen=-",
263 });262 });
264263