1const builtin = @import("builtin");
2
3const std = @import("std");
4const Io = std.Io;
5const Dir = std.Io.Dir;
6const print = std.debug.print;
7const mem = std.mem;
8const testing = std.testing;
9const Allocator = std.mem.Allocator;
10const fatal = std.process.fatal;
11
12const max_doc_file_size = 10 * 1024 * 1024;
13
14pub fn main(init: std.process.Init) !void {
15 const arena = init.arena.allocator();
16 const io = init.io;
17 const args = try init.minimal.args.toSlice(arena);
18
19 const input_file = args[1];
20 const output_file = args[2];
21
22 var in_file = try Dir.cwd().openFile(io, input_file, .{ .mode = .read_only });
23 defer in_file.close(io);
24
25 var out_file = try Dir.cwd().createFile(io, output_file, .{});
26 defer out_file.close(io);
27 var out_file_buffer: [4096]u8 = undefined;
28 var out_file_writer = out_file.writer(io, &out_file_buffer);
29
30 var out_dir = try Dir.cwd().openDir(io, Dir.path.dirname(output_file).?, .{});
31 defer out_dir.close(io);
32
33 var in_file_reader = in_file.reader(io, &.{});
34 const input_file_bytes = try in_file_reader.interface.allocRemaining(arena, .unlimited);
35
36 var tokenizer = Tokenizer.init(input_file, input_file_bytes);
37
38 try walk(arena, io, &tokenizer, out_dir, &out_file_writer.interface);
39
40 try out_file_writer.end();
41}
42
43const Token = struct {
44 id: Id,
45 start: usize,
46 end: usize,
47
48 const Id = enum {
49 invalid,
50 content,
51 bracket_open,
52 tag_content,
53 separator,
54 bracket_close,
55 eof,
56 };
57};
58
59const Tokenizer = struct {
60 buffer: []const u8,
61 index: usize,
62 state: State,
63 source_file_name: []const u8,
64
65 const State = enum {
66 start,
67 l_bracket,
68 hash,
69 tag_name,
70 eof,
71 };
72
73 fn init(source_file_name: []const u8, buffer: []const u8) Tokenizer {
74 return Tokenizer{
75 .buffer = buffer,
76 .index = 0,
77 .state = .start,
78 .source_file_name = source_file_name,
79 };
80 }
81
82 fn next(self: *Tokenizer) Token {
83 var result = Token{
84 .id = .eof,
85 .start = self.index,
86 .end = undefined,
87 };
88 while (self.index < self.buffer.len) : (self.index += 1) {
89 const c = self.buffer[self.index];
90 switch (self.state) {
91 .start => switch (c) {
92 '{' => {
93 self.state = .l_bracket;
94 },
95 else => {
96 result.id = .content;
97 },
98 },
99 .l_bracket => switch (c) {
100 '#' => {
101 if (result.id != .eof) {
102 self.index -= 1;
103 self.state = .start;
104 break;
105 } else {
106 result.id = .bracket_open;
107 self.index += 1;
108 self.state = .tag_name;
109 break;
110 }
111 },
112 else => {
113 result.id = .content;
114 self.state = .start;
115 },
116 },
117 .tag_name => switch (c) {
118 '|' => {
119 if (result.id != .eof) {
120 break;
121 } else {
122 result.id = .separator;
123 self.index += 1;
124 break;
125 }
126 },
127 '#' => {
128 self.state = .hash;
129 },
130 else => {
131 result.id = .tag_content;
132 },
133 },
134 .hash => switch (c) {
135 '}' => {
136 if (result.id != .eof) {
137 self.index -= 1;
138 self.state = .tag_name;
139 break;
140 } else {
141 result.id = .bracket_close;
142 self.index += 1;
143 self.state = .start;
144 break;
145 }
146 },
147 else => {
148 result.id = .tag_content;
149 self.state = .tag_name;
150 },
151 },
152 .eof => unreachable,
153 }
154 } else {
155 switch (self.state) {
156 .start, .l_bracket, .eof => {},
157 else => {
158 result.id = .invalid;
159 },
160 }
161 self.state = .eof;
162 }
163 result.end = self.index;
164 return result;
165 }
166
167 const Location = struct {
168 line: usize,
169 column: usize,
170 line_start: usize,
171 line_end: usize,
172 };
173
174 fn getTokenLocation(self: *Tokenizer, token: Token) Location {
175 var loc = Location{
176 .line = 0,
177 .column = 0,
178 .line_start = 0,
179 .line_end = 0,
180 };
181 for (self.buffer, 0..) |c, i| {
182 if (i == token.start) {
183 loc.line_end = i;
184 while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
185 return loc;
186 }
187 if (c == '\n') {
188 loc.line += 1;
189 loc.column = 0;
190 loc.line_start = i + 1;
191 } else {
192 loc.column += 1;
193 }
194 }
195 return loc;
196 }
197};
198
199fn parseError(tokenizer: *Tokenizer, token: Token, comptime fmt: []const u8, args: anytype) anyerror {
200 const loc = tokenizer.getTokenLocation(token);
201 const args_prefix = .{ tokenizer.source_file_name, loc.line + 1, loc.column + 1 };
202 print("{s}:{d}:{d}: error: " ++ fmt ++ "\n", args_prefix ++ args);
203 if (loc.line_start <= loc.line_end) {
204 print("{s}\n", .{tokenizer.buffer[loc.line_start..loc.line_end]});
205 {
206 var i: usize = 0;
207 while (i < loc.column) : (i += 1) {
208 print(" ", .{});
209 }
210 }
211 {
212 const caret_count = @min(token.end, loc.line_end) - token.start;
213 var i: usize = 0;
214 while (i < caret_count) : (i += 1) {
215 print("~", .{});
216 }
217 }
218 print("\n", .{});
219 }
220 return error.ParseError;
221}
222
223fn assertToken(tokenizer: *Tokenizer, token: Token, id: Token.Id) !void {
224 if (token.id != id) {
225 return parseError(tokenizer, token, "expected {s}, found {s}", .{ @tagName(id), @tagName(token.id) });
226 }
227}
228
229fn eatToken(tokenizer: *Tokenizer, id: Token.Id) !Token {
230 const token = tokenizer.next();
231 try assertToken(tokenizer, token, id);
232 return token;
233}
234
235const ExpectedOutcome = enum {
236 succeed,
237 fail,
238 build_fail,
239};
240
241const Code = struct {
242 id: Id,
243 name: []const u8,
244 source_token: Token,
245 just_check_syntax: bool,
246 mode: std.builtin.Optimize,
247 link_objects: []const []const u8,
248 target_str: ?[]const u8,
249 link_libc: bool,
250 link_mode: ?std.builtin.LinkMode,
251 disable_cache: bool,
252 additional_options: []const []const u8,
253
254 const Id = union(enum) {
255 @"test",
256 test_error: []const u8,
257 test_safety: []const u8,
258 exe: ExpectedOutcome,
259 obj: ?[]const u8,
260 lib,
261 };
262};
263
264fn walk(arena: Allocator, io: Io, tokenizer: *Tokenizer, out_dir: Dir, w: anytype) !void {
265 while (true) {
266 const token = tokenizer.next();
267 switch (token.id) {
268 .eof => break,
269 .content,
270 => {
271 try w.writeAll(tokenizer.buffer[token.start..token.end]);
272 },
273 .bracket_open => {
274 const tag_token = try eatToken(tokenizer, .tag_content);
275 const tag_name = tokenizer.buffer[tag_token.start..tag_token.end];
276
277 if (mem.eql(u8, tag_name, "code_begin")) {
278 _ = try eatToken(tokenizer, .separator);
279 const code_kind_tok = try eatToken(tokenizer, .tag_content);
280 _ = try eatToken(tokenizer, .separator);
281 const name_tok = try eatToken(tokenizer, .tag_content);
282 const name = tokenizer.buffer[name_tok.start..name_tok.end];
283 var error_str: []const u8 = "";
284 const maybe_sep = tokenizer.next();
285 switch (maybe_sep.id) {
286 .separator => {
287 const error_tok = try eatToken(tokenizer, .tag_content);
288 error_str = tokenizer.buffer[error_tok.start..error_tok.end];
289 _ = try eatToken(tokenizer, .bracket_close);
290 },
291 .bracket_close => {},
292 else => return parseError(tokenizer, token, "invalid token", .{}),
293 }
294 const code_kind_str = tokenizer.buffer[code_kind_tok.start..code_kind_tok.end];
295 var code_kind_id: Code.Id = undefined;
296 var just_check_syntax = false;
297 if (mem.eql(u8, code_kind_str, "exe")) {
298 code_kind_id = Code.Id{ .exe = .succeed };
299 } else if (mem.eql(u8, code_kind_str, "exe_err")) {
300 code_kind_id = Code.Id{ .exe = .fail };
301 } else if (mem.eql(u8, code_kind_str, "exe_build_err")) {
302 code_kind_id = Code.Id{ .exe = .build_fail };
303 } else if (mem.eql(u8, code_kind_str, "test")) {
304 code_kind_id = .@"test";
305 } else if (mem.eql(u8, code_kind_str, "test_err")) {
306 code_kind_id = Code.Id{ .test_error = error_str };
307 } else if (mem.eql(u8, code_kind_str, "test_safety")) {
308 code_kind_id = Code.Id{ .test_safety = error_str };
309 } else if (mem.eql(u8, code_kind_str, "obj")) {
310 code_kind_id = Code.Id{ .obj = null };
311 } else if (mem.eql(u8, code_kind_str, "obj_err")) {
312 code_kind_id = Code.Id{ .obj = error_str };
313 } else if (mem.eql(u8, code_kind_str, "lib")) {
314 code_kind_id = Code.Id.lib;
315 } else if (mem.eql(u8, code_kind_str, "syntax")) {
316 code_kind_id = Code.Id{ .obj = null };
317 just_check_syntax = true;
318 } else {
319 return parseError(tokenizer, code_kind_tok, "unrecognized code kind: {s}", .{code_kind_str});
320 }
321
322 var mode: std.builtin.Optimize = .debug;
323 var link_objects = std.array_list.Managed([]const u8).init(arena);
324 var target_str: ?[]const u8 = null;
325 var link_libc = false;
326 var link_mode: ?std.builtin.LinkMode = null;
327 var disable_cache = false;
328 var additional_options = std.array_list.Managed([]const u8).init(arena);
329
330 const source_token = while (true) {
331 const content_tok = try eatToken(tokenizer, .content);
332 _ = try eatToken(tokenizer, .bracket_open);
333 const end_code_tag = try eatToken(tokenizer, .tag_content);
334 const end_tag_name = tokenizer.buffer[end_code_tag.start..end_code_tag.end];
335 if (mem.eql(u8, end_tag_name, "code_release_fast")) {
336 mode = .fast;
337 } else if (mem.eql(u8, end_tag_name, "code_release_safe")) {
338 mode = .safe;
339 } else if (mem.eql(u8, end_tag_name, "code_disable_cache")) {
340 disable_cache = true;
341 } else if (mem.eql(u8, end_tag_name, "code_link_object")) {
342 _ = try eatToken(tokenizer, .separator);
343 const obj_tok = try eatToken(tokenizer, .tag_content);
344 try link_objects.append(tokenizer.buffer[obj_tok.start..obj_tok.end]);
345 } else if (mem.eql(u8, end_tag_name, "target_windows")) {
346 target_str = "x86_64-windows";
347 } else if (mem.eql(u8, end_tag_name, "target_linux_x86_64")) {
348 target_str = "x86_64-linux";
349 } else if (mem.eql(u8, end_tag_name, "target_linux_riscv64")) {
350 target_str = "riscv64-linux";
351 } else if (mem.eql(u8, end_tag_name, "target_wasm")) {
352 target_str = "wasm32-freestanding";
353 } else if (mem.eql(u8, end_tag_name, "target_wasi")) {
354 target_str = "wasm32-wasi";
355 } else if (mem.eql(u8, end_tag_name, "link_libc")) {
356 link_libc = true;
357 } else if (mem.eql(u8, end_tag_name, "link_mode_dynamic")) {
358 link_mode = .dynamic;
359 } else if (mem.eql(u8, end_tag_name, "additonal_option")) {
360 _ = try eatToken(tokenizer, .separator);
361 const option = try eatToken(tokenizer, .tag_content);
362 try additional_options.append(tokenizer.buffer[option.start..option.end]);
363 } else if (mem.eql(u8, end_tag_name, "code_end")) {
364 _ = try eatToken(tokenizer, .bracket_close);
365 break content_tok;
366 } else {
367 return parseError(
368 tokenizer,
369 end_code_tag,
370 "invalid token inside code_begin: {s}",
371 .{end_tag_name},
372 );
373 }
374 _ = try eatToken(tokenizer, .bracket_close);
375 };
376
377 const basename = try std.fmt.allocPrint(arena, "{s}.zig", .{name});
378
379 var file = out_dir.createFile(io, basename, .{ .exclusive = true }) catch |err| {
380 fatal("unable to create file '{s}': {s}", .{ name, @errorName(err) });
381 };
382 defer file.close(io);
383 var file_buffer: [1024]u8 = undefined;
384 var file_writer = file.writer(io, &file_buffer);
385 const code = &file_writer.interface;
386
387 const source = tokenizer.buffer[source_token.start..source_token.end];
388 try code.writeAll(std.mem.trim(u8, source[1..], " \t\r\n"));
389 try code.writeAll("\n\n");
390
391 if (just_check_syntax) {
392 try code.print("// syntax\n", .{});
393 } else switch (code_kind_id) {
394 .@"test" => try code.print("// test\n", .{}),
395 .lib => try code.print("// lib\n", .{}),
396 .test_error => |s| try code.print("// test_error={s}\n", .{s}),
397 .test_safety => |s| try code.print("// test_safety={s}\n", .{s}),
398 .exe => |s| try code.print("// exe={s}\n", .{@tagName(s)}),
399 .obj => |opt| if (opt) |s| {
400 try code.print("// obj={s}\n", .{s});
401 } else {
402 try code.print("// obj\n", .{});
403 },
404 }
405
406 if (mode != .debug)
407 try code.print("// optimize={s}\n", .{@tagName(mode)});
408
409 for (link_objects.items) |link_object| {
410 try code.print("// link_object={s}\n", .{link_object});
411 }
412
413 if (target_str) |s|
414 try code.print("// target={s}\n", .{s});
415
416 if (link_libc) try code.print("// link_libc\n", .{});
417 if (disable_cache) try code.print("// disable_cache\n", .{});
418
419 if (link_mode) |m|
420 try code.print("// link_mode={s}\n", .{@tagName(m)});
421
422 for (additional_options.items) |o| {
423 try code.print("// additional_option={s}\n", .{o});
424 }
425 try code.flush();
426 try w.print("{{#code|{s}#}}\n", .{basename});
427 } else {
428 const close_bracket = while (true) {
429 const next = tokenizer.next();
430 if (next.id == .bracket_close) break next;
431 };
432 try w.writeAll(tokenizer.buffer[token.start..close_bracket.end]);
433 }
434 },
435 else => return parseError(tokenizer, token, "invalid token", .{}),
436 }
437 }
438}
439
440fn urlize(allocator: Allocator, input: []const u8) ![]u8 {
441 var buf = std.array_list.Managed(u8).init(allocator);
442 defer buf.deinit();
443
444 const out = buf.writer();
445 for (input) |c| {
446 switch (c) {
447 'a'...'z', 'A'...'Z', '_', '-', '0'...'9' => {
448 try out.writeByte(c);
449 },
450 ' ' => {
451 try out.writeByte('-');
452 },
453 else => {},
454 }
455 }
456 return try buf.toOwnedSlice();
457}