authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-05 20:19:36-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:29-07:00
log576702ca40cef65964709ad70595ad913f1944ab
tree6799082eb1b0dc5c674d215696e22d9ec61bf736
parentaf4bb996f0e1f022c25a28db021a536b92e80bf3

get autodocs wasm compiling again

it's triggering an error.EndOfStream error in the browser tho

11 files changed, 591 insertions(+), 579 deletions(-)

lib/compiler/std-docs.zig+4-2
...@@ -334,7 +334,7 @@ fn buildWasmBinary(...@@ -334,7 +334,7 @@ fn buildWasmBinary(
334 var result_error_bundle = std.zig.ErrorBundle.empty;334 var result_error_bundle = std.zig.ErrorBundle.empty;
335335
336 while (true) {336 while (true) {
337 receiveWasmMessage(arena, context, poller.reader(.stdout), &result, &result_error_bundle) catch |err| switch (err) {337 receiveWasmMessage(gpa, arena, context, poller.reader(.stdout), &result, &result_error_bundle) catch |err| switch (err) {
338 error.EndOfStream => break,338 error.EndOfStream => break,
339 error.ReadFailed => if (!(try poller.poll())) break,339 error.ReadFailed => if (!(try poller.poll())) break,
340 else => |e| return e,340 else => |e| return e,
...@@ -387,6 +387,7 @@ fn buildWasmBinary(...@@ -387,6 +387,7 @@ fn buildWasmBinary(
387}387}
388388
389fn receiveWasmMessage(389fn receiveWasmMessage(
390 gpa: Allocator,
390 arena: Allocator,391 arena: Allocator,
391 context: *Context,392 context: *Context,
392 br: *std.io.BufferedReader,393 br: *std.io.BufferedReader,
...@@ -394,8 +395,9 @@ fn receiveWasmMessage(...@@ -394,8 +395,9 @@ fn receiveWasmMessage(
394 result_error_bundle: *std.zig.ErrorBundle,395 result_error_bundle: *std.zig.ErrorBundle,
395) !void {396) !void {
396 // Ensure that we will be able to read the entire message without blocking.397 // Ensure that we will be able to read the entire message without blocking.
398 try br.fillAlloc(gpa, @sizeOf(std.zig.Server.Message.Header));
397 const header = try br.peekStructEndian(std.zig.Server.Message.Header, .little);399 const header = try br.peekStructEndian(std.zig.Server.Message.Header, .little);
398 try br.fill(@sizeOf(std.zig.Server.Message.Header) + header.bytes_len);400 try br.fillAlloc(gpa, @sizeOf(std.zig.Server.Message.Header) + header.bytes_len);
399 br.toss(@sizeOf(std.zig.Server.Message.Header));401 br.toss(@sizeOf(std.zig.Server.Message.Header));
400 switch (header.tag) {402 switch (header.tag) {
401 .zig_version => {403 .zig_version => {
lib/docs/wasm/Walk.zig+7-1
...@@ -443,7 +443,13 @@ fn parse(file_name: []const u8, source: []u8) Oom!Ast {...@@ -443,7 +443,13 @@ fn parse(file_name: []const u8, source: []u8) Oom!Ast {
443 const err_offset = token_offsets[err.token] + ast.errorOffset(err);443 const err_offset = token_offsets[err.token] + ast.errorOffset(err);
444 const err_loc = std.zig.findLineColumn(ast.source, err_offset);444 const err_loc = std.zig.findLineColumn(ast.source, err_offset);
445 rendered_err.clearRetainingCapacity();445 rendered_err.clearRetainingCapacity();
446 try ast.renderError(err, rendered_err.writer(gpa));446 {
447 var aw: std.io.AllocatingWriter = undefined;
448 defer rendered_err = aw.toArrayList();
449 ast.renderError(err, aw.fromArrayList(gpa, &rendered_err)) catch |e| switch (e) {
450 error.WriteFailed => return error.OutOfMemory,
451 };
452 }
447 log.err("{s}:{}:{}: {s}", .{ file_name, err_loc.line + 1, err_loc.column + 1, rendered_err.items });453 log.err("{s}:{}:{}: {s}", .{ file_name, err_loc.line + 1, err_loc.column + 1, rendered_err.items });
448 }454 }
449 return Ast.parse(gpa, "", .zig);455 return Ast.parse(gpa, "", .zig);
lib/docs/wasm/html_render.zig+1-1
...@@ -88,7 +88,7 @@ pub fn fileSourceHtml(...@@ -88,7 +88,7 @@ pub fn fileSourceHtml(
88 if (next_annotate_index >= options.source_location_annotations.len) break;88 if (next_annotate_index >= options.source_location_annotations.len) break;
89 const next_annotation = options.source_location_annotations[next_annotate_index];89 const next_annotation = options.source_location_annotations[next_annotate_index];
90 if (cursor <= next_annotation.file_byte_offset) break;90 if (cursor <= next_annotation.file_byte_offset) break;
91 try out.writer(gpa).print("<span id=\"{s}{d}\"></span>", .{91 try out.print(gpa, "<span id=\"{s}{d}\"></span>", .{
92 options.annotation_prefix, next_annotation.dom_id,92 options.annotation_prefix, next_annotation.dom_id,
93 });93 });
94 next_annotate_index += 1;94 next_annotate_index += 1;
lib/docs/wasm/main.zig+22-15
...@@ -694,43 +694,49 @@ fn render_docs(...@@ -694,43 +694,49 @@ fn render_docs(
694 var link_buffer: std.ArrayListUnmanaged(u8) = .empty;694 var link_buffer: std.ArrayListUnmanaged(u8) = .empty;
695 };695 };
696696
697 const Writer = std.ArrayListUnmanaged(u8).Writer;697 var render: markdown.Render = .{
698 const Renderer = markdown.Renderer(Writer, Decl.Index);698 .context = &decl_index,
699 const renderer: Renderer = .{
700 .context = decl_index,
701 .renderFn = struct {699 .renderFn = struct {
700 const fmtHtml = markdown.Render.fmtHtml;
702 fn render(701 fn render(
703 r: Renderer,702 r: markdown.Render,
704 doc: markdown.Document,703 doc: markdown.Document,
705 node: markdown.Document.Node.Index,704 node: markdown.Document.Node.Index,
706 writer: Writer,705 writer: *std.io.BufferedWriter,
707 ) !void {706 ) !void {
707 const decl_index_ptr: *const Decl.Index = @alignCast(@ptrCast(r.context));
708 const data = doc.nodes.items(.data)[@intFromEnum(node)];708 const data = doc.nodes.items(.data)[@intFromEnum(node)];
709 switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {709 switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
710 .code_span => {710 .code_span => {
711 try writer.writeAll("<code>");711 try writer.writeAll("<code>");
712 const content = doc.string(data.text.content);712 const content = doc.string(data.text.content);
713 if (resolve_decl_path(r.context, content)) |resolved_decl_index| {713 if (resolve_decl_path(decl_index_ptr.*, content)) |resolved_decl_index| {
714 g.link_buffer.clearRetainingCapacity();714 g.link_buffer.clearRetainingCapacity();
715 try resolveDeclLink(resolved_decl_index, &g.link_buffer);715 resolveDeclLink(resolved_decl_index, &g.link_buffer) catch |err| switch (err) {
716 error.OutOfMemory => return error.WriteFailed,
717 };
716718
717 try writer.writeAll("<a href=\"#");719 try writer.writeAll("<a href=\"#");
718 _ = missing_feature_url_escape;720 _ = missing_feature_url_escape;
719 try writer.writeAll(g.link_buffer.items);721 try writer.writeAll(g.link_buffer.items);
720 try writer.print("\">{}</a>", .{markdown.fmtHtml(content)});722 try writer.print("\">{f}</a>", .{fmtHtml(content)});
721 } else {723 } else {
722 try writer.print("{}", .{markdown.fmtHtml(content)});724 try writer.print("{f}", .{fmtHtml(content)});
723 }725 }
724726
725 try writer.writeAll("</code>");727 try writer.writeAll("</code>");
726 },728 },
727729
728 else => try Renderer.renderDefault(r, doc, node, writer),730 else => try markdown.Render.renderDefault(r, doc, node, writer),
729 }731 }
730 }732 }
731 }.render,733 }.render,
732 };734 };
733 try renderer.render(parsed_doc, out.writer(gpa));735 var aw: std.io.AllocatingWriter = undefined;
736 defer out.* = aw.toArrayList();
737 render.render(parsed_doc, aw.fromArrayList(gpa, out)) catch |err| switch (err) {
738 error.WriteFailed => return error.OutOfMemory,
739 };
734}740}
735741
736fn resolve_decl_path(decl_index: Decl.Index, path: []const u8) ?Decl.Index {742fn resolve_decl_path(decl_index: Decl.Index, path: []const u8) ?Decl.Index {
...@@ -772,10 +778,11 @@ export fn decl_type_html(decl_index: Decl.Index) String {...@@ -772,10 +778,11 @@ export fn decl_type_html(decl_index: Decl.Index) String {
772const Oom = error{OutOfMemory};778const Oom = error{OutOfMemory};
773779
774fn unpackInner(tar_bytes: []u8) !void {780fn unpackInner(tar_bytes: []u8) !void {
775 var fbs = std.io.fixedBufferStream(tar_bytes);781 var br: std.io.BufferedReader = undefined;
782 br.initFixed(tar_bytes);
776 var file_name_buffer: [1024]u8 = undefined;783 var file_name_buffer: [1024]u8 = undefined;
777 var link_name_buffer: [1024]u8 = undefined;784 var link_name_buffer: [1024]u8 = undefined;
778 var it = std.tar.iterator(fbs.reader(), .{785 var it = std.tar.Iterator.init(&br, .{
779 .file_name_buffer = &file_name_buffer,786 .file_name_buffer = &file_name_buffer,
780 .link_name_buffer = &link_name_buffer,787 .link_name_buffer = &link_name_buffer,
781 });788 });
...@@ -796,7 +803,7 @@ fn unpackInner(tar_bytes: []u8) !void {...@@ -796,7 +803,7 @@ fn unpackInner(tar_bytes: []u8) !void {
796 {803 {
797 gop.value_ptr.* = file;804 gop.value_ptr.* = file;
798 }805 }
799 const file_bytes = tar_bytes[fbs.pos..][0..@intCast(tar_file.size)];806 const file_bytes = tar_bytes[br.seek..][0..@intCast(tar_file.size)];
800 assert(file == try Walk.add_file(file_name, file_bytes));807 assert(file == try Walk.add_file(file_name, file_bytes));
801 }808 }
802 } else {809 } else {
lib/docs/wasm/markdown.zig+1-3
...@@ -129,9 +129,7 @@ const testing = std.testing;...@@ -129,9 +129,7 @@ const testing = std.testing;
129129
130pub const Document = @import("markdown/Document.zig");130pub const Document = @import("markdown/Document.zig");
131pub const Parser = @import("markdown/Parser.zig");131pub const Parser = @import("markdown/Parser.zig");
132pub const Renderer = @import("markdown/renderer.zig").Renderer;132pub const Render = @import("markdown/Render.zig");
133pub const renderNodeInlineText = @import("markdown/renderer.zig").renderNodeInlineText;
134pub const fmtHtml = @import("markdown/renderer.zig").fmtHtml;
135133
136// Avoid exposing main to other files merely importing this one.134// Avoid exposing main to other files merely importing this one.
137pub const main = if (@import("root") == @This())135pub const main = if (@import("root") == @This())
lib/docs/wasm/markdown/Document.zig+3-3
...@@ -4,7 +4,7 @@ const std = @import("std");...@@ -4,7 +4,7 @@ const std = @import("std");
4const builtin = @import("builtin");4const builtin = @import("builtin");
5const assert = std.debug.assert;5const assert = std.debug.assert;
6const Allocator = std.mem.Allocator;6const Allocator = std.mem.Allocator;
7const Renderer = @import("renderer.zig").Renderer;7const Render = @import("Render.zig");
88
9nodes: Node.List.Slice,9nodes: Node.List.Slice,
10extra: []u32,10extra: []u32,
...@@ -160,8 +160,8 @@ pub fn deinit(doc: *Document, allocator: Allocator) void {...@@ -160,8 +160,8 @@ pub fn deinit(doc: *Document, allocator: Allocator) void {
160}160}
161161
162/// Renders a document directly to a writer using the default renderer.162/// Renders a document directly to a writer using the default renderer.
163pub fn render(doc: Document, writer: anytype) @TypeOf(writer).Error!void {163pub fn render(doc: Document, writer: *std.io.BufferedWriter) @TypeOf(writer).Error!void {
164 const renderer: Renderer(@TypeOf(writer), void) = .{ .context = {} };164 const renderer: Render(@TypeOf(writer), void) = .{ .context = {} };
165 try renderer.render(doc, writer);165 try renderer.render(doc, writer);
166}166}
167167
lib/docs/wasm/markdown/Render.zig created+243
...@@ -0,0 +1,243 @@
1//! A Markdown document renderer.
2//!
3//! Each concrete `Renderer` type has a `renderDefault` function, with the
4//! intention that custom `renderFn` implementations can call `renderDefault`
5//! for node types for which they require no special rendering.
6
7const std = @import("std");
8const Document = @import("Document.zig");
9const Node = Document.Node;
10const Render = @This();
11
12context: ?*const anyopaque,
13renderFn: *const fn (
14 r: Render,
15 doc: Document,
16 node: Node.Index,
17 writer: *std.io.BufferedWriter,
18) std.io.Writer.Error!void = renderDefault,
19
20pub fn render(r: Render, doc: Document, writer: *std.io.BufferedWriter) std.io.Writer.Error!void {
21 try r.renderFn(r, doc, .root, writer);
22}
23
24pub fn renderDefault(
25 r: Render,
26 doc: Document,
27 node: Node.Index,
28 writer: *std.io.BufferedWriter,
29) std.io.Writer.Error!void {
30 const data = doc.nodes.items(.data)[@intFromEnum(node)];
31 switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
32 .root => {
33 for (doc.extraChildren(data.container.children)) |child| {
34 try r.renderFn(r, doc, child, writer);
35 }
36 },
37 .list => {
38 if (data.list.start.asNumber()) |start| {
39 if (start == 1) {
40 try writer.writeAll("<ol>\n");
41 } else {
42 try writer.print("<ol start=\"{}\">\n", .{start});
43 }
44 } else {
45 try writer.writeAll("<ul>\n");
46 }
47 for (doc.extraChildren(data.list.children)) |child| {
48 try r.renderFn(r, doc, child, writer);
49 }
50 if (data.list.start.asNumber() != null) {
51 try writer.writeAll("</ol>\n");
52 } else {
53 try writer.writeAll("</ul>\n");
54 }
55 },
56 .list_item => {
57 try writer.writeAll("<li>");
58 for (doc.extraChildren(data.list_item.children)) |child| {
59 if (data.list_item.tight and doc.nodes.items(.tag)[@intFromEnum(child)] == .paragraph) {
60 const para_data = doc.nodes.items(.data)[@intFromEnum(child)];
61 for (doc.extraChildren(para_data.container.children)) |para_child| {
62 try r.renderFn(r, doc, para_child, writer);
63 }
64 } else {
65 try r.renderFn(r, doc, child, writer);
66 }
67 }
68 try writer.writeAll("</li>\n");
69 },
70 .table => {
71 try writer.writeAll("<table>\n");
72 for (doc.extraChildren(data.container.children)) |child| {
73 try r.renderFn(r, doc, child, writer);
74 }
75 try writer.writeAll("</table>\n");
76 },
77 .table_row => {
78 try writer.writeAll("<tr>\n");
79 for (doc.extraChildren(data.container.children)) |child| {
80 try r.renderFn(r, doc, child, writer);
81 }
82 try writer.writeAll("</tr>\n");
83 },
84 .table_cell => {
85 if (data.table_cell.info.header) {
86 try writer.writeAll("<th");
87 } else {
88 try writer.writeAll("<td");
89 }
90 switch (data.table_cell.info.alignment) {
91 .unset => try writer.writeAll(">"),
92 else => |a| try writer.print(" style=\"text-align: {s}\">", .{@tagName(a)}),
93 }
94
95 for (doc.extraChildren(data.table_cell.children)) |child| {
96 try r.renderFn(r, doc, child, writer);
97 }
98
99 if (data.table_cell.info.header) {
100 try writer.writeAll("</th>\n");
101 } else {
102 try writer.writeAll("</td>\n");
103 }
104 },
105 .heading => {
106 try writer.print("<h{}>", .{data.heading.level});
107 for (doc.extraChildren(data.heading.children)) |child| {
108 try r.renderFn(r, doc, child, writer);
109 }
110 try writer.print("</h{}>\n", .{data.heading.level});
111 },
112 .code_block => {
113 const content = doc.string(data.code_block.content);
114 try writer.print("<pre><code>{f}</code></pre>\n", .{fmtHtml(content)});
115 },
116 .blockquote => {
117 try writer.writeAll("<blockquote>\n");
118 for (doc.extraChildren(data.container.children)) |child| {
119 try r.renderFn(r, doc, child, writer);
120 }
121 try writer.writeAll("</blockquote>\n");
122 },
123 .paragraph => {
124 try writer.writeAll("<p>");
125 for (doc.extraChildren(data.container.children)) |child| {
126 try r.renderFn(r, doc, child, writer);
127 }
128 try writer.writeAll("</p>\n");
129 },
130 .thematic_break => {
131 try writer.writeAll("<hr />\n");
132 },
133 .link => {
134 const target = doc.string(data.link.target);
135 try writer.print("<a href=\"{f}\">", .{fmtHtml(target)});
136 for (doc.extraChildren(data.link.children)) |child| {
137 try r.renderFn(r, doc, child, writer);
138 }
139 try writer.writeAll("</a>");
140 },
141 .autolink => {
142 const target = doc.string(data.text.content);
143 try writer.print("<a href=\"{0f}\">{0f}</a>", .{fmtHtml(target)});
144 },
145 .image => {
146 const target = doc.string(data.link.target);
147 try writer.print("<img src=\"{f}\" alt=\"", .{fmtHtml(target)});
148 for (doc.extraChildren(data.link.children)) |child| {
149 try renderInlineNodeText(doc, child, writer);
150 }
151 try writer.writeAll("\" />");
152 },
153 .strong => {
154 try writer.writeAll("<strong>");
155 for (doc.extraChildren(data.container.children)) |child| {
156 try r.renderFn(r, doc, child, writer);
157 }
158 try writer.writeAll("</strong>");
159 },
160 .emphasis => {
161 try writer.writeAll("<em>");
162 for (doc.extraChildren(data.container.children)) |child| {
163 try r.renderFn(r, doc, child, writer);
164 }
165 try writer.writeAll("</em>");
166 },
167 .code_span => {
168 const content = doc.string(data.text.content);
169 try writer.print("<code>{f}</code>", .{fmtHtml(content)});
170 },
171 .text => {
172 const content = doc.string(data.text.content);
173 try writer.print("{f}", .{fmtHtml(content)});
174 },
175 .line_break => {
176 try writer.writeAll("<br />\n");
177 },
178 }
179}
180
181/// Renders an inline node as plain text. Asserts that the node is an inline and
182/// has no non-inline children.
183pub fn renderInlineNodeText(
184 doc: Document,
185 node: Node.Index,
186 writer: *std.io.BufferedWriter,
187) std.io.Writer.Error!void {
188 const data = doc.nodes.items(.data)[@intFromEnum(node)];
189 switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
190 .root,
191 .list,
192 .list_item,
193 .table,
194 .table_row,
195 .table_cell,
196 .heading,
197 .code_block,
198 .blockquote,
199 .paragraph,
200 .thematic_break,
201 => unreachable, // Blocks
202
203 .link, .image => {
204 for (doc.extraChildren(data.link.children)) |child| {
205 try renderInlineNodeText(doc, child, writer);
206 }
207 },
208 .strong => {
209 for (doc.extraChildren(data.container.children)) |child| {
210 try renderInlineNodeText(doc, child, writer);
211 }
212 },
213 .emphasis => {
214 for (doc.extraChildren(data.container.children)) |child| {
215 try renderInlineNodeText(doc, child, writer);
216 }
217 },
218 .autolink, .code_span, .text => {
219 const content = doc.string(data.text.content);
220 try writer.print("{f}", .{fmtHtml(content)});
221 },
222 .line_break => {
223 try writer.writeAll("\n");
224 },
225 }
226}
227
228pub fn fmtHtml(bytes: []const u8) std.fmt.Formatter(formatHtml) {
229 return .{ .data = bytes };
230}
231
232fn formatHtml(bytes: []const u8, writer: *std.io.BufferedWriter, comptime fmt: []const u8) !void {
233 _ = fmt;
234 for (bytes) |b| {
235 switch (b) {
236 '<' => try writer.writeAll("&lt;"),
237 '>' => try writer.writeAll("&gt;"),
238 '&' => try writer.writeAll("&amp;"),
239 '"' => try writer.writeAll("&quot;"),
240 else => try writer.writeByte(b),
241 }
242 }
243}
lib/docs/wasm/markdown/renderer.zig deleted-253
...@@ -1,253 +0,0 @@
1const std = @import("std");
2const Document = @import("Document.zig");
3const Node = Document.Node;
4
5/// A Markdown document renderer.
6///
7/// Each concrete `Renderer` type has a `renderDefault` function, with the
8/// intention that custom `renderFn` implementations can call `renderDefault`
9/// for node types for which they require no special rendering.
10pub fn Renderer(comptime Writer: type, comptime Context: type) type {
11 return struct {
12 renderFn: *const fn (
13 r: Self,
14 doc: Document,
15 node: Node.Index,
16 writer: Writer,
17 ) Writer.Error!void = renderDefault,
18 context: Context,
19
20 const Self = @This();
21
22 pub fn render(r: Self, doc: Document, writer: Writer) Writer.Error!void {
23 try r.renderFn(r, doc, .root, writer);
24 }
25
26 pub fn renderDefault(
27 r: Self,
28 doc: Document,
29 node: Node.Index,
30 writer: Writer,
31 ) Writer.Error!void {
32 const data = doc.nodes.items(.data)[@intFromEnum(node)];
33 switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
34 .root => {
35 for (doc.extraChildren(data.container.children)) |child| {
36 try r.renderFn(r, doc, child, writer);
37 }
38 },
39 .list => {
40 if (data.list.start.asNumber()) |start| {
41 if (start == 1) {
42 try writer.writeAll("<ol>\n");
43 } else {
44 try writer.print("<ol start=\"{}\">\n", .{start});
45 }
46 } else {
47 try writer.writeAll("<ul>\n");
48 }
49 for (doc.extraChildren(data.list.children)) |child| {
50 try r.renderFn(r, doc, child, writer);
51 }
52 if (data.list.start.asNumber() != null) {
53 try writer.writeAll("</ol>\n");
54 } else {
55 try writer.writeAll("</ul>\n");
56 }
57 },
58 .list_item => {
59 try writer.writeAll("<li>");
60 for (doc.extraChildren(data.list_item.children)) |child| {
61 if (data.list_item.tight and doc.nodes.items(.tag)[@intFromEnum(child)] == .paragraph) {
62 const para_data = doc.nodes.items(.data)[@intFromEnum(child)];
63 for (doc.extraChildren(para_data.container.children)) |para_child| {
64 try r.renderFn(r, doc, para_child, writer);
65 }
66 } else {
67 try r.renderFn(r, doc, child, writer);
68 }
69 }
70 try writer.writeAll("</li>\n");
71 },
72 .table => {
73 try writer.writeAll("<table>\n");
74 for (doc.extraChildren(data.container.children)) |child| {
75 try r.renderFn(r, doc, child, writer);
76 }
77 try writer.writeAll("</table>\n");
78 },
79 .table_row => {
80 try writer.writeAll("<tr>\n");
81 for (doc.extraChildren(data.container.children)) |child| {
82 try r.renderFn(r, doc, child, writer);
83 }
84 try writer.writeAll("</tr>\n");
85 },
86 .table_cell => {
87 if (data.table_cell.info.header) {
88 try writer.writeAll("<th");
89 } else {
90 try writer.writeAll("<td");
91 }
92 switch (data.table_cell.info.alignment) {
93 .unset => try writer.writeAll(">"),
94 else => |a| try writer.print(" style=\"text-align: {s}\">", .{@tagName(a)}),
95 }
96
97 for (doc.extraChildren(data.table_cell.children)) |child| {
98 try r.renderFn(r, doc, child, writer);
99 }
100
101 if (data.table_cell.info.header) {
102 try writer.writeAll("</th>\n");
103 } else {
104 try writer.writeAll("</td>\n");
105 }
106 },
107 .heading => {
108 try writer.print("<h{}>", .{data.heading.level});
109 for (doc.extraChildren(data.heading.children)) |child| {
110 try r.renderFn(r, doc, child, writer);
111 }
112 try writer.print("</h{}>\n", .{data.heading.level});
113 },
114 .code_block => {
115 const content = doc.string(data.code_block.content);
116 try writer.print("<pre><code>{}</code></pre>\n", .{fmtHtml(content)});
117 },
118 .blockquote => {
119 try writer.writeAll("<blockquote>\n");
120 for (doc.extraChildren(data.container.children)) |child| {
121 try r.renderFn(r, doc, child, writer);
122 }
123 try writer.writeAll("</blockquote>\n");
124 },
125 .paragraph => {
126 try writer.writeAll("<p>");
127 for (doc.extraChildren(data.container.children)) |child| {
128 try r.renderFn(r, doc, child, writer);
129 }
130 try writer.writeAll("</p>\n");
131 },
132 .thematic_break => {
133 try writer.writeAll("<hr />\n");
134 },
135 .link => {
136 const target = doc.string(data.link.target);
137 try writer.print("<a href=\"{}\">", .{fmtHtml(target)});
138 for (doc.extraChildren(data.link.children)) |child| {
139 try r.renderFn(r, doc, child, writer);
140 }
141 try writer.writeAll("</a>");
142 },
143 .autolink => {
144 const target = doc.string(data.text.content);
145 try writer.print("<a href=\"{0}\">{0}</a>", .{fmtHtml(target)});
146 },
147 .image => {
148 const target = doc.string(data.link.target);
149 try writer.print("<img src=\"{}\" alt=\"", .{fmtHtml(target)});
150 for (doc.extraChildren(data.link.children)) |child| {
151 try renderInlineNodeText(doc, child, writer);
152 }
153 try writer.writeAll("\" />");
154 },
155 .strong => {
156 try writer.writeAll("<strong>");
157 for (doc.extraChildren(data.container.children)) |child| {
158 try r.renderFn(r, doc, child, writer);
159 }
160 try writer.writeAll("</strong>");
161 },
162 .emphasis => {
163 try writer.writeAll("<em>");
164 for (doc.extraChildren(data.container.children)) |child| {
165 try r.renderFn(r, doc, child, writer);
166 }
167 try writer.writeAll("</em>");
168 },
169 .code_span => {
170 const content = doc.string(data.text.content);
171 try writer.print("<code>{}</code>", .{fmtHtml(content)});
172 },
173 .text => {
174 const content = doc.string(data.text.content);
175 try writer.print("{}", .{fmtHtml(content)});
176 },
177 .line_break => {
178 try writer.writeAll("<br />\n");
179 },
180 }
181 }
182 };
183}
184
185/// Renders an inline node as plain text. Asserts that the node is an inline and
186/// has no non-inline children.
187pub fn renderInlineNodeText(
188 doc: Document,
189 node: Node.Index,
190 writer: anytype,
191) @TypeOf(writer).Error!void {
192 const data = doc.nodes.items(.data)[@intFromEnum(node)];
193 switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
194 .root,
195 .list,
196 .list_item,
197 .table,
198 .table_row,
199 .table_cell,
200 .heading,
201 .code_block,
202 .blockquote,
203 .paragraph,
204 .thematic_break,
205 => unreachable, // Blocks
206
207 .link, .image => {
208 for (doc.extraChildren(data.link.children)) |child| {
209 try renderInlineNodeText(doc, child, writer);
210 }
211 },
212 .strong => {
213 for (doc.extraChildren(data.container.children)) |child| {
214 try renderInlineNodeText(doc, child, writer);
215 }
216 },
217 .emphasis => {
218 for (doc.extraChildren(data.container.children)) |child| {
219 try renderInlineNodeText(doc, child, writer);
220 }
221 },
222 .autolink, .code_span, .text => {
223 const content = doc.string(data.text.content);
224 try writer.print("{}", .{fmtHtml(content)});
225 },
226 .line_break => {
227 try writer.writeAll("\n");
228 },
229 }
230}
231
232pub fn fmtHtml(bytes: []const u8) std.fmt.Formatter(formatHtml) {
233 return .{ .data = bytes };
234}
235
236fn formatHtml(
237 bytes: []const u8,
238 comptime fmt: []const u8,
239 options: std.fmt.FormatOptions,
240 writer: anytype,
241) !void {
242 _ = fmt;
243 _ = options;
244 for (bytes) |b| {
245 switch (b) {
246 '<' => try writer.writeAll("&lt;"),
247 '>' => try writer.writeAll("&gt;"),
248 '&' => try writer.writeAll("&amp;"),
249 '"' => try writer.writeAll("&quot;"),
250 else => try writer.writeByte(b),
251 }
252 }
253}
lib/std/io/AllocatingWriter.zig+1
...@@ -166,6 +166,7 @@ fn writeFile(...@@ -166,6 +166,7 @@ fn writeFile(
166 headers_and_trailers_full: []const []const u8,166 headers_and_trailers_full: []const []const u8,
167 headers_len_full: usize,167 headers_len_full: usize,
168) std.io.Writer.FileError!usize {168) std.io.Writer.FileError!usize {
169 if (std.fs.File.Handle == void) unreachable;
169 const aw: *AllocatingWriter = @alignCast(@ptrCast(context));170 const aw: *AllocatingWriter = @alignCast(@ptrCast(context));
170 const gpa = aw.allocator;171 const gpa = aw.allocator;
171 var list = aw.toArrayList();172 var list = aw.toArrayList();
lib/std/io/BufferedReader.zig+29-13
...@@ -316,6 +316,15 @@ pub fn discardAll(br: *BufferedReader, n: usize) Reader.Error!void {...@@ -316,6 +316,15 @@ pub fn discardAll(br: *BufferedReader, n: usize) Reader.Error!void {
316 if ((try br.discardShort(n)) != n) return error.EndOfStream;316 if ((try br.discardShort(n)) != n) return error.EndOfStream;
317}317}
318318
319pub fn discardAll64(br: *BufferedReader, n: u64) Reader.Error!void {
320 var remaining: u64 = n;
321 while (remaining > 0) {
322 const limited = std.math.cast(usize, remaining) orelse std.math.maxInt(usize);
323 try discardAll(br, limited);
324 remaining -= limited;
325 }
326}
327
319/// Skips the next `n` bytes from the stream, advancing the seek position.328/// Skips the next `n` bytes from the stream, advancing the seek position.
320///329///
321/// Unlike `toss` which is infallible, in this function `n` can be any amount.330/// Unlike `toss` which is infallible, in this function `n` can be any amount.
...@@ -896,28 +905,35 @@ pub fn takeLeb128(br: *BufferedReader, comptime Result: type) TakeLeb128Error!Re...@@ -896,28 +905,35 @@ pub fn takeLeb128(br: *BufferedReader, comptime Result: type) TakeLeb128Error!Re
896 } }))) orelse error.Overflow;905 } }))) orelse error.Overflow;
897}906}
898907
908pub fn expandTotalCapacity(br: *BufferedReader, allocator: Allocator, n: usize) Allocator.Error!void {
909 if (n <= br.buffer.len) return;
910 if (br.seek > 0) rebase(br);
911 var list: ArrayList(u8) = .{
912 .items = br.buffer[0..br.end],
913 .capacity = br.buffer.len,
914 };
915 defer br.buffer = list.allocatedSlice();
916 try list.ensureTotalCapacity(allocator, n);
917}
918
919pub const FillAllocError = Reader.Error || Allocator.Error;
920
921pub fn fillAlloc(br: *BufferedReader, allocator: Allocator, n: usize) FillAllocError!void {
922 try expandTotalCapacity(br, allocator, n);
923 return fill(br, n);
924}
925
899/// Returns a slice into the unused capacity of `buffer` with at least926/// Returns a slice into the unused capacity of `buffer` with at least
900/// `min_len` bytes, extending `buffer` by resizing it with `gpa` as necessary.927/// `min_len` bytes, extending `buffer` by resizing it with `gpa` as necessary.
901///928///
902/// After calling this function, typically the caller will follow up with a929/// After calling this function, typically the caller will follow up with a
903/// call to `advanceBufferEnd` to report the actual number of bytes buffered.930/// call to `advanceBufferEnd` to report the actual number of bytes buffered.
904pub fn writableSliceGreedyAlloc(931pub fn writableSliceGreedyAlloc(br: *BufferedReader, allocator: Allocator, min_len: usize) Allocator.Error![]u8 {
905 br: *BufferedReader,
906 allocator: Allocator,
907 min_len: usize,
908) error{OutOfMemory}![]u8 {
909 {932 {
910 const unused = br.buffer[br.end..];933 const unused = br.buffer[br.end..];
911 if (unused.len >= min_len) return unused;934 if (unused.len >= min_len) return unused;
912 }935 }
913 const seek = br.seek;936 if (br.seek > 0) rebase(br);
914 if (seek > 0) {
915 const buffer = br.buffer[0..br.end];
916 const remainder = buffer[seek..];
917 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
918 br.end = remainder.len;
919 br.seek = 0;
920 }
921 {937 {
922 var list: ArrayList(u8) = .{938 var list: ArrayList(u8) = .{
923 .items = br.buffer[0..br.end],939 .items = br.buffer[0..br.end],
lib/std/tar.zig+280-288
...@@ -293,28 +293,6 @@ fn nullStr(str: []const u8) []const u8 {...@@ -293,28 +293,6 @@ fn nullStr(str: []const u8) []const u8 {
293 return str;293 return str;
294}294}
295295
296/// Options for iterator.
297/// Buffers should be provided by the caller.
298pub const IteratorOptions = struct {
299 /// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
300 file_name_buffer: []u8,
301 /// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
302 link_name_buffer: []u8,
303 /// Collects error messages during unpacking
304 diagnostics: ?*Diagnostics = null,
305};
306
307/// Iterates over files in tar archive.
308/// `next` returns each file in tar archive.
309pub fn iterator(reader: anytype, options: IteratorOptions) Iterator(@TypeOf(reader)) {
310 return .{
311 .reader = reader,
312 .diagnostics = options.diagnostics,
313 .file_name_buffer = options.file_name_buffer,
314 .link_name_buffer = options.link_name_buffer,
315 };
316}
317
318/// Type of the file returned by iterator `next` method.296/// Type of the file returned by iterator `next` method.
319pub const FileKind = enum {297pub const FileKind = enum {
320 directory,298 directory,
...@@ -323,206 +301,230 @@ pub const FileKind = enum {...@@ -323,206 +301,230 @@ pub const FileKind = enum {
323};301};
324302
325/// Iterator over entries in the tar file represented by reader.303/// Iterator over entries in the tar file represented by reader.
326pub fn Iterator(comptime ReaderType: type) type {304pub const Iterator = struct {
327 return struct {305 reader: *std.io.BufferedReader,
328 reader: ReaderType,306 diagnostics: ?*Diagnostics = null,
329 diagnostics: ?*Diagnostics = null,
330
331 // buffers for heeader and file attributes
332 header_buffer: [Header.SIZE]u8 = undefined,
333 file_name_buffer: []u8,
334 link_name_buffer: []u8,
335
336 // bytes of padding to the end of the block
337 padding: usize = 0,
338 // not consumed bytes of file from last next iteration
339 unread_file_bytes: u64 = 0,
340307
341 pub const File = struct {308 // buffers for heeader and file attributes
342 name: []const u8, // name of file, symlink or directory309 header_buffer: [Header.SIZE]u8 = undefined,
343 link_name: []const u8, // target name of symlink310 file_name_buffer: []u8,
344 size: u64 = 0, // size of the file in bytes311 link_name_buffer: []u8,
345 mode: u32 = 0,
346 kind: FileKind = .file,
347312
348 unread_bytes: *u64,313 // bytes of padding to the end of the block
349 parent_reader: ReaderType,314 padding: usize = 0,
315 // not consumed bytes of file from last next iteration
316 unread_file_bytes: u64 = 0,
350317
351 pub const Reader = std.io.Reader(File, ReaderType.Error, File.read);318 /// Options for iterator.
319 /// Buffers should be provided by the caller.
320 pub const Options = struct {
321 /// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
322 file_name_buffer: []u8,
323 /// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
324 link_name_buffer: []u8,
325 /// Collects error messages during unpacking
326 diagnostics: ?*Diagnostics = null,
327 };
352328
353 pub fn reader(self: File) Reader {329 /// Iterates over files in tar archive.
354 return .{ .context = self };330 /// `next` returns each file in tar archive.
355 }331 pub fn init(reader: *std.io.BufferedReader, options: Options) Iterator {
332 return .{
333 .reader = reader,
334 .diagnostics = options.diagnostics,
335 .file_name_buffer = options.file_name_buffer,
336 .link_name_buffer = options.link_name_buffer,
337 };
338 }
356339
357 pub fn read(self: File, dest: []u8) ReaderType.Error!usize {340 pub const File = struct {
358 const buf = dest[0..@min(dest.len, self.unread_bytes.*)];341 name: []const u8, // name of file, symlink or directory
359 const n = try self.parent_reader.read(buf);342 link_name: []const u8, // target name of symlink
360 self.unread_bytes.* -= n;343 size: u64 = 0, // size of the file in bytes
361 return n;344 mode: u32 = 0,
362 }345 kind: FileKind = .file,
363346
364 // Writes file content to writer.347 unread_bytes: *u64,
365 pub fn writeAll(self: File, out_writer: anytype) !void {348 parent_reader: *std.io.BufferedReader,
366 var buffer: [4096]u8 = undefined;
367349
368 while (self.unread_bytes.* > 0) {350 pub fn reader(self: *File) std.io.Reader {
369 const buf = buffer[0..@min(buffer.len, self.unread_bytes.*)];351 return .{
370 try self.parent_reader.readNoEof(buf);352 .context = self,
371 try out_writer.writeAll(buf);353 .vtable = &.{
372 self.unread_bytes.* -= buf.len;354 .read = read,
373 }355 .readVec = readVec,
374 }356 .discard = discard,
375 };357 },
358 };
359 }
376360
377 const Self = @This();361 fn read(context: ?*anyopaque, bw: *std.io.BufferedWriter, limit: std.io.Reader.Limit) std.io.Reader.RwError!usize {
362 const file: *File = @ptrCast(@alignCast(context));
363 if (file.unread_bytes.* == 0) return error.EndOfStream;
364 const n = try file.parent_reader.read(bw, limit.min(.limited(file.unread_bytes.*)));
365 file.unread_bytes.* -= n;
366 return n;
367 }
378368
379 fn readHeader(self: *Self) !?Header {369 fn readVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
380 if (self.padding > 0) {370 const file: *File = @ptrCast(@alignCast(context));
381 try self.reader.skipBytes(self.padding, .{});371 if (file.unread_bytes.* == 0) return error.EndOfStream;
382 }372 const n = try file.parent_reader.readVecLimit(data, .limited(file.unread_bytes.*));
383 const n = try self.reader.readAll(&self.header_buffer);373 file.unread_bytes.* -= n;
384 if (n == 0) return null;374 return n;
385 if (n < Header.SIZE) return error.UnexpectedEndOfStream;
386 const header = Header{ .bytes = self.header_buffer[0..Header.SIZE] };
387 if (try header.checkChksum() == 0) return null;
388 return header;
389 }375 }
390376
391 fn readString(self: *Self, size: usize, buffer: []u8) ![]const u8 {377 fn discard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
392 if (size > buffer.len) return error.TarInsufficientBuffer;378 const file: *File = @ptrCast(@alignCast(context));
393 const buf = buffer[0..size];379 const n = limit.minInt(file.unread_bytes.*);
394 try self.reader.readNoEof(buf);380 file.unread_bytes.* -= n;
395 return nullStr(buf);381 return n;
396 }382 }
397383
398 fn newFile(self: *Self) File {384 pub fn readRemaining(file: *File, out: *std.io.BufferedWriter) std.io.Reader.RwRemainingError!void {
399 return .{385 return file.reader().readRemaining(out);
400 .name = self.file_name_buffer[0..0],
401 .link_name = self.link_name_buffer[0..0],
402 .parent_reader = self.reader,
403 .unread_bytes = &self.unread_file_bytes,
404 };
405 }386 }
387 };
406388
407 // Number of padding bytes in the last file block.389 fn readHeader(self: *Iterator) !?Header {
408 fn blockPadding(size: u64) usize {390 if (self.padding > 0) {
409 const block_rounded = std.mem.alignForward(u64, size, Header.SIZE); // size rounded to te block boundary391 try self.reader.discardAll(self.padding);
410 return @intCast(block_rounded - size);
411 }392 }
393 const n = try self.reader.readSliceShort(&self.header_buffer);
394 if (n == 0) return null;
395 if (n < Header.SIZE) return error.UnexpectedEndOfStream;
396 const header = Header{ .bytes = self.header_buffer[0..Header.SIZE] };
397 if (try header.checkChksum() == 0) return null;
398 return header;
399 }
412400
413 /// Iterates through the tar archive as if it is a series of files.401 fn readString(self: *Iterator, size: usize, buffer: []u8) ![]const u8 {
414 /// Internally, the tar format often uses entries (header with optional402 if (size > buffer.len) return error.TarInsufficientBuffer;
415 /// content) to add meta data that describes the next file. These403 const buf = buffer[0..size];
416 /// entries should not normally be visible to the outside. As such, this404 try self.reader.readSlice(buf);
417 /// loop iterates through one or more entries until it collects a all405 return nullStr(buf);
418 /// file attributes.406 }
419 pub fn next(self: *Self) !?File {
420 if (self.unread_file_bytes > 0) {
421 // If file content was not consumed by caller
422 try self.reader.skipBytes(self.unread_file_bytes, .{});
423 self.unread_file_bytes = 0;
424 }
425 var file: File = self.newFile();
426
427 while (try self.readHeader()) |header| {
428 const kind = header.kind();
429 const size: u64 = try header.size();
430 self.padding = blockPadding(size);
431
432 switch (kind) {
433 // File types to return upstream
434 .directory, .normal, .symbolic_link => {
435 file.kind = switch (kind) {
436 .directory => .directory,
437 .normal => .file,
438 .symbolic_link => .sym_link,
439 else => unreachable,
440 };
441 file.mode = try header.mode();
442
443 // set file attributes if not already set by prefix/extended headers
444 if (file.size == 0) {
445 file.size = size;
446 }
447 if (file.link_name.len == 0) {
448 file.link_name = try header.linkName(self.link_name_buffer);
449 }
450 if (file.name.len == 0) {
451 file.name = try header.fullName(self.file_name_buffer);
452 }
453407
454 self.padding = blockPadding(file.size);408 fn newFile(self: *Iterator) File {
455 self.unread_file_bytes = file.size;409 return .{
456 return file;410 .name = self.file_name_buffer[0..0],
457 },411 .link_name = self.link_name_buffer[0..0],
458 // Prefix header types412 .parent_reader = self.reader,
459 .gnu_long_name => {413 .unread_bytes = &self.unread_file_bytes,
460 file.name = try self.readString(@intCast(size), self.file_name_buffer);414 };
461 },415 }
462 .gnu_long_link => {416
463 file.link_name = try self.readString(@intCast(size), self.link_name_buffer);417 // Number of padding bytes in the last file block.
464 },418 fn blockPadding(size: u64) usize {
465 .extended_header => {419 const block_rounded = std.mem.alignForward(u64, size, Header.SIZE); // size rounded to te block boundary
466 // Use just attributes from last extended header.420 return @intCast(block_rounded - size);
467 file = self.newFile();421 }
468422
469 var rdr = paxIterator(self.reader, @intCast(size));423 /// Iterates through the tar archive as if it is a series of files.
470 while (try rdr.next()) |attr| {424 /// Internally, the tar format often uses entries (header with optional
471 switch (attr.kind) {425 /// content) to add meta data that describes the next file. These
472 .path => {426 /// entries should not normally be visible to the outside. As such, this
473 file.name = try attr.value(self.file_name_buffer);427 /// loop iterates through one or more entries until it collects a all
474 },428 /// file attributes.
475 .linkpath => {429 pub fn next(self: *Iterator) !?File {
476 file.link_name = try attr.value(self.link_name_buffer);430 if (self.unread_file_bytes > 0) {
477 },431 // If file content was not consumed by caller
478 .size => {432 try self.reader.discardAll64(self.unread_file_bytes);
479 var buf: [pax_max_size_attr_len]u8 = undefined;433 self.unread_file_bytes = 0;
480 file.size = try std.fmt.parseInt(u64, try attr.value(&buf), 10);434 }
481 },435 var file: File = self.newFile();
482 }436
483 }437 while (try self.readHeader()) |header| {
484 },438 const kind = header.kind();
485 // Ignored header type439 const size: u64 = try header.size();
486 .global_extended_header => {440 self.padding = blockPadding(size);
487 self.reader.skipBytes(size, .{}) catch return error.TarHeadersTooBig;441
488 },442 switch (kind) {
489 // All other are unsupported header types443 // File types to return upstream
490 else => {444 .directory, .normal, .symbolic_link => {
491 const d = self.diagnostics orelse return error.TarUnsupportedHeader;445 file.kind = switch (kind) {
492 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{446 .directory => .directory,
493 .file_name = try d.allocator.dupe(u8, header.name()),447 .normal => .file,
494 .file_type = kind,448 .symbolic_link => .sym_link,
495 } });449 else => unreachable,
496 if (kind == .gnu_sparse) {450 };
497 try self.skipGnuSparseExtendedHeaders(header);451 file.mode = try header.mode();
452
453 // set file attributes if not already set by prefix/extended headers
454 if (file.size == 0) {
455 file.size = size;
456 }
457 if (file.link_name.len == 0) {
458 file.link_name = try header.linkName(self.link_name_buffer);
459 }
460 if (file.name.len == 0) {
461 file.name = try header.fullName(self.file_name_buffer);
462 }
463
464 self.padding = blockPadding(file.size);
465 self.unread_file_bytes = file.size;
466 return file;
467 },
468 // Prefix header types
469 .gnu_long_name => {
470 file.name = try self.readString(@intCast(size), self.file_name_buffer);
471 },
472 .gnu_long_link => {
473 file.link_name = try self.readString(@intCast(size), self.link_name_buffer);
474 },
475 .extended_header => {
476 // Use just attributes from last extended header.
477 file = self.newFile();
478
479 var rdr: PaxIterator = .{
480 .reader = self.reader,
481 .size = @intCast(size),
482 };
483 while (try rdr.next()) |attr| {
484 switch (attr.kind) {
485 .path => {
486 file.name = try attr.value(self.file_name_buffer);
487 },
488 .linkpath => {
489 file.link_name = try attr.value(self.link_name_buffer);
490 },
491 .size => {
492 var buf: [pax_max_size_attr_len]u8 = undefined;
493 file.size = try std.fmt.parseInt(u64, try attr.value(&buf), 10);
494 },
498 }495 }
499 self.reader.skipBytes(size, .{}) catch return error.TarHeadersTooBig;496 }
500 },497 },
501 }498 // Ignored header type
499 .global_extended_header => {
500 self.reader.discardAll64(size) catch return error.TarHeadersTooBig;
501 },
502 // All other are unsupported header types
503 else => {
504 const d = self.diagnostics orelse return error.TarUnsupportedHeader;
505 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
506 .file_name = try d.allocator.dupe(u8, header.name()),
507 .file_type = kind,
508 } });
509 if (kind == .gnu_sparse) {
510 try self.skipGnuSparseExtendedHeaders(header);
511 }
512 self.reader.discardAll64(size) catch return error.TarHeadersTooBig;
513 },
502 }514 }
503 return null;
504 }515 }
516 return null;
517 }
505518
506 fn skipGnuSparseExtendedHeaders(self: *Self, header: Header) !void {519 fn skipGnuSparseExtendedHeaders(self: *Iterator, header: Header) !void {
507 var is_extended = header.bytes[482] > 0;520 var is_extended = header.bytes[482] > 0;
508 while (is_extended) {521 while (is_extended) {
509 var buf: [Header.SIZE]u8 = undefined;522 var buf: [Header.SIZE]u8 = undefined;
510 const n = try self.reader.readAll(&buf);523 try self.reader.readSlice(&buf);
511 if (n < Header.SIZE) return error.UnexpectedEndOfStream;524 is_extended = buf[504] > 0;
512 is_extended = buf[504] > 0;
513 }
514 }525 }
515 };526 }
516}527};
517
518/// Pax attributes iterator.
519/// Size is length of pax extended header in reader.
520fn paxIterator(reader: anytype, size: usize) PaxIterator(@TypeOf(reader)) {
521 return PaxIterator(@TypeOf(reader)){
522 .reader = reader,
523 .size = size,
524 };
525}
526528
527const PaxAttributeKind = enum {529const PaxAttributeKind = enum {
528 path,530 path,
...@@ -533,103 +535,92 @@ const PaxAttributeKind = enum {...@@ -533,103 +535,92 @@ const PaxAttributeKind = enum {
533// maxInt(u64) has 20 chars, base 10 in practice we got 24 chars535// maxInt(u64) has 20 chars, base 10 in practice we got 24 chars
534const pax_max_size_attr_len = 64;536const pax_max_size_attr_len = 64;
535537
536fn PaxIterator(comptime ReaderType: type) type {538pub const PaxIterator = struct {
537 return struct {539 size: usize, // cumulative size of all pax attributes
538 size: usize, // cumulative size of all pax attributes540 reader: *std.io.BufferedReader,
539 reader: ReaderType,
540 // scratch buffer used for reading attribute length and keyword
541 scratch: [128]u8 = undefined,
542
543 const Self = @This();
544
545 const Attribute = struct {
546 kind: PaxAttributeKind,
547 len: usize, // length of the attribute value
548 reader: ReaderType, // reader positioned at value start
549
550 // Copies pax attribute value into destination buffer.
551 // Must be called with destination buffer of size at least Attribute.len.
552 pub fn value(self: Attribute, dst: []u8) ![]const u8 {
553 if (self.len > dst.len) return error.TarInsufficientBuffer;
554 // assert(self.len <= dst.len);
555 const buf = dst[0..self.len];
556 const n = try self.reader.readAll(buf);
557 if (n < self.len) return error.UnexpectedEndOfStream;
558 try validateAttributeEnding(self.reader);
559 if (hasNull(buf)) return error.PaxNullInValue;
560 return buf;
561 }
562 };
563541
564 // Iterates over pax attributes. Returns known only known attributes.542 const Self = @This();
565 // Caller has to call value in Attribute, to advance reader across value.
566 pub fn next(self: *Self) !?Attribute {
567 // Pax extended header consists of one or more attributes, each constructed as follows:
568 // "%d %s=%s\n", <length>, <keyword>, <value>
569 while (self.size > 0) {
570 const length_buf = try self.readUntil(' ');
571 const length = try std.fmt.parseInt(usize, length_buf, 10); // record length in bytes
572
573 const keyword = try self.readUntil('=');
574 if (hasNull(keyword)) return error.PaxNullInKeyword;
575
576 // calculate value_len
577 const value_start = length_buf.len + keyword.len + 2; // 2 separators
578 if (length < value_start + 1 or self.size < length) return error.UnexpectedEndOfStream;
579 const value_len = length - value_start - 1; // \n separator at end
580 self.size -= length;
581
582 const kind: PaxAttributeKind = if (eql(keyword, "path"))
583 .path
584 else if (eql(keyword, "linkpath"))
585 .linkpath
586 else if (eql(keyword, "size"))
587 .size
588 else {
589 try self.reader.skipBytes(value_len, .{});
590 try validateAttributeEnding(self.reader);
591 continue;
592 };
593 if (kind == .size and value_len > pax_max_size_attr_len) {
594 return error.PaxSizeAttrOverflow;
595 }
596 return Attribute{
597 .kind = kind,
598 .len = value_len,
599 .reader = self.reader,
600 };
601 }
602543
603 return null;544 const Attribute = struct {
545 kind: PaxAttributeKind,
546 len: usize, // length of the attribute value
547 reader: *std.io.BufferedReader, // reader positioned at value start
548
549 // Copies pax attribute value into destination buffer.
550 // Must be called with destination buffer of size at least Attribute.len.
551 pub fn value(self: Attribute, dst: []u8) ![]const u8 {
552 if (self.len > dst.len) return error.TarInsufficientBuffer;
553 // assert(self.len <= dst.len);
554 const buf = dst[0..self.len];
555 const n = try self.reader.readSliceShort(buf);
556 if (n < self.len) return error.UnexpectedEndOfStream;
557 try validateAttributeEnding(self.reader);
558 if (hasNull(buf)) return error.PaxNullInValue;
559 return buf;
604 }560 }
561 };
605562
606 fn readUntil(self: *Self, delimiter: u8) ![]const u8 {563 // Iterates over pax attributes. Returns known only known attributes.
607 var fbs: std.io.BufferedWriter = undefined;564 // Caller has to call value in Attribute, to advance reader across value.
608 fbs.initFixed(&self.scratch);565 pub fn next(self: *Self) !?Attribute {
609 try self.reader.streamUntilDelimiter(&fbs, delimiter, null);566 // Pax extended header consists of one or more attributes, each constructed as follows:
610 return fbs.getWritten();567 // "%d %s=%s\n", <length>, <keyword>, <value>
568 while (self.size > 0) {
569 const length_buf = try self.reader.takeSentinel(' ');
570 const length = try std.fmt.parseInt(usize, length_buf, 10); // record length in bytes
571
572 const keyword = try self.reader.takeSentinel('=');
573 if (hasNull(keyword)) return error.PaxNullInKeyword;
574
575 // calculate value_len
576 const value_start = length_buf.len + keyword.len + 2; // 2 separators
577 if (length < value_start + 1 or self.size < length) return error.UnexpectedEndOfStream;
578 const value_len = length - value_start - 1; // \n separator at end
579 self.size -= length;
580
581 const kind: PaxAttributeKind = if (eql(keyword, "path"))
582 .path
583 else if (eql(keyword, "linkpath"))
584 .linkpath
585 else if (eql(keyword, "size"))
586 .size
587 else {
588 try self.reader.discardAll(value_len);
589 try validateAttributeEnding(self.reader);
590 continue;
591 };
592 if (kind == .size and value_len > pax_max_size_attr_len) {
593 return error.PaxSizeAttrOverflow;
594 }
595 return .{
596 .kind = kind,
597 .len = value_len,
598 .reader = self.reader,
599 };
611 }600 }
612601
613 fn eql(a: []const u8, b: []const u8) bool {602 return null;
614 return std.mem.eql(u8, a, b);603 }
615 }
616604
617 fn hasNull(str: []const u8) bool {605 fn eql(a: []const u8, b: []const u8) bool {
618 return (std.mem.indexOfScalar(u8, str, 0)) != null;606 return std.mem.eql(u8, a, b);
619 }607 }
620608
621 // Checks that each record ends with new line.609 fn hasNull(str: []const u8) bool {
622 fn validateAttributeEnding(reader: ReaderType) !void {610 return (std.mem.indexOfScalar(u8, str, 0)) != null;
623 if (try reader.readByte() != '\n') return error.PaxInvalidAttributeEnd;611 }
624 }612
625 };613 // Checks that each record ends with new line.
626}614 fn validateAttributeEnding(reader: *std.io.BufferedReader) !void {
615 if (try reader.takeByte() != '\n') return error.PaxInvalidAttributeEnd;
616 }
617};
627618
628/// Saves tar file content to the file systems.619/// Saves tar file content to the file systems.
629pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions) !void {620pub fn pipeToFileSystem(dir: std.fs.Dir, reader: *std.io.BufferedReader, options: PipeOptions) !void {
630 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;621 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
631 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;622 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
632 var iter = iterator(reader, .{623 var iter: Iterator = .init(reader, .{
633 .file_name_buffer = &file_name_buffer,624 .file_name_buffer = &file_name_buffer,
634 .link_name_buffer = &link_name_buffer,625 .link_name_buffer = &link_name_buffer,
635 .diagnostics = options.diagnostics,626 .diagnostics = options.diagnostics,
...@@ -827,8 +818,9 @@ test PaxIterator {...@@ -827,8 +818,9 @@ test PaxIterator {
827 var buffer: [1024]u8 = undefined;818 var buffer: [1024]u8 = undefined;
828819
829 outer: for (cases) |case| {820 outer: for (cases) |case| {
830 var stream = std.io.fixedBufferStream(case.data);821 var br: std.io.BufferedReader = undefined;
831 var iter = paxIterator(stream.reader(), case.data.len);822 br.initFixed(case.data);
823 var iter: PaxIterator = .init(&br, case.data.len);
832824
833 var i: usize = 0;825 var i: usize = 0;
834 while (iter.next() catch |err| {826 while (iter.next() catch |err| {
...@@ -942,7 +934,7 @@ test "create file and symlink" {...@@ -942,7 +934,7 @@ test "create file and symlink" {
942 file.close();934 file.close();
943}935}
944936
945test iterator {937test Iterator {
946 // Example tar file is created from this tree structure:938 // Example tar file is created from this tree structure:
947 // $ tree example939 // $ tree example
948 // example940 // example
...@@ -969,7 +961,7 @@ test iterator {...@@ -969,7 +961,7 @@ test iterator {
969 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;961 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
970 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;962 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
971 // Create iterator963 // Create iterator
972 var iter = iterator(fbs.reader(), .{964 var iter: Iterator = .init(fbs.reader(), .{
973 .file_name_buffer = &file_name_buffer,965 .file_name_buffer = &file_name_buffer,
974 .link_name_buffer = &link_name_buffer,966 .link_name_buffer = &link_name_buffer,
975 });967 });