diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index e2c51cc6fe91216ba8f8188324a4b30c50d62443..40845f75c31ccbfe5b931fabbb7f40c997212ffb 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -527,9 +527,6 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build. const arena = b.allocator; const io = b.graph.io; - var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, zp.child.stderr.?, .unlimited }); - defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {}; - var timer = try std.time.Timer.start(); try sendMessage(io, zp.child.stdin.?, .update); @@ -537,19 +534,18 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build. var result: ?Path = null; - var stdout_buffer: [512]u8 = undefined; - var stdout_reader: Io.File.Reader = .initStreaming(zp.child.stdout.?, io, &stdout_buffer); - const stdout = &stdout_reader.interface; + var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; + var multi_reader: Io.File.MultiReader = undefined; + multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ zp.child.stdout.?, zp.child.stderr.? }); + defer multi_reader.deinit(); - var body_buffer: std.ArrayList(u8) = .empty; - defer body_buffer.deinit(gpa); + const stdout = multi_reader.reader(0); + const stderr = multi_reader.reader(1); while (true) { const Header = std.zig.Server.Message.Header; const header = try stdout.takeStruct(Header, .little); - body_buffer.clearRetainingCapacity(); - try stdout.appendExact(gpa, &body_buffer, header.bytes_len); - const body = body_buffer.items; + const body = try stdout.take(header.bytes_len); switch (header.tag) { .zig_version => { if (!std.mem.eql(u8, builtin.zig_version_string, body)) { @@ -640,8 +636,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build. s.result_duration_ns = timer.read(); - const stderr_contents = try stderr_task.await(io); - defer gpa.free(stderr_contents); + const stderr_contents = stderr.buffered(); if (stderr_contents.len > 0) { try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); } @@ -649,14 +644,6 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build. return result; } -fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 { - var file_reader: Io.File.Reader = .initStreaming(file, io, &.{}); - return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - else => |e| return e, - }; -} - pub fn getZigProcess(s: *Step) ?*ZigProcess { return switch (s.id) { .compile => s.cast(Compile).?.zig_process, diff --git a/lib/std/Io.zig b/lib/std/Io.zig index b503979fda68d5db200da632970f114f40de819d..980379b72b985fdd5107e522eda9bdbee475caf0 100644 --- a/lib/std/Io.zig +++ b/lib/std/Io.zig @@ -350,8 +350,6 @@ pub const Batch = struct { } }; - pub const WaitError = ConcurrentError || Cancelable || Timeout.Error; - pub fn init(operations: []Operation, ring: []u32) Batch { const len: u31 = @intCast(operations.len); assert(ring.len == len); @@ -405,6 +403,8 @@ pub const Batch = struct { return b.ring[0..len][head.index(len)]; } + pub const WaitError = ConcurrentError || Cancelable || Timeout.Error; + /// Starts work on any submitted operations and returns when at least one has completeed. /// /// Returns `error.Timeout` if `timeout` expires first. diff --git a/lib/std/Io/File.zig b/lib/std/Io/File.zig index cc7042d443d0ca250e3a853012d4241462e908b5..c545b6022278d5043d8890a39a91a7baae81d1dc 100644 --- a/lib/std/Io/File.zig +++ b/lib/std/Io/File.zig @@ -18,6 +18,9 @@ pub const Writer = @import("File/Writer.zig"); pub const Atomic = @import("File/Atomic.zig"); /// Memory intended to remain consistent with file contents. pub const MemoryMap = @import("File/MemoryMap.zig"); +/// Concurrently read from multiple file streams, eliminating risk of +/// deadlocking. +pub const MultiReader = @import("File/MultiReader.zig"); pub const INode = std.posix.ino_t; pub const NLink = std.posix.nlink_t; diff --git a/lib/std/Io/File/MultiReader.zig b/lib/std/Io/File/MultiReader.zig new file mode 100644 index 0000000000000000000000000000000000000000..1cf3f7b4042e01c2038575d69c583ea0afe23f7b --- /dev/null +++ b/lib/std/Io/File/MultiReader.zig @@ -0,0 +1,240 @@ +const MultiReader = @This(); + +const std = @import("../../std.zig"); +const Io = std.Io; +const File = Io.File; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; + +gpa: Allocator, +streams: *Streams, +batch: Io.Batch, + +pub const Context = struct { + mr: *MultiReader, + fr: File.Reader, + vec: [1][]u8, + err: ?Error, + eos: bool, +}; + +pub const Error = Allocator.Error || File.Reader.Error || Io.ConcurrentError; + +/// Trailing: +/// * `contexts: [len]Context` +/// * `ring: [len]u32` +/// * `operations: [len]Io.Operation` +pub const Streams = extern struct { + len: u32, + + pub fn contexts(s: *Streams) []Context { + _ = s; + @panic("TODO"); + } + + pub fn ring(s: *Streams) []u32 { + _ = s; + @panic("TODO"); + } + + pub fn operations(s: *Streams) []Io.Operation { + _ = s; + @panic("TODO"); + } +}; + +pub fn Buffer(comptime n: usize) type { + return extern struct { + len: u32, + contexts: [n][@sizeOf(Context)]u8 align(@alignOf(Context)), + ring: [n]u32, + operations: [n][@sizeOf(Io.Operation)]u8 align(@alignOf(Io.Operation)), + + pub fn toStreams(b: *@This()) *Streams { + return @ptrCast(b); + } + }; +} + +/// See `Streams.Buffer` for convenience API to obtain the `streams` parameter. +pub fn init(mr: *MultiReader, gpa: Allocator, io: Io, streams: *Streams, files: []const File) void { + const contexts = streams.contexts(); + for (contexts, files) |*context, file| context.* = .{ + .mr = mr, + .fr = .{ + .io = io, + .file = file, + .mode = .streaming, + .interface = .{ + .vtable = &.{ + .stream = stream, + .discard = discard, + .readVec = readVec, + .rebase = rebase, + }, + .buffer = &.{}, + .seek = 0, + .end = 0, + }, + }, + .vec = .{&.{}}, + .err = null, + .eos = false, + }; + const operations = streams.operations(); + const ring = streams.ring(); + mr.* = .{ + .gpa = gpa, + .streams = streams, + .batch = .init(operations, ring), + }; + for (operations, contexts, files, 0..) |*op, *context, file, i| { + const r = &context.fr.interface; + op.* = .{ .file_read_streaming = .{ + .file = file, + .data = &context.vec, + } }; + rebaseGrowing(mr, context, 1) catch |err| { + context.err = err; + continue; + }; + context.vec[0] = r.buffer; + mr.batch.add(i); + } +} + +pub fn deinit(mr: *MultiReader) void { + const gpa = mr.gpa; + const contexts = mr.streams.contexts(); + const io = contexts[0].fr.io; + mr.batch.cancel(io); + for (contexts) |*context| { + gpa.free(context.fr.interface.buffer); + } +} + +pub fn reader(mr: *MultiReader, index: usize) *Io.Reader { + return &mr.streams.contexts()[index].fr.interface; +} + +pub fn toOwnedSlice(mr: *MultiReader, index: usize) Allocator.Error![]u8 { + const gpa = mr.gpa; + const r: *Io.Reader = reader(mr, index); + if (r.seek == 0) { + const new = try gpa.realloc(r.buffer, r.end); + r.buffer = &.{}; + r.end = 0; + return new; + } + const new = try gpa.dupe(u8, r.buffered()); + gpa.free(r.buffer); + r.buffer = &.{}; + r.seek = 0; + r.end = 0; + return new; +} + +fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize { + _ = limit; + _ = w; + const fr: *File.Reader = @alignCast(@fieldParentPtr("interface", r)); + const context: *Context = @fieldParentPtr("fr", fr); + const mr = context.mr; + return fill(mr, context); +} + +fn discard(r: *Io.Reader, limit: Io.Limit) Io.Reader.Error!usize { + _ = limit; + const fr: *File.Reader = @alignCast(@fieldParentPtr("interface", r)); + const context: *Context = @fieldParentPtr("fr", fr); + const mr = context.mr; + return fill(mr, context); +} + +fn readVec(r: *Io.Reader, data: [][]u8) Io.Reader.Error!usize { + _ = data; + const fr: *File.Reader = @alignCast(@fieldParentPtr("interface", r)); + const context: *Context = @fieldParentPtr("fr", fr); + const mr = context.mr; + return fill(mr, context); +} + +fn rebase(r: *Io.Reader, capacity: usize) Io.Reader.RebaseError!void { + const fr: *File.Reader = @alignCast(@fieldParentPtr("interface", r)); + const context: *Context = @fieldParentPtr("fr", fr); + const mr = context.mr; + + return rebaseGrowing(mr, context, capacity) catch |err| { + context.err = err; + return error.ReadFailed; + }; +} + +fn rebaseGrowing(mr: *MultiReader, context: *Context, capacity: usize) Allocator.Error!void { + const gpa = mr.gpa; + const r = &context.fr.interface; + if (r.buffer.len >= capacity) { + const data = r.buffer[r.seek..r.end]; + @memmove(r.buffer[0..data.len], data); + r.seek = 0; + r.end = data.len; + } else { + const adjusted_capacity = std.ArrayList(u8).growCapacity(capacity); + + if (r.seek == 0) { + if (gpa.remap(r.buffer, adjusted_capacity)) |new_memory| { + r.buffer = new_memory; + return; + } + } + + const data = r.buffer[r.seek..r.end]; + const new = try gpa.alloc(u8, adjusted_capacity); + @memcpy(new[0..data.len], data); + r.seek = 0; + r.end = data.len; + } +} + +fn fill(mr: *MultiReader, original_context: *Context) Io.Reader.Error!usize { + const contexts = mr.streams.contexts(); + const operations = mr.streams.operations(); + const io = contexts[0].fr.io; + + mr.batch.wait(io, .none) catch |err| switch (err) { + error.Timeout, error.UnsupportedClock => unreachable, + else => |e| { + original_context.err = e; + return error.ReadFailed; + }, + }; + + while (mr.batch.next()) |i| { + const context = &contexts[i]; + const operation = &operations[i]; + const n = operation.file_read_streaming.status.result catch |err| { + context.err = err; + continue; + }; + if (n == 0) { + context.eos = true; + continue; + } + const r = &context.fr.interface; + r.end += n; + if (r.buffer.len - r.end == 0) { + rebaseGrowing(mr, context, r.bufferedLen() + 1) catch |err| { + context.err = err; + continue; + }; + assert(r.seek == 0); + context.vec[0] = r.buffer; + } + operation.file_read_streaming.status = .{ .unstarted = {} }; + mr.batch.add(i); + } + + if (original_context.err != null) return error.ReadFailed; + if (original_context.eos) return error.EndOfStream; + return 0; +} diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 9c5c762844c0f1e296fbd389ed016ebb991abfcf..9ff025a637c87e15f852d09234fd8d6570ad2ee4 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -127,9 +127,7 @@ pub const ShortError = error{ ReadFailed, }; -pub const RebaseError = error{ - EndOfStream, -}; +pub const RebaseError = Error; pub const failing: Reader = .{ .vtable = &.{ @@ -1402,7 +1400,7 @@ pub fn takeLeb128(r: *Reader, comptime T: type) TakeLeb128Error!T { } /// Ensures `capacity` data can be buffered without rebasing. -pub fn rebase(r: *Reader, capacity: usize) RebaseError!void { +pub fn rebase(r: *Reader, capacity: usize) Error!void { if (r.buffer.len - r.seek >= capacity) { @branchHint(.likely); return; @@ -1410,7 +1408,7 @@ pub fn rebase(r: *Reader, capacity: usize) RebaseError!void { return r.vtable.rebase(r, capacity); } -pub fn defaultRebase(r: *Reader, capacity: usize) RebaseError!void { +pub fn defaultRebase(r: *Reader, capacity: usize) Error!void { assert(r.buffer.len - r.seek < capacity); const data = r.buffer[r.seek..r.end]; @memmove(r.buffer[0..data.len], data);