diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index b5db0e68f7597a1e951900e8b741afb20521bcda..e692343bd2a8e1e052b0b536d7bf03d5f03fa054 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -11,6 +11,7 @@ const File = std.Io.File; const Io = std.Io; const Dir = std.Io.Dir; const Path = std.Build.Cache.Path; +const Reader = std.Io.Reader; const Writer = std.Io.Writer; const assert = std.debug.assert; const fatal = std.process.fatal; @@ -19,6 +20,8 @@ const log = std.log; const mem = std.mem; const process = std.process; const Color = std.zig.Color; +const Client = std.zig.Client; +const Server = std.zig.Server; const EnvVar = std.zig.EnvVar; const default_local_zig_cache_basename = std.zig.default_local_zig_cache_basename; const stringToEnum = std.meta.stringToEnum; @@ -51,6 +54,8 @@ max_rss_mutex: Io.Mutex, skip_oom_steps: bool, unit_test_timeout_ns: ?u64, watch: bool, +protocol_server: ?*AvoidableServer, +protocol_server_mutex: Io.Mutex, web_server: ?*AvoidableWebServer, /// Allocated into `gpa`. memory_blocked_steps: std.ArrayList(Configuration.Step.Index), @@ -67,6 +72,7 @@ var stdio_buffer_allocation: [256]u8 = undefined; var stdout_writer_allocation: Io.File.Writer = undefined; var debug_maker_leaks: bool = false; +const AvoidableServer = if (builtin.single_threaded) void else Server; const AvoidableWebServer = if (builtin.single_threaded) void else WebServer; const is_debug_mode = builtin.mode == .debug; @@ -216,6 +222,7 @@ pub fn main(init: process.Init.Minimal) !void { var watch = false; var fuzz: ?Fuzz.Mode = null; var debounce_interval_ms: u16 = 50; + var listen: bool = false; var webui_listen: ?Io.net.IpAddress = null; var debug_pkg_config = false; var run_args: ?[]const []const u8 = null; @@ -416,6 +423,8 @@ pub fn main(init: process.Init.Minimal) !void { next_arg, err, }); }; + } else if (mem.eql(u8, arg, "--listen=-")) { + listen = true; } else if (mem.eql(u8, arg, "--webui")) { if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; } else if (mem.startsWith(u8, arg, "--webui=")) { @@ -553,7 +562,7 @@ pub fn main(init: process.Init.Minimal) !void { } const early_exit_mode = fetch_only or help_menu or steps_menu or print_configuration != .none; - const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null); + const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null or listen); process.raiseFileDescriptorLimit(); @@ -661,6 +670,25 @@ pub fn main(init: process.Init.Minimal) !void { break :ws &web_server_allocation; } else null; + var stdin_buffer: [256]u8 = undefined; + var stdout_buffer: [256]u8 = undefined; + var stdin_reader = Io.File.stdin().reader(io, &stdin_buffer); + var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer); + + var protocol_server_allocation: AvoidableServer = undefined; + const protocol_server: ?*AvoidableServer = if (listen) s: { + if (builtin.single_threaded) fatal("--listen is not yet supported on single-threaded hosts", .{}); + if (watch) fatal("using '--watch' and '--listen' together is not supported", .{}); + if (fuzz != null) fatal("using '--fuzz' and '--listen' together is not supported", .{}); + if (step_names.items.len > 0) fatal("build steps must be provided over the protocol instead of using CLI arguments", .{}); + protocol_server_allocation = .{ + .in = &stdin_reader.interface, + .out = &stdout_writer.interface, + }; + try serveBSPHandshake(&protocol_server_allocation); + break :s &protocol_server_allocation; + } else null; + while (true) { // If this fails, we can still start the server and wait for user // to request a rebuild. If it returns error.FailedButCacheIntact @@ -731,13 +759,20 @@ pub fn main(init: process.Init.Minimal) !void { .watch = watch, .web_server = web_server, + .protocol_server = protocol_server, + .protocol_server_mutex = .init, .memory_blocked_steps = .empty, .step_stack = .empty, .pkg_config = .{ .debug = debug_pkg_config }, .error_style = error_style, .multiline_errors = multiline_errors, - .summary = summary orelse if (watch or webui_listen != null) .new else .failures, + .summary = summary orelse if (listen) + .none + else if (watch or webui_listen != null) + .new + else + .failures, }; defer { maker.memory_blocked_steps.deinit(gpa); @@ -749,6 +784,52 @@ pub fn main(init: process.Init.Minimal) !void { maker.max_rss_is_default = true; } + if (protocol_server) |s| { + try s.serveStringMessage(.bsp_configuration, try arena.print("{f}", .{scanned_config.path})); + + var w: ?Watch = null; + + const Event = union(enum) { + message: Reader.Error!Client.Message.Header, + fs_event: if (Watch.have_impl) @typeInfo(@TypeOf(Watch.wait)).@"fn".return_type.? else noreturn, + }; + + var select_buffer: [2]Event = undefined; + var select: Io.Select(Event) = .init(io, &select_buffer); + defer select.cancelDiscard(); + + try select.concurrent(.message, Server.receiveMessage, .{s}); + + var in_debounce = false; + loop: switch (try select.await()) { + .message => |payload| { + const header: Client.Message.Header = try payload; + switch (header.tag) { + .exit => { + cleanExit(io, &scanned_config); + process.exit(0); + }, + else => fatal("unsupported message: {t}", .{header.tag}), + } + }, + .fs_event => |payload| { + if (!Watch.have_impl) unreachable; + switch (try payload) { + .timeout => { + assert(in_debounce); + markFailedStepsDirty(&maker); + if (true) @panic("TODO run steps that were previous specified over the build system protocol"); + in_debounce = false; + }, + .dirty => in_debounce = true, + .clean => {}, + } + try select.concurrent(.fs_event, Watch.wait, .{ &w.?, if (in_debounce) .{ .ms = debounce_interval_ms } else .none }); + continue :loop try select.await(); + }, + } + } + maker.prepare(step_names.items) catch |err| switch (err) { error.DependencyLoopDetected, error.InsufficientMemory => { // TODO handle DependencyLoopDetected as error.FailedButCacheIntact @@ -850,6 +931,9 @@ pub fn main(init: process.Init.Minimal) !void { _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(1); } + if (protocol_server != null) { + fatal("(zig build system) TODO send error messages to client when build.zig compilation fails", .{}); + } if (watch and can_fs_watch) { fatal("(zig build system) TODO set up fs watching even when build.zig compilation fails", .{}); } else { @@ -2990,6 +3074,21 @@ fn cleanTmpFiles(maker: *Maker, steps: []const Configuration.Step.Index) void { } } +fn serveBSPHandshake(s: *const std.zig.Server) !void { + const handshake_header: Server.Message.Handshake = .{ + .version = Server.build_system_version, + .flags = .{ + .file_system_watch_supported = Watch.have_impl, + }, + }; + try s.serveMessageHeader(.{ + .tag = .bsp_handshake, + .bytes_len = @sizeOf(Server.Message.Handshake), + }); + try s.out.writeStruct(handshake_header, .little); + try s.out.flush(); +} + fn initStdoutWriter(io: Io) *Writer { stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); return &stdout_writer_allocation.interface; diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 5c1cb20b36c713a1dad9353833d67037626739e6..1da63f95310b561f4264771a271fe70346c82fe7 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -12,6 +12,14 @@ const Writer = std.Io.Writer; in: *Reader, out: *Writer, +/// The ABI version of the build system protocol. Will be bumped whenever a +/// backwards incompatible changes to the protocol is made. +/// +/// Does not apply to the internal compiler protocol or test runner. +/// +/// See `version` in `Message.Handshake`. +pub const build_system_version: u32 = 1; + pub const Message = struct { pub const Header = extern struct { tag: Tag, @@ -66,9 +74,31 @@ pub const Message = struct { /// Body is a TimeReport. time_report, + /// The first message sent by the server over the build system protocol. + /// Body is a `Handshake`. + /// This message only applies to the build system protocol. + bsp_handshake = 0x80000000, + /// Notifies that a new configuration file is available. + /// Body is a cwd relative path to the configuration file. + /// This message only applies to the build system protocol. + bsp_configuration, + _, }; + /// Trailing: + /// * base_paths: BasePaths, + pub const Handshake = extern struct { + /// See `build_system_version`. + version: u32, + flags: Flags, + + pub const Flags = packed struct(u32) { + file_system_watch_supported: bool, + _: u31 = 0, + }; + }; + pub const PathPrefix = enum(u8) { cwd, zig_lib, diff --git a/test/standalone/build.zig b/test/standalone/build.zig index dc8d85d399477256b65c60dc1b4850144c641738..1679c32af85839baa16a7305ea53b22aacc75149 100644 --- a/test/standalone/build.zig +++ b/test/standalone/build.zig @@ -31,6 +31,7 @@ pub fn build(b: *std.Build) void { const tools_target = b.resolveTargetQuery(.{}); for ([_][]const u8{ // Alphabetically sorted. No need to build `tools/spirv/grammar.zig`. + "../../tools/bsp.zig", "../../tools/check_mingw.zig", "../../tools/dump-cov.zig", "../../tools/fetch_them_macos_headers.zig", diff --git a/tools/bsp.zig b/tools/bsp.zig new file mode 100644 index 0000000000000000000000000000000000000000..875fda3e772739f6c337355027c84784e8e9a2d3 --- /dev/null +++ b/tools/bsp.zig @@ -0,0 +1,196 @@ +//! CLI tool to interface with the build system protocol (zig build --listen=-) + +const std = @import("std"); +const Io = std.Io; +const Allocator = std.mem.Allocator; +const Configuration = std.Build.Configuration; +const Client = std.zig.Client; +const Server = std.zig.Server; +const log = std.log.scoped(.bsp); + +pub fn main(init: std.process.Init) !void { + const io = init.io; + const gpa = init.gpa; + const arena = init.arena.allocator(); + + var maker_args: std.ArrayList([]const u8) = .empty; + + const args = try init.minimal.args.toSlice(arena); + for (args[1..]) |arg| { + try maker_args.append(arena, try arena.dupe(u8, arg)); + } + if (maker_args.items.len < 1) try maker_args.append(arena, "zig"); + if (maker_args.items.len < 2) try maker_args.append(arena, "build"); + if (!std.mem.eql(u8, maker_args.last().?.*, "--listen=-")) try maker_args.append(arena, "--listen=-"); + + log.debug("cmd: {f}", .{std.zig.SubprocessCommand{ + .argv = maker_args.items, + }}); + + var child_process = std.process.spawn(io, .{ + .argv = maker_args.items, + .stdin = .pipe, + .stdout = .pipe, + .stderr = .pipe, + }) catch |err| std.debug.panic("failed to spawn process: {}", .{err}); + errdefer child_process.kill(io); + + var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; + var multi_reader: Io.File.MultiReader = undefined; + defer multi_reader.deinit(); + multi_reader.init( + gpa, + io, + multi_reader_buffer.toStreams(), + &.{ child_process.stdout.?, child_process.stderr.? }, + ); + const client_stdout = multi_reader.reader(0); + const client_stderr = multi_reader.reader(1); + + var client_stdout_buffer: [256]u8 = undefined; + var client_stdout_writer = child_process.stdin.?.writerStreaming(io, &client_stdout_buffer); + + var client: Client = .{ + .in = client_stdout, + .out = &client_stdout_writer.interface, + }; + + const err = blk: { + const handshake: Server.Message.Handshake = handshake: { + const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch |err| switch (err) { + error.Canceled, error.ConcurrencyUnavailable => |e| return e, + error.Timeout => unreachable, + else => |e| { + log.err("failed to receive message: {t}", .{err}); + break :blk e; + }, + }; + const body = client_stdout.take(header.bytes_len) catch unreachable; + log.debug("received {f} ({d} bytes)", .{ fmtEnum(header.tag), body.len }); + + if (header.tag != .bsp_handshake) { + log.err("received unexpected message: {f}", .{fmtEnum(header.tag)}); + return error.UnexpectedMessage; + } + + var r: Io.Reader = .fixed(body); + break :handshake try r.takeStruct(Server.Message.Handshake, .little); + }; + _ = handshake; + + var conf_arena_allocator: std.heap.ArenaAllocator = .init(gpa); + defer conf_arena_allocator.deinit(); + const conf_arena = conf_arena_allocator.allocator(); + + const configuration = configuration: { + const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch |err| switch (err) { + error.Canceled, error.ConcurrencyUnavailable => |e| return e, + error.Timeout => unreachable, + else => |e| { + log.err("failed to receive message: {t}", .{err}); + break :blk e; + }, + }; + const body = client_stdout.take(header.bytes_len) catch unreachable; + log.debug("received {t} ({d} bytes)", .{ header.tag, body.len }); + + if (header.tag != .bsp_configuration) { + log.err("received unexpected message: {f}", .{fmtEnum(header.tag)}); + return error.UnexpectedMessage; + } + + const configuration_path = body; + var file = Io.Dir.cwd().openFile(io, configuration_path, .{}) catch |err| + std.debug.panic("failed to open configuration file {q}: {t}", .{ configuration_path, err }); + defer file.close(io); + break :configuration Configuration.loadFile(conf_arena, io, file) catch |err| + std.debug.panic("failed to load configuration file {q}: {t}", .{ configuration_path, err }); + }; + const c = &configuration; + + var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty; + defer top_level_steps.deinit(gpa); + + for (c.steps, 0..) |*conf_step, step_index_usize| { + if (conf_step.owner != .root) continue; + const step_index: Configuration.Step.Index = @fromBackingInt(@intCast(step_index_usize)); + const flags = conf_step.flags(c); + if (flags.tag != .top_level) continue; + const name = step_index.ptr(c).name.slice(c); + try top_level_steps.putNoClobber(gpa, name, step_index); + } + + std.debug.print("Steps:\n", .{}); + for (top_level_steps.keys()) |name| { + std.debug.print(" - {q}\n", .{name}); + } + std.debug.print( + \\Available Commands: + \\ - build [step names / step indices] + \\ - watch [step names / step indices] + \\ - exit + \\ + , .{}); + + var stdin_reader_buffer: [256]u8 = undefined; + var stdin_reader = Io.File.stdin().reader(io, &stdin_reader_buffer); + const stdin = &stdin_reader.interface; + + while (true) { + try Io.File.stdout().writeStreamingAll(io, "> "); + const command = try stdin.takeDelimiterExclusive('\n'); + stdin.toss(1); + if (std.mem.startsWith(u8, command, "build") or + std.mem.startsWith(u8, command, "watch")) + { + @panic("TODO"); + } else if (std.mem.eql(u8, command, "exit")) { + try client.serveBodylessMessage(.exit); + break; + } else { + log.err("unknown command: {q}", .{command}); + continue; + } + } + }; + + try multi_reader.fillRemaining(.none); + + if (client_stderr.bufferedLen() > 0) { + log.err("stderr:\n{s}\n", .{client_stderr.buffered()}); + } + + try err; + + const term = try child_process.wait(io); + + if (!term.success()) { + log.err("maker {f}", .{term}); + } +} + +const FormatEnum = union(enum) { + named: []const u8, + unnamed: usize, + + pub fn format( + e: FormatEnum, + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + switch (e) { + .named => |name| { + try writer.writeByte('.'); + try writer.writeAll(name); + }, + .unnamed => |number| try writer.print("0x{x}", .{number}), + } + } +}; + +fn fmtEnum(e: anytype) FormatEnum { + if (std.enums.tagName(@TypeOf(e), e)) |name| { + return .{ .named = name }; + } else { + return .{ .unnamed = @backingInt(e) }; + } +}