authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-14 17:04:03-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-21 16:25:25-07:00
log131cbcbdac353477af9ee119a735397ae993a2c4
treeb1f64bd924ff297ced0828fdd4cb1b4430a54c0e
parentc9ae24503dc8da2e59f46619695bf4eb863fb3ac

stage2: hot code swapping PoC

* CLI supports --listen to accept commands on a socket * make it able to produce an updated executable while it is running

4 files changed, 214 insertions(+), 2 deletions(-)

lib/std/child_process.zig+4
...@@ -19,6 +19,8 @@ const maxInt = std.math.maxInt;...@@ -19,6 +19,8 @@ const maxInt = std.math.maxInt;
19const assert = std.debug.assert;19const assert = std.debug.assert;
2020
21pub const ChildProcess = struct {21pub const ChildProcess = struct {
22 /// Available after calling `spawn()`. It is a race condition to use this
23 /// value after `wait()`.
22 pid: if (builtin.os.tag == .windows) void else i32,24 pid: if (builtin.os.tag == .windows) void else i32,
23 handle: if (builtin.os.tag == .windows) windows.HANDLE else void,25 handle: if (builtin.os.tag == .windows) windows.HANDLE else void,
24 thread_handle: if (builtin.os.tag == .windows) windows.HANDLE else void,26 thread_handle: if (builtin.os.tag == .windows) windows.HANDLE else void,
...@@ -123,6 +125,7 @@ pub const ChildProcess = struct {...@@ -123,6 +125,7 @@ pub const ChildProcess = struct {
123 }125 }
124126
125 /// On success must call `kill` or `wait`.127 /// On success must call `kill` or `wait`.
128 /// After spawning the `pid` is available.
126 pub fn spawn(self: *ChildProcess) SpawnError!void {129 pub fn spawn(self: *ChildProcess) SpawnError!void {
127 if (builtin.os.tag == .windows) {130 if (builtin.os.tag == .windows) {
128 return self.spawnWindows();131 return self.spawnWindows();
...@@ -167,6 +170,7 @@ pub const ChildProcess = struct {...@@ -167,6 +170,7 @@ pub const ChildProcess = struct {
167 }170 }
168171
169 /// Blocks until child process terminates and then cleans up all resources.172 /// Blocks until child process terminates and then cleans up all resources.
173 /// TODO: set the pid to undefined in this function.
170 pub fn wait(self: *ChildProcess) !Term {174 pub fn wait(self: *ChildProcess) !Term {
171 if (builtin.os.tag == .windows) {175 if (builtin.os.tag == .windows) {
172 return self.waitWindows();176 return self.waitWindows();
src/Compilation.zig+9-2
...@@ -358,10 +358,10 @@ pub const AllErrors = struct {...@@ -358,10 +358,10 @@ pub const AllErrors = struct {
358 return msg.renderToStdErrInner(ttyconf, stderr, "error:", .Red, 0) catch return;358 return msg.renderToStdErrInner(ttyconf, stderr, "error:", .Red, 0) catch return;
359 }359 }
360360
361 fn renderToStdErrInner(361 pub fn renderToStdErrInner(
362 msg: Message,362 msg: Message,
363 ttyconf: std.debug.TTY.Config,363 ttyconf: std.debug.TTY.Config,
364 stderr_file: std.fs.File,364 stderr_file: anytype,
365 kind: []const u8,365 kind: []const u8,
366 color: std.debug.TTY.Color,366 color: std.debug.TTY.Color,
367 indent: usize,367 indent: usize,
...@@ -5221,3 +5221,10 @@ pub fn compilerRtStrip(comp: Compilation) bool {...@@ -5221,3 +5221,10 @@ pub fn compilerRtStrip(comp: Compilation) bool {
5221 return true;5221 return true;
5222 }5222 }
5223}5223}
5224
5225pub fn hotCodeSwap(comp: *Compilation, pid: std.os.pid_t) !void {
5226 comp.bin_file.child_pid = pid;
5227 try comp.makeBinFileWritable();
5228 try comp.update();
5229 try comp.makeBinFileExecutable();
5230}
src/link.zig+13
...@@ -204,6 +204,8 @@ pub const File = struct {...@@ -204,6 +204,8 @@ pub const File = struct {
204 /// of this linking operation.204 /// of this linking operation.
205 lock: ?Cache.Lock = null,205 lock: ?Cache.Lock = null,
206206
207 child_pid: ?std.os.pid_t = null,
208
207 pub const LinkBlock = union {209 pub const LinkBlock = union {
208 elf: Elf.TextBlock,210 elf: Elf.TextBlock,
209 coff: Coff.TextBlock,211 coff: Coff.TextBlock,
...@@ -330,6 +332,17 @@ pub const File = struct {...@@ -330,6 +332,17 @@ pub const File = struct {
330 .coff, .elf, .macho, .plan9 => {332 .coff, .elf, .macho, .plan9 => {
331 if (base.file != null) return;333 if (base.file != null) return;
332 const emit = base.options.emit orelse return;334 const emit = base.options.emit orelse return;
335 if (base.child_pid != null) {
336 // If we try to open the output file in write mode while it is running,
337 // it will return ETXTBSY. So instead, we copy the file, atomically rename it
338 // over top of the exe path, and then proceed normally. This changes the inode,
339 // avoiding the error.
340 const tmp_sub_path = try std.fmt.allocPrint(base.allocator, "{s}-{x}", .{
341 emit.sub_path, std.crypto.random.int(u32),
342 });
343 try emit.directory.handle.copyFile(emit.sub_path, emit.directory.handle, tmp_sub_path, .{});
344 try emit.directory.handle.rename(tmp_sub_path, emit.sub_path);
345 }
333 base.file = try emit.directory.handle.createFile(emit.sub_path, .{346 base.file = try emit.directory.handle.createFile(emit.sub_path, .{
334 .truncate = false,347 .truncate = false,
335 .read = true,348 .read = true,
src/main.zig+188
...@@ -578,6 +578,7 @@ fn buildOutputType(...@@ -578,6 +578,7 @@ fn buildOutputType(
578 var strip = false;578 var strip = false;
579 var function_sections = false;579 var function_sections = false;
580 var watch = false;580 var watch = false;
581 var listen_addr: ?std.net.Ip4Address = null;
581 var debug_compile_errors = false;582 var debug_compile_errors = false;
582 var verbose_link = std.process.hasEnvVarConstant("ZIG_VERBOSE_LINK");583 var verbose_link = std.process.hasEnvVarConstant("ZIG_VERBOSE_LINK");
583 var verbose_cc = std.process.hasEnvVarConstant("ZIG_VERBOSE_CC");584 var verbose_cc = std.process.hasEnvVarConstant("ZIG_VERBOSE_CC");
...@@ -994,6 +995,18 @@ fn buildOutputType(...@@ -994,6 +995,18 @@ fn buildOutputType(
994 } else {995 } else {
995 try log_scopes.append(gpa, args[i]);996 try log_scopes.append(gpa, args[i]);
996 }997 }
998 } else if (mem.eql(u8, arg, "--listen")) {
999 if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
1000 i += 1;
1001 // example: --listen 127.0.0.1:9000
1002 var it = std.mem.split(u8, args[i], ":");
1003 const host = it.next().?;
1004 const port_text = it.next() orelse "14735";
1005 const port = std.fmt.parseInt(u16, port_text, 10) catch |err|
1006 fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) });
1007 listen_addr = std.net.Ip4Address.parse(host, port) catch |err|
1008 fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) });
1009 watch = true;
997 } else if (mem.eql(u8, arg, "--debug-link-snapshot")) {1010 } else if (mem.eql(u8, arg, "--debug-link-snapshot")) {
998 if (!build_options.enable_link_snapshots) {1011 if (!build_options.enable_link_snapshots) {
999 std.log.warn("Zig was compiled without linker snapshots enabled (-Dlink-snapshot). --debug-link-snapshot has no effect.", .{});1012 std.log.warn("Zig was compiled without linker snapshots enabled (-Dlink-snapshot). --debug-link-snapshot has no effect.", .{});
...@@ -2649,6 +2662,125 @@ fn buildOutputType(...@@ -2649,6 +2662,125 @@ fn buildOutputType(
26492662
2650 var last_cmd: ReplCmd = .help;2663 var last_cmd: ReplCmd = .help;
26512664
2665 if (listen_addr) |ip4_addr| {
2666 var server = std.net.StreamServer.init(.{
2667 .reuse_address = true,
2668 });
2669 defer server.deinit();
2670
2671 try server.listen(.{ .in = ip4_addr });
2672
2673 while (true) {
2674 const conn = try server.accept();
2675 defer conn.stream.close();
2676
2677 var buf: [100]u8 = undefined;
2678 var child_pid: ?i32 = null;
2679
2680 while (true) {
2681 try comp.makeBinFileExecutable();
2682
2683 const amt = try conn.stream.read(&buf);
2684 const line = buf[0..amt];
2685 const actual_line = mem.trimRight(u8, line, "\r\n ");
2686
2687 const cmd: ReplCmd = blk: {
2688 if (mem.eql(u8, actual_line, "update")) {
2689 break :blk .update;
2690 } else if (mem.eql(u8, actual_line, "exit")) {
2691 break;
2692 } else if (mem.eql(u8, actual_line, "help")) {
2693 break :blk .help;
2694 } else if (mem.eql(u8, actual_line, "run")) {
2695 break :blk .run;
2696 } else if (mem.eql(u8, actual_line, "update-and-run")) {
2697 break :blk .update_and_run;
2698 } else if (actual_line.len == 0) {
2699 break :blk last_cmd;
2700 } else {
2701 try stderr.print("unknown command: {s}\n", .{actual_line});
2702 continue;
2703 }
2704 };
2705 last_cmd = cmd;
2706 switch (cmd) {
2707 .update => {
2708 tracy.frameMark();
2709 if (output_mode == .Exe) {
2710 try comp.makeBinFileWritable();
2711 }
2712 updateModule(gpa, comp, hook) catch |err| switch (err) {
2713 error.SemanticAnalyzeFail => continue,
2714 else => |e| return e,
2715 };
2716 },
2717 .help => {
2718 try stderr.writeAll(repl_help);
2719 },
2720 .run => {
2721 tracy.frameMark();
2722 try runOrTest(
2723 comp,
2724 gpa,
2725 arena,
2726 test_exec_args.items,
2727 self_exe_path,
2728 arg_mode,
2729 target_info,
2730 watch,
2731 &comp_destroyed,
2732 all_args,
2733 runtime_args_start,
2734 link_libc,
2735 );
2736 },
2737 .update_and_run => {
2738 tracy.frameMark();
2739 if (child_pid) |pid| {
2740 try conn.stream.writer().print("hot code swap requested for pid {d}", .{pid});
2741 try comp.hotCodeSwap(pid);
2742
2743 var errors = try comp.getAllErrorsAlloc();
2744 defer errors.deinit(comp.gpa);
2745
2746 if (errors.list.len != 0) {
2747 const ttyconf: std.debug.TTY.Config = switch (comp.color) {
2748 .auto => std.debug.detectTTYConfig(),
2749 .on => .escape_codes,
2750 .off => .no_color,
2751 };
2752 for (errors.list) |full_err_msg| {
2753 try full_err_msg.renderToStdErrInner(ttyconf, conn.stream, "error:", .Red, 0);
2754 }
2755 continue;
2756 }
2757 } else {
2758 if (output_mode == .Exe) {
2759 try comp.makeBinFileWritable();
2760 }
2761 updateModule(gpa, comp, hook) catch |err| switch (err) {
2762 error.SemanticAnalyzeFail => continue,
2763 else => |e| return e,
2764 };
2765 try comp.makeBinFileExecutable();
2766
2767 child_pid = try runOrTestHotSwap(
2768 comp,
2769 gpa,
2770 arena,
2771 test_exec_args.items,
2772 self_exe_path,
2773 arg_mode,
2774 all_args,
2775 runtime_args_start,
2776 );
2777 }
2778 },
2779 }
2780 }
2781 }
2782 }
2783
2652 while (watch) {2784 while (watch) {
2653 try stderr.print("(zig) ", .{});2785 try stderr.print("(zig) ", .{});
2654 try comp.makeBinFileExecutable();2786 try comp.makeBinFileExecutable();
...@@ -2901,6 +3033,62 @@ fn runOrTest(...@@ -2901,6 +3033,62 @@ fn runOrTest(
2901 }3033 }
2902}3034}
29033035
3036fn runOrTestHotSwap(
3037 comp: *Compilation,
3038 gpa: Allocator,
3039 arena: Allocator,
3040 test_exec_args: []const ?[]const u8,
3041 self_exe_path: []const u8,
3042 arg_mode: ArgMode,
3043 all_args: []const []const u8,
3044 runtime_args_start: ?usize,
3045) !i32 {
3046 const exe_emit = comp.bin_file.options.emit.?;
3047 // A naive `directory.join` here will indeed get the correct path to the binary,
3048 // however, in the case of cwd, we actually want `./foo` so that the path can be executed.
3049 const exe_path = try fs.path.join(arena, &[_][]const u8{
3050 exe_emit.directory.path orelse ".", exe_emit.sub_path,
3051 });
3052
3053 var argv = std.ArrayList([]const u8).init(gpa);
3054 defer argv.deinit();
3055
3056 if (test_exec_args.len == 0) {
3057 // when testing pass the zig_exe_path to argv
3058 if (arg_mode == .zig_test)
3059 try argv.appendSlice(&[_][]const u8{
3060 exe_path, self_exe_path,
3061 })
3062 // when running just pass the current exe
3063 else
3064 try argv.appendSlice(&[_][]const u8{
3065 exe_path,
3066 });
3067 } else {
3068 for (test_exec_args) |arg| {
3069 if (arg) |a| {
3070 try argv.append(a);
3071 } else {
3072 try argv.appendSlice(&[_][]const u8{
3073 exe_path, self_exe_path,
3074 });
3075 }
3076 }
3077 }
3078 if (runtime_args_start) |i| {
3079 try argv.appendSlice(all_args[i..]);
3080 }
3081 const child = try std.ChildProcess.init(argv.items, arena);
3082
3083 child.stdin_behavior = .Inherit;
3084 child.stdout_behavior = .Inherit;
3085 child.stderr_behavior = .Inherit;
3086
3087 try child.spawn();
3088
3089 return child.pid;
3090}
3091
2904const AfterUpdateHook = union(enum) {3092const AfterUpdateHook = union(enum) {
2905 none,3093 none,
2906 print_emit_bin_dir_path,3094 print_emit_bin_dir_path,