authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-16 20:18:06-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-16 20:18:06-07:00
log01a7affb87f74e33e674891d77d4b720d3014013
treedbaaff1962b0db60e3c45f8d061075551294b374
parent07eb2c65f6ee3d2ea81a59f776689b1d919d3280

stage2: ask for a higher open fd limit

Here's the doc comment from the commit: For one example of why this is handy, consider the case of building musl libc. We keep a lock open for each of the object files in the form of a file descriptor until they are finally put into an archive file. This is to allow a zig-cache garbage collector to run concurrently to zig processes, and to allow multiple zig processes to run concurrently with each other, without clobbering each other. This code is disabled until #6361 is implemented (getrlimit/setrlimit are not yet added to the standard library).

2 files changed, 45 insertions(+), 1 deletions(-)

BRANCH_TODO-1
...@@ -12,7 +12,6 @@...@@ -12,7 +12,6 @@
12 * capture lld stdout/stderr better12 * capture lld stdout/stderr better
13 * musl13 * musl
14 * mingw-w6414 * mingw-w64
15 * port the stage1 os.cpp code that raises the open fd limit
16 * use global zig-cache dir for crt files15 * use global zig-cache dir for crt files
17 * `zig translate-c`16 * `zig translate-c`
18 * MachO LLD linking17 * MachO LLD linking
src-self-hosted/main.zig+45
...@@ -1103,6 +1103,8 @@ pub fn buildOutputType(...@@ -1103,6 +1103,8 @@ pub fn buildOutputType(
1103 },1103 },
1104 };1104 };
11051105
1106 gimmeMoreOfThoseSweetSweetFileDescriptors();
1107
1106 const comp = Compilation.create(gpa, .{1108 const comp = Compilation.create(gpa, .{
1107 .zig_lib_directory = zig_lib_directory,1109 .zig_lib_directory = zig_lib_directory,
1108 .zig_cache_directory = zig_cache_directory,1110 .zig_cache_directory = zig_cache_directory,
...@@ -1930,3 +1932,46 @@ fn parseCodeModel(arg: []const u8) std.builtin.CodeModel {...@@ -1930,3 +1932,46 @@ fn parseCodeModel(arg: []const u8) std.builtin.CodeModel {
1930 return std.meta.stringToEnum(std.builtin.CodeModel, arg) orelse1932 return std.meta.stringToEnum(std.builtin.CodeModel, arg) orelse
1931 fatal("unsupported machine code model: '{}'", .{arg});1933 fatal("unsupported machine code model: '{}'", .{arg});
1932}1934}
1935
1936/// Raise the open file descriptor limit. Ask and ye shall receive.
1937/// For one example of why this is handy, consider the case of building musl libc.
1938/// We keep a lock open for each of the object files in the form of a file descriptor
1939/// until they are finally put into an archive file. This is to allow a zig-cache
1940/// garbage collector to run concurrently to zig processes, and to allow multiple
1941/// zig processes to run concurrently with each other, without clobbering each other.
1942fn gimmeMoreOfThoseSweetSweetFileDescriptors() void {
1943 switch (std.Target.current.os.tag) {
1944 .windows, .wasi, .uefi, .other, .freestanding => return,
1945 // std lib is missing getrlimit/setrlimit.
1946 // https://github.com/ziglang/zig/issues/6361
1947 //else => {},
1948 else => return,
1949 }
1950 const posix = std.os;
1951 var lim = posix.getrlimit(posix.RLIMIT_NOFILE, &lim) catch return; // Oh well; we tried.
1952 if (lim.cur == lim.max) return;
1953 while (true) {
1954 // Do a binary search for the limit.
1955 var min: posix.rlim_t = lim.cur;
1956 var max: posix.rlim_t = 1 << 20;
1957 // But if there's a defined upper bound, don't search, just set it.
1958 if (lim.max != posix.RLIM_INFINITY) {
1959 min = lim.max;
1960 max = lim.max;
1961 }
1962 while (true) {
1963 lim.cur = min + (max - min) / 2;
1964 if (posix.setrlimit(posix.RLIMIT_NOFILE, lim)) |_| {
1965 min = lim.cur;
1966 } else |_| {
1967 max = lim.cur;
1968 }
1969 if (min + 1 < max) continue;
1970 return;
1971 }
1972 }
1973}
1974
1975test "fds" {
1976 gimmeMoreOfThoseSweetSweetFileDescriptors();
1977}