Commit e657b73f30e3380b14caf3a115a4c7a97c510d77 | zig

author	Andrew Kelley <andrew@ziglang.org> 2019-09-10 10:39:27-04:00
committer	Andrew Kelley <andrew@ziglang.org> 2019-09-10 10:39:27-04:00
log	e657b73f30e3380b14caf3a115a4c7a97c510d77
tree	26c3f3b7a56f215b56c4e92cef5559dd5b1964b2
parent	a29ce78651c05029dbd72064752a099885edfd0c
parent	ff051f8f5de47f4c5033c61fb70a5c5260f34dff
signature	Commit is signed but in an unrecognized format.

Merge branch 'async-std-lib'

This introduces the concept of "IO mode" which is configurable by the
root source file (e.g. next to `pub fn main`). Applications can put this
in their root source file:

```
pub const io_mode = .evented;
```

This will populate `std.io.mode` to be `std.io.Mode.evented`. When I/O
mode is evented, `std.os.read` handles EAGAIN by suspending until the
file descriptor becomes available for reading. Although the std lib
event loop supports epoll, kqueue, and Windows I/O Completion Ports,
this integration with `std.os.read` currently only works on Linux.

This integration is currently only hooked up to `std.os.read`, and not,
for example, `std.os.write`, child processes, and timers. The fact that
we can do this and still have a working master branch is thanks to Zig's
lazy analysis, comptime, and inferred async. We can continue to make
incremental progress on async std lib features, enabling more and more
test cases and coverage.

In addition to `std.io.mode` there is `std.io.is_async` which is equal
to `std.io.mode == .evented`. In case I/O mode is async, `std.io.InStream`
notices this and the read function pointer becomes an async function
pointer rather than a blocking function pointer. Even in this case,
`std.io.InStream` can *still be used as a blocking input stream*.
Users of the API control whether it is blocking or async at runtime by whether
or not the read function suspends. In case of file descriptors, for
example, this might correspond to whether it was opened with `O_NONBLOCK`.
The `noasync` keyword makes a function call or `await` assert that no
suspension happens. This assertion has runtime safety enabled.

`std.io.InStream`, in the case of async I/O, uses by default a 4 MiB
frame size for calling the read function. If this is too large or too
small, the application can globally increase the frame size used by
declaring `pub const stack_size_std_io_InStream = 1234;` in their root
source file. This way, `std.io.InStream` will only be generated once,
avoiding bloat, and as long as this number is configured to be high
enough, everything works fine. Zig has runtime safety to detect when
`@asyncCall` is given too small of a buffer for the frame size.

This merge introduces -fstack-report which can help identify large async
function frame sizes and explain what is making them so big. Until #3069 is
solved, it's recommended to stick with blocking IO mode.

-fstack-report outputs JSON format, which can then be viewed in a GUI
that represents the tree structure. As an example, Firefox does a decent
job of this.

One feature that is currently missing is detecting that the call stack
upper bound is greater than the default for a given target, and passing
this upper bound to the linker. As an example, if Zig detects that 20
MiB stack upper bound is needed - which would be quite reasonable -
currently on Linux the application would only be given the default of 16
MiB.

Unrelated miscellaneous change: added std.c.readv

14 files changed, 435 insertions(+), 307 deletions(-)

CMakeLists.txt
src
std
- c.zig
- debug.zig
- event.zig
- event
  - io.zig
  - loop.zig
- io.zig
- io
  - in_stream.zig
- os.zig

CMakeLists.txt+1

...	...	@@ -449,6 +449,7 @@ set(ZIG_SOURCES
449	449	"${CMAKE_SOURCE_DIR}/src/os.cpp"
450	450	"${CMAKE_SOURCE_DIR}/src/parser.cpp"
451	451	"${CMAKE_SOURCE_DIR}/src/range_set.cpp"
	452	"${CMAKE_SOURCE_DIR}/src/stack_report.cpp"
452	453	"${CMAKE_SOURCE_DIR}/src/target.cpp"
453	454	"${CMAKE_SOURCE_DIR}/src/tokenizer.cpp"
454	455	"${CMAKE_SOURCE_DIR}/src/translate_c.cpp"

src/all_types.hpp+3

...	...	@@ -1972,6 +1972,8 @@ struct CodeGen {
1972	1972	ZigFn *panic_fn;
1973	1973	TldFn *panic_tld_fn;
1974	1974
	1975	ZigFn *largest_frame_fn;
	1976
1975	1977	WantPIC want_pic;
1976	1978	WantStackCheck want_stack_check;
1977	1979	CacheHash cache_hash;
...	...	@@ -2004,6 +2006,7 @@ struct CodeGen {
2004	2006	bool generate_error_name_table;
2005	2007	bool enable_cache; // mutually exclusive with output_dir
2006	2008	bool enable_time_report;
	2009	bool enable_stack_report;
2007	2010	bool system_linker_hack;
2008	2011	bool reported_bad_link_libc_error;
2009	2012	bool is_dynamic; // shared library rather than static library. dynamic musl rather than static musl.

src/analyze.cpp+14-9

...	...	@@ -5737,11 +5737,19 @@ static void mark_suspension_point(Scope *scope) {
5737	5737	return;
5738	5738	case ScopeIdVarDecl:
5739	5739	case ScopeIdDefer:
	5740	case ScopeIdBlock:
5740	5741	looking_for_exprs = false;
5741	5742	continue;
5742		case ScopeIdLoop:
5743	5743	case ScopeIdRuntime:
5744	5744	continue;
	5745	case ScopeIdLoop: {
	5746	ScopeLoop loop_scope = reinterpret_cast<ScopeLoop >(scope);
	5747	if (loop_scope->spill_scope != nullptr) {
	5748	loop_scope->spill_scope->need_spill = MemoizedBoolTrue;
	5749	}
	5750	looking_for_exprs = false;
	5751	continue;
	5752	}
5745	5753	case ScopeIdExpr: {
5746	5754	if (!looking_for_exprs) {
5747	5755	// Now we're only looking for a block, to see if it's in a loop (see the case ScopeIdBlock)
...	...	@@ -5758,14 +5766,6 @@ static void mark_suspension_point(Scope *scope) {
5758	5766	child_expr_scope = parent_expr_scope;
5759	5767	continue;
5760	5768	}
5761		case ScopeIdBlock:
5762		if (scope->parent->parent->id == ScopeIdLoop) {
5763		ScopeLoop loop_scope = reinterpret_cast<ScopeLoop >(scope->parent->parent);
5764		if (loop_scope->spill_scope != nullptr) {
5765		loop_scope->spill_scope->need_spill = MemoizedBoolTrue;
5766		}
5767		}
5768		return;
5769	5769	}
5770	5770	}
5771	5771	}
...	...	@@ -6082,6 +6082,11 @@ static Error resolve_async_frame(CodeGen g, ZigType frame_type) {
6082	6082	frame_type->abi_size = frame_type->data.frame.locals_struct->abi_size;
6083	6083	frame_type->abi_align = frame_type->data.frame.locals_struct->abi_align;
6084	6084	frame_type->size_in_bits = frame_type->data.frame.locals_struct->size_in_bits;
	6085
	6086	if (g->largest_frame_fn == nullptr \|\| frame_type->abi_size > g->largest_frame_fn->frame_type->abi_size) {
	6087	g->largest_frame_fn = fn;
	6088	}
	6089
6085	6090	return ErrorNone;
6086	6091	}
6087	6092

src/main.cpp+12

...	...	@@ -16,6 +16,7 @@
16	16	#include "libc_installation.hpp"
17	17	#include "userland.h"
18	18	#include "glibc.hpp"
	19	#include "stack_report.hpp"
19	20
20	21	#include <stdio.h>
21	22
...	...	@@ -62,6 +63,7 @@ static int print_full_usage(const char arg0, FILE file, int return_code) {
62	63	" -fPIC enable Position Independent Code\n"
63	64	" -fno-PIC disable Position Independent Code\n"
64	65	" -ftime-report print timing diagnostics\n"
	66	" -fstack-report print stack size diagnostics\n"
65	67	" --libc [file] Provide a file which specifies libc paths\n"
66	68	" --name [name] override output name\n"
67	69	" --output-dir [dir] override output directory (defaults to cwd)\n"
...	...	@@ -476,6 +478,7 @@ int main(int argc, char **argv) {
476	478	size_t ver_minor = 0;
477	479	size_t ver_patch = 0;
478	480	bool timing_info = false;
	481	bool stack_report = false;
479	482	const char *cache_dir = nullptr;
480	483	CliPkg *cur_pkg = allocate<CliPkg>(1);
481	484	BuildMode build_mode = BuildModeDebug;
...	...	@@ -664,6 +667,8 @@ int main(int argc, char **argv) {
664	667	each_lib_rpath = true;
665	668	} else if (strcmp(arg, "-ftime-report") == 0) {
666	669	timing_info = true;
	670	} else if (strcmp(arg, "-fstack-report") == 0) {
	671	stack_report = true;
667	672	} else if (strcmp(arg, "--enable-valgrind") == 0) {
668	673	valgrind_support = ValgrindSupportEnabled;
669	674	} else if (strcmp(arg, "--disable-valgrind") == 0) {
...	...	@@ -1136,6 +1141,7 @@ int main(int argc, char **argv) {
1136	1141	g->subsystem = subsystem;
1137	1142
1138	1143	g->enable_time_report = timing_info;
	1144	g->enable_stack_report = stack_report;
1139	1145	codegen_set_out_name(g, buf_out_name);
1140	1146	codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
1141	1147	g->want_single_threaded = want_single_threaded;
...	...	@@ -1223,6 +1229,8 @@ int main(int argc, char **argv) {
1223	1229	codegen_build_and_link(g);
1224	1230	if (timing_info)
1225	1231	codegen_print_timing_report(g, stdout);
	1232	if (stack_report)
	1233	zig_print_stack_report(g, stdout);
1226	1234
1227	1235	if (cmd == CmdRun) {
1228	1236	const char *exec_path = buf_ptr(&g->output_file_path);
...	...	@@ -1272,6 +1280,10 @@ int main(int argc, char **argv) {
1272	1280	codegen_print_timing_report(g, stdout);
1273	1281	}
1274	1282
	1283	if (stack_report) {
	1284	zig_print_stack_report(g, stdout);
	1285	}
	1286
1275	1287	Buf *test_exe_path_unresolved = &g->output_file_path;
1276	1288	Buf *test_exe_path = buf_alloc();
1277	1289	*test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1);

src/stack_report.cpp created+121

...	...	@@ -0,0 +1,121 @@
	1	/*
	2	* Copyright (c) 2019 Andrew Kelley
	3	*
	4	* This file is part of zig, which is MIT licensed.
	5	* See http://opensource.org/licenses/MIT
	6	*/
	7
	8	#include "stack_report.hpp"
	9
	10	static void tree_print(FILE f, ZigType ty, size_t indent);
	11
	12	static void pretty_print_bytes(FILE *f, double n) {
	13	if (n > 1024.0 * 1024.0 * 1024.0) {
	14	fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0);
	15	return;
	16	}
	17	if (n > 1024.0 * 1024.0) {
	18	fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0);
	19	return;
	20	}
	21	if (n > 1024.0) {
	22	fprintf(f, "%.02f KiB", n / 1024.0);
	23	return;
	24	}
	25	fprintf(f, "%.02f bytes", n );
	26	return;
	27	}
	28
	29	static int compare_type_abi_sizes_desc(const void a, const void b) {
	30	uint64_t size_a = ((ZigType const*)(a))->abi_size;
	31	uint64_t size_b = ((ZigType const*)(b))->abi_size;
	32	if (size_a > size_b)
	33	return -1;
	34	if (size_a < size_b)
	35	return 1;
	36	return 0;
	37	}
	38
	39	static void start_child(FILE *f, size_t indent) {
	40	fprintf(f, "\n");
	41	for (size_t i = 0; i < indent; i += 1) {
	42	fprintf(f, " ");
	43	}
	44	}
	45
	46	static void start_peer(FILE *f, size_t indent) {
	47	fprintf(f, ",\n");
	48	for (size_t i = 0; i < indent; i += 1) {
	49	fprintf(f, " ");
	50	}
	51	}
	52
	53	static void tree_print_struct(FILE f, ZigType struct_type, size_t indent) {
	54	ZigList<ZigType *> children = {};
	55	uint64_t sum_from_fields = 0;
	56	for (size_t i = 0; i < struct_type->data.structure.src_field_count; i += 1) {
	57	TypeStructField *field = &struct_type->data.structure.fields[i];
	58	children.append(field->type_entry);
	59	sum_from_fields += field->type_entry->abi_size;
	60	}
	61	qsort(children.items, children.length, sizeof(ZigType *), compare_type_abi_sizes_desc);
	62
	63	start_peer(f, indent);
	64	fprintf(f, "\"padding\": \"%" ZIG_PRI_u64 "\"", struct_type->abi_size - sum_from_fields);
	65
	66	start_peer(f, indent);
	67	fprintf(f, "\"fields\": [");
	68
	69	for (size_t i = 0; i < children.length; i += 1) {
	70	if (i == 0) {
	71	start_child(f, indent + 1);
	72	} else {
	73	start_peer(f, indent + 1);
	74	}
	75	fprintf(f, "{");
	76
	77	ZigType *child_type = children.at(i);
	78	tree_print(f, child_type, indent + 2);
	79
	80	start_child(f, indent + 1);
	81	fprintf(f, "}");
	82	}
	83
	84	start_child(f, indent);
	85	fprintf(f, "]");
	86	}
	87
	88	static void tree_print(FILE f, ZigType ty, size_t indent) {
	89	start_child(f, indent);
	90	fprintf(f, "\"type\": \"%s\"", buf_ptr(&ty->name));
	91
	92	start_peer(f, indent);
	93	fprintf(f, "\"sizef\": \"");
	94	pretty_print_bytes(f, ty->abi_size);
	95	fprintf(f, "\"");
	96
	97	start_peer(f, indent);
	98	fprintf(f, "\"size\": \"%" ZIG_PRI_u64 "\"", ty->abi_size);
	99
	100	switch (ty->id) {
	101	case ZigTypeIdFnFrame:
	102	return tree_print_struct(f, ty->data.frame.locals_struct, indent);
	103	case ZigTypeIdStruct:
	104	return tree_print_struct(f, ty, indent);
	105	default:
	106	start_child(f, indent);
	107	return;
	108	}
	109	}
	110
	111	void zig_print_stack_report(CodeGen g, FILE f) {
	112	if (g->largest_frame_fn == nullptr) {
	113	fprintf(f, "{\"error\": \"No async function frames in entire compilation.\"}\n");
	114	return;
	115	}
	116	fprintf(f, "{");
	117	tree_print(f, g->largest_frame_fn->frame_type, 1);
	118
	119	start_child(f, 0);
	120	fprintf(f, "}\n");
	121	}

src/stack_report.hpp created+16

...	...	@@ -0,0 +1,16 @@
	1	/*
	2	* Copyright (c) 2019 Andrew Kelley
	3	*
	4	* This file is part of zig, which is MIT licensed.
	5	* See http://opensource.org/licenses/MIT
	6	*/
	7
	8	#ifndef ZIG_STACK_REPORT_HPP
	9	#define ZIG_STACK_REPORT_HPP
	10
	11	#include "all_types.hpp"
	12	#include <stdio.h>
	13
	14	void zig_print_stack_report(CodeGen g, FILE f);
	15
	16	#endif

std/c.zig+1

...	...	@@ -68,6 +68,7 @@ pub extern "c" fn open(path: [*]const u8, oflag: c_uint, ...) c_int;
68	68	pub extern "c" fn openat(fd: c_int, path: [*]const u8, oflag: c_uint, ...) c_int;
69	69	pub extern "c" fn raise(sig: c_int) c_int;
70	70	pub extern "c" fn read(fd: fd_t, buf: [*]u8, nbyte: usize) isize;
	71	pub extern "c" fn readv(fd: c_int, iov: [*]const iovec, iovcnt: c_uint) isize;
71	72	pub extern "c" fn pread(fd: fd_t, buf: [*]u8, nbyte: usize, offset: u64) isize;
72	73	pub extern "c" fn preadv(fd: c_int, iov: [*]const iovec, iovcnt: c_uint, offset: usize) isize;
73	74	pub extern "c" fn writev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint) isize;

std/debug.zig+30-13

...	...	@@ -330,14 +330,16 @@ pub fn writeCurrentStackTraceWindows(
330	330	}
331	331	}
332	332
	333	/// TODO once https://github.com/ziglang/zig/issues/3157 is fully implemented,
	334	/// make this `noasync fn` and remove the individual noasync calls.
333	335	pub fn printSourceAtAddress(debug_info: *DebugInfo, out_stream: var, address: usize, tty_color: bool) !void {
334	336	if (windows.is_the_target) {
335		return printSourceAtAddressWindows(debug_info, out_stream, address, tty_color);
	337	return noasync printSourceAtAddressWindows(debug_info, out_stream, address, tty_color);
336	338	}
337	339	if (os.darwin.is_the_target) {
338		return printSourceAtAddressMacOs(debug_info, out_stream, address, tty_color);
	340	return noasync printSourceAtAddressMacOs(debug_info, out_stream, address, tty_color);
339	341	}
340		return printSourceAtAddressPosix(debug_info, out_stream, address, tty_color);
	342	return noasync printSourceAtAddressPosix(debug_info, out_stream, address, tty_color);
341	343	}
342	344
343	345	fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, relocated_address: usize, tty_color: bool) !void {
...	...	@@ -793,7 +795,7 @@ fn printLineInfo(
793	795	try out_stream.write(GREEN ++ "^" ++ RESET ++ "\n");
794	796	}
795	797	} else \|err\| switch (err) {
796		error.EndOfFile, error.FileNotFound => {},
	798	error.EndOfFile, error.FileNotFound => {},
797	799	else => return err,
798	800	}
799	801	} else {
...	...	@@ -816,16 +818,18 @@ pub const OpenSelfDebugInfoError = error{
816	818	UnsupportedOperatingSystem,
817	819	};
818	820
	821	/// TODO once https://github.com/ziglang/zig/issues/3157 is fully implemented,
	822	/// make this `noasync fn` and remove the individual noasync calls.
819	823	pub fn openSelfDebugInfo(allocator: *mem.Allocator) !DebugInfo {
820	824	if (builtin.strip_debug_info)
821	825	return error.MissingDebugInfo;
822	826	if (windows.is_the_target) {
823		return openSelfDebugInfoWindows(allocator);
	827	return noasync openSelfDebugInfoWindows(allocator);
824	828	}
825	829	if (os.darwin.is_the_target) {
826		return openSelfDebugInfoMacOs(allocator);
	830	return noasync openSelfDebugInfoMacOs(allocator);
827	831	}
828		return openSelfDebugInfoPosix(allocator);
	832	return noasync openSelfDebugInfoPosix(allocator);
829	833	}
830	834
831	835	fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo {
...	...	@@ -1508,15 +1512,25 @@ fn parseFormValueBlock(allocator: *mem.Allocator, in_stream: var, size: usize) !
1508	1512	}
1509	1513
1510	1514	fn parseFormValueConstant(allocator: *mem.Allocator, in_stream: var, signed: bool, comptime size: i32) !FormValue {
	1515	// TODO: Please forgive me, I've worked around zig not properly spilling some intermediate values here.
	1516	// `noasync` should be removed from all the function calls once it is fixed.
1511	1517	return FormValue{
1512	1518	.Const = Constant{
1513	1519	.signed = signed,
1514	1520	.payload = switch (size) {
1515		1 => try in_stream.readIntLittle(u8),
1516		2 => try in_stream.readIntLittle(u16),
1517		4 => try in_stream.readIntLittle(u32),
1518		8 => try in_stream.readIntLittle(u64),
1519		-1 => if (signed) @bitCast(u64, try leb.readILEB128(i64, in_stream)) else try leb.readULEB128(u64, in_stream),
	1521	1 => try noasync in_stream.readIntLittle(u8),
	1522	2 => try noasync in_stream.readIntLittle(u16),
	1523	4 => try noasync in_stream.readIntLittle(u32),
	1524	8 => try noasync in_stream.readIntLittle(u64),
	1525	-1 => blk: {
	1526	if (signed) {
	1527	const x = try noasync leb.readILEB128(i64, in_stream);
	1528	break :blk @bitCast(u64, x);
	1529	} else {
	1530	const x = try noasync leb.readULEB128(u64, in_stream);
	1531	break :blk x;
	1532	}
	1533	},
1520	1534	else => @compileError("Invalid size"),
1521	1535	},
1522	1536	},
...	...	@@ -1584,7 +1598,10 @@ fn parseFormValue(allocator: *mem.Allocator, in_stream: var, form_id: u64, is_64
1584	1598	DW.FORM_strp => FormValue{ .StrPtr = try parseFormValueDwarfOffsetSize(in_stream, is_64) },
1585	1599	DW.FORM_indirect => {
1586	1600	const child_form_id = try leb.readULEB128(u64, in_stream);
1587		return parseFormValue(allocator, in_stream, child_form_id, is_64);
	1601	const F = @typeOf(async parseFormValue(allocator, in_stream, child_form_id, is_64));
	1602	var frame = try allocator.create(F);
	1603	defer allocator.destroy(frame);
	1604	return await @asyncCall(frame, {}, parseFormValue, allocator, in_stream, child_form_id, is_64);
1588	1605	},
1589	1606	else => error.InvalidDebugInfo,
1590	1607	};

std/event.zig-2

...	...	@@ -6,7 +6,6 @@ pub const Locked = @import("event/locked.zig").Locked;
6	6	pub const RwLock = @import("event/rwlock.zig").RwLock;
7	7	pub const RwLocked = @import("event/rwlocked.zig").RwLocked;
8	8	pub const Loop = @import("event/loop.zig").Loop;
9		pub const io = @import("event/io.zig");
10	9	pub const fs = @import("event/fs.zig");
11	10	pub const net = @import("event/net.zig");
12	11
...	...	@@ -15,7 +14,6 @@ test "import event tests" {
15	14	_ = @import("event/fs.zig");
16	15	_ = @import("event/future.zig");
17	16	_ = @import("event/group.zig");
18		_ = @import("event/io.zig");
19	17	_ = @import("event/lock.zig");
20	18	_ = @import("event/locked.zig");
21	19	_ = @import("event/rwlock.zig");

std/event/io.zig deleted-76

...	...	@@ -1,76 +0,0 @@
1		const std = @import("../std.zig");
2		const builtin = @import("builtin");
3		const assert = std.debug.assert;
4		const mem = std.mem;
5
6		pub fn InStream(comptime ReadError: type) type {
7		return struct {
8		const Self = @This();
9		pub const Error = ReadError;
10
11		/// Return the number of bytes read. It may be less than buffer.len.
12		/// If the number of bytes read is 0, it means end of stream.
13		/// End of stream is not an error condition.
14		readFn: async fn (self: *Self, buffer: []u8) Error!usize,
15
16		/// Return the number of bytes read. It may be less than buffer.len.
17		/// If the number of bytes read is 0, it means end of stream.
18		/// End of stream is not an error condition.
19		pub async fn read(self: *Self, buffer: []u8) !usize {
20		return self.readFn(self, buffer);
21		}
22
23		/// Return the number of bytes read. If it is less than buffer.len
24		/// it means end of stream.
25		pub async fn readFull(self: *Self, buffer: []u8) !usize {
26		var index: usize = 0;
27		while (index != buf.len) {
28		const amt_read = try self.read(buf[index..]);
29		if (amt_read == 0) return index;
30		index += amt_read;
31		}
32		return index;
33		}
34
35		/// Same as `readFull` but end of stream returns `error.EndOfStream`.
36		pub async fn readNoEof(self: *Self, buf: []u8) !void {
37		const amt_read = try self.readFull(buf[index..]);
38		if (amt_read < buf.len) return error.EndOfStream;
39		}
40
41		pub async fn readIntLittle(self: *Self, comptime T: type) !T {
42		var bytes: [@sizeOf(T)]u8 = undefined;
43		try self.readNoEof(bytes[0..]);
44		return mem.readIntLittle(T, &bytes);
45		}
46
47		pub async fn readIntBe(self: *Self, comptime T: type) !T {
48		var bytes: [@sizeOf(T)]u8 = undefined;
49		try self.readNoEof(bytes[0..]);
50		return mem.readIntBig(T, &bytes);
51		}
52
53		pub async fn readInt(self: *Self, comptime T: type, endian: builtin.Endian) !T {
54		var bytes: [@sizeOf(T)]u8 = undefined;
55		try self.readNoEof(bytes[0..]);
56		return mem.readInt(T, &bytes, endian);
57		}
58
59		pub async fn readStruct(self: *Self, comptime T: type) !T {
60		// Only extern and packed structs have defined in-memory layout.
61		comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);
62		var res: [1]T = undefined;
63		try self.readNoEof(@sliceToBytes(res[0..]));
64		return res[0];
65		}
66		};
67		}
68
69		pub fn OutStream(comptime WriteError: type) type {
70		return struct {
71		const Self = @This();
72		pub const Error = WriteError;
73
74		writeFn: async fn (self: *Self, buffer: []u8) Error!void,
75		};
76		}

std/event/loop.zig+5-9

...	...	@@ -86,18 +86,10 @@ pub const Loop = struct {
86	86	};
87	87	};
88	88
89		pub const IoMode = enum {
90		blocking,
91		evented,
92		mixed,
93		};
94		pub const io_mode: IoMode = if (@hasDecl(root, "io_mode")) root.io_mode else IoMode.blocking;
95	89	var global_instance_state: Loop = undefined;
96		threadlocal var per_thread_instance: ?*Loop = null;
97		const default_instance: ?*Loop = switch (io_mode) {
	90	const default_instance: ?*Loop = switch (std.io.mode) {
98	91	.blocking => null,
99	92	.evented => &global_instance_state,
100		.mixed => per_thread_instance,
101	93	};
102	94	pub const instance: ?*Loop = if (@hasDecl(root, "event_loop")) root.event_loop else default_instance;
103	95
...	...	@@ -470,6 +462,10 @@ pub const Loop = struct {
470	462	}
471	463	}
472	464
	465	pub fn waitUntilFdReadable(self: *Loop, fd: os.fd_t) !void {
	466	return self.linuxWaitFd(fd, os.EPOLLET \| os.EPOLLIN);
	467	}
	468
473	469	pub async fn bsdWaitKev(self: *Loop, ident: usize, filter: i16, fflags: u32) !os.Kevent {
474	470	var resume_node = ResumeNode.Basic{
475	471	.base = ResumeNode{

std/io.zig+14-174

...	...	@@ -1,5 +1,6 @@
1	1	const std = @import("std.zig");
2	2	const builtin = @import("builtin");
	3	const root = @import("root");
3	4	const c = std.c;
4	5
5	6	const math = std.math;
...	...	@@ -15,6 +16,18 @@ const fmt = std.fmt;
15	16	const File = std.fs.File;
16	17	const testing = std.testing;
17	18
	19	pub const Mode = enum {
	20	blocking,
	21	evented,
	22	};
	23	pub const mode: Mode = if (@hasDecl(root, "io_mode"))
	24	root.io_mode
	25	else if (@hasDecl(root, "event_loop"))
	26	Mode.evented
	27	else
	28	Mode.blocking;
	29	pub const is_async = mode != .blocking;
	30
18	31	pub const GetStdIoError = os.windows.GetStdHandleError;
19	32
20	33	pub fn getStdOut() GetStdIoError!File {
...	...	@@ -44,180 +57,7 @@ pub fn getStdIn() GetStdIoError!File {
44	57	pub const SeekableStream = @import("io/seekable_stream.zig").SeekableStream;
45	58	pub const SliceSeekableInStream = @import("io/seekable_stream.zig").SliceSeekableInStream;
46	59	pub const COutStream = @import("io/c_out_stream.zig").COutStream;
47
48		pub fn InStream(comptime ReadError: type) type {
49		return struct {
50		const Self = @This();
51		pub const Error = ReadError;
52
53		/// Return the number of bytes read. If the number read is smaller than buf.len, it
54		/// means the stream reached the end. Reaching the end of a stream is not an error
55		/// condition.
56		readFn: fn (self: *Self, buffer: []u8) Error!usize,
57
58		/// Replaces `buffer` contents by reading from the stream until it is finished.
59		/// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and
60		/// the contents read from the stream are lost.
61		pub fn readAllBuffer(self: Self, buffer: Buffer, max_size: usize) !void {
62		try buffer.resize(0);
63
64		var actual_buf_len: usize = 0;
65		while (true) {
66		const dest_slice = buffer.toSlice()[actual_buf_len..];
67		const bytes_read = try self.readFull(dest_slice);
68		actual_buf_len += bytes_read;
69
70		if (bytes_read != dest_slice.len) {
71		buffer.shrink(actual_buf_len);
72		return;
73		}
74
75		const new_buf_size = math.min(max_size, actual_buf_len + mem.page_size);
76		if (new_buf_size == actual_buf_len) return error.StreamTooLong;
77		try buffer.resize(new_buf_size);
78		}
79		}
80
81		/// Allocates enough memory to hold all the contents of the stream. If the allocated
82		/// memory would be greater than `max_size`, returns `error.StreamTooLong`.
83		/// Caller owns returned memory.
84		/// If this function returns an error, the contents from the stream read so far are lost.
85		pub fn readAllAlloc(self: Self, allocator: mem.Allocator, max_size: usize) ![]u8 {
86		var buf = Buffer.initNull(allocator);
87		defer buf.deinit();
88
89		try self.readAllBuffer(&buf, max_size);
90		return buf.toOwnedSlice();
91		}
92
93		/// Replaces `buffer` contents by reading from the stream until `delimiter` is found.
94		/// Does not include the delimiter in the result.
95		/// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and the contents
96		/// read from the stream so far are lost.
97		pub fn readUntilDelimiterBuffer(self: Self, buffer: Buffer, delimiter: u8, max_size: usize) !void {
98		try buffer.resize(0);
99
100		while (true) {
101		var byte: u8 = try self.readByte();
102
103		if (byte == delimiter) {
104		return;
105		}
106
107		if (buffer.len() == max_size) {
108		return error.StreamTooLong;
109		}
110
111		try buffer.appendByte(byte);
112		}
113		}
114
115		/// Allocates enough memory to read until `delimiter`. If the allocated
116		/// memory would be greater than `max_size`, returns `error.StreamTooLong`.
117		/// Caller owns returned memory.
118		/// If this function returns an error, the contents from the stream read so far are lost.
119		pub fn readUntilDelimiterAlloc(self: Self, allocator: mem.Allocator, delimiter: u8, max_size: usize) ![]u8 {
120		var buf = Buffer.initNull(allocator);
121		defer buf.deinit();
122
123		try self.readUntilDelimiterBuffer(&buf, delimiter, max_size);
124		return buf.toOwnedSlice();
125		}
126
127		/// Returns the number of bytes read. It may be less than buffer.len.
128		/// If the number of bytes read is 0, it means end of stream.
129		/// End of stream is not an error condition.
130		pub fn read(self: *Self, buffer: []u8) Error!usize {
131		return self.readFn(self, buffer);
132		}
133
134		/// Returns the number of bytes read. If the number read is smaller than buf.len, it
135		/// means the stream reached the end. Reaching the end of a stream is not an error
136		/// condition.
137		pub fn readFull(self: *Self, buffer: []u8) Error!usize {
138		var index: usize = 0;
139		while (index != buffer.len) {
140		const amt = try self.read(buffer[index..]);
141		if (amt == 0) return index;
142		index += amt;
143		}
144		return index;
145		}
146
147		/// Same as `readFull` but end of stream returns `error.EndOfStream`.
148		pub fn readNoEof(self: *Self, buf: []u8) !void {
149		const amt_read = try self.readFull(buf);
150		if (amt_read < buf.len) return error.EndOfStream;
151		}
152
153		/// Reads 1 byte from the stream or returns `error.EndOfStream`.
154		pub fn readByte(self: *Self) !u8 {
155		var result: [1]u8 = undefined;
156		try self.readNoEof(result[0..]);
157		return result[0];
158		}
159
160		/// Same as `readByte` except the returned byte is signed.
161		pub fn readByteSigned(self: *Self) !i8 {
162		return @bitCast(i8, try self.readByte());
163		}
164
165		/// Reads a native-endian integer
166		pub fn readIntNative(self: *Self, comptime T: type) !T {
167		var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
168		try self.readNoEof(bytes[0..]);
169		return mem.readIntNative(T, &bytes);
170		}
171
172		/// Reads a foreign-endian integer
173		pub fn readIntForeign(self: *Self, comptime T: type) !T {
174		var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
175		try self.readNoEof(bytes[0..]);
176		return mem.readIntForeign(T, &bytes);
177		}
178
179		pub fn readIntLittle(self: *Self, comptime T: type) !T {
180		var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
181		try self.readNoEof(bytes[0..]);
182		return mem.readIntLittle(T, &bytes);
183		}
184
185		pub fn readIntBig(self: *Self, comptime T: type) !T {
186		var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
187		try self.readNoEof(bytes[0..]);
188		return mem.readIntBig(T, &bytes);
189		}
190
191		pub fn readInt(self: *Self, comptime T: type, endian: builtin.Endian) !T {
192		var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
193		try self.readNoEof(bytes[0..]);
194		return mem.readInt(T, &bytes, endian);
195		}
196
197		pub fn readVarInt(self: *Self, comptime ReturnType: type, endian: builtin.Endian, size: usize) !ReturnType {
198		assert(size <= @sizeOf(ReturnType));
199		var bytes_buf: [@sizeOf(ReturnType)]u8 = undefined;
200		const bytes = bytes_buf[0..size];
201		try self.readNoEof(bytes);
202		return mem.readVarInt(ReturnType, bytes, endian);
203		}
204
205		pub fn skipBytes(self: *Self, num_bytes: u64) !void {
206		var i: u64 = 0;
207		while (i < num_bytes) : (i += 1) {
208		_ = try self.readByte();
209		}
210		}
211
212		pub fn readStruct(self: *Self, comptime T: type) !T {
213		// Only extern and packed structs have defined in-memory layout.
214		comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);
215		var res: [1]T = undefined;
216		try self.readNoEof(@sliceToBytes(res[0..]));
217		return res[0];
218		}
219		};
220		}
	60	pub const InStream = @import("io/in_stream.zig").InStream;
221	61
222	62	pub fn OutStream(comptime WriteError: type) type {
223	63	return struct {

std/io/in_stream.zig created+200

...	...	@@ -0,0 +1,200 @@
	1	const std = @import("../std.zig");
	2	const builtin = @import("builtin");
	3	const root = @import("root");
	4	const math = std.math;
	5	const assert = std.debug.assert;
	6	const mem = std.mem;
	7	const Buffer = std.Buffer;
	8
	9	pub const default_stack_size = 4 * 1024 * 1024;
	10	pub const stack_size: usize = if (@hasDecl(root, "stack_size_std_io_InStream"))
	11	root.stack_size_std_io_InStream
	12	else
	13	default_stack_size;
	14	pub const stack_align = 16;
	15
	16	pub fn InStream(comptime ReadError: type) type {
	17	return struct {
	18	const Self = @This();
	19	pub const Error = ReadError;
	20	pub const ReadFn = if (std.io.is_async)
	21	async fn (self: *Self, buffer: []u8) Error!usize
	22	else
	23	fn (self: *Self, buffer: []u8) Error!usize;
	24
	25	/// Returns the number of bytes read. It may be less than buffer.len.
	26	/// If the number of bytes read is 0, it means end of stream.
	27	/// End of stream is not an error condition.
	28	readFn: ReadFn,
	29
	30	/// Returns the number of bytes read. It may be less than buffer.len.
	31	/// If the number of bytes read is 0, it means end of stream.
	32	/// End of stream is not an error condition.
	33	pub fn read(self: *Self, buffer: []u8) Error!usize {
	34	if (std.io.is_async) {
	35	var stack_frame: [stack_size]u8 align(stack_align) = undefined;
	36	// TODO https://github.com/ziglang/zig/issues/3068
	37	var result: Error!usize = undefined;
	38	return await @asyncCall(&stack_frame, &result, self.readFn, self, buffer);
	39	} else {
	40	return self.readFn(self, buffer);
	41	}
	42	}
	43
	44	/// Returns the number of bytes read. If the number read is smaller than buf.len, it
	45	/// means the stream reached the end. Reaching the end of a stream is not an error
	46	/// condition.
	47	pub fn readFull(self: *Self, buffer: []u8) Error!usize {
	48	var index: usize = 0;
	49	while (index != buffer.len) {
	50	const amt = try self.read(buffer[index..]);
	51	if (amt == 0) return index;
	52	index += amt;
	53	}
	54	return index;
	55	}
	56
	57	/// Returns the number of bytes read. If the number read would be smaller than buf.len,
	58	/// error.EndOfStream is returned instead.
	59	pub fn readNoEof(self: *Self, buf: []u8) !void {
	60	const amt_read = try self.readFull(buf);
	61	if (amt_read < buf.len) return error.EndOfStream;
	62	}
	63
	64	/// Replaces `buffer` contents by reading from the stream until it is finished.
	65	/// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and
	66	/// the contents read from the stream are lost.
	67	pub fn readAllBuffer(self: Self, buffer: Buffer, max_size: usize) !void {
	68	try buffer.resize(0);
	69
	70	var actual_buf_len: usize = 0;
	71	while (true) {
	72	const dest_slice = buffer.toSlice()[actual_buf_len..];
	73	const bytes_read = try self.readFull(dest_slice);
	74	actual_buf_len += bytes_read;
	75
	76	if (bytes_read != dest_slice.len) {
	77	buffer.shrink(actual_buf_len);
	78	return;
	79	}
	80
	81	const new_buf_size = math.min(max_size, actual_buf_len + mem.page_size);
	82	if (new_buf_size == actual_buf_len) return error.StreamTooLong;
	83	try buffer.resize(new_buf_size);
	84	}
	85	}
	86
	87	/// Allocates enough memory to hold all the contents of the stream. If the allocated
	88	/// memory would be greater than `max_size`, returns `error.StreamTooLong`.
	89	/// Caller owns returned memory.
	90	/// If this function returns an error, the contents from the stream read so far are lost.
	91	pub fn readAllAlloc(self: Self, allocator: mem.Allocator, max_size: usize) ![]u8 {
	92	var buf = Buffer.initNull(allocator);
	93	defer buf.deinit();
	94
	95	try self.readAllBuffer(&buf, max_size);
	96	return buf.toOwnedSlice();
	97	}
	98
	99	/// Replaces `buffer` contents by reading from the stream until `delimiter` is found.
	100	/// Does not include the delimiter in the result.
	101	/// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and the contents
	102	/// read from the stream so far are lost.
	103	pub fn readUntilDelimiterBuffer(self: Self, buffer: Buffer, delimiter: u8, max_size: usize) !void {
	104	try buffer.resize(0);
	105
	106	while (true) {
	107	var byte: u8 = try self.readByte();
	108
	109	if (byte == delimiter) {
	110	return;
	111	}
	112
	113	if (buffer.len() == max_size) {
	114	return error.StreamTooLong;
	115	}
	116
	117	try buffer.appendByte(byte);
	118	}
	119	}
	120
	121	/// Allocates enough memory to read until `delimiter`. If the allocated
	122	/// memory would be greater than `max_size`, returns `error.StreamTooLong`.
	123	/// Caller owns returned memory.
	124	/// If this function returns an error, the contents from the stream read so far are lost.
	125	pub fn readUntilDelimiterAlloc(self: Self, allocator: mem.Allocator, delimiter: u8, max_size: usize) ![]u8 {
	126	var buf = Buffer.initNull(allocator);
	127	defer buf.deinit();
	128
	129	try self.readUntilDelimiterBuffer(&buf, delimiter, max_size);
	130	return buf.toOwnedSlice();
	131	}
	132
	133	/// Reads 1 byte from the stream or returns `error.EndOfStream`.
	134	pub fn readByte(self: *Self) !u8 {
	135	var result: [1]u8 = undefined;
	136	try self.readNoEof(result[0..]);
	137	return result[0];
	138	}
	139
	140	/// Same as `readByte` except the returned byte is signed.
	141	pub fn readByteSigned(self: *Self) !i8 {
	142	return @bitCast(i8, try self.readByte());
	143	}
	144
	145	/// Reads a native-endian integer
	146	pub fn readIntNative(self: *Self, comptime T: type) !T {
	147	var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
	148	try self.readNoEof(bytes[0..]);
	149	return mem.readIntNative(T, &bytes);
	150	}
	151
	152	/// Reads a foreign-endian integer
	153	pub fn readIntForeign(self: *Self, comptime T: type) !T {
	154	var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
	155	try self.readNoEof(bytes[0..]);
	156	return mem.readIntForeign(T, &bytes);
	157	}
	158
	159	pub fn readIntLittle(self: *Self, comptime T: type) !T {
	160	var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
	161	try self.readNoEof(bytes[0..]);
	162	return mem.readIntLittle(T, &bytes);
	163	}
	164
	165	pub fn readIntBig(self: *Self, comptime T: type) !T {
	166	var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
	167	try self.readNoEof(bytes[0..]);
	168	return mem.readIntBig(T, &bytes);
	169	}
	170
	171	pub fn readInt(self: *Self, comptime T: type, endian: builtin.Endian) !T {
	172	var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
	173	try self.readNoEof(bytes[0..]);
	174	return mem.readInt(T, &bytes, endian);
	175	}
	176
	177	pub fn readVarInt(self: *Self, comptime ReturnType: type, endian: builtin.Endian, size: usize) !ReturnType {
	178	assert(size <= @sizeOf(ReturnType));
	179	var bytes_buf: [@sizeOf(ReturnType)]u8 = undefined;
	180	const bytes = bytes_buf[0..size];
	181	try self.readNoEof(bytes);
	182	return mem.readVarInt(ReturnType, bytes, endian);
	183	}
	184
	185	pub fn skipBytes(self: *Self, num_bytes: u64) !void {
	186	var i: u64 = 0;
	187	while (i < num_bytes) : (i += 1) {
	188	_ = try self.readByte();
	189	}
	190	}
	191
	192	pub fn readStruct(self: *Self, comptime T: type) !T {
	193	// Only extern and packed structs have defined in-memory layout.
	194	comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);
	195	var res: [1]T = undefined;
	196	try self.readNoEof(@sliceToBytes(res[0..]));
	197	return res[0];
	198	}
	199	};
	200	}

std/os.zig+18-24

...	...	@@ -254,13 +254,18 @@ pub const ReadError = error{
254	254	IsDir,
255	255	OperationAborted,
256	256	BrokenPipe,
	257
	258	/// This error occurs when no global event loop is configured,
	259	/// and reading from the file descriptor would block.
	260	WouldBlock,
	261
257	262	Unexpected,
258	263	};
259	264
260	265	/// Returns the number of bytes that were read, which can be less than
261	266	/// buf.len. If 0 bytes were read, that means EOF.
262		/// This function is for blocking file descriptors only. For non-blocking, see
263		/// `readAsync`.
	267	/// If the application has a global event loop enabled, EAGAIN is handled
	268	/// via the event loop. Otherwise EAGAIN results in error.WouldBlock.
264	269	pub fn read(fd: fd_t, buf: []u8) ReadError!usize {
265	270	if (windows.is_the_target) {
266	271	return windows.ReadFile(fd, buf);
...	...	@@ -279,28 +284,19 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize {
279	284	}
280	285	}
281	286
282		// Linux can return EINVAL when read amount is > 0x7ffff000
283		// See https://github.com/ziglang/zig/pull/743#issuecomment-363158274
284		// TODO audit this. Shawn Landden says that this is not actually true.
285		// if this logic should stay, move it to std.os.linux
286		const max_buf_len = 0x7ffff000;
287
288		var index: usize = 0;
289		while (index < buf.len) {
290		const want_to_read = math.min(buf.len - index, usize(max_buf_len));
291		const rc = system.read(fd, buf.ptr + index, want_to_read);
	287	while (true) {
	288	const rc = system.read(fd, buf.ptr, buf.len);
292	289	switch (errno(rc)) {
293		0 => {
294		const amt_read = @intCast(usize, rc);
295		index += amt_read;
296		if (amt_read == want_to_read) continue;
297		// Read returned less than buf.len.
298		return index;
299		},
	290	0 => return @intCast(usize, rc),
300	291	EINTR => continue,
301	292	EINVAL => unreachable,
302	293	EFAULT => unreachable,
303		EAGAIN => unreachable, // This function is for blocking reads.
	294	EAGAIN => if (std.event.Loop.instance) \|loop\| {
	295	loop.waitUntilFdReadable(fd) catch return error.WouldBlock;
	296	continue;
	297	} else {
	298	return error.WouldBlock;
	299	},
304	300	EBADF => unreachable, // Always a race condition.
305	301	EIO => return error.InputOutput,
306	302	EISDIR => return error.IsDir,
...	...	@@ -313,8 +309,7 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize {
313	309	}
314	310
315	311	/// Number of bytes read is returned. Upon reading end-of-file, zero is returned.
316		/// This function is for blocking file descriptors only. For non-blocking, see
317		/// `preadvAsync`.
	312	/// This function is for blocking file descriptors only.
318	313	pub fn preadv(fd: fd_t, iov: []const iovec, offset: u64) ReadError!usize {
319	314	if (darwin.is_the_target) {
320	315	// Darwin does not have preadv but it does have pread.
...	...	@@ -386,8 +381,7 @@ pub const WriteError = error{
386	381	};
387	382
388	383	/// Write to a file descriptor. Keeps trying if it gets interrupted.
389		/// This function is for blocking file descriptors only. For non-blocking, see
390		/// `writeAsync`.
	384	/// This function is for blocking file descriptors only.
391	385	pub fn write(fd: fd_t, bytes: []const u8) WriteError!void {
392	386	if (windows.is_the_target) {
393	387	return windows.WriteFile(fd, bytes);