Commit ccef60a64033a25dbe2351c27f28257546b2ae5b | zig

author	Andrew Kelley <andrew@ziglang.org> 2018-07-09 22:06:47-04:00
committer	GitHub <noreply@github.com> 2018-07-09 22:06:47-04:00
log	ccef60a64033a25dbe2351c27f28257546b2ae5b
tree	67390c7e43f9852cf3786f2eed35ebf04e15510d
parent	10cc49db1ca1f9b3ac63277c0742e05f6412f3c6
parent	c89aac85c440ea4cbccf1abdbd6acf84a33077e3
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #1198 from ziglang/m-n-threading

M:N threading

16 files changed, 1774 insertions(+), 142 deletions(-)

src-self-hosted
- main.zig
- module.zig
src
- ir.cpp
std
- atomic
  - queue_mpsc.zig
- c
  - darwin.zig
- debug
  - index.zig
- event.zig
- heap.zig
- mem.zig
- os
  - darwin.zig
  - index.zig
  - linux
    index.zig
  - test.zig
  - windows
    index.zig
    util.zig
- special
  - compiler_rt
    extendXfYf2_test.zig

src-self-hosted/main.zig+2-3

...	...	@@ -384,7 +384,8 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Mo
384	384	const zig_lib_dir = introspect.resolveZigLibDir(allocator) catch os.exit(1);
385	385	defer allocator.free(zig_lib_dir);
386	386
387		var loop = try event.Loop.init(allocator);
	387	var loop: event.Loop = undefined;
	388	try loop.initMultiThreaded(allocator);
388	389
389	390	var module = try Module.create(
390	391	&loop,
...	...	@@ -493,8 +494,6 @@ async fn processBuildEvents(module: *Module, watch: bool) void {
493	494	switch (build_event) {
494	495	Module.Event.Ok => {
495	496	std.debug.warn("Build succeeded\n");
496		// for now we stop after 1
497		module.loop.stop();
498	497	return;
499	498	},
500	499	Module.Event.Error => \|err\| {

src-self-hosted/module.zig+242-15

...	...	@@ -2,6 +2,7 @@ const std = @import("std");
2	2	const os = std.os;
3	3	const io = std.io;
4	4	const mem = std.mem;
	5	const Allocator = mem.Allocator;
5	6	const Buffer = std.Buffer;
6	7	const llvm = @import("llvm.zig");
7	8	const c = @import("c.zig");
...	...	@@ -13,6 +14,7 @@ const ArrayList = std.ArrayList;
13	14	const errmsg = @import("errmsg.zig");
14	15	const ast = std.zig.ast;
15	16	const event = std.event;
	17	const assert = std.debug.assert;
16	18
17	19	pub const Module = struct {
18	20	loop: *event.Loop,
...	...	@@ -81,6 +83,8 @@ pub const Module = struct {
81	83	link_out_file: ?[]const u8,
82	84	events: *event.Channel(Event),
83	85
	86	exported_symbol_names: event.Locked(Decl.Table),
	87
84	88	// TODO handle some of these earlier and report them in a way other than error codes
85	89	pub const BuildError = error{
86	90	OutOfMemory,
...	...	@@ -232,6 +236,7 @@ pub const Module = struct {
232	236	.test_name_prefix = null,
233	237	.emit_file_type = Emit.Binary,
234	238	.link_out_file = null,
	239	.exported_symbol_names = event.Locked(Decl.Table).init(loop, Decl.Table.init(loop.allocator)),
235	240	});
236	241	}
237	242
...	...	@@ -272,38 +277,91 @@ pub const Module = struct {
272	277	return;
273	278	};
274	279	await (async self.events.put(Event.Ok) catch unreachable);
	280	// for now we stop after 1
	281	return;
275	282	}
276	283	}
277	284
278	285	async fn addRootSrc(self: *Module) !void {
279	286	const root_src_path = self.root_src_path orelse @panic("TODO handle null root src path");
	287	// TODO async/await os.path.real
280	288	const root_src_real_path = os.path.real(self.a(), root_src_path) catch \|err\| {
281	289	try printError("unable to get real path '{}': {}", root_src_path, err);
282	290	return err;
283	291	};
284	292	errdefer self.a().free(root_src_real_path);
285	293
	294	// TODO async/await readFileAlloc()
286	295	const source_code = io.readFileAlloc(self.a(), root_src_real_path) catch \|err\| {
287	296	try printError("unable to open '{}': {}", root_src_real_path, err);
288	297	return err;
289	298	};
290	299	errdefer self.a().free(source_code);
291	300
292		var tree = try std.zig.parse(self.a(), source_code);
293		defer tree.deinit();
294
295		//var it = tree.root_node.decls.iterator();
296		//while (it.next()) \|decl_ptr\| {
297		// const decl = decl_ptr.*;
298		// switch (decl.id) {
299		// ast.Node.Comptime => @panic("TODO"),
300		// ast.Node.VarDecl => @panic("TODO"),
301		// ast.Node.UseDecl => @panic("TODO"),
302		// ast.Node.FnDef => @panic("TODO"),
303		// ast.Node.TestDecl => @panic("TODO"),
304		// else => unreachable,
305		// }
306		//}
	301	var parsed_file = ParsedFile{
	302	.tree = try std.zig.parse(self.a(), source_code),
	303	.realpath = root_src_real_path,
	304	};
	305	errdefer parsed_file.tree.deinit();
	306
	307	const tree = &parsed_file.tree;
	308
	309	// create empty struct for it
	310	const decls = try Scope.Decls.create(self.a(), null);
	311	errdefer decls.destroy();
	312
	313	var it = tree.root_node.decls.iterator(0);
	314	while (it.next()) \|decl_ptr\| {
	315	const decl = decl_ptr.*;
	316	switch (decl.id) {
	317	ast.Node.Id.Comptime => @panic("TODO"),
	318	ast.Node.Id.VarDecl => @panic("TODO"),
	319	ast.Node.Id.FnProto => {
	320	const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", decl);
	321
	322	const name = if (fn_proto.name_token) \|name_token\| tree.tokenSlice(name_token) else {
	323	@panic("TODO add compile error");
	324	//try self.addCompileError(
	325	// &parsed_file,
	326	// fn_proto.fn_token,
	327	// fn_proto.fn_token + 1,
	328	// "missing function name",
	329	//);
	330	continue;
	331	};
	332
	333	const fn_decl = try self.a().create(Decl.Fn{
	334	.base = Decl{
	335	.id = Decl.Id.Fn,
	336	.name = name,
	337	.visib = parseVisibToken(tree, fn_proto.visib_token),
	338	.resolution = Decl.Resolution.Unresolved,
	339	},
	340	.value = Decl.Fn.Val{ .Unresolved = {} },
	341	.fn_proto = fn_proto,
	342	});
	343	errdefer self.a().destroy(fn_decl);
	344
	345	// TODO make this parallel
	346	try await try async self.addTopLevelDecl(tree, &fn_decl.base);
	347	},
	348	ast.Node.Id.TestDecl => @panic("TODO"),
	349	else => unreachable,
	350	}
	351	}
	352	}
	353
	354	async fn addTopLevelDecl(self: Module, tree: ast.Tree, decl: *Decl) !void {
	355	const is_export = decl.isExported(tree);
	356
	357	{
	358	const exported_symbol_names = await try async self.exported_symbol_names.acquire();
	359	defer exported_symbol_names.release();
	360
	361	if (try exported_symbol_names.value.put(decl.name, decl)) \|other_decl\| {
	362	@panic("TODO report compile error");
	363	}
	364	}
307	365	}
308	366
309	367	pub fn link(self: *Module, out_file: ?[]const u8) !void {
...	...	@@ -350,3 +408,172 @@ fn printError(comptime format: []const u8, args: ...) !void {
350	408	const out_stream = &stderr_file_out_stream.stream;
351	409	try out_stream.print(format, args);
352	410	}
	411
	412	fn parseVisibToken(tree: *ast.Tree, optional_token_index: ?ast.TokenIndex) Visib {
	413	if (optional_token_index) \|token_index\| {
	414	const token = tree.tokens.at(token_index);
	415	assert(token.id == Token.Id.Keyword_pub);
	416	return Visib.Pub;
	417	} else {
	418	return Visib.Private;
	419	}
	420	}
	421
	422	pub const Scope = struct {
	423	id: Id,
	424	parent: ?*Scope,
	425
	426	pub const Id = enum {
	427	Decls,
	428	Block,
	429	};
	430
	431	pub const Decls = struct {
	432	base: Scope,
	433	table: Decl.Table,
	434
	435	pub fn create(a: Allocator, parent: ?Scope) !*Decls {
	436	const self = try a.create(Decls{
	437	.base = Scope{
	438	.id = Id.Decls,
	439	.parent = parent,
	440	},
	441	.table = undefined,
	442	});
	443	errdefer a.destroy(self);
	444
	445	self.table = Decl.Table.init(a);
	446	errdefer self.table.deinit();
	447
	448	return self;
	449	}
	450
	451	pub fn destroy(self: *Decls) void {
	452	self.table.deinit();
	453	self.table.allocator.destroy(self);
	454	self.* = undefined;
	455	}
	456	};
	457
	458	pub const Block = struct {
	459	base: Scope,
	460	};
	461	};
	462
	463	pub const Visib = enum {
	464	Private,
	465	Pub,
	466	};
	467
	468	pub const Decl = struct {
	469	id: Id,
	470	name: []const u8,
	471	visib: Visib,
	472	resolution: Resolution,
	473
	474	pub const Table = std.HashMap([]const u8, *Decl, mem.hash_slice_u8, mem.eql_slice_u8);
	475
	476	pub fn isExported(base: const Decl, tree: ast.Tree) bool {
	477	switch (base.id) {
	478	Id.Fn => {
	479	const fn_decl = @fieldParentPtr(Fn, "base", base);
	480	return fn_decl.isExported(tree);
	481	},
	482	else => return false,
	483	}
	484	}
	485
	486	pub const Resolution = enum {
	487	Unresolved,
	488	InProgress,
	489	Invalid,
	490	Ok,
	491	};
	492
	493	pub const Id = enum {
	494	Var,
	495	Fn,
	496	CompTime,
	497	};
	498
	499	pub const Var = struct {
	500	base: Decl,
	501	};
	502
	503	pub const Fn = struct {
	504	base: Decl,
	505	value: Val,
	506	fn_proto: *const ast.Node.FnProto,
	507
	508	// TODO https://github.com/ziglang/zig/issues/683 and then make this anonymous
	509	pub const Val = union {
	510	Unresolved: void,
	511	Ok: *Value.Fn,
	512	};
	513
	514	pub fn externLibName(self: Fn, tree: *ast.Tree) ?[]const u8 {
	515	return if (self.fn_proto.extern_export_inline_token) \|tok_index\| x: {
	516	const token = tree.tokens.at(tok_index);
	517	break :x switch (token.id) {
	518	Token.Id.Extern => tree.tokenSlicePtr(token),
	519	else => null,
	520	};
	521	} else null;
	522	}
	523
	524	pub fn isExported(self: Fn, tree: *ast.Tree) bool {
	525	if (self.fn_proto.extern_export_inline_token) \|tok_index\| {
	526	const token = tree.tokens.at(tok_index);
	527	return token.id == Token.Id.Keyword_export;
	528	} else {
	529	return false;
	530	}
	531	}
	532	};
	533
	534	pub const CompTime = struct {
	535	base: Decl,
	536	};
	537	};
	538
	539	pub const Value = struct {
	540	pub const Fn = struct {};
	541	};
	542
	543	pub const Type = struct {
	544	id: Id,
	545
	546	pub const Id = enum {
	547	Type,
	548	Void,
	549	Bool,
	550	NoReturn,
	551	Int,
	552	Float,
	553	Pointer,
	554	Array,
	555	Struct,
	556	ComptimeFloat,
	557	ComptimeInt,
	558	Undefined,
	559	Null,
	560	Optional,
	561	ErrorUnion,
	562	ErrorSet,
	563	Enum,
	564	Union,
	565	Fn,
	566	Opaque,
	567	Promise,
	568	};
	569
	570	pub const Struct = struct {
	571	base: Type,
	572	decls: *Scope.Decls,
	573	};
	574	};
	575
	576	pub const ParsedFile = struct {
	577	tree: ast.Tree,
	578	realpath: []const u8,
	579	};

src/ir.cpp+1-1

...	...	@@ -13278,7 +13278,7 @@ static TypeTableEntry ir_analyze_instruction_call(IrAnalyze ira, IrInstruction
13278	13278	FnTableEntry *fn_table_entry = fn_ref->value.data.x_bound_fn.fn;
13279	13279	IrInstruction *first_arg_ptr = fn_ref->value.data.x_bound_fn.first_arg;
13280	13280	return ir_analyze_fn_call(ira, call_instruction, fn_table_entry, fn_table_entry->type_entry,
13281		nullptr, first_arg_ptr, is_comptime, call_instruction->fn_inline);
	13281	fn_ref, first_arg_ptr, is_comptime, call_instruction->fn_inline);
13282	13282	} else {
13283	13283	ir_add_error_node(ira, fn_ref->source_node,
13284	13284	buf_sprintf("type '%s' not a function", buf_ptr(&fn_ref->value.type->name)));

std/atomic/queue_mpsc.zig+42

...	...	@@ -15,6 +15,8 @@ pub fn QueueMpsc(comptime T: type) type {
15	15
16	16	pub const Node = std.atomic.Stack(T).Node;
17	17
	18	/// Not thread-safe. The call to init() must complete before any other functions are called.
	19	/// No deinitialization required.
18	20	pub fn init() Self {
19	21	return Self{
20	22	.inboxes = []std.atomic.Stack(T){
...	...	@@ -26,12 +28,15 @@ pub fn QueueMpsc(comptime T: type) type {
26	28	};
27	29	}
28	30
	31	/// Fully thread-safe. put() may be called from any thread at any time.
29	32	pub fn put(self: Self, node: Node) void {
30	33	const inbox_index = @atomicLoad(usize, &self.inbox_index, AtomicOrder.SeqCst);
31	34	const inbox = &self.inboxes[inbox_index];
32	35	inbox.push(node);
33	36	}
34	37
	38	/// Must be called by only 1 consumer at a time. Every call to get() and isEmpty() must complete before
	39	/// the next call to get().
35	40	pub fn get(self: Self) ?Node {
36	41	if (self.outbox.pop()) \|node\| {
37	42	return node;
...	...	@@ -43,6 +48,43 @@ pub fn QueueMpsc(comptime T: type) type {
43	48	}
44	49	return self.outbox.pop();
45	50	}
	51
	52	/// Must be called by only 1 consumer at a time. Every call to get() and isEmpty() must complete before
	53	/// the next call to isEmpty().
	54	pub fn isEmpty(self: *Self) bool {
	55	if (!self.outbox.isEmpty()) return false;
	56	const prev_inbox_index = @atomicRmw(usize, &self.inbox_index, AtomicRmwOp.Xor, 0x1, AtomicOrder.SeqCst);
	57	const prev_inbox = &self.inboxes[prev_inbox_index];
	58	while (prev_inbox.pop()) \|node\| {
	59	self.outbox.push(node);
	60	}
	61	return self.outbox.isEmpty();
	62	}
	63
	64	/// For debugging only. No API guarantees about what this does.
	65	pub fn dump(self: *Self) void {
	66	{
	67	var it = self.outbox.root;
	68	while (it) \|node\| {
	69	std.debug.warn("0x{x} -> ", @ptrToInt(node));
	70	it = node.next;
	71	}
	72	}
	73	const inbox_index = self.inbox_index;
	74	const inboxes = []*std.atomic.Stack(T){
	75	&self.inboxes[self.inbox_index],
	76	&self.inboxes[1 - self.inbox_index],
	77	};
	78	for (inboxes) \|inbox\| {
	79	var it = inbox.root;
	80	while (it) \|node\| {
	81	std.debug.warn("0x{x} -> ", @ptrToInt(node));
	82	it = node.next;
	83	}
	84	}
	85
	86	std.debug.warn("null\n");
	87	}
46	88	};
47	89	}
48	90

std/c/darwin.zig+72

...	...	@@ -6,6 +6,30 @@ pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: [*]u8, buf_len: usize, b
6	6	pub extern "c" fn mach_absolute_time() u64;
7	7	pub extern "c" fn mach_timebase_info(tinfo: ?*mach_timebase_info_data) void;
8	8
	9	pub extern "c" fn kqueue() c_int;
	10	pub extern "c" fn kevent(
	11	kq: c_int,
	12	changelist: [*]const Kevent,
	13	nchanges: c_int,
	14	eventlist: [*]Kevent,
	15	nevents: c_int,
	16	timeout: ?*const timespec,
	17	) c_int;
	18
	19	pub extern "c" fn kevent64(
	20	kq: c_int,
	21	changelist: [*]const kevent64_s,
	22	nchanges: c_int,
	23	eventlist: [*]kevent64_s,
	24	nevents: c_int,
	25	flags: c_uint,
	26	timeout: ?*const timespec,
	27	) c_int;
	28
	29	pub extern "c" fn sysctl(name: []c_int, namelen: c_uint, oldp: ?c_void, oldlenp: ?usize, newp: ?c_void, newlen: usize) c_int;
	30	pub extern "c" fn sysctlbyname(name: []const u8, oldp: ?c_void, oldlenp: ?usize, newp: ?c_void, newlen: usize) c_int;
	31	pub extern "c" fn sysctlnametomib(name: []const u8, mibp: ?c_int, sizep: ?*usize) c_int;
	32
9	33	pub use @import("../os/darwin_errno.zig");
10	34
11	35	pub const _errno = __error;
...	...	@@ -86,3 +110,51 @@ pub const pthread_attr_t = extern struct {
86	110	__sig: c_long,
87	111	__opaque: [56]u8,
88	112	};
	113
	114	/// Renamed from `kevent` to `Kevent` to avoid conflict with function name.
	115	pub const Kevent = extern struct {
	116	ident: usize,
	117	filter: i16,
	118	flags: u16,
	119	fflags: u32,
	120	data: isize,
	121	udata: usize,
	122	};
	123
	124	// sys/types.h on macos uses #pragma pack(4) so these checks are
	125	// to make sure the struct is laid out the same. These values were
	126	// produced from C code using the offsetof macro.
	127	const std = @import("../index.zig");
	128	const assert = std.debug.assert;
	129
	130	comptime {
	131	assert(@offsetOf(Kevent, "ident") == 0);
	132	assert(@offsetOf(Kevent, "filter") == 8);
	133	assert(@offsetOf(Kevent, "flags") == 10);
	134	assert(@offsetOf(Kevent, "fflags") == 12);
	135	assert(@offsetOf(Kevent, "data") == 16);
	136	assert(@offsetOf(Kevent, "udata") == 24);
	137	}
	138
	139	pub const kevent64_s = extern struct {
	140	ident: u64,
	141	filter: i16,
	142	flags: u16,
	143	fflags: u32,
	144	data: i64,
	145	udata: u64,
	146	ext: [2]u64,
	147	};
	148
	149	// sys/types.h on macos uses #pragma pack() so these checks are
	150	// to make sure the struct is laid out the same. These values were
	151	// produced from C code using the offsetof macro.
	152	comptime {
	153	assert(@offsetOf(kevent64_s, "ident") == 0);
	154	assert(@offsetOf(kevent64_s, "filter") == 8);
	155	assert(@offsetOf(kevent64_s, "flags") == 10);
	156	assert(@offsetOf(kevent64_s, "fflags") == 12);
	157	assert(@offsetOf(kevent64_s, "data") == 16);
	158	assert(@offsetOf(kevent64_s, "udata") == 24);
	159	assert(@offsetOf(kevent64_s, "ext") == 32);
	160	}

std/debug/index.zig+6-1

...	...	@@ -12,6 +12,11 @@ const builtin = @import("builtin");
12	12	pub const FailingAllocator = @import("failing_allocator.zig").FailingAllocator;
13	13	pub const failing_allocator = FailingAllocator.init(global_allocator, 0);
14	14
	15	pub const runtime_safety = switch (builtin.mode) {
	16	builtin.Mode.Debug, builtin.Mode.ReleaseSafe => true,
	17	builtin.Mode.ReleaseFast, builtin.Mode.ReleaseSmall => false,
	18	};
	19
15	20	/// Tries to write to stderr, unbuffered, and ignores any error returned.
16	21	/// Does not append a newline.
17	22	/// TODO atomic/multithread support
...	...	@@ -1125,7 +1130,7 @@ fn readILeb128(in_stream: var) !i64 {
1125	1130
1126	1131	/// This should only be used in temporary test programs.
1127	1132	pub const global_allocator = &global_fixed_allocator.allocator;
1128		var global_fixed_allocator = std.heap.FixedBufferAllocator.init(global_allocator_mem[0..]);
	1133	var global_fixed_allocator = std.heap.ThreadSafeFixedBufferAllocator.init(global_allocator_mem[0..]);
1129	1134	var global_allocator_mem: [100 * 1024]u8 = undefined;
1130	1135
1131	1136	// TODO make thread safe

std/event.zig+774-76

...	...	@@ -4,6 +4,7 @@ const assert = std.debug.assert;
4	4	const event = this;
5	5	const mem = std.mem;
6	6	const posix = std.os.posix;
	7	const windows = std.os.windows;
7	8	const AtomicRmwOp = builtin.AtomicRmwOp;
8	9	const AtomicOrder = builtin.AtomicOrder;
9	10
...	...	@@ -11,53 +12,69 @@ pub const TcpServer = struct {
11	12	handleRequestFn: async<mem.Allocator> fn (TcpServer, const std.net.Address, const std.os.File) void,
12	13
13	14	loop: *Loop,
14		sockfd: i32,
	15	sockfd: ?i32,
15	16	accept_coro: ?promise,
16	17	listen_address: std.net.Address,
17	18
18	19	waiting_for_emfile_node: PromiseNode,
	20	listen_resume_node: event.Loop.ResumeNode,
19	21
20	22	const PromiseNode = std.LinkedList(promise).Node;
21	23
22		pub fn init(loop: *Loop) !TcpServer {
23		const sockfd = try std.os.posixSocket(posix.AF_INET, posix.SOCK_STREAM \| posix.SOCK_CLOEXEC \| posix.SOCK_NONBLOCK, posix.PROTO_tcp);
24		errdefer std.os.close(sockfd);
25
	24	pub fn init(loop: *Loop) TcpServer {
26	25	// TODO can't initialize handler coroutine here because we need well defined copy elision
27	26	return TcpServer{
28	27	.loop = loop,
29		.sockfd = sockfd,
	28	.sockfd = null,
30	29	.accept_coro = null,
31	30	.handleRequestFn = undefined,
32	31	.waiting_for_emfile_node = undefined,
33	32	.listen_address = undefined,
	33	.listen_resume_node = event.Loop.ResumeNode{
	34	.id = event.Loop.ResumeNode.Id.Basic,
	35	.handle = undefined,
	36	},
34	37	};
35	38	}
36	39
37		pub fn listen(self: TcpServer, address: const std.net.Address, handleRequestFn: async<mem.Allocator> fn (TcpServer, const std.net.Address, const std.os.File) void) !void {
	40	pub fn listen(
	41	self: *TcpServer,
	42	address: *const std.net.Address,
	43	handleRequestFn: async<mem.Allocator> fn (TcpServer, const std.net.Address, const std.os.File) void,
	44	) !void {
38	45	self.handleRequestFn = handleRequestFn;
39	46
40		try std.os.posixBind(self.sockfd, &address.os_addr);
41		try std.os.posixListen(self.sockfd, posix.SOMAXCONN);
42		self.listen_address = std.net.Address.initPosix(try std.os.posixGetSockName(self.sockfd));
	47	const sockfd = try std.os.posixSocket(posix.AF_INET, posix.SOCK_STREAM \| posix.SOCK_CLOEXEC \| posix.SOCK_NONBLOCK, posix.PROTO_tcp);
	48	errdefer std.os.close(sockfd);
	49	self.sockfd = sockfd;
	50
	51	try std.os.posixBind(sockfd, &address.os_addr);
	52	try std.os.posixListen(sockfd, posix.SOMAXCONN);
	53	self.listen_address = std.net.Address.initPosix(try std.os.posixGetSockName(sockfd));
43	54
44	55	self.accept_coro = try async<self.loop.allocator> TcpServer.handler(self);
45	56	errdefer cancel self.accept_coro.?;
46	57
47		try self.loop.addFd(self.sockfd, self.accept_coro.?);
48		errdefer self.loop.removeFd(self.sockfd);
	58	self.listen_resume_node.handle = self.accept_coro.?;
	59	try self.loop.addFd(sockfd, &self.listen_resume_node);
	60	errdefer self.loop.removeFd(sockfd);
	61	}
	62
	63	/// Stop listening
	64	pub fn close(self: *TcpServer) void {
	65	self.loop.removeFd(self.sockfd.?);
	66	std.os.close(self.sockfd.?);
49	67	}
50	68
51	69	pub fn deinit(self: *TcpServer) void {
52		self.loop.removeFd(self.sockfd);
53	70	if (self.accept_coro) \|accept_coro\| cancel accept_coro;
54		std.os.close(self.sockfd);
	71	if (self.sockfd) \|sockfd\| std.os.close(sockfd);
55	72	}
56	73
57	74	pub async fn handler(self: *TcpServer) void {
58	75	while (true) {
59	76	var accepted_addr: std.net.Address = undefined;
60		if (std.os.posixAccept(self.sockfd, &accepted_addr.os_addr, posix.SOCK_NONBLOCK \| posix.SOCK_CLOEXEC)) \|accepted_fd\| {
	77	if (std.os.posixAccept(self.sockfd.?, &accepted_addr.os_addr, posix.SOCK_NONBLOCK \| posix.SOCK_CLOEXEC)) \|accepted_fd\| {
61	78	var socket = std.os.File.openHandle(accepted_fd);
62	79	_ = async<self.loop.allocator> self.handleRequestFn(self, accepted_addr, socket) catch \|err\| switch (err) {
63	80	error.OutOfMemory => {
...	...	@@ -95,46 +112,276 @@ pub const TcpServer = struct {
95	112
96	113	pub const Loop = struct {
97	114	allocator: *mem.Allocator,
98		keep_running: bool,
99	115	next_tick_queue: std.atomic.QueueMpsc(promise),
100	116	os_data: OsData,
	117	final_resume_node: ResumeNode,
	118	dispatch_lock: u8, // TODO make this a bool
	119	pending_event_count: usize,
	120	extra_threads: []*std.os.Thread,
101	121
102		const OsData = switch (builtin.os) {
103		builtin.Os.linux => struct {
104		epollfd: i32,
105		},
106		else => struct {},
107		};
	122	// pre-allocated eventfds. all permanently active.
	123	// this is how we send promises to be resumed on other threads.
	124	available_eventfd_resume_nodes: std.atomic.Stack(ResumeNode.EventFd),
	125	eventfd_resume_nodes: []std.atomic.Stack(ResumeNode.EventFd).Node,
108	126
109	127	pub const NextTickNode = std.atomic.QueueMpsc(promise).Node;
110	128
	129	pub const ResumeNode = struct {
	130	id: Id,
	131	handle: promise,
	132
	133	pub const Id = enum {
	134	Basic,
	135	Stop,
	136	EventFd,
	137	};
	138
	139	pub const EventFd = switch (builtin.os) {
	140	builtin.Os.macosx => MacOsEventFd,
	141	builtin.Os.linux => struct {
	142	base: ResumeNode,
	143	epoll_op: u32,
	144	eventfd: i32,
	145	},
	146	builtin.Os.windows => struct {
	147	base: ResumeNode,
	148	completion_key: usize,
	149	},
	150	else => @compileError("unsupported OS"),
	151	};
	152
	153	const MacOsEventFd = struct {
	154	base: ResumeNode,
	155	kevent: posix.Kevent,
	156	};
	157	};
	158
	159	/// After initialization, call run().
	160	/// TODO copy elision / named return values so that the threads referencing *Loop
	161	/// have the correct pointer value.
	162	fn initSingleThreaded(self: Loop, allocator: mem.Allocator) !void {
	163	return self.initInternal(allocator, 1);
	164	}
	165
111	166	/// The allocator must be thread-safe because we use it for multiplexing
112	167	/// coroutines onto kernel threads.
113		pub fn init(allocator: *mem.Allocator) !Loop {
114		var self = Loop{
115		.keep_running = true,
	168	/// After initialization, call run().
	169	/// TODO copy elision / named return values so that the threads referencing *Loop
	170	/// have the correct pointer value.
	171	fn initMultiThreaded(self: Loop, allocator: mem.Allocator) !void {
	172	const core_count = try std.os.cpuCount(allocator);
	173	return self.initInternal(allocator, core_count);
	174	}
	175
	176	/// Thread count is the total thread count. The thread pool size will be
	177	/// max(thread_count - 1, 0)
	178	fn initInternal(self: Loop, allocator: mem.Allocator, thread_count: usize) !void {
	179	self.* = Loop{
	180	.pending_event_count = 0,
116	181	.allocator = allocator,
117	182	.os_data = undefined,
118	183	.next_tick_queue = std.atomic.QueueMpsc(promise).init(),
	184	.dispatch_lock = 1, // start locked so threads go directly into epoll wait
	185	.extra_threads = undefined,
	186	.available_eventfd_resume_nodes = std.atomic.Stack(ResumeNode.EventFd).init(),
	187	.eventfd_resume_nodes = undefined,
	188	.final_resume_node = ResumeNode{
	189	.id = ResumeNode.Id.Stop,
	190	.handle = undefined,
	191	},
119	192	};
120		try self.initOsData();
121		errdefer self.deinitOsData();
	193	const extra_thread_count = thread_count - 1;
	194	self.eventfd_resume_nodes = try self.allocator.alloc(
	195	std.atomic.Stack(ResumeNode.EventFd).Node,
	196	extra_thread_count,
	197	);
	198	errdefer self.allocator.free(self.eventfd_resume_nodes);
	199
	200	self.extra_threads = try self.allocator.alloc(*std.os.Thread, extra_thread_count);
	201	errdefer self.allocator.free(self.extra_threads);
122	202
123		return self;
	203	try self.initOsData(extra_thread_count);
	204	errdefer self.deinitOsData();
124	205	}
125	206
126	207	/// must call stop before deinit
127	208	pub fn deinit(self: *Loop) void {
128	209	self.deinitOsData();
	210	self.allocator.free(self.extra_threads);
129	211	}
130	212
131		const InitOsDataError = std.os.LinuxEpollCreateError;
	213	const InitOsDataError = std.os.LinuxEpollCreateError \|\| mem.Allocator.Error \|\| std.os.LinuxEventFdError \|\|
	214	std.os.SpawnThreadError \|\| std.os.LinuxEpollCtlError \|\| std.os.BsdKEventError \|\|
	215	std.os.WindowsCreateIoCompletionPortError;
132	216
133		fn initOsData(self: *Loop) InitOsDataError!void {
	217	const wakeup_bytes = []u8{0x1} ** 8;
	218
	219	fn initOsData(self: *Loop, extra_thread_count: usize) InitOsDataError!void {
134	220	switch (builtin.os) {
135	221	builtin.Os.linux => {
136		self.os_data.epollfd = try std.os.linuxEpollCreate(std.os.linux.EPOLL_CLOEXEC);
	222	errdefer {
	223	while (self.available_eventfd_resume_nodes.pop()) \|node\| std.os.close(node.data.eventfd);
	224	}
	225	for (self.eventfd_resume_nodes) \|*eventfd_node\| {
	226	eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{
	227	.data = ResumeNode.EventFd{
	228	.base = ResumeNode{
	229	.id = ResumeNode.Id.EventFd,
	230	.handle = undefined,
	231	},
	232	.eventfd = try std.os.linuxEventFd(1, posix.EFD_CLOEXEC \| posix.EFD_NONBLOCK),
	233	.epoll_op = posix.EPOLL_CTL_ADD,
	234	},
	235	.next = undefined,
	236	};
	237	self.available_eventfd_resume_nodes.push(eventfd_node);
	238	}
	239
	240	self.os_data.epollfd = try std.os.linuxEpollCreate(posix.EPOLL_CLOEXEC);
137	241	errdefer std.os.close(self.os_data.epollfd);
	242
	243	self.os_data.final_eventfd = try std.os.linuxEventFd(0, posix.EFD_CLOEXEC \| posix.EFD_NONBLOCK);
	244	errdefer std.os.close(self.os_data.final_eventfd);
	245
	246	self.os_data.final_eventfd_event = posix.epoll_event{
	247	.events = posix.EPOLLIN,
	248	.data = posix.epoll_data{ .ptr = @ptrToInt(&self.final_resume_node) },
	249	};
	250	try std.os.linuxEpollCtl(
	251	self.os_data.epollfd,
	252	posix.EPOLL_CTL_ADD,
	253	self.os_data.final_eventfd,
	254	&self.os_data.final_eventfd_event,
	255	);
	256
	257	var extra_thread_index: usize = 0;
	258	errdefer {
	259	// writing 8 bytes to an eventfd cannot fail
	260	std.os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable;
	261	while (extra_thread_index != 0) {
	262	extra_thread_index -= 1;
	263	self.extra_threads[extra_thread_index].wait();
	264	}
	265	}
	266	while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) {
	267	self.extra_threads[extra_thread_index] = try std.os.spawnThread(self, workerRun);
	268	}
	269	},
	270	builtin.Os.macosx => {
	271	self.os_data.kqfd = try std.os.bsdKQueue();
	272	errdefer std.os.close(self.os_data.kqfd);
	273
	274	self.os_data.kevents = try self.allocator.alloc(posix.Kevent, extra_thread_count);
	275	errdefer self.allocator.free(self.os_data.kevents);
	276
	277	const eventlist = ([*]posix.Kevent)(undefined)[0..0];
	278
	279	for (self.eventfd_resume_nodes) \|*eventfd_node, i\| {
	280	eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{
	281	.data = ResumeNode.EventFd{
	282	.base = ResumeNode{
	283	.id = ResumeNode.Id.EventFd,
	284	.handle = undefined,
	285	},
	286	// this one is for sending events
	287	.kevent = posix.Kevent{
	288	.ident = i,
	289	.filter = posix.EVFILT_USER,
	290	.flags = posix.EV_CLEAR \| posix.EV_ADD \| posix.EV_DISABLE,
	291	.fflags = 0,
	292	.data = 0,
	293	.udata = @ptrToInt(&eventfd_node.data.base),
	294	},
	295	},
	296	.next = undefined,
	297	};
	298	self.available_eventfd_resume_nodes.push(eventfd_node);
	299	const kevent_array = (*[1]posix.Kevent)(&eventfd_node.data.kevent);
	300	_ = try std.os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null);
	301	eventfd_node.data.kevent.flags = posix.EV_CLEAR \| posix.EV_ENABLE;
	302	eventfd_node.data.kevent.fflags = posix.NOTE_TRIGGER;
	303	// this one is for waiting for events
	304	self.os_data.kevents[i] = posix.Kevent{
	305	.ident = i,
	306	.filter = posix.EVFILT_USER,
	307	.flags = 0,
	308	.fflags = 0,
	309	.data = 0,
	310	.udata = @ptrToInt(&eventfd_node.data.base),
	311	};
	312	}
	313
	314	// Pre-add so that we cannot get error.SystemResources
	315	// later when we try to activate it.
	316	self.os_data.final_kevent = posix.Kevent{
	317	.ident = extra_thread_count,
	318	.filter = posix.EVFILT_USER,
	319	.flags = posix.EV_ADD \| posix.EV_DISABLE,
	320	.fflags = 0,
	321	.data = 0,
	322	.udata = @ptrToInt(&self.final_resume_node),
	323	};
	324	const kevent_array = (*[1]posix.Kevent)(&self.os_data.final_kevent);
	325	_ = try std.os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null);
	326	self.os_data.final_kevent.flags = posix.EV_ENABLE;
	327	self.os_data.final_kevent.fflags = posix.NOTE_TRIGGER;
	328
	329	var extra_thread_index: usize = 0;
	330	errdefer {
	331	_ = std.os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null) catch unreachable;
	332	while (extra_thread_index != 0) {
	333	extra_thread_index -= 1;
	334	self.extra_threads[extra_thread_index].wait();
	335	}
	336	}
	337	while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) {
	338	self.extra_threads[extra_thread_index] = try std.os.spawnThread(self, workerRun);
	339	}
	340	},
	341	builtin.Os.windows => {
	342	self.os_data.extra_thread_count = extra_thread_count;
	343
	344	self.os_data.io_port = try std.os.windowsCreateIoCompletionPort(
	345	windows.INVALID_HANDLE_VALUE,
	346	null,
	347	undefined,
	348	undefined,
	349	);
	350	errdefer std.os.close(self.os_data.io_port);
	351
	352	for (self.eventfd_resume_nodes) \|*eventfd_node, i\| {
	353	eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{
	354	.data = ResumeNode.EventFd{
	355	.base = ResumeNode{
	356	.id = ResumeNode.Id.EventFd,
	357	.handle = undefined,
	358	},
	359	// this one is for sending events
	360	.completion_key = @ptrToInt(&eventfd_node.data.base),
	361	},
	362	.next = undefined,
	363	};
	364	self.available_eventfd_resume_nodes.push(eventfd_node);
	365	}
	366
	367	var extra_thread_index: usize = 0;
	368	errdefer {
	369	var i: usize = 0;
	370	while (i < extra_thread_index) : (i += 1) {
	371	while (true) {
	372	const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1);
	373	std.os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, @ptrToInt(&self.final_resume_node), overlapped) catch continue;
	374	break;
	375	}
	376	}
	377	while (extra_thread_index != 0) {
	378	extra_thread_index -= 1;
	379	self.extra_threads[extra_thread_index].wait();
	380	}
	381	}
	382	while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) {
	383	self.extra_threads[extra_thread_index] = try std.os.spawnThread(self, workerRun);
	384	}
138	385	},
139	386	else => {},
140	387	}
...	...	@@ -142,65 +389,281 @@ pub const Loop = struct {
142	389
143	390	fn deinitOsData(self: *Loop) void {
144	391	switch (builtin.os) {
145		builtin.Os.linux => std.os.close(self.os_data.epollfd),
	392	builtin.Os.linux => {
	393	std.os.close(self.os_data.final_eventfd);
	394	while (self.available_eventfd_resume_nodes.pop()) \|node\| std.os.close(node.data.eventfd);
	395	std.os.close(self.os_data.epollfd);
	396	self.allocator.free(self.eventfd_resume_nodes);
	397	},
	398	builtin.Os.macosx => {
	399	self.allocator.free(self.os_data.kevents);
	400	std.os.close(self.os_data.kqfd);
	401	},
	402	builtin.Os.windows => {
	403	std.os.close(self.os_data.io_port);
	404	},
146	405	else => {},
147	406	}
148	407	}
149	408
150		pub fn addFd(self: *Loop, fd: i32, prom: promise) !void {
	409	/// resume_node must live longer than the promise that it holds a reference to.
	410	pub fn addFd(self: Loop, fd: i32, resume_node: ResumeNode) !void {
	411	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	412	errdefer {
	413	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	414	}
	415	try self.modFd(
	416	fd,
	417	posix.EPOLL_CTL_ADD,
	418	std.os.linux.EPOLLIN \| std.os.linux.EPOLLOUT \| std.os.linux.EPOLLET,
	419	resume_node,
	420	);
	421	}
	422
	423	pub fn modFd(self: Loop, fd: i32, op: u32, events: u32, resume_node: ResumeNode) !void {
151	424	var ev = std.os.linux.epoll_event{
152		.events = std.os.linux.EPOLLIN \| std.os.linux.EPOLLOUT \| std.os.linux.EPOLLET,
153		.data = std.os.linux.epoll_data{ .ptr = @ptrToInt(prom) },
	425	.events = events,
	426	.data = std.os.linux.epoll_data{ .ptr = @ptrToInt(resume_node) },
154	427	};
155		try std.os.linuxEpollCtl(self.os_data.epollfd, std.os.linux.EPOLL_CTL_ADD, fd, &ev);
	428	try std.os.linuxEpollCtl(self.os_data.epollfd, op, fd, &ev);
156	429	}
157	430
158	431	pub fn removeFd(self: *Loop, fd: i32) void {
	432	self.removeFdNoCounter(fd);
	433	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	434	}
	435
	436	fn removeFdNoCounter(self: *Loop, fd: i32) void {
159	437	std.os.linuxEpollCtl(self.os_data.epollfd, std.os.linux.EPOLL_CTL_DEL, fd, undefined) catch {};
160	438	}
161		async fn waitFd(self: *Loop, fd: i32) !void {
	439
	440	pub async fn waitFd(self: *Loop, fd: i32) !void {
162	441	defer self.removeFd(fd);
163	442	suspend \|p\| {
164		try self.addFd(fd, p);
	443	// TODO explicitly put this memory in the coroutine frame #1194
	444	var resume_node = ResumeNode{
	445	.id = ResumeNode.Id.Basic,
	446	.handle = p,
	447	};
	448	try self.addFd(fd, &resume_node);
165	449	}
166	450	}
167	451
168		pub fn stop(self: *Loop) void {
169		// TODO make atomic
170		self.keep_running = false;
171		// TODO activate an fd in the epoll set which should cancel all the promises
172		}
173
174		/// bring your own linked list node. this means it can't fail.
	452	/// Bring your own linked list node. This means it can't fail.
175	453	pub fn onNextTick(self: Loop, node: NextTickNode) void {
	454	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
176	455	self.next_tick_queue.put(node);
177	456	}
178	457
179	458	pub fn run(self: *Loop) void {
180		while (self.keep_running) {
181		// TODO multiplex the next tick queue and the epoll event results onto a thread pool
182		while (self.next_tick_queue.get()) \|node\| {
183		resume node.data;
184		}
185		if (!self.keep_running) break;
186
187		self.dispatchOsEvents();
	459	_ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	460	self.workerRun();
	461	for (self.extra_threads) \|extra_thread\| {
	462	extra_thread.wait();
188	463	}
189	464	}
190	465
191		fn dispatchOsEvents(self: *Loop) void {
192		switch (builtin.os) {
193		builtin.Os.linux => {
194		var events: [16]std.os.linux.epoll_event = undefined;
195		const count = std.os.linuxEpollWait(self.os_data.epollfd, events[0..], -1);
196		for (events[0..count]) \|ev\| {
197		const p = @intToPtr(promise, ev.data.ptr);
198		resume p;
	466	fn workerRun(self: *Loop) void {
	467	start_over: while (true) {
	468	if (@atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) == 0) {
	469	while (self.next_tick_queue.get()) \|next_tick_node\| {
	470	const handle = next_tick_node.data;
	471	if (self.next_tick_queue.isEmpty()) {
	472	// last node, just resume it
	473	_ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	474	resume handle;
	475	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	476	continue :start_over;
	477	}
	478
	479	// non-last node, stick it in the epoll/kqueue set so that
	480	// other threads can get to it
	481	if (self.available_eventfd_resume_nodes.pop()) \|resume_stack_node\| {
	482	const eventfd_node = &resume_stack_node.data;
	483	eventfd_node.base.handle = handle;
	484	switch (builtin.os) {
	485	builtin.Os.macosx => {
	486	const kevent_array = (*[1]posix.Kevent)(&eventfd_node.kevent);
	487	const eventlist = ([*]posix.Kevent)(undefined)[0..0];
	488	_ = std.os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null) catch {
	489	// fine, we didn't need it anyway
	490	_ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	491	self.available_eventfd_resume_nodes.push(resume_stack_node);
	492	resume handle;
	493	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	494	continue :start_over;
	495	};
	496	},
	497	builtin.Os.linux => {
	498	// the pending count is already accounted for
	499	const epoll_events = posix.EPOLLONESHOT \| std.os.linux.EPOLLIN \| std.os.linux.EPOLLOUT \| std.os.linux.EPOLLET;
	500	self.modFd(eventfd_node.eventfd, eventfd_node.epoll_op, epoll_events, &eventfd_node.base) catch {
	501	// fine, we didn't need it anyway
	502	_ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	503	self.available_eventfd_resume_nodes.push(resume_stack_node);
	504	resume handle;
	505	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	506	continue :start_over;
	507	};
	508	},
	509	builtin.Os.windows => {
	510	// this value is never dereferenced but we need it to be non-null so that
	511	// the consumer code can decide whether to read the completion key.
	512	// it has to do this for normal I/O, so we match that behavior here.
	513	const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1);
	514	std.os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, eventfd_node.completion_key, overlapped) catch {
	515	// fine, we didn't need it anyway
	516	_ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	517	self.available_eventfd_resume_nodes.push(resume_stack_node);
	518	resume handle;
	519	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	520	continue :start_over;
	521	};
	522	},
	523	else => @compileError("unsupported OS"),
	524	}
	525	} else {
	526	// threads are too busy, can't add another eventfd to wake one up
	527	_ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	528	resume handle;
	529	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	530	continue :start_over;
	531	}
199	532	}
200		},
201		else => {},
	533
	534	const pending_event_count = @atomicLoad(usize, &self.pending_event_count, AtomicOrder.SeqCst);
	535	if (pending_event_count == 0) {
	536	// cause all the threads to stop
	537	switch (builtin.os) {
	538	builtin.Os.linux => {
	539	// writing 8 bytes to an eventfd cannot fail
	540	std.os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable;
	541	return;
	542	},
	543	builtin.Os.macosx => {
	544	const final_kevent = (*[1]posix.Kevent)(&self.os_data.final_kevent);
	545	const eventlist = ([*]posix.Kevent)(undefined)[0..0];
	546	// cannot fail because we already added it and this just enables it
	547	_ = std.os.bsdKEvent(self.os_data.kqfd, final_kevent, eventlist, null) catch unreachable;
	548	return;
	549	},
	550	builtin.Os.windows => {
	551	var i: usize = 0;
	552	while (i < self.os_data.extra_thread_count) : (i += 1) {
	553	while (true) {
	554	const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1);
	555	std.os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, @ptrToInt(&self.final_resume_node), overlapped) catch continue;
	556	break;
	557	}
	558	}
	559	return;
	560	},
	561	else => @compileError("unsupported OS"),
	562	}
	563	}
	564
	565	_ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	566	}
	567
	568	switch (builtin.os) {
	569	builtin.Os.linux => {
	570	// only process 1 event so we don't steal from other threads
	571	var events: [1]std.os.linux.epoll_event = undefined;
	572	const count = std.os.linuxEpollWait(self.os_data.epollfd, events[0..], -1);
	573	for (events[0..count]) \|ev\| {
	574	const resume_node = @intToPtr(*ResumeNode, ev.data.ptr);
	575	const handle = resume_node.handle;
	576	const resume_node_id = resume_node.id;
	577	switch (resume_node_id) {
	578	ResumeNode.Id.Basic => {},
	579	ResumeNode.Id.Stop => return,
	580	ResumeNode.Id.EventFd => {
	581	const event_fd_node = @fieldParentPtr(ResumeNode.EventFd, "base", resume_node);
	582	event_fd_node.epoll_op = posix.EPOLL_CTL_MOD;
	583	const stack_node = @fieldParentPtr(std.atomic.Stack(ResumeNode.EventFd).Node, "data", event_fd_node);
	584	self.available_eventfd_resume_nodes.push(stack_node);
	585	},
	586	}
	587	resume handle;
	588	if (resume_node_id == ResumeNode.Id.EventFd) {
	589	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	590	}
	591	}
	592	},
	593	builtin.Os.macosx => {
	594	var eventlist: [1]posix.Kevent = undefined;
	595	const count = std.os.bsdKEvent(self.os_data.kqfd, self.os_data.kevents, eventlist[0..], null) catch unreachable;
	596	for (eventlist[0..count]) \|ev\| {
	597	const resume_node = @intToPtr(*ResumeNode, ev.udata);
	598	const handle = resume_node.handle;
	599	const resume_node_id = resume_node.id;
	600	switch (resume_node_id) {
	601	ResumeNode.Id.Basic => {},
	602	ResumeNode.Id.Stop => return,
	603	ResumeNode.Id.EventFd => {
	604	const event_fd_node = @fieldParentPtr(ResumeNode.EventFd, "base", resume_node);
	605	const stack_node = @fieldParentPtr(std.atomic.Stack(ResumeNode.EventFd).Node, "data", event_fd_node);
	606	self.available_eventfd_resume_nodes.push(stack_node);
	607	},
	608	}
	609	resume handle;
	610	if (resume_node_id == ResumeNode.Id.EventFd) {
	611	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	612	}
	613	}
	614	},
	615	builtin.Os.windows => {
	616	var completion_key: usize = undefined;
	617	while (true) {
	618	var nbytes: windows.DWORD = undefined;
	619	var overlapped: ?*windows.OVERLAPPED = undefined;
	620	switch (std.os.windowsGetQueuedCompletionStatus(self.os_data.io_port, &nbytes, &completion_key, &overlapped, windows.INFINITE)) {
	621	std.os.WindowsWaitResult.Aborted => return,
	622	std.os.WindowsWaitResult.Normal => {},
	623	}
	624	if (overlapped != null) break;
	625	}
	626	const resume_node = @intToPtr(*ResumeNode, completion_key);
	627	const handle = resume_node.handle;
	628	const resume_node_id = resume_node.id;
	629	switch (resume_node_id) {
	630	ResumeNode.Id.Basic => {},
	631	ResumeNode.Id.Stop => return,
	632	ResumeNode.Id.EventFd => {
	633	const event_fd_node = @fieldParentPtr(ResumeNode.EventFd, "base", resume_node);
	634	const stack_node = @fieldParentPtr(std.atomic.Stack(ResumeNode.EventFd).Node, "data", event_fd_node);
	635	self.available_eventfd_resume_nodes.push(stack_node);
	636	},
	637	}
	638	resume handle;
	639	if (resume_node_id == ResumeNode.Id.EventFd) {
	640	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	641	}
	642	},
	643	else => @compileError("unsupported OS"),
	644	}
202	645	}
203	646	}
	647
	648	const OsData = switch (builtin.os) {
	649	builtin.Os.linux => struct {
	650	epollfd: i32,
	651	final_eventfd: i32,
	652	final_eventfd_event: std.os.linux.epoll_event,
	653	},
	654	builtin.Os.macosx => MacOsData,
	655	builtin.Os.windows => struct {
	656	io_port: windows.HANDLE,
	657	extra_thread_count: usize,
	658	},
	659	else => struct {},
	660	};
	661
	662	const MacOsData = struct {
	663	kqfd: i32,
	664	final_kevent: posix.Kevent,
	665	kevents: []posix.Kevent,
	666	};
204	667	};
205	668
206	669	/// many producer, many consumer, thread-safe, lock-free, runtime configurable buffer size
...	...	@@ -304,9 +767,7 @@ pub fn Channel(comptime T: type) type {
304	767	// TODO integrate this function with named return values
305	768	// so we can get rid of this extra result copy
306	769	var result: T = undefined;
307		var debug_handle: usize = undefined;
308	770	suspend \|handle\| {
309		debug_handle = @ptrToInt(handle);
310	771	var my_tick_node = Loop.NextTickNode{
311	772	.next = undefined,
312	773	.data = handle,
...	...	@@ -438,9 +899,8 @@ test "listen on a port, send bytes, receive bytes" {
438	899	const self = @fieldParentPtr(Self, "tcp_server", tcp_server);
439	900	var socket = _socket.*; // TODO https://github.com/ziglang/zig/issues/733
440	901	defer socket.close();
441		const next_handler = async errorableHandler(self, _addr, socket) catch \|err\| switch (err) {
442		error.OutOfMemory => @panic("unable to handle connection: out of memory"),
443		};
	902	// TODO guarantee elision of this allocation
	903	const next_handler = async errorableHandler(self, _addr, socket) catch unreachable;
444	904	(await next_handler) catch \|err\| {
445	905	std.debug.panic("unable to handle connection: {}\n", err);
446	906	};
...	...	@@ -461,17 +921,18 @@ test "listen on a port, send bytes, receive bytes" {
461	921	const ip4addr = std.net.parseIp4("127.0.0.1") catch unreachable;
462	922	const addr = std.net.Address.initIp4(ip4addr, 0);
463	923
464		var loop = try Loop.init(std.debug.global_allocator);
465		var server = MyServer{ .tcp_server = try TcpServer.init(&loop) };
	924	var loop: Loop = undefined;
	925	try loop.initSingleThreaded(std.debug.global_allocator);
	926	var server = MyServer{ .tcp_server = TcpServer.init(&loop) };
466	927	defer server.tcp_server.deinit();
467	928	try server.tcp_server.listen(addr, MyServer.handler);
468	929
469		const p = try async<std.debug.global_allocator> doAsyncTest(&loop, server.tcp_server.listen_address);
	930	const p = try async<std.debug.global_allocator> doAsyncTest(&loop, server.tcp_server.listen_address, &server.tcp_server);
470	931	defer cancel p;
471	932	loop.run();
472	933	}
473	934
474		async fn doAsyncTest(loop: Loop, address: const std.net.Address) void {
	935	async fn doAsyncTest(loop: Loop, address: const std.net.Address, server: *TcpServer) void {
475	936	errdefer @panic("test failure");
476	937
477	938	var socket_file = try await try async event.connect(loop, address);
...	...	@@ -481,7 +942,7 @@ async fn doAsyncTest(loop: Loop, address: const std.net.Address) void {
481	942	const amt_read = try socket_file.read(buf[0..]);
482	943	const msg = buf[0..amt_read];
483	944	assert(mem.eql(u8, msg, "hello from server\n"));
484		loop.stop();
	945	server.close();
485	946	}
486	947
487	948	test "std.event.Channel" {
...	...	@@ -490,7 +951,9 @@ test "std.event.Channel" {
490	951
491	952	const allocator = &da.allocator;
492	953
493		var loop = try Loop.init(allocator);
	954	var loop: Loop = undefined;
	955	// TODO make a multi threaded test
	956	try loop.initSingleThreaded(allocator);
494	957	defer loop.deinit();
495	958
496	959	const channel = try Channel(i32).create(&loop, 0);
...	...	@@ -515,11 +978,246 @@ async fn testChannelGetter(loop: Loop, channel: Channel(i32)) void {
515	978	const value2_promise = try async channel.get();
516	979	const value2 = await value2_promise;
517	980	assert(value2 == 4567);
518
519		loop.stop();
520	981	}
521	982
522	983	async fn testChannelPutter(channel: *Channel(i32)) void {
523	984	await (async channel.put(1234) catch @panic("out of memory"));
524	985	await (async channel.put(4567) catch @panic("out of memory"));
525	986	}
	987
	988	/// Thread-safe async/await lock.
	989	/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and
	990	/// are resumed when the lock is released, in order.
	991	pub const Lock = struct {
	992	loop: *Loop,
	993	shared_bit: u8, // TODO make this a bool
	994	queue: Queue,
	995	queue_empty_bit: u8, // TODO make this a bool
	996
	997	const Queue = std.atomic.QueueMpsc(promise);
	998
	999	pub const Held = struct {
	1000	lock: *Lock,
	1001
	1002	pub fn release(self: Held) void {
	1003	// Resume the next item from the queue.
	1004	if (self.lock.queue.get()) \|node\| {
	1005	self.lock.loop.onNextTick(node);
	1006	return;
	1007	}
	1008
	1009	// We need to release the lock.
	1010	_ = @atomicRmw(u8, &self.lock.queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	1011	_ = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	1012
	1013	// There might be a queue item. If we know the queue is empty, we can be done,
	1014	// because the other actor will try to obtain the lock.
	1015	// But if there's a queue item, we are the actor which must loop and attempt
	1016	// to grab the lock again.
	1017	if (@atomicLoad(u8, &self.lock.queue_empty_bit, AtomicOrder.SeqCst) == 1) {
	1018	return;
	1019	}
	1020
	1021	while (true) {
	1022	const old_bit = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	1023	if (old_bit != 0) {
	1024	// We did not obtain the lock. Great, the queue is someone else's problem.
	1025	return;
	1026	}
	1027
	1028	// Resume the next item from the queue.
	1029	if (self.lock.queue.get()) \|node\| {
	1030	self.lock.loop.onNextTick(node);
	1031	return;
	1032	}
	1033
	1034	// Release the lock again.
	1035	_ = @atomicRmw(u8, &self.lock.queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	1036	_ = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	1037
	1038	// Find out if we can be done.
	1039	if (@atomicLoad(u8, &self.lock.queue_empty_bit, AtomicOrder.SeqCst) == 1) {
	1040	return;
	1041	}
	1042	}
	1043	}
	1044	};
	1045
	1046	pub fn init(loop: *Loop) Lock {
	1047	return Lock{
	1048	.loop = loop,
	1049	.shared_bit = 0,
	1050	.queue = Queue.init(),
	1051	.queue_empty_bit = 1,
	1052	};
	1053	}
	1054
	1055	/// Must be called when not locked. Not thread safe.
	1056	/// All calls to acquire() and release() must complete before calling deinit().
	1057	pub fn deinit(self: *Lock) void {
	1058	assert(self.shared_bit == 0);
	1059	while (self.queue.get()) \|node\| cancel node.data;
	1060	}
	1061
	1062	pub async fn acquire(self: *Lock) Held {
	1063	s: suspend \|handle\| {
	1064	// TODO explicitly put this memory in the coroutine frame #1194
	1065	var my_tick_node = Loop.NextTickNode{
	1066	.data = handle,
	1067	.next = undefined,
	1068	};
	1069
	1070	self.queue.put(&my_tick_node);
	1071
	1072	// At this point, we are in the queue, so we might have already been resumed and this coroutine
	1073	// frame might be destroyed. For the rest of the suspend block we cannot access the coroutine frame.
	1074
	1075	// We set this bit so that later we can rely on the fact, that if queue_empty_bit is 1, some actor
	1076	// will attempt to grab the lock.
	1077	_ = @atomicRmw(u8, &self.queue_empty_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	1078
	1079	while (true) {
	1080	const old_bit = @atomicRmw(u8, &self.shared_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	1081	if (old_bit != 0) {
	1082	// We did not obtain the lock. Trust that our queue entry will resume us, and allow
	1083	// suspend to complete.
	1084	break;
	1085	}
	1086	// We got the lock. However we might have already been resumed from the queue.
	1087	if (self.queue.get()) \|node\| {
	1088	// Whether this node is us or someone else, we tail resume it.
	1089	resume node.data;
	1090	break;
	1091	} else {
	1092	// We already got resumed, and there are none left in the queue, which means that
	1093	// we aren't even supposed to hold the lock right now.
	1094	_ = @atomicRmw(u8, &self.queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	1095	_ = @atomicRmw(u8, &self.shared_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	1096
	1097	// There might be a queue item. If we know the queue is empty, we can be done,
	1098	// because the other actor will try to obtain the lock.
	1099	// But if there's a queue item, we are the actor which must loop and attempt
	1100	// to grab the lock again.
	1101	if (@atomicLoad(u8, &self.queue_empty_bit, AtomicOrder.SeqCst) == 1) {
	1102	break;
	1103	} else {
	1104	continue;
	1105	}
	1106	}
	1107	unreachable;
	1108	}
	1109	}
	1110
	1111	return Held{ .lock = self };
	1112	}
	1113	};
	1114
	1115	/// Thread-safe async/await lock that protects one piece of data.
	1116	/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and
	1117	/// are resumed when the lock is released, in order.
	1118	pub fn Locked(comptime T: type) type {
	1119	return struct {
	1120	lock: Lock,
	1121	private_data: T,
	1122
	1123	const Self = this;
	1124
	1125	pub const HeldLock = struct {
	1126	value: *T,
	1127	held: Lock.Held,
	1128
	1129	pub fn release(self: HeldLock) void {
	1130	self.held.release();
	1131	}
	1132	};
	1133
	1134	pub fn init(loop: *Loop, data: T) Self {
	1135	return Self{
	1136	.lock = Lock.init(loop),
	1137	.private_data = data,
	1138	};
	1139	}
	1140
	1141	pub fn deinit(self: *Self) void {
	1142	self.lock.deinit();
	1143	}
	1144
	1145	pub async fn acquire(self: *Self) HeldLock {
	1146	return HeldLock{
	1147	// TODO guaranteed allocation elision
	1148	.held = await (async self.lock.acquire() catch unreachable),
	1149	.value = &self.private_data,
	1150	};
	1151	}
	1152	};
	1153	}
	1154
	1155	test "std.event.Lock" {
	1156	var da = std.heap.DirectAllocator.init();
	1157	defer da.deinit();
	1158
	1159	const allocator = &da.allocator;
	1160
	1161	var loop: Loop = undefined;
	1162	try loop.initMultiThreaded(allocator);
	1163	defer loop.deinit();
	1164
	1165	var lock = Lock.init(&loop);
	1166	defer lock.deinit();
	1167
	1168	const handle = try async<allocator> testLock(&loop, &lock);
	1169	defer cancel handle;
	1170	loop.run();
	1171
	1172	assert(mem.eql(i32, shared_test_data, [1]i32{3 * @intCast(i32, shared_test_data.len)} ** shared_test_data.len));
	1173	}
	1174
	1175	async fn testLock(loop: Loop, lock: Lock) void {
	1176	// TODO explicitly put next tick node memory in the coroutine frame #1194
	1177	suspend \|p\| {
	1178	resume p;
	1179	}
	1180	const handle1 = async lockRunner(lock) catch @panic("out of memory");
	1181	var tick_node1 = Loop.NextTickNode{
	1182	.next = undefined,
	1183	.data = handle1,
	1184	};
	1185	loop.onNextTick(&tick_node1);
	1186
	1187	const handle2 = async lockRunner(lock) catch @panic("out of memory");
	1188	var tick_node2 = Loop.NextTickNode{
	1189	.next = undefined,
	1190	.data = handle2,
	1191	};
	1192	loop.onNextTick(&tick_node2);
	1193
	1194	const handle3 = async lockRunner(lock) catch @panic("out of memory");
	1195	var tick_node3 = Loop.NextTickNode{
	1196	.next = undefined,
	1197	.data = handle3,
	1198	};
	1199	loop.onNextTick(&tick_node3);
	1200
	1201	await handle1;
	1202	await handle2;
	1203	await handle3;
	1204	}
	1205
	1206	var shared_test_data = [1]i32{0} ** 10;
	1207	var shared_test_index: usize = 0;
	1208
	1209	async fn lockRunner(lock: *Lock) void {
	1210	suspend; // resumed by onNextTick
	1211
	1212	var i: usize = 0;
	1213	while (i < shared_test_data.len) : (i += 1) {
	1214	const lock_promise = async lock.acquire() catch @panic("out of memory");
	1215	const handle = await lock_promise;
	1216	defer handle.release();
	1217
	1218	shared_test_index = 0;
	1219	while (shared_test_index < shared_test_data.len) : (shared_test_index += 1) {
	1220	shared_test_data[shared_test_index] = shared_test_data[shared_test_index] + 1;
	1221	}
	1222	}
	1223	}

std/heap.zig+83-16

...	...	@@ -38,7 +38,7 @@ fn cFree(self: *Allocator, old_mem: []u8) void {
38	38	}
39	39
40	40	/// This allocator makes a syscall directly for every allocation and free.
41		/// TODO make this thread-safe. The windows implementation will need some atomics.
	41	/// Thread-safe and lock-free.
42	42	pub const DirectAllocator = struct {
43	43	allocator: Allocator,
44	44	heap_handle: ?HeapHandle,
...	...	@@ -74,34 +74,34 @@ pub const DirectAllocator = struct {
74	74	const alloc_size = if (alignment <= os.page_size) n else n + alignment;
75	75	const addr = p.mmap(null, alloc_size, p.PROT_READ \| p.PROT_WRITE, p.MAP_PRIVATE \| p.MAP_ANONYMOUS, -1, 0);
76	76	if (addr == p.MAP_FAILED) return error.OutOfMemory;
77
78	77	if (alloc_size == n) return @intToPtr([*]u8, addr)[0..n];
79	78
80		var aligned_addr = addr & ~usize(alignment - 1);
81		aligned_addr += alignment;
	79	const aligned_addr = (addr & ~usize(alignment - 1)) + alignment;
82	80
83		//We can unmap the unused portions of our mmap, but we must only
84		// pass munmap bytes that exist outside our allocated pages or it
85		// will happily eat us too
	81	// We can unmap the unused portions of our mmap, but we must only
	82	// pass munmap bytes that exist outside our allocated pages or it
	83	// will happily eat us too.
86	84
87		//Since alignment > page_size, we are by definition on a page boundry
	85	// Since alignment > page_size, we are by definition on a page boundary.
88	86	const unused_start = addr;
89	87	const unused_len = aligned_addr - 1 - unused_start;
90	88
91		var err = p.munmap(unused_start, unused_len);
92		debug.assert(p.getErrno(err) == 0);
	89	const err = p.munmap(unused_start, unused_len);
	90	assert(p.getErrno(err) == 0);
93	91
94		//It is impossible that there is an unoccupied page at the top of our
95		// mmap.
	92	// It is impossible that there is an unoccupied page at the top of our
	93	// mmap.
96	94
97	95	return @intToPtr([*]u8, aligned_addr)[0..n];
98	96	},
99	97	Os.windows => {
100	98	const amt = n + alignment + @sizeOf(usize);
101		const heap_handle = self.heap_handle orelse blk: {
102		const hh = os.windows.HeapCreate(os.windows.HEAP_NO_SERIALIZE, amt, 0) orelse return error.OutOfMemory;
103		self.heap_handle = hh;
104		break :blk hh;
	99	const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, builtin.AtomicOrder.SeqCst);
	100	const heap_handle = optional_heap_handle orelse blk: {
	101	const hh = os.windows.HeapCreate(0, amt, 0) orelse return error.OutOfMemory;
	102	const other_hh = @cmpxchgStrong(?HeapHandle, &self.heap_handle, null, hh, builtin.AtomicOrder.SeqCst, builtin.AtomicOrder.SeqCst) orelse break :blk hh;
	103	_ = os.windows.HeapDestroy(hh);
	104	break :blk other_hh.?; // can't be null because of the cmpxchg
105	105	};
106	106	const ptr = os.windows.HeapAlloc(heap_handle, 0, amt) orelse return error.OutOfMemory;
107	107	const root_addr = @ptrToInt(ptr);
...	...	@@ -361,6 +361,73 @@ pub const ThreadSafeFixedBufferAllocator = struct {
361	361	fn free(allocator: *Allocator, bytes: []u8) void {}
362	362	};
363	363
	364	pub fn stackFallback(comptime size: usize, fallback_allocator: *Allocator) StackFallbackAllocator(size) {
	365	return StackFallbackAllocator(size){
	366	.buffer = undefined,
	367	.fallback_allocator = fallback_allocator,
	368	.fixed_buffer_allocator = undefined,
	369	.allocator = Allocator{
	370	.allocFn = StackFallbackAllocator(size).alloc,
	371	.reallocFn = StackFallbackAllocator(size).realloc,
	372	.freeFn = StackFallbackAllocator(size).free,
	373	},
	374	};
	375	}
	376
	377	pub fn StackFallbackAllocator(comptime size: usize) type {
	378	return struct {
	379	const Self = this;
	380
	381	buffer: [size]u8,
	382	allocator: Allocator,
	383	fallback_allocator: *Allocator,
	384	fixed_buffer_allocator: FixedBufferAllocator,
	385
	386	pub fn get(self: Self) Allocator {
	387	self.fixed_buffer_allocator = FixedBufferAllocator.init(self.buffer[0..]);
	388	return &self.allocator;
	389	}
	390
	391	fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {
	392	const self = @fieldParentPtr(Self, "allocator", allocator);
	393	return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator.allocator, n, alignment) catch
	394	self.fallback_allocator.allocFn(self.fallback_allocator, n, alignment);
	395	}
	396
	397	fn realloc(allocator: *Allocator, old_mem: []u8, new_size: usize, alignment: u29) ![]u8 {
	398	const self = @fieldParentPtr(Self, "allocator", allocator);
	399	const in_buffer = @ptrToInt(old_mem.ptr) >= @ptrToInt(&self.buffer) and
	400	@ptrToInt(old_mem.ptr) < @ptrToInt(&self.buffer) + self.buffer.len;
	401	if (in_buffer) {
	402	return FixedBufferAllocator.realloc(
	403	&self.fixed_buffer_allocator.allocator,
	404	old_mem,
	405	new_size,
	406	alignment,
	407	) catch {
	408	const result = try self.fallback_allocator.allocFn(
	409	self.fallback_allocator,
	410	new_size,
	411	alignment,
	412	);
	413	mem.copy(u8, result, old_mem);
	414	return result;
	415	};
	416	}
	417	return self.fallback_allocator.reallocFn(self.fallback_allocator, old_mem, new_size, alignment);
	418	}
	419
	420	fn free(allocator: *Allocator, bytes: []u8) void {
	421	const self = @fieldParentPtr(Self, "allocator", allocator);
	422	const in_buffer = @ptrToInt(bytes.ptr) >= @ptrToInt(&self.buffer) and
	423	@ptrToInt(bytes.ptr) < @ptrToInt(&self.buffer) + self.buffer.len;
	424	if (!in_buffer) {
	425	return self.fallback_allocator.freeFn(self.fallback_allocator, bytes);
	426	}
	427	}
	428	};
	429	}
	430
364	431	test "c_allocator" {
365	432	if (builtin.link_libc) {
366	433	var slice = c_allocator.alloc(u8, 50) catch return;

std/mem.zig+1-1

...	...	@@ -6,7 +6,7 @@ const builtin = @import("builtin");
6	6	const mem = this;
7	7
8	8	pub const Allocator = struct {
9		const Error = error{OutOfMemory};
	9	pub const Error = error{OutOfMemory};
10	10
11	11	/// Allocate byte_count bytes and return them in a slice, with the
12	12	/// slice's pointer aligned at least to alignment bytes.

std/os/darwin.zig+259

...	...	@@ -264,6 +264,224 @@ pub const SIGUSR1 = 30;
264	264	/// user defined signal 2
265	265	pub const SIGUSR2 = 31;
266	266
	267	/// no flag value
	268	pub const KEVENT_FLAG_NONE = 0x000;
	269
	270	/// immediate timeout
	271	pub const KEVENT_FLAG_IMMEDIATE = 0x001;
	272
	273	/// output events only include change
	274	pub const KEVENT_FLAG_ERROR_EVENTS = 0x002;
	275
	276	/// add event to kq (implies enable)
	277	pub const EV_ADD = 0x0001;
	278
	279	/// delete event from kq
	280	pub const EV_DELETE = 0x0002;
	281
	282	/// enable event
	283	pub const EV_ENABLE = 0x0004;
	284
	285	/// disable event (not reported)
	286	pub const EV_DISABLE = 0x0008;
	287
	288	/// only report one occurrence
	289	pub const EV_ONESHOT = 0x0010;
	290
	291	/// clear event state after reporting
	292	pub const EV_CLEAR = 0x0020;
	293
	294	/// force immediate event output
	295	/// ... with or without EV_ERROR
	296	/// ... use KEVENT_FLAG_ERROR_EVENTS
	297	/// on syscalls supporting flags
	298	pub const EV_RECEIPT = 0x0040;
	299
	300	/// disable event after reporting
	301	pub const EV_DISPATCH = 0x0080;
	302
	303	/// unique kevent per udata value
	304	pub const EV_UDATA_SPECIFIC = 0x0100;
	305
	306	/// ... in combination with EV_DELETE
	307	/// will defer delete until udata-specific
	308	/// event enabled. EINPROGRESS will be
	309	/// returned to indicate the deferral
	310	pub const EV_DISPATCH2 = EV_DISPATCH \| EV_UDATA_SPECIFIC;
	311
	312	/// report that source has vanished
	313	/// ... only valid with EV_DISPATCH2
	314	pub const EV_VANISHED = 0x0200;
	315
	316	/// reserved by system
	317	pub const EV_SYSFLAGS = 0xF000;
	318
	319	/// filter-specific flag
	320	pub const EV_FLAG0 = 0x1000;
	321
	322	/// filter-specific flag
	323	pub const EV_FLAG1 = 0x2000;
	324
	325	/// EOF detected
	326	pub const EV_EOF = 0x8000;
	327
	328	/// error, data contains errno
	329	pub const EV_ERROR = 0x4000;
	330
	331	pub const EV_POLL = EV_FLAG0;
	332	pub const EV_OOBAND = EV_FLAG1;
	333
	334	pub const EVFILT_READ = -1;
	335	pub const EVFILT_WRITE = -2;
	336
	337	/// attached to aio requests
	338	pub const EVFILT_AIO = -3;
	339
	340	/// attached to vnodes
	341	pub const EVFILT_VNODE = -4;
	342
	343	/// attached to struct proc
	344	pub const EVFILT_PROC = -5;
	345
	346	/// attached to struct proc
	347	pub const EVFILT_SIGNAL = -6;
	348
	349	/// timers
	350	pub const EVFILT_TIMER = -7;
	351
	352	/// Mach portsets
	353	pub const EVFILT_MACHPORT = -8;
	354
	355	/// Filesystem events
	356	pub const EVFILT_FS = -9;
	357
	358	/// User events
	359	pub const EVFILT_USER = -10;
	360
	361	/// Virtual memory events
	362	pub const EVFILT_VM = -12;
	363
	364	/// Exception events
	365	pub const EVFILT_EXCEPT = -15;
	366
	367	pub const EVFILT_SYSCOUNT = 17;
	368
	369	/// On input, NOTE_TRIGGER causes the event to be triggered for output.
	370	pub const NOTE_TRIGGER = 0x01000000;
	371
	372	/// ignore input fflags
	373	pub const NOTE_FFNOP = 0x00000000;
	374
	375	/// and fflags
	376	pub const NOTE_FFAND = 0x40000000;
	377
	378	/// or fflags
	379	pub const NOTE_FFOR = 0x80000000;
	380
	381	/// copy fflags
	382	pub const NOTE_FFCOPY = 0xc0000000;
	383
	384	/// mask for operations
	385	pub const NOTE_FFCTRLMASK = 0xc0000000;
	386	pub const NOTE_FFLAGSMASK = 0x00ffffff;
	387
	388	/// low water mark
	389	pub const NOTE_LOWAT = 0x00000001;
	390
	391	/// OOB data
	392	pub const NOTE_OOB = 0x00000002;
	393
	394	/// vnode was removed
	395	pub const NOTE_DELETE = 0x00000001;
	396
	397	/// data contents changed
	398	pub const NOTE_WRITE = 0x00000002;
	399
	400	/// size increased
	401	pub const NOTE_EXTEND = 0x00000004;
	402
	403	/// attributes changed
	404	pub const NOTE_ATTRIB = 0x00000008;
	405
	406	/// link count changed
	407	pub const NOTE_LINK = 0x00000010;
	408
	409	/// vnode was renamed
	410	pub const NOTE_RENAME = 0x00000020;
	411
	412	/// vnode access was revoked
	413	pub const NOTE_REVOKE = 0x00000040;
	414
	415	/// No specific vnode event: to test for EVFILT_READ activation
	416	pub const NOTE_NONE = 0x00000080;
	417
	418	/// vnode was unlocked by flock(2)
	419	pub const NOTE_FUNLOCK = 0x00000100;
	420
	421	/// process exited
	422	pub const NOTE_EXIT = 0x80000000;
	423
	424	/// process forked
	425	pub const NOTE_FORK = 0x40000000;
	426
	427	/// process exec'd
	428	pub const NOTE_EXEC = 0x20000000;
	429
	430	/// shared with EVFILT_SIGNAL
	431	pub const NOTE_SIGNAL = 0x08000000;
	432
	433	/// exit status to be returned, valid for child process only
	434	pub const NOTE_EXITSTATUS = 0x04000000;
	435
	436	/// provide details on reasons for exit
	437	pub const NOTE_EXIT_DETAIL = 0x02000000;
	438
	439	/// mask for signal & exit status
	440	pub const NOTE_PDATAMASK = 0x000fffff;
	441	pub const NOTE_PCTRLMASK = (~NOTE_PDATAMASK);
	442
	443	pub const NOTE_EXIT_DETAIL_MASK = 0x00070000;
	444	pub const NOTE_EXIT_DECRYPTFAIL = 0x00010000;
	445	pub const NOTE_EXIT_MEMORY = 0x00020000;
	446	pub const NOTE_EXIT_CSERROR = 0x00040000;
	447
	448	/// will react on memory pressure
	449	pub const NOTE_VM_PRESSURE = 0x80000000;
	450
	451	/// will quit on memory pressure, possibly after cleaning up dirty state
	452	pub const NOTE_VM_PRESSURE_TERMINATE = 0x40000000;
	453
	454	/// will quit immediately on memory pressure
	455	pub const NOTE_VM_PRESSURE_SUDDEN_TERMINATE = 0x20000000;
	456
	457	/// there was an error
	458	pub const NOTE_VM_ERROR = 0x10000000;
	459
	460	/// data is seconds
	461	pub const NOTE_SECONDS = 0x00000001;
	462
	463	/// data is microseconds
	464	pub const NOTE_USECONDS = 0x00000002;
	465
	466	/// data is nanoseconds
	467	pub const NOTE_NSECONDS = 0x00000004;
	468
	469	/// absolute timeout
	470	pub const NOTE_ABSOLUTE = 0x00000008;
	471
	472	/// ext[1] holds leeway for power aware timers
	473	pub const NOTE_LEEWAY = 0x00000010;
	474
	475	/// system does minimal timer coalescing
	476	pub const NOTE_CRITICAL = 0x00000020;
	477
	478	/// system does maximum timer coalescing
	479	pub const NOTE_BACKGROUND = 0x00000040;
	480	pub const NOTE_MACH_CONTINUOUS_TIME = 0x00000080;
	481
	482	/// data is mach absolute time units
	483	pub const NOTE_MACHTIME = 0x00000100;
	484
267	485	fn wstatus(x: i32) i32 {
268	486	return x & 0o177;
269	487	}
...	...	@@ -385,6 +603,31 @@ pub fn getdirentries64(fd: i32, buf_ptr: []u8, buf_len: usize, basep: i64) usi
385	603	return errnoWrap(@bitCast(isize, c.__getdirentries64(fd, buf_ptr, buf_len, basep)));
386	604	}
387	605
	606	pub fn kqueue() usize {
	607	return errnoWrap(c.kqueue());
	608	}
	609
	610	pub fn kevent(kq: i32, changelist: []const Kevent, eventlist: []Kevent, timeout: ?*const timespec) usize {
	611	return errnoWrap(c.kevent(
	612	kq,
	613	changelist.ptr,
	614	@intCast(c_int, changelist.len),
	615	eventlist.ptr,
	616	@intCast(c_int, eventlist.len),
	617	timeout,
	618	));
	619	}
	620
	621	pub fn kevent64(
	622	kq: i32,
	623	changelist: []const kevent64_s,
	624	eventlist: []kevent64_s,
	625	flags: u32,
	626	timeout: ?*const timespec,
	627	) usize {
	628	return errnoWrap(c.kevent64(kq, changelist.ptr, changelist.len, eventlist.ptr, eventlist.len, flags, timeout));
	629	}
	630
388	631	pub fn mkdir(path: [*]const u8, mode: u32) usize {
389	632	return errnoWrap(c.mkdir(path, mode));
390	633	}
...	...	@@ -393,6 +636,18 @@ pub fn symlink(existing: []const u8, new: []const u8) usize {
393	636	return errnoWrap(c.symlink(existing, new));
394	637	}
395	638
	639	pub fn sysctl(name: []c_int, namelen: c_uint, oldp: ?c_void, oldlenp: ?usize, newp: ?c_void, newlen: usize) usize {
	640	return errnoWrap(c.sysctl(name, namelen, oldp, oldlenp, newp, newlen));
	641	}
	642
	643	pub fn sysctlbyname(name: []const u8, oldp: ?c_void, oldlenp: ?usize, newp: ?c_void, newlen: usize) usize {
	644	return errnoWrap(c.sysctlbyname(name, oldp, oldlenp, newp, newlen));
	645	}
	646
	647	pub fn sysctlnametomib(name: []const u8, mibp: ?c_int, sizep: ?*usize) usize {
	648	return errnoWrap(c.sysctlnametomib(name, wibp, sizep));
	649	}
	650
396	651	pub fn rename(old: []const u8, new: []const u8) usize {
397	652	return errnoWrap(c.rename(old, new));
398	653	}
...	...	@@ -474,6 +729,10 @@ pub const dirent = c.dirent;
474	729	pub const sa_family_t = c.sa_family_t;
475	730	pub const sockaddr = c.sockaddr;
476	731
	732	/// Renamed from `kevent` to `Kevent` to avoid conflict with the syscall.
	733	pub const Kevent = c.Kevent;
	734	pub const kevent64_s = c.kevent64_s;
	735
477	736	/// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall.
478	737	pub const Sigaction = struct {
479	738	handler: extern fn (i32) void,

std/os/index.zig+180-9

...	...	@@ -61,6 +61,15 @@ pub const windowsLoadDll = windows_util.windowsLoadDll;
61	61	pub const windowsUnloadDll = windows_util.windowsUnloadDll;
62	62	pub const createWindowsEnvBlock = windows_util.createWindowsEnvBlock;
63	63
	64	pub const WindowsCreateIoCompletionPortError = windows_util.WindowsCreateIoCompletionPortError;
	65	pub const windowsCreateIoCompletionPort = windows_util.windowsCreateIoCompletionPort;
	66
	67	pub const WindowsPostQueuedCompletionStatusError = windows_util.WindowsPostQueuedCompletionStatusError;
	68	pub const windowsPostQueuedCompletionStatus = windows_util.windowsPostQueuedCompletionStatus;
	69
	70	pub const WindowsWaitResult = windows_util.WindowsWaitResult;
	71	pub const windowsGetQueuedCompletionStatus = windows_util.windowsGetQueuedCompletionStatus;
	72
64	73	pub const WindowsWaitError = windows_util.WaitError;
65	74	pub const WindowsOpenError = windows_util.OpenError;
66	75	pub const WindowsWriteError = windows_util.WriteError;
...	...	@@ -2317,6 +2326,30 @@ pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usiz
2317	2326	}
2318	2327	}
2319	2328
	2329	pub const LinuxEventFdError = error{
	2330	InvalidFlagValue,
	2331	SystemResources,
	2332	ProcessFdQuotaExceeded,
	2333	SystemFdQuotaExceeded,
	2334
	2335	Unexpected,
	2336	};
	2337
	2338	pub fn linuxEventFd(initval: u32, flags: u32) LinuxEventFdError!i32 {
	2339	const rc = posix.eventfd(initval, flags);
	2340	const err = posix.getErrno(rc);
	2341	switch (err) {
	2342	0 => return @intCast(i32, rc),
	2343	else => return unexpectedErrorPosix(err),
	2344
	2345	posix.EINVAL => return LinuxEventFdError.InvalidFlagValue,
	2346	posix.EMFILE => return LinuxEventFdError.ProcessFdQuotaExceeded,
	2347	posix.ENFILE => return LinuxEventFdError.SystemFdQuotaExceeded,
	2348	posix.ENODEV => return LinuxEventFdError.SystemResources,
	2349	posix.ENOMEM => return LinuxEventFdError.SystemResources,
	2350	}
	2351	}
	2352
2320	2353	pub const PosixGetSockNameError = error{
2321	2354	/// Insufficient resources were available in the system to perform the operation.
2322	2355	SystemResources,
...	...	@@ -2576,11 +2609,17 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
2576	2609	thread: Thread,
2577	2610	inner: Context,
2578	2611	};
2579		extern fn threadMain(arg: windows.LPVOID) windows.DWORD {
2580		if (@sizeOf(Context) == 0) {
2581		return startFn({});
2582		} else {
2583		return startFn(@ptrCast(Context, @alignCast(@alignOf(Context), arg)).);
	2612	extern fn threadMain(raw_arg: windows.LPVOID) windows.DWORD {
	2613	const arg = if (@sizeOf(Context) == 0) {} else @ptrCast(Context, @alignCast(@alignOf(Context), raw_arg)).;
	2614	switch (@typeId(@typeOf(startFn).ReturnType)) {
	2615	builtin.TypeId.Int => {
	2616	return startFn(arg);
	2617	},
	2618	builtin.TypeId.Void => {
	2619	startFn(arg);
	2620	return 0;
	2621	},
	2622	else => @compileError("expected return type of startFn to be 'u8', 'noreturn', 'void', or '!void'"),
2584	2623	}
2585	2624	}
2586	2625	};
...	...	@@ -2613,10 +2652,17 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
2613	2652
2614	2653	const MainFuncs = struct {
2615	2654	extern fn linuxThreadMain(ctx_addr: usize) u8 {
2616		if (@sizeOf(Context) == 0) {
2617		return startFn({});
2618		} else {
2619		return startFn(@intToPtr(const Context, ctx_addr).);
	2655	const arg = if (@sizeOf(Context) == 0) {} else @intToPtr(const Context, ctx_addr).;
	2656
	2657	switch (@typeId(@typeOf(startFn).ReturnType)) {
	2658	builtin.TypeId.Int => {
	2659	return startFn(arg);
	2660	},
	2661	builtin.TypeId.Void => {
	2662	startFn(arg);
	2663	return 0;
	2664	},
	2665	else => @compileError("expected return type of startFn to be 'u8', 'noreturn', 'void', or '!void'"),
2620	2666	}
2621	2667	}
2622	2668	extern fn posixThreadMain(ctx: ?c_void) ?c_void {
...	...	@@ -2725,3 +2771,128 @@ pub fn posixFStat(fd: i32) !posix.Stat {
2725	2771
2726	2772	return stat;
2727	2773	}
	2774
	2775	pub const CpuCountError = error{
	2776	OutOfMemory,
	2777	PermissionDenied,
	2778	Unexpected,
	2779	};
	2780
	2781	pub fn cpuCount(fallback_allocator: *mem.Allocator) CpuCountError!usize {
	2782	switch (builtin.os) {
	2783	builtin.Os.macosx => {
	2784	var count: c_int = undefined;
	2785	var count_len: usize = @sizeOf(c_int);
	2786	const rc = posix.sysctlbyname(c"hw.ncpu", @ptrCast(*c_void, &count), &count_len, null, 0);
	2787	const err = posix.getErrno(rc);
	2788	switch (err) {
	2789	0 => return @intCast(usize, count),
	2790	posix.EFAULT => unreachable,
	2791	posix.EINVAL => unreachable,
	2792	posix.ENOMEM => return CpuCountError.OutOfMemory,
	2793	posix.ENOTDIR => unreachable,
	2794	posix.EISDIR => unreachable,
	2795	posix.ENOENT => unreachable,
	2796	posix.EPERM => unreachable,
	2797	else => return os.unexpectedErrorPosix(err),
	2798	}
	2799	},
	2800	builtin.Os.linux => {
	2801	const usize_count = 16;
	2802	const allocator = std.heap.stackFallback(usize_count * @sizeOf(usize), fallback_allocator).get();
	2803
	2804	var set = try allocator.alloc(usize, usize_count);
	2805	defer allocator.free(set);
	2806
	2807	while (true) {
	2808	const rc = posix.sched_getaffinity(0, set);
	2809	const err = posix.getErrno(rc);
	2810	switch (err) {
	2811	0 => {
	2812	if (rc < set.len * @sizeOf(usize)) {
	2813	const result = set[0 .. rc / @sizeOf(usize)];
	2814	var sum: usize = 0;
	2815	for (result) \|x\| {
	2816	sum += @popCount(x);
	2817	}
	2818	return sum;
	2819	} else {
	2820	set = try allocator.realloc(usize, set, set.len * 2);
	2821	continue;
	2822	}
	2823	},
	2824	posix.EFAULT => unreachable,
	2825	posix.EINVAL => unreachable,
	2826	posix.EPERM => return CpuCountError.PermissionDenied,
	2827	posix.ESRCH => unreachable,
	2828	else => return os.unexpectedErrorPosix(err),
	2829	}
	2830	}
	2831	},
	2832	builtin.Os.windows => {
	2833	var system_info: windows.SYSTEM_INFO = undefined;
	2834	windows.GetSystemInfo(&system_info);
	2835	return @intCast(usize, system_info.dwNumberOfProcessors);
	2836	},
	2837	else => @compileError("unsupported OS"),
	2838	}
	2839	}
	2840
	2841	pub const BsdKQueueError = error{
	2842	/// The per-process limit on the number of open file descriptors has been reached.
	2843	ProcessFdQuotaExceeded,
	2844
	2845	/// The system-wide limit on the total number of open files has been reached.
	2846	SystemFdQuotaExceeded,
	2847
	2848	Unexpected,
	2849	};
	2850
	2851	pub fn bsdKQueue() BsdKQueueError!i32 {
	2852	const rc = posix.kqueue();
	2853	const err = posix.getErrno(rc);
	2854	switch (err) {
	2855	0 => return @intCast(i32, rc),
	2856	posix.EMFILE => return BsdKQueueError.ProcessFdQuotaExceeded,
	2857	posix.ENFILE => return BsdKQueueError.SystemFdQuotaExceeded,
	2858	else => return unexpectedErrorPosix(err),
	2859	}
	2860	}
	2861
	2862	pub const BsdKEventError = error{
	2863	/// The process does not have permission to register a filter.
	2864	AccessDenied,
	2865
	2866	/// The event could not be found to be modified or deleted.
	2867	EventNotFound,
	2868
	2869	/// No memory was available to register the event.
	2870	SystemResources,
	2871
	2872	/// The specified process to attach to does not exist.
	2873	ProcessNotFound,
	2874	};
	2875
	2876	pub fn bsdKEvent(
	2877	kq: i32,
	2878	changelist: []const posix.Kevent,
	2879	eventlist: []posix.Kevent,
	2880	timeout: ?*const posix.timespec,
	2881	) BsdKEventError!usize {
	2882	while (true) {
	2883	const rc = posix.kevent(kq, changelist, eventlist, timeout);
	2884	const err = posix.getErrno(rc);
	2885	switch (err) {
	2886	0 => return rc,
	2887	posix.EACCES => return BsdKEventError.AccessDenied,
	2888	posix.EFAULT => unreachable,
	2889	posix.EBADF => unreachable,
	2890	posix.EINTR => continue,
	2891	posix.EINVAL => unreachable,
	2892	posix.ENOENT => return BsdKEventError.EventNotFound,
	2893	posix.ENOMEM => return BsdKEventError.SystemResources,
	2894	posix.ESRCH => return BsdKEventError.ProcessNotFound,
	2895	else => unreachable,
	2896	}
	2897	}
	2898	}

std/os/linux/index.zig+12

...	...	@@ -523,6 +523,10 @@ pub const CLONE_NEWPID = 0x20000000;
523	523	pub const CLONE_NEWNET = 0x40000000;
524	524	pub const CLONE_IO = 0x80000000;
525	525
	526	pub const EFD_SEMAPHORE = 1;
	527	pub const EFD_CLOEXEC = O_CLOEXEC;
	528	pub const EFD_NONBLOCK = O_NONBLOCK;
	529
526	530	pub const MS_RDONLY = 1;
527	531	pub const MS_NOSUID = 2;
528	532	pub const MS_NODEV = 4;
...	...	@@ -1193,6 +1197,10 @@ pub fn fremovexattr(fd: usize, name: [*]const u8) usize {
1193	1197	return syscall2(SYS_fremovexattr, fd, @ptrToInt(name));
1194	1198	}
1195	1199
	1200	pub fn sched_getaffinity(pid: i32, set: []usize) usize {
	1201	return syscall3(SYS_sched_getaffinity, @bitCast(usize, isize(pid)), set.len * @sizeOf(usize), @ptrToInt(set.ptr));
	1202	}
	1203
1196	1204	pub const epoll_data = packed union {
1197	1205	ptr: usize,
1198	1206	fd: i32,
...	...	@@ -1221,6 +1229,10 @@ pub fn epoll_wait(epoll_fd: i32, events: [*]epoll_event, maxevents: u32, timeout
1221	1229	return syscall4(SYS_epoll_wait, @intCast(usize, epoll_fd), @ptrToInt(events), @intCast(usize, maxevents), @intCast(usize, timeout));
1222	1230	}
1223	1231
	1232	pub fn eventfd(count: u32, flags: u32) usize {
	1233	return syscall2(SYS_eventfd2, count, flags);
	1234	}
	1235
1224	1236	pub fn timerfd_create(clockid: i32, flags: u32) usize {
1225	1237	return syscall2(SYS_timerfd_create, @intCast(usize, clockid), @intCast(usize, flags));
1226	1238	}

std/os/test.zig+5

...	...	@@ -58,3 +58,8 @@ fn start2(ctx: *i32) u8 {
58	58	_ = @atomicRmw(i32, ctx, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
59	59	return 0;
60	60	}
	61
	62	test "cpu count" {
	63	const cpu_count = try std.os.cpuCount(a);
	64	assert(cpu_count >= 1);
	65	}

std/os/windows/index.zig+28

...	...	@@ -59,6 +59,9 @@ pub extern "kernel32" stdcallcc fn CreateSymbolicLinkA(
59	59	dwFlags: DWORD,
60	60	) BOOLEAN;
61	61
	62
	63	pub extern "kernel32" stdcallcc fn CreateIoCompletionPort(FileHandle: HANDLE, ExistingCompletionPort: ?HANDLE, CompletionKey: ULONG_PTR, NumberOfConcurrentThreads: DWORD) ?HANDLE;
	64
62	65	pub extern "kernel32" stdcallcc fn CreateThread(lpThreadAttributes: ?LPSECURITY_ATTRIBUTES, dwStackSize: SIZE_T, lpStartAddress: LPTHREAD_START_ROUTINE, lpParameter: ?LPVOID, dwCreationFlags: DWORD, lpThreadId: ?LPDWORD) ?HANDLE;
63	66
64	67	pub extern "kernel32" stdcallcc fn DeleteFileA(lpFileName: LPCSTR) BOOL;
...	...	@@ -106,7 +109,9 @@ pub extern "kernel32" stdcallcc fn GetFinalPathNameByHandleA(
106	109	) DWORD;
107	110
108	111	pub extern "kernel32" stdcallcc fn GetProcessHeap() ?HANDLE;
	112	pub extern "kernel32" stdcallcc fn GetQueuedCompletionStatus(CompletionPort: HANDLE, lpNumberOfBytesTransferred: LPDWORD, lpCompletionKey: ULONG_PTR, lpOverlapped: ?*OVERLAPPED, dwMilliseconds: DWORD) BOOL;
109	113
	114	pub extern "kernel32" stdcallcc fn GetSystemInfo(lpSystemInfo: *SYSTEM_INFO) void;
110	115	pub extern "kernel32" stdcallcc fn GetSystemTimeAsFileTime(*FILETIME) void;
111	116
112	117	pub extern "kernel32" stdcallcc fn HeapCreate(flOptions: DWORD, dwInitialSize: SIZE_T, dwMaximumSize: SIZE_T) ?HANDLE;
...	...	@@ -129,6 +134,9 @@ pub extern "kernel32" stdcallcc fn MoveFileExA(
129	134	dwFlags: DWORD,
130	135	) BOOL;
131	136
	137
	138	pub extern "kernel32" stdcallcc fn PostQueuedCompletionStatus(CompletionPort: HANDLE, dwNumberOfBytesTransferred: DWORD, dwCompletionKey: ULONG_PTR, lpOverlapped: ?*OVERLAPPED) BOOL;
	139
132	140	pub extern "kernel32" stdcallcc fn QueryPerformanceCounter(lpPerformanceCount: *LARGE_INTEGER) BOOL;
133	141
134	142	pub extern "kernel32" stdcallcc fn QueryPerformanceFrequency(lpFrequency: *LARGE_INTEGER) BOOL;
...	...	@@ -204,6 +212,7 @@ pub const SIZE_T = usize;
204	212	pub const TCHAR = if (UNICODE) WCHAR else u8;
205	213	pub const UINT = c_uint;
206	214	pub const ULONG_PTR = usize;
	215	pub const DWORD_PTR = ULONG_PTR;
207	216	pub const UNICODE = false;
208	217	pub const WCHAR = u16;
209	218	pub const WORD = u16;
...	...	@@ -413,3 +422,22 @@ pub const FILETIME = extern struct {
413	422	dwLowDateTime: DWORD,
414	423	dwHighDateTime: DWORD,
415	424	};
	425
	426	pub const SYSTEM_INFO = extern struct {
	427	anon1: extern union {
	428	dwOemId: DWORD,
	429	anon2: extern struct {
	430	wProcessorArchitecture: WORD,
	431	wReserved: WORD,
	432	},
	433	},
	434	dwPageSize: DWORD,
	435	lpMinimumApplicationAddress: LPVOID,
	436	lpMaximumApplicationAddress: LPVOID,
	437	dwActiveProcessorMask: DWORD_PTR,
	438	dwNumberOfProcessors: DWORD,
	439	dwProcessorType: DWORD,
	440	dwAllocationGranularity: DWORD,
	441	wProcessorLevel: WORD,
	442	wProcessorRevision: WORD,
	443	};

std/os/windows/util.zig+47

...	...	@@ -214,3 +214,50 @@ pub fn windowsFindNextFile(handle: windows.HANDLE, find_file_data: *windows.WIN3
214	214	}
215	215	return true;
216	216	}
	217
	218
	219	pub const WindowsCreateIoCompletionPortError = error {
	220	Unexpected,
	221	};
	222
	223	pub fn windowsCreateIoCompletionPort(file_handle: windows.HANDLE, existing_completion_port: ?windows.HANDLE, completion_key: usize, concurrent_thread_count: windows.DWORD) !windows.HANDLE {
	224	const handle = windows.CreateIoCompletionPort(file_handle, existing_completion_port, completion_key, concurrent_thread_count) orelse {
	225	const err = windows.GetLastError();
	226	switch (err) {
	227	else => return os.unexpectedErrorWindows(err),
	228	}
	229	};
	230	return handle;
	231	}
	232
	233	pub const WindowsPostQueuedCompletionStatusError = error {
	234	Unexpected,
	235	};
	236
	237	pub fn windowsPostQueuedCompletionStatus(completion_port: windows.HANDLE, bytes_transferred_count: windows.DWORD, completion_key: usize, lpOverlapped: ?*windows.OVERLAPPED) WindowsPostQueuedCompletionStatusError!void {
	238	if (windows.PostQueuedCompletionStatus(completion_port, bytes_transferred_count, completion_key, lpOverlapped) == 0) {
	239	const err = windows.GetLastError();
	240	switch (err) {
	241	else => return os.unexpectedErrorWindows(err),
	242	}
	243	}
	244	}
	245
	246	pub const WindowsWaitResult = error {
	247	Normal,
	248	Aborted,
	249	};
	250
	251	pub fn windowsGetQueuedCompletionStatus(completion_port: windows.HANDLE, bytes_transferred_count: windows.DWORD, lpCompletionKey: usize, lpOverlapped: ?windows.OVERLAPPED, dwMilliseconds: windows.DWORD) WindowsWaitResult {
	252	if (windows.GetQueuedCompletionStatus(completion_port, bytes_transferred_count, lpCompletionKey, lpOverlapped, dwMilliseconds) == windows.FALSE) {
	253	if (std.debug.runtime_safety) {
	254	const err = windows.GetLastError();
	255	if (err != windows.ERROR.ABANDONED_WAIT_0) {
	256	std.debug.warn("err: {}\n", err);
	257	}
	258	assert(err == windows.ERROR.ABANDONED_WAIT_0);
	259	}
	260	return WindowsWaitResult.Aborted;
	261	}
	262	return WindowsWaitResult.Normal;
	263	}

std/special/compiler_rt/extendXfYf2_test.zig+20-20

...	...	@@ -31,7 +31,7 @@ fn test__extendhfsf2(a: u16, expected: u32) void {
31	31
32	32	if (rep == expected) {
33	33	if (rep & 0x7fffffff > 0x7f800000) {
34		return; // NaN is always unequal.
	34	return; // NaN is always unequal.
35	35	}
36	36	if (x == @bitCast(f32, expected)) {
37	37	return;
...	...	@@ -86,33 +86,33 @@ test "extenddftf2" {
86	86	}
87	87
88	88	test "extendhfsf2" {
89		test__extendhfsf2(0x7e00, 0x7fc00000); // qNaN
90		test__extendhfsf2(0x7f00, 0x7fe00000); // sNaN
91		test__extendhfsf2(0x7c01, 0x7f802000); // sNaN
	89	test__extendhfsf2(0x7e00, 0x7fc00000); // qNaN
	90	test__extendhfsf2(0x7f00, 0x7fe00000); // sNaN
	91	test__extendhfsf2(0x7c01, 0x7f802000); // sNaN
92	92
93		test__extendhfsf2(0, 0); // 0
94		test__extendhfsf2(0x8000, 0x80000000); // -0
	93	test__extendhfsf2(0, 0); // 0
	94	test__extendhfsf2(0x8000, 0x80000000); // -0
95	95
96		test__extendhfsf2(0x7c00, 0x7f800000); // inf
97		test__extendhfsf2(0xfc00, 0xff800000); // -inf
	96	test__extendhfsf2(0x7c00, 0x7f800000); // inf
	97	test__extendhfsf2(0xfc00, 0xff800000); // -inf
98	98
99		test__extendhfsf2(0x0001, 0x33800000); // denormal (min), 2**-24
100		test__extendhfsf2(0x8001, 0xb3800000); // denormal (min), -2**-24
	99	test__extendhfsf2(0x0001, 0x33800000); // denormal (min), 2**-24
	100	test__extendhfsf2(0x8001, 0xb3800000); // denormal (min), -2**-24
101	101
102		test__extendhfsf2(0x03ff, 0x387fc000); // denormal (max), 2-14 - 2-24
103		test__extendhfsf2(0x83ff, 0xb87fc000); // denormal (max), -2-14 + 2-24
	102	test__extendhfsf2(0x03ff, 0x387fc000); // denormal (max), 2-14 - 2-24
	103	test__extendhfsf2(0x83ff, 0xb87fc000); // denormal (max), -2-14 + 2-24
104	104
105		test__extendhfsf2(0x0400, 0x38800000); // normal (min), 2**-14
106		test__extendhfsf2(0x8400, 0xb8800000); // normal (min), -2**-14
	105	test__extendhfsf2(0x0400, 0x38800000); // normal (min), 2**-14
	106	test__extendhfsf2(0x8400, 0xb8800000); // normal (min), -2**-14
107	107
108		test__extendhfsf2(0x7bff, 0x477fe000); // normal (max), 65504
109		test__extendhfsf2(0xfbff, 0xc77fe000); // normal (max), -65504
	108	test__extendhfsf2(0x7bff, 0x477fe000); // normal (max), 65504
	109	test__extendhfsf2(0xfbff, 0xc77fe000); // normal (max), -65504
110	110
111		test__extendhfsf2(0x3c01, 0x3f802000); // normal, 1 + 2**-10
112		test__extendhfsf2(0xbc01, 0xbf802000); // normal, -1 - 2**-10
	111	test__extendhfsf2(0x3c01, 0x3f802000); // normal, 1 + 2**-10
	112	test__extendhfsf2(0xbc01, 0xbf802000); // normal, -1 - 2**-10
113	113
114		test__extendhfsf2(0x3555, 0x3eaaa000); // normal, approx. 1/3
115		test__extendhfsf2(0xb555, 0xbeaaa000); // normal, approx. -1/3
	114	test__extendhfsf2(0x3555, 0x3eaaa000); // normal, approx. 1/3
	115	test__extendhfsf2(0xb555, 0xbeaaa000); // normal, approx. -1/3
116	116	}
117	117
118	118	test "extendsftf2" {