authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-07-05 15:09:02-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-07-07 00:32:19-04:00
logeb326e15530dd6dca4ccbe7dbfde7bf048de813e
treea20438803ab35a874750906281dc19a463be0acc
parentd8295c188946b0f07d62420c2f08c940f70b03ac

M:N threading

* add std.atomic.QueueMpsc.isEmpty * make std.debug.global_allocator thread-safe * std.event.Loop: now you have to choose between - initSingleThreaded - initMultiThreaded * std.event.Loop multiplexes coroutines onto kernel threads * Remove std.event.Loop.stop. Instead the event loop run() function returns once there are no pending coroutines. * fix crash in ir.cpp for calling methods under some conditions * small progress self-hosted compiler, analyzing top level declarations * Introduce std.event.Lock for synchronizing coroutines * introduce std.event.Locked(T) for data that only 1 coroutine should modify at once. * make the self hosted compiler use multi threaded event loop * make std.heap.DirectAllocator thread-safe See #174 TODO: * call sched_getaffinity instead of hard coding thread pool size 4 * support for Windows and MacOS * #1194 * #1197

10 files changed, 833 insertions(+), 114 deletions(-)

src-self-hosted/main.zig+2-3
...@@ -384,7 +384,8 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Mo...@@ -384,7 +384,8 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Mo
384 const zig_lib_dir = introspect.resolveZigLibDir(allocator) catch os.exit(1);384 const zig_lib_dir = introspect.resolveZigLibDir(allocator) catch os.exit(1);
385 defer allocator.free(zig_lib_dir);385 defer allocator.free(zig_lib_dir);
386386
387 var loop = try event.Loop.init(allocator);387 var loop: event.Loop = undefined;
388 try loop.initMultiThreaded(allocator);
388389
389 var module = try Module.create(390 var module = try Module.create(
390 &loop,391 &loop,
...@@ -493,8 +494,6 @@ async fn processBuildEvents(module: *Module, watch: bool) void {...@@ -493,8 +494,6 @@ async fn processBuildEvents(module: *Module, watch: bool) void {
493 switch (build_event) {494 switch (build_event) {
494 Module.Event.Ok => {495 Module.Event.Ok => {
495 std.debug.warn("Build succeeded\n");496 std.debug.warn("Build succeeded\n");
496 // for now we stop after 1
497 module.loop.stop();
498 return;497 return;
499 },498 },
500 Module.Event.Error => |err| {499 Module.Event.Error => |err| {
src-self-hosted/module.zig+242-15
...@@ -2,6 +2,7 @@ const std = @import("std");...@@ -2,6 +2,7 @@ const std = @import("std");
2const os = std.os;2const os = std.os;
3const io = std.io;3const io = std.io;
4const mem = std.mem;4const mem = std.mem;
5const Allocator = mem.Allocator;
5const Buffer = std.Buffer;6const Buffer = std.Buffer;
6const llvm = @import("llvm.zig");7const llvm = @import("llvm.zig");
7const c = @import("c.zig");8const c = @import("c.zig");
...@@ -13,6 +14,7 @@ const ArrayList = std.ArrayList;...@@ -13,6 +14,7 @@ const ArrayList = std.ArrayList;
13const errmsg = @import("errmsg.zig");14const errmsg = @import("errmsg.zig");
14const ast = std.zig.ast;15const ast = std.zig.ast;
15const event = std.event;16const event = std.event;
17const assert = std.debug.assert;
1618
17pub const Module = struct {19pub const Module = struct {
18 loop: *event.Loop,20 loop: *event.Loop,
...@@ -81,6 +83,8 @@ pub const Module = struct {...@@ -81,6 +83,8 @@ pub const Module = struct {
81 link_out_file: ?[]const u8,83 link_out_file: ?[]const u8,
82 events: *event.Channel(Event),84 events: *event.Channel(Event),
8385
86 exported_symbol_names: event.Locked(Decl.Table),
87
84 // TODO handle some of these earlier and report them in a way other than error codes88 // TODO handle some of these earlier and report them in a way other than error codes
85 pub const BuildError = error{89 pub const BuildError = error{
86 OutOfMemory,90 OutOfMemory,
...@@ -232,6 +236,7 @@ pub const Module = struct {...@@ -232,6 +236,7 @@ pub const Module = struct {
232 .test_name_prefix = null,236 .test_name_prefix = null,
233 .emit_file_type = Emit.Binary,237 .emit_file_type = Emit.Binary,
234 .link_out_file = null,238 .link_out_file = null,
239 .exported_symbol_names = event.Locked(Decl.Table).init(loop, Decl.Table.init(loop.allocator)),
235 });240 });
236 }241 }
237242
...@@ -272,38 +277,91 @@ pub const Module = struct {...@@ -272,38 +277,91 @@ pub const Module = struct {
272 return;277 return;
273 };278 };
274 await (async self.events.put(Event.Ok) catch unreachable);279 await (async self.events.put(Event.Ok) catch unreachable);
280 // for now we stop after 1
281 return;
275 }282 }
276 }283 }
277284
278 async fn addRootSrc(self: *Module) !void {285 async fn addRootSrc(self: *Module) !void {
279 const root_src_path = self.root_src_path orelse @panic("TODO handle null root src path");286 const root_src_path = self.root_src_path orelse @panic("TODO handle null root src path");
287 // TODO async/await os.path.real
280 const root_src_real_path = os.path.real(self.a(), root_src_path) catch |err| {288 const root_src_real_path = os.path.real(self.a(), root_src_path) catch |err| {
281 try printError("unable to get real path '{}': {}", root_src_path, err);289 try printError("unable to get real path '{}': {}", root_src_path, err);
282 return err;290 return err;
283 };291 };
284 errdefer self.a().free(root_src_real_path);292 errdefer self.a().free(root_src_real_path);
285293
294 // TODO async/await readFileAlloc()
286 const source_code = io.readFileAlloc(self.a(), root_src_real_path) catch |err| {295 const source_code = io.readFileAlloc(self.a(), root_src_real_path) catch |err| {
287 try printError("unable to open '{}': {}", root_src_real_path, err);296 try printError("unable to open '{}': {}", root_src_real_path, err);
288 return err;297 return err;
289 };298 };
290 errdefer self.a().free(source_code);299 errdefer self.a().free(source_code);
291300
292 var tree = try std.zig.parse(self.a(), source_code);301 var parsed_file = ParsedFile{
293 defer tree.deinit();302 .tree = try std.zig.parse(self.a(), source_code),
294303 .realpath = root_src_real_path,
295 //var it = tree.root_node.decls.iterator();304 };
296 //while (it.next()) |decl_ptr| {305 errdefer parsed_file.tree.deinit();
297 // const decl = decl_ptr.*;306
298 // switch (decl.id) {307 const tree = &parsed_file.tree;
299 // ast.Node.Comptime => @panic("TODO"),308
300 // ast.Node.VarDecl => @panic("TODO"),309 // create empty struct for it
301 // ast.Node.UseDecl => @panic("TODO"),310 const decls = try Scope.Decls.create(self.a(), null);
302 // ast.Node.FnDef => @panic("TODO"),311 errdefer decls.destroy();
303 // ast.Node.TestDecl => @panic("TODO"),312
304 // else => unreachable,313 var it = tree.root_node.decls.iterator(0);
305 // }314 while (it.next()) |decl_ptr| {
306 //}315 const decl = decl_ptr.*;
316 switch (decl.id) {
317 ast.Node.Id.Comptime => @panic("TODO"),
318 ast.Node.Id.VarDecl => @panic("TODO"),
319 ast.Node.Id.FnProto => {
320 const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", decl);
321
322 const name = if (fn_proto.name_token) |name_token| tree.tokenSlice(name_token) else {
323 @panic("TODO add compile error");
324 //try self.addCompileError(
325 // &parsed_file,
326 // fn_proto.fn_token,
327 // fn_proto.fn_token + 1,
328 // "missing function name",
329 //);
330 continue;
331 };
332
333 const fn_decl = try self.a().create(Decl.Fn{
334 .base = Decl{
335 .id = Decl.Id.Fn,
336 .name = name,
337 .visib = parseVisibToken(tree, fn_proto.visib_token),
338 .resolution = Decl.Resolution.Unresolved,
339 },
340 .value = Decl.Fn.Val{ .Unresolved = {} },
341 .fn_proto = fn_proto,
342 });
343 errdefer self.a().destroy(fn_decl);
344
345 // TODO make this parallel
346 try await try async self.addTopLevelDecl(tree, &fn_decl.base);
347 },
348 ast.Node.Id.TestDecl => @panic("TODO"),
349 else => unreachable,
350 }
351 }
352 }
353
354 async fn addTopLevelDecl(self: *Module, tree: *ast.Tree, decl: *Decl) !void {
355 const is_export = decl.isExported(tree);
356
357 {
358 const exported_symbol_names = await try async self.exported_symbol_names.acquire();
359 defer exported_symbol_names.release();
360
361 if (try exported_symbol_names.value.put(decl.name, decl)) |other_decl| {
362 @panic("TODO report compile error");
363 }
364 }
307 }365 }
308366
309 pub fn link(self: *Module, out_file: ?[]const u8) !void {367 pub fn link(self: *Module, out_file: ?[]const u8) !void {
...@@ -350,3 +408,172 @@ fn printError(comptime format: []const u8, args: ...) !void {...@@ -350,3 +408,172 @@ fn printError(comptime format: []const u8, args: ...) !void {
350 const out_stream = &stderr_file_out_stream.stream;408 const out_stream = &stderr_file_out_stream.stream;
351 try out_stream.print(format, args);409 try out_stream.print(format, args);
352}410}
411
412fn parseVisibToken(tree: *ast.Tree, optional_token_index: ?ast.TokenIndex) Visib {
413 if (optional_token_index) |token_index| {
414 const token = tree.tokens.at(token_index);
415 assert(token.id == Token.Id.Keyword_pub);
416 return Visib.Pub;
417 } else {
418 return Visib.Private;
419 }
420}
421
422pub const Scope = struct {
423 id: Id,
424 parent: ?*Scope,
425
426 pub const Id = enum {
427 Decls,
428 Block,
429 };
430
431 pub const Decls = struct {
432 base: Scope,
433 table: Decl.Table,
434
435 pub fn create(a: *Allocator, parent: ?*Scope) !*Decls {
436 const self = try a.create(Decls{
437 .base = Scope{
438 .id = Id.Decls,
439 .parent = parent,
440 },
441 .table = undefined,
442 });
443 errdefer a.destroy(self);
444
445 self.table = Decl.Table.init(a);
446 errdefer self.table.deinit();
447
448 return self;
449 }
450
451 pub fn destroy(self: *Decls) void {
452 self.table.deinit();
453 self.table.allocator.destroy(self);
454 self.* = undefined;
455 }
456 };
457
458 pub const Block = struct {
459 base: Scope,
460 };
461};
462
463pub const Visib = enum {
464 Private,
465 Pub,
466};
467
468pub const Decl = struct {
469 id: Id,
470 name: []const u8,
471 visib: Visib,
472 resolution: Resolution,
473
474 pub const Table = std.HashMap([]const u8, *Decl, mem.hash_slice_u8, mem.eql_slice_u8);
475
476 pub fn isExported(base: *const Decl, tree: *ast.Tree) bool {
477 switch (base.id) {
478 Id.Fn => {
479 const fn_decl = @fieldParentPtr(Fn, "base", base);
480 return fn_decl.isExported(tree);
481 },
482 else => return false,
483 }
484 }
485
486 pub const Resolution = enum {
487 Unresolved,
488 InProgress,
489 Invalid,
490 Ok,
491 };
492
493 pub const Id = enum {
494 Var,
495 Fn,
496 CompTime,
497 };
498
499 pub const Var = struct {
500 base: Decl,
501 };
502
503 pub const Fn = struct {
504 base: Decl,
505 value: Val,
506 fn_proto: *const ast.Node.FnProto,
507
508 // TODO https://github.com/ziglang/zig/issues/683 and then make this anonymous
509 pub const Val = union {
510 Unresolved: void,
511 Ok: *Value.Fn,
512 };
513
514 pub fn externLibName(self: Fn, tree: *ast.Tree) ?[]const u8 {
515 return if (self.fn_proto.extern_export_inline_token) |tok_index| x: {
516 const token = tree.tokens.at(tok_index);
517 break :x switch (token.id) {
518 Token.Id.Extern => tree.tokenSlicePtr(token),
519 else => null,
520 };
521 } else null;
522 }
523
524 pub fn isExported(self: Fn, tree: *ast.Tree) bool {
525 if (self.fn_proto.extern_export_inline_token) |tok_index| {
526 const token = tree.tokens.at(tok_index);
527 return token.id == Token.Id.Keyword_export;
528 } else {
529 return false;
530 }
531 }
532 };
533
534 pub const CompTime = struct {
535 base: Decl,
536 };
537};
538
539pub const Value = struct {
540 pub const Fn = struct {};
541};
542
543pub const Type = struct {
544 id: Id,
545
546 pub const Id = enum {
547 Type,
548 Void,
549 Bool,
550 NoReturn,
551 Int,
552 Float,
553 Pointer,
554 Array,
555 Struct,
556 ComptimeFloat,
557 ComptimeInt,
558 Undefined,
559 Null,
560 Optional,
561 ErrorUnion,
562 ErrorSet,
563 Enum,
564 Union,
565 Fn,
566 Opaque,
567 Promise,
568 };
569
570 pub const Struct = struct {
571 base: Type,
572 decls: *Scope.Decls,
573 };
574};
575
576pub const ParsedFile = struct {
577 tree: ast.Tree,
578 realpath: []const u8,
579};
src/ir.cpp+1-1
...@@ -13278,7 +13278,7 @@ static TypeTableEntry *ir_analyze_instruction_call(IrAnalyze *ira, IrInstruction...@@ -13278,7 +13278,7 @@ static TypeTableEntry *ir_analyze_instruction_call(IrAnalyze *ira, IrInstruction
13278 FnTableEntry *fn_table_entry = fn_ref->value.data.x_bound_fn.fn;13278 FnTableEntry *fn_table_entry = fn_ref->value.data.x_bound_fn.fn;
13279 IrInstruction *first_arg_ptr = fn_ref->value.data.x_bound_fn.first_arg;13279 IrInstruction *first_arg_ptr = fn_ref->value.data.x_bound_fn.first_arg;
13280 return ir_analyze_fn_call(ira, call_instruction, fn_table_entry, fn_table_entry->type_entry,13280 return ir_analyze_fn_call(ira, call_instruction, fn_table_entry, fn_table_entry->type_entry,
13281 nullptr, first_arg_ptr, is_comptime, call_instruction->fn_inline);13281 fn_ref, first_arg_ptr, is_comptime, call_instruction->fn_inline);
13282 } else {13282 } else {
13283 ir_add_error_node(ira, fn_ref->source_node,13283 ir_add_error_node(ira, fn_ref->source_node,
13284 buf_sprintf("type '%s' not a function", buf_ptr(&fn_ref->value.type->name)));13284 buf_sprintf("type '%s' not a function", buf_ptr(&fn_ref->value.type->name)));
std/atomic/queue_mpsc.zig+17
...@@ -15,6 +15,8 @@ pub fn QueueMpsc(comptime T: type) type {...@@ -15,6 +15,8 @@ pub fn QueueMpsc(comptime T: type) type {
1515
16 pub const Node = std.atomic.Stack(T).Node;16 pub const Node = std.atomic.Stack(T).Node;
1717
18 /// Not thread-safe. The call to init() must complete before any other functions are called.
19 /// No deinitialization required.
18 pub fn init() Self {20 pub fn init() Self {
19 return Self{21 return Self{
20 .inboxes = []std.atomic.Stack(T){22 .inboxes = []std.atomic.Stack(T){
...@@ -26,12 +28,15 @@ pub fn QueueMpsc(comptime T: type) type {...@@ -26,12 +28,15 @@ pub fn QueueMpsc(comptime T: type) type {
26 };28 };
27 }29 }
2830
31 /// Fully thread-safe. put() may be called from any thread at any time.
29 pub fn put(self: *Self, node: *Node) void {32 pub fn put(self: *Self, node: *Node) void {
30 const inbox_index = @atomicLoad(usize, &self.inbox_index, AtomicOrder.SeqCst);33 const inbox_index = @atomicLoad(usize, &self.inbox_index, AtomicOrder.SeqCst);
31 const inbox = &self.inboxes[inbox_index];34 const inbox = &self.inboxes[inbox_index];
32 inbox.push(node);35 inbox.push(node);
33 }36 }
3437
38 /// Must be called by only 1 consumer at a time. Every call to get() and isEmpty() must complete before
39 /// the next call to get().
35 pub fn get(self: *Self) ?*Node {40 pub fn get(self: *Self) ?*Node {
36 if (self.outbox.pop()) |node| {41 if (self.outbox.pop()) |node| {
37 return node;42 return node;
...@@ -43,6 +48,18 @@ pub fn QueueMpsc(comptime T: type) type {...@@ -43,6 +48,18 @@ pub fn QueueMpsc(comptime T: type) type {
43 }48 }
44 return self.outbox.pop();49 return self.outbox.pop();
45 }50 }
51
52 /// Must be called by only 1 consumer at a time. Every call to get() and isEmpty() must complete before
53 /// the next call to isEmpty().
54 pub fn isEmpty(self: *Self) bool {
55 if (!self.outbox.isEmpty()) return false;
56 const prev_inbox_index = @atomicRmw(usize, &self.inbox_index, AtomicRmwOp.Xor, 0x1, AtomicOrder.SeqCst);
57 const prev_inbox = &self.inboxes[prev_inbox_index];
58 while (prev_inbox.pop()) |node| {
59 self.outbox.push(node);
60 }
61 return self.outbox.isEmpty();
62 }
46 };63 };
47}64}
4865
std/debug/index.zig+6-1
...@@ -11,6 +11,11 @@ const builtin = @import("builtin");...@@ -11,6 +11,11 @@ const builtin = @import("builtin");
1111
12pub const FailingAllocator = @import("failing_allocator.zig").FailingAllocator;12pub const FailingAllocator = @import("failing_allocator.zig").FailingAllocator;
1313
14pub const runtime_safety = switch (builtin.mode) {
15 builtin.Mode.Debug, builtin.Mode.ReleaseSafe => true,
16 builtin.Mode.ReleaseFast, builtin.Mode.ReleaseSmall => false,
17};
18
14/// Tries to write to stderr, unbuffered, and ignores any error returned.19/// Tries to write to stderr, unbuffered, and ignores any error returned.
15/// Does not append a newline.20/// Does not append a newline.
16/// TODO atomic/multithread support21/// TODO atomic/multithread support
...@@ -1098,7 +1103,7 @@ fn readILeb128(in_stream: var) !i64 {...@@ -1098,7 +1103,7 @@ fn readILeb128(in_stream: var) !i64 {
10981103
1099/// This should only be used in temporary test programs.1104/// This should only be used in temporary test programs.
1100pub const global_allocator = &global_fixed_allocator.allocator;1105pub const global_allocator = &global_fixed_allocator.allocator;
1101var global_fixed_allocator = std.heap.FixedBufferAllocator.init(global_allocator_mem[0..]);1106var global_fixed_allocator = std.heap.ThreadSafeFixedBufferAllocator.init(global_allocator_mem[0..]);
1102var global_allocator_mem: [100 * 1024]u8 = undefined;1107var global_allocator_mem: [100 * 1024]u8 = undefined;
11031108
1104// TODO make thread safe1109// TODO make thread safe
std/event.zig+506-74
...@@ -11,53 +11,69 @@ pub const TcpServer = struct {...@@ -11,53 +11,69 @@ pub const TcpServer = struct {
11 handleRequestFn: async<*mem.Allocator> fn (*TcpServer, *const std.net.Address, *const std.os.File) void,11 handleRequestFn: async<*mem.Allocator> fn (*TcpServer, *const std.net.Address, *const std.os.File) void,
1212
13 loop: *Loop,13 loop: *Loop,
14 sockfd: i32,14 sockfd: ?i32,
15 accept_coro: ?promise,15 accept_coro: ?promise,
16 listen_address: std.net.Address,16 listen_address: std.net.Address,
1717
18 waiting_for_emfile_node: PromiseNode,18 waiting_for_emfile_node: PromiseNode,
19 listen_resume_node: event.Loop.ResumeNode,
1920
20 const PromiseNode = std.LinkedList(promise).Node;21 const PromiseNode = std.LinkedList(promise).Node;
2122
22 pub fn init(loop: *Loop) !TcpServer {23 pub fn init(loop: *Loop) TcpServer {
23 const sockfd = try std.os.posixSocket(posix.AF_INET, posix.SOCK_STREAM | posix.SOCK_CLOEXEC | posix.SOCK_NONBLOCK, posix.PROTO_tcp);
24 errdefer std.os.close(sockfd);
25
26 // TODO can't initialize handler coroutine here because we need well defined copy elision24 // TODO can't initialize handler coroutine here because we need well defined copy elision
27 return TcpServer{25 return TcpServer{
28 .loop = loop,26 .loop = loop,
29 .sockfd = sockfd,27 .sockfd = null,
30 .accept_coro = null,28 .accept_coro = null,
31 .handleRequestFn = undefined,29 .handleRequestFn = undefined,
32 .waiting_for_emfile_node = undefined,30 .waiting_for_emfile_node = undefined,
33 .listen_address = undefined,31 .listen_address = undefined,
32 .listen_resume_node = event.Loop.ResumeNode{
33 .id = event.Loop.ResumeNode.Id.Basic,
34 .handle = undefined,
35 },
34 };36 };
35 }37 }
3638
37 pub fn listen(self: *TcpServer, address: *const std.net.Address, handleRequestFn: async<*mem.Allocator> fn (*TcpServer, *const std.net.Address, *const std.os.File) void) !void {39 pub fn listen(
40 self: *TcpServer,
41 address: *const std.net.Address,
42 handleRequestFn: async<*mem.Allocator> fn (*TcpServer, *const std.net.Address, *const std.os.File) void,
43 ) !void {
38 self.handleRequestFn = handleRequestFn;44 self.handleRequestFn = handleRequestFn;
3945
40 try std.os.posixBind(self.sockfd, &address.os_addr);46 const sockfd = try std.os.posixSocket(posix.AF_INET, posix.SOCK_STREAM | posix.SOCK_CLOEXEC | posix.SOCK_NONBLOCK, posix.PROTO_tcp);
41 try std.os.posixListen(self.sockfd, posix.SOMAXCONN);47 errdefer std.os.close(sockfd);
42 self.listen_address = std.net.Address.initPosix(try std.os.posixGetSockName(self.sockfd));48 self.sockfd = sockfd;
49
50 try std.os.posixBind(sockfd, &address.os_addr);
51 try std.os.posixListen(sockfd, posix.SOMAXCONN);
52 self.listen_address = std.net.Address.initPosix(try std.os.posixGetSockName(sockfd));
4353
44 self.accept_coro = try async<self.loop.allocator> TcpServer.handler(self);54 self.accept_coro = try async<self.loop.allocator> TcpServer.handler(self);
45 errdefer cancel self.accept_coro.?;55 errdefer cancel self.accept_coro.?;
4656
47 try self.loop.addFd(self.sockfd, self.accept_coro.?);57 self.listen_resume_node.handle = self.accept_coro.?;
48 errdefer self.loop.removeFd(self.sockfd);58 try self.loop.addFd(sockfd, &self.listen_resume_node);
59 errdefer self.loop.removeFd(sockfd);
60 }
61
62 /// Stop listening
63 pub fn close(self: *TcpServer) void {
64 self.loop.removeFd(self.sockfd.?);
65 std.os.close(self.sockfd.?);
49 }66 }
5067
51 pub fn deinit(self: *TcpServer) void {68 pub fn deinit(self: *TcpServer) void {
52 self.loop.removeFd(self.sockfd);
53 if (self.accept_coro) |accept_coro| cancel accept_coro;69 if (self.accept_coro) |accept_coro| cancel accept_coro;
54 std.os.close(self.sockfd);70 if (self.sockfd) |sockfd| std.os.close(sockfd);
55 }71 }
5672
57 pub async fn handler(self: *TcpServer) void {73 pub async fn handler(self: *TcpServer) void {
58 while (true) {74 while (true) {
59 var accepted_addr: std.net.Address = undefined;75 var accepted_addr: std.net.Address = undefined;
60 if (std.os.posixAccept(self.sockfd, &accepted_addr.os_addr, posix.SOCK_NONBLOCK | posix.SOCK_CLOEXEC)) |accepted_fd| {76 if (std.os.posixAccept(self.sockfd.?, &accepted_addr.os_addr, posix.SOCK_NONBLOCK | posix.SOCK_CLOEXEC)) |accepted_fd| {
61 var socket = std.os.File.openHandle(accepted_fd);77 var socket = std.os.File.openHandle(accepted_fd);
62 _ = async<self.loop.allocator> self.handleRequestFn(self, accepted_addr, socket) catch |err| switch (err) {78 _ = async<self.loop.allocator> self.handleRequestFn(self, accepted_addr, socket) catch |err| switch (err) {
63 error.OutOfMemory => {79 error.OutOfMemory => {
...@@ -95,32 +111,65 @@ pub const TcpServer = struct {...@@ -95,32 +111,65 @@ pub const TcpServer = struct {
95111
96pub const Loop = struct {112pub const Loop = struct {
97 allocator: *mem.Allocator,113 allocator: *mem.Allocator,
98 keep_running: bool,
99 next_tick_queue: std.atomic.QueueMpsc(promise),114 next_tick_queue: std.atomic.QueueMpsc(promise),
100 os_data: OsData,115 os_data: OsData,
116 dispatch_lock: u8, // TODO make this a bool
117 pending_event_count: usize,
118 extra_threads: []*std.os.Thread,
119 final_resume_node: ResumeNode,
101120
102 const OsData = switch (builtin.os) {121 pub const NextTickNode = std.atomic.QueueMpsc(promise).Node;
103 builtin.Os.linux => struct {122
104 epollfd: i32,123 pub const ResumeNode = struct {
105 },124 id: Id,
106 else => struct {},125 handle: promise,
126
127 pub const Id = enum {
128 Basic,
129 Stop,
130 EventFd,
131 };
132
133 pub const EventFd = struct {
134 base: ResumeNode,
135 eventfd: i32,
136 };
107 };137 };
108138
109 pub const NextTickNode = std.atomic.QueueMpsc(promise).Node;139 /// After initialization, call run().
140 /// TODO copy elision / named return values so that the threads referencing *Loop
141 /// have the correct pointer value.
142 fn initSingleThreaded(self: *Loop, allocator: *mem.Allocator) !void {
143 return self.initInternal(allocator, 1);
144 }
110145
111 /// The allocator must be thread-safe because we use it for multiplexing146 /// The allocator must be thread-safe because we use it for multiplexing
112 /// coroutines onto kernel threads.147 /// coroutines onto kernel threads.
113 pub fn init(allocator: *mem.Allocator) !Loop {148 /// After initialization, call run().
114 var self = Loop{149 /// TODO copy elision / named return values so that the threads referencing *Loop
115 .keep_running = true,150 /// have the correct pointer value.
151 fn initMultiThreaded(self: *Loop, allocator: *mem.Allocator) !void {
152 // TODO check the actual cpu core count
153 return self.initInternal(allocator, 4);
154 }
155
156 /// Thread count is the total thread count. The thread pool size will be
157 /// max(thread_count - 1, 0)
158 fn initInternal(self: *Loop, allocator: *mem.Allocator, thread_count: usize) !void {
159 self.* = Loop{
160 .pending_event_count = 0,
116 .allocator = allocator,161 .allocator = allocator,
117 .os_data = undefined,162 .os_data = undefined,
118 .next_tick_queue = std.atomic.QueueMpsc(promise).init(),163 .next_tick_queue = std.atomic.QueueMpsc(promise).init(),
164 .dispatch_lock = 1, // start locked so threads go directly into epoll wait
165 .extra_threads = undefined,
166 .final_resume_node = ResumeNode{
167 .id = ResumeNode.Id.Stop,
168 .handle = undefined,
169 },
119 };170 };
120 try self.initOsData();171 try self.initOsData(thread_count);
121 errdefer self.deinitOsData();172 errdefer self.deinitOsData();
122
123 return self;
124 }173 }
125174
126 /// must call stop before deinit175 /// must call stop before deinit
...@@ -128,13 +177,70 @@ pub const Loop = struct {...@@ -128,13 +177,70 @@ pub const Loop = struct {
128 self.deinitOsData();177 self.deinitOsData();
129 }178 }
130179
131 const InitOsDataError = std.os.LinuxEpollCreateError;180 const InitOsDataError = std.os.LinuxEpollCreateError || mem.Allocator.Error || std.os.LinuxEventFdError ||
181 std.os.SpawnThreadError || std.os.LinuxEpollCtlError;
182
183 const wakeup_bytes = []u8{0x1} ** 8;
132184
133 fn initOsData(self: *Loop) InitOsDataError!void {185 fn initOsData(self: *Loop, thread_count: usize) InitOsDataError!void {
134 switch (builtin.os) {186 switch (builtin.os) {
135 builtin.Os.linux => {187 builtin.Os.linux => {
136 self.os_data.epollfd = try std.os.linuxEpollCreate(std.os.linux.EPOLL_CLOEXEC);188 const extra_thread_count = thread_count - 1;
189 self.os_data.available_eventfd_resume_nodes = std.atomic.Stack(ResumeNode.EventFd).init();
190 self.os_data.eventfd_resume_nodes = try self.allocator.alloc(
191 std.atomic.Stack(ResumeNode.EventFd).Node,
192 extra_thread_count,
193 );
194 errdefer self.allocator.free(self.os_data.eventfd_resume_nodes);
195
196 errdefer {
197 while (self.os_data.available_eventfd_resume_nodes.pop()) |node| std.os.close(node.data.eventfd);
198 }
199 for (self.os_data.eventfd_resume_nodes) |*eventfd_node| {
200 eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{
201 .data = ResumeNode.EventFd{
202 .base = ResumeNode{
203 .id = ResumeNode.Id.EventFd,
204 .handle = undefined,
205 },
206 .eventfd = try std.os.linuxEventFd(1, posix.EFD_CLOEXEC | posix.EFD_NONBLOCK),
207 },
208 .next = undefined,
209 };
210 self.os_data.available_eventfd_resume_nodes.push(eventfd_node);
211 }
212
213 self.os_data.epollfd = try std.os.linuxEpollCreate(posix.EPOLL_CLOEXEC);
137 errdefer std.os.close(self.os_data.epollfd);214 errdefer std.os.close(self.os_data.epollfd);
215
216 self.os_data.final_eventfd = try std.os.linuxEventFd(0, posix.EFD_CLOEXEC | posix.EFD_NONBLOCK);
217 errdefer std.os.close(self.os_data.final_eventfd);
218
219 self.os_data.final_eventfd_event = posix.epoll_event{
220 .events = posix.EPOLLIN,
221 .data = posix.epoll_data{ .ptr = @ptrToInt(&self.final_resume_node) },
222 };
223 try std.os.linuxEpollCtl(
224 self.os_data.epollfd,
225 posix.EPOLL_CTL_ADD,
226 self.os_data.final_eventfd,
227 &self.os_data.final_eventfd_event,
228 );
229 self.extra_threads = try self.allocator.alloc(*std.os.Thread, extra_thread_count);
230 errdefer self.allocator.free(self.extra_threads);
231
232 var extra_thread_index: usize = 0;
233 errdefer {
234 while (extra_thread_index != 0) {
235 extra_thread_index -= 1;
236 // writing 8 bytes to an eventfd cannot fail
237 std.os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable;
238 self.extra_threads[extra_thread_index].wait();
239 }
240 }
241 while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) {
242 self.extra_threads[extra_thread_index] = try std.os.spawnThread(self, workerRun);
243 }
138 },244 },
139 else => {},245 else => {},
140 }246 }
...@@ -142,65 +248,154 @@ pub const Loop = struct {...@@ -142,65 +248,154 @@ pub const Loop = struct {
142248
143 fn deinitOsData(self: *Loop) void {249 fn deinitOsData(self: *Loop) void {
144 switch (builtin.os) {250 switch (builtin.os) {
145 builtin.Os.linux => std.os.close(self.os_data.epollfd),251 builtin.Os.linux => {
252 std.os.close(self.os_data.final_eventfd);
253 while (self.os_data.available_eventfd_resume_nodes.pop()) |node| std.os.close(node.data.eventfd);
254 std.os.close(self.os_data.epollfd);
255 self.allocator.free(self.os_data.eventfd_resume_nodes);
256 self.allocator.free(self.extra_threads);
257 },
146 else => {},258 else => {},
147 }259 }
148 }260 }
149261
150 pub fn addFd(self: *Loop, fd: i32, prom: promise) !void {262 /// resume_node must live longer than the promise that it holds a reference to.
263 pub fn addFd(self: *Loop, fd: i32, resume_node: *ResumeNode) !void {
264 _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
265 errdefer {
266 _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
267 }
268 try self.addFdNoCounter(fd, resume_node);
269 }
270
271 fn addFdNoCounter(self: *Loop, fd: i32, resume_node: *ResumeNode) !void {
151 var ev = std.os.linux.epoll_event{272 var ev = std.os.linux.epoll_event{
152 .events = std.os.linux.EPOLLIN | std.os.linux.EPOLLOUT | std.os.linux.EPOLLET,273 .events = std.os.linux.EPOLLIN | std.os.linux.EPOLLOUT | std.os.linux.EPOLLET,
153 .data = std.os.linux.epoll_data{ .ptr = @ptrToInt(prom) },274 .data = std.os.linux.epoll_data{ .ptr = @ptrToInt(resume_node) },
154 };275 };
155 try std.os.linuxEpollCtl(self.os_data.epollfd, std.os.linux.EPOLL_CTL_ADD, fd, &ev);276 try std.os.linuxEpollCtl(self.os_data.epollfd, std.os.linux.EPOLL_CTL_ADD, fd, &ev);
156 }277 }
157278
158 pub fn removeFd(self: *Loop, fd: i32) void {279 pub fn removeFd(self: *Loop, fd: i32) void {
280 self.removeFdNoCounter(fd);
281 _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
282 }
283
284 fn removeFdNoCounter(self: *Loop, fd: i32) void {
159 std.os.linuxEpollCtl(self.os_data.epollfd, std.os.linux.EPOLL_CTL_DEL, fd, undefined) catch {};285 std.os.linuxEpollCtl(self.os_data.epollfd, std.os.linux.EPOLL_CTL_DEL, fd, undefined) catch {};
160 }286 }
161 async fn waitFd(self: *Loop, fd: i32) !void {287
288 pub async fn waitFd(self: *Loop, fd: i32) !void {
162 defer self.removeFd(fd);289 defer self.removeFd(fd);
290 var resume_node = ResumeNode{
291 .id = ResumeNode.Id.Basic,
292 .handle = undefined,
293 };
163 suspend |p| {294 suspend |p| {
164 try self.addFd(fd, p);295 resume_node.handle = p;
296 try self.addFd(fd, &resume_node);
165 }297 }
298 var a = &resume_node; // TODO better way to explicitly put memory in coro frame
166 }299 }
167300
168 pub fn stop(self: *Loop) void {301 /// Bring your own linked list node. This means it can't fail.
169 // TODO make atomic
170 self.keep_running = false;
171 // TODO activate an fd in the epoll set which should cancel all the promises
172 }
173
174 /// bring your own linked list node. this means it can't fail.
175 pub fn onNextTick(self: *Loop, node: *NextTickNode) void {302 pub fn onNextTick(self: *Loop, node: *NextTickNode) void {
303 _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
176 self.next_tick_queue.put(node);304 self.next_tick_queue.put(node);
177 }305 }
178306
179 pub fn run(self: *Loop) void {307 pub fn run(self: *Loop) void {
180 while (self.keep_running) {308 _ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
181 // TODO multiplex the next tick queue and the epoll event results onto a thread pool309 self.workerRun();
182 while (self.next_tick_queue.get()) |node| {310 for (self.extra_threads) |extra_thread| {
183 resume node.data;311 extra_thread.wait();
184 }
185 if (!self.keep_running) break;
186
187 self.dispatchOsEvents();
188 }312 }
189 }313 }
190314
191 fn dispatchOsEvents(self: *Loop) void {315 fn workerRun(self: *Loop) void {
192 switch (builtin.os) {316 start_over: while (true) {
193 builtin.Os.linux => {317 if (@atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) == 0) {
194 var events: [16]std.os.linux.epoll_event = undefined;318 while (self.next_tick_queue.get()) |next_tick_node| {
195 const count = std.os.linuxEpollWait(self.os_data.epollfd, events[0..], -1);319 const handle = next_tick_node.data;
196 for (events[0..count]) |ev| {320 if (self.next_tick_queue.isEmpty()) {
197 const p = @intToPtr(promise, ev.data.ptr);321 // last node, just resume it
198 resume p;322 _ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
323 resume handle;
324 _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
325 continue :start_over;
326 }
327
328 // non-last node, stick it in the epoll set so that
329 // other threads can get to it
330 if (self.os_data.available_eventfd_resume_nodes.pop()) |resume_stack_node| {
331 const eventfd_node = &resume_stack_node.data;
332 eventfd_node.base.handle = handle;
333 // the pending count is already accounted for
334 self.addFdNoCounter(eventfd_node.eventfd, &eventfd_node.base) catch |_| {
335 // fine, we didn't need it anyway
336 _ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
337 self.os_data.available_eventfd_resume_nodes.push(resume_stack_node);
338 resume handle;
339 _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
340 continue :start_over;
341 };
342 } else {
343 // threads are too busy, can't add another eventfd to wake one up
344 _ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
345 resume handle;
346 _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
347 continue :start_over;
348 }
199 }349 }
200 },350
201 else => {},351 const pending_event_count = @atomicLoad(usize, &self.pending_event_count, AtomicOrder.SeqCst);
352 if (pending_event_count == 0) {
353 // cause all the threads to stop
354 // writing 8 bytes to an eventfd cannot fail
355 std.os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable;
356 return;
357 }
358
359 _ = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
360 }
361
362 // only process 1 event so we don't steal from other threads
363 var events: [1]std.os.linux.epoll_event = undefined;
364 const count = std.os.linuxEpollWait(self.os_data.epollfd, events[0..], -1);
365 for (events[0..count]) |ev| {
366 const resume_node = @intToPtr(*ResumeNode, ev.data.ptr);
367 const handle = resume_node.handle;
368 const resume_node_id = resume_node.id;
369 switch (resume_node_id) {
370 ResumeNode.Id.Basic => {},
371 ResumeNode.Id.Stop => return,
372 ResumeNode.Id.EventFd => {
373 const event_fd_node = @fieldParentPtr(ResumeNode.EventFd, "base", resume_node);
374 self.removeFdNoCounter(event_fd_node.eventfd);
375 const stack_node = @fieldParentPtr(std.atomic.Stack(ResumeNode.EventFd).Node, "data", event_fd_node);
376 self.os_data.available_eventfd_resume_nodes.push(stack_node);
377 },
378 }
379 resume handle;
380 if (resume_node_id == ResumeNode.Id.EventFd) {
381 _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
382 }
383 }
202 }384 }
203 }385 }
386
387 const OsData = switch (builtin.os) {
388 builtin.Os.linux => struct {
389 epollfd: i32,
390 // pre-allocated eventfds. all permanently active.
391 // this is how we send promises to be resumed on other threads.
392 available_eventfd_resume_nodes: std.atomic.Stack(ResumeNode.EventFd),
393 eventfd_resume_nodes: []std.atomic.Stack(ResumeNode.EventFd).Node,
394 final_eventfd: i32,
395 final_eventfd_event: posix.epoll_event,
396 },
397 else => struct {},
398 };
204};399};
205400
206/// many producer, many consumer, thread-safe, lock-free, runtime configurable buffer size401/// many producer, many consumer, thread-safe, lock-free, runtime configurable buffer size
...@@ -304,9 +499,7 @@ pub fn Channel(comptime T: type) type {...@@ -304,9 +499,7 @@ pub fn Channel(comptime T: type) type {
304 // TODO integrate this function with named return values499 // TODO integrate this function with named return values
305 // so we can get rid of this extra result copy500 // so we can get rid of this extra result copy
306 var result: T = undefined;501 var result: T = undefined;
307 var debug_handle: usize = undefined;
308 suspend |handle| {502 suspend |handle| {
309 debug_handle = @ptrToInt(handle);
310 var my_tick_node = Loop.NextTickNode{503 var my_tick_node = Loop.NextTickNode{
311 .next = undefined,504 .next = undefined,
312 .data = handle,505 .data = handle,
...@@ -438,9 +631,8 @@ test "listen on a port, send bytes, receive bytes" {...@@ -438,9 +631,8 @@ test "listen on a port, send bytes, receive bytes" {
438 const self = @fieldParentPtr(Self, "tcp_server", tcp_server);631 const self = @fieldParentPtr(Self, "tcp_server", tcp_server);
439 var socket = _socket.*; // TODO https://github.com/ziglang/zig/issues/733632 var socket = _socket.*; // TODO https://github.com/ziglang/zig/issues/733
440 defer socket.close();633 defer socket.close();
441 const next_handler = async errorableHandler(self, _addr, socket) catch |err| switch (err) {634 // TODO guarantee elision of this allocation
442 error.OutOfMemory => @panic("unable to handle connection: out of memory"),635 const next_handler = async errorableHandler(self, _addr, socket) catch unreachable;
443 };
444 (await next_handler) catch |err| {636 (await next_handler) catch |err| {
445 std.debug.panic("unable to handle connection: {}\n", err);637 std.debug.panic("unable to handle connection: {}\n", err);
446 };638 };
...@@ -461,17 +653,18 @@ test "listen on a port, send bytes, receive bytes" {...@@ -461,17 +653,18 @@ test "listen on a port, send bytes, receive bytes" {
461 const ip4addr = std.net.parseIp4("127.0.0.1") catch unreachable;653 const ip4addr = std.net.parseIp4("127.0.0.1") catch unreachable;
462 const addr = std.net.Address.initIp4(ip4addr, 0);654 const addr = std.net.Address.initIp4(ip4addr, 0);
463655
464 var loop = try Loop.init(std.debug.global_allocator);656 var loop: Loop = undefined;
465 var server = MyServer{ .tcp_server = try TcpServer.init(&loop) };657 try loop.initSingleThreaded(std.debug.global_allocator);
658 var server = MyServer{ .tcp_server = TcpServer.init(&loop) };
466 defer server.tcp_server.deinit();659 defer server.tcp_server.deinit();
467 try server.tcp_server.listen(addr, MyServer.handler);660 try server.tcp_server.listen(addr, MyServer.handler);
468661
469 const p = try async<std.debug.global_allocator> doAsyncTest(&loop, server.tcp_server.listen_address);662 const p = try async<std.debug.global_allocator> doAsyncTest(&loop, server.tcp_server.listen_address, &server.tcp_server);
470 defer cancel p;663 defer cancel p;
471 loop.run();664 loop.run();
472}665}
473666
474async fn doAsyncTest(loop: *Loop, address: *const std.net.Address) void {667async fn doAsyncTest(loop: *Loop, address: *const std.net.Address, server: *TcpServer) void {
475 errdefer @panic("test failure");668 errdefer @panic("test failure");
476669
477 var socket_file = try await try async event.connect(loop, address);670 var socket_file = try await try async event.connect(loop, address);
...@@ -481,7 +674,7 @@ async fn doAsyncTest(loop: *Loop, address: *const std.net.Address) void {...@@ -481,7 +674,7 @@ async fn doAsyncTest(loop: *Loop, address: *const std.net.Address) void {
481 const amt_read = try socket_file.read(buf[0..]);674 const amt_read = try socket_file.read(buf[0..]);
482 const msg = buf[0..amt_read];675 const msg = buf[0..amt_read];
483 assert(mem.eql(u8, msg, "hello from server\n"));676 assert(mem.eql(u8, msg, "hello from server\n"));
484 loop.stop();677 server.close();
485}678}
486679
487test "std.event.Channel" {680test "std.event.Channel" {
...@@ -490,7 +683,9 @@ test "std.event.Channel" {...@@ -490,7 +683,9 @@ test "std.event.Channel" {
490683
491 const allocator = &da.allocator;684 const allocator = &da.allocator;
492685
493 var loop = try Loop.init(allocator);686 var loop: Loop = undefined;
687 // TODO make a multi threaded test
688 try loop.initSingleThreaded(allocator);
494 defer loop.deinit();689 defer loop.deinit();
495690
496 const channel = try Channel(i32).create(&loop, 0);691 const channel = try Channel(i32).create(&loop, 0);
...@@ -515,11 +710,248 @@ async fn testChannelGetter(loop: *Loop, channel: *Channel(i32)) void {...@@ -515,11 +710,248 @@ async fn testChannelGetter(loop: *Loop, channel: *Channel(i32)) void {
515 const value2_promise = try async channel.get();710 const value2_promise = try async channel.get();
516 const value2 = await value2_promise;711 const value2 = await value2_promise;
517 assert(value2 == 4567);712 assert(value2 == 4567);
518
519 loop.stop();
520}713}
521714
522async fn testChannelPutter(channel: *Channel(i32)) void {715async fn testChannelPutter(channel: *Channel(i32)) void {
523 await (async channel.put(1234) catch @panic("out of memory"));716 await (async channel.put(1234) catch @panic("out of memory"));
524 await (async channel.put(4567) catch @panic("out of memory"));717 await (async channel.put(4567) catch @panic("out of memory"));
525}718}
719
720/// Thread-safe async/await lock.
721/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and
722/// are resumed when the lock is released, in order.
723pub const Lock = struct {
724 loop: *Loop,
725 shared_bit: u8, // TODO make this a bool
726 queue: Queue,
727 queue_empty_bit: u8, // TODO make this a bool
728
729 const Queue = std.atomic.QueueMpsc(promise);
730
731 pub const Held = struct {
732 lock: *Lock,
733
734 pub fn release(self: Held) void {
735 // Resume the next item from the queue.
736 if (self.lock.queue.get()) |node| {
737 self.lock.loop.onNextTick(node);
738 return;
739 }
740
741 // We need to release the lock.
742 _ = @atomicRmw(u8, &self.lock.queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
743 _ = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
744
745 // There might be a queue item. If we know the queue is empty, we can be done,
746 // because the other actor will try to obtain the lock.
747 // But if there's a queue item, we are the actor which must loop and attempt
748 // to grab the lock again.
749 if (@atomicLoad(u8, &self.lock.queue_empty_bit, AtomicOrder.SeqCst) == 1) {
750 return;
751 }
752
753 while (true) {
754 const old_bit = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
755 if (old_bit != 0) {
756 // We did not obtain the lock. Great, the queue is someone else's problem.
757 return;
758 }
759
760 // Resume the next item from the queue.
761 if (self.lock.queue.get()) |node| {
762 self.lock.loop.onNextTick(node);
763 return;
764 }
765
766 // Release the lock again.
767 _ = @atomicRmw(u8, &self.lock.queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
768 _ = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
769
770 // Find out if we can be done.
771 if (@atomicLoad(u8, &self.lock.queue_empty_bit, AtomicOrder.SeqCst) == 1) {
772 return;
773 }
774 }
775 }
776 };
777
778 pub fn init(loop: *Loop) Lock {
779 return Lock{
780 .loop = loop,
781 .shared_bit = 0,
782 .queue = Queue.init(),
783 .queue_empty_bit = 1,
784 };
785 }
786
787 /// Must be called when not locked. Not thread safe.
788 /// All calls to acquire() and release() must complete before calling deinit().
789 pub fn deinit(self: *Lock) void {
790 assert(self.shared_bit == 0);
791 while (self.queue.get()) |node| cancel node.data;
792 }
793
794 pub async fn acquire(self: *Lock) Held {
795 var my_tick_node: Loop.NextTickNode = undefined;
796
797 s: suspend |handle| {
798 my_tick_node.data = handle;
799 self.queue.put(&my_tick_node);
800
801 // At this point, we are in the queue, so we might have already been resumed and this coroutine
802 // frame might be destroyed. For the rest of the suspend block we cannot access the coroutine frame.
803
804 // We set this bit so that later we can rely on the fact, that if queue_empty_bit is 1, some actor
805 // will attempt to grab the lock.
806 _ = @atomicRmw(u8, &self.queue_empty_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
807
808 while (true) {
809 const old_bit = @atomicRmw(u8, &self.shared_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
810 if (old_bit != 0) {
811 // We did not obtain the lock. Trust that our queue entry will resume us, and allow
812 // suspend to complete.
813 break;
814 }
815 // We got the lock. However we might have already been resumed from the queue.
816 if (self.queue.get()) |node| {
817 // Whether this node is us or someone else, we tail resume it.
818 resume node.data;
819 break;
820 } else {
821 // We already got resumed, and there are none left in the queue, which means that
822 // we aren't even supposed to hold the lock right now.
823 _ = @atomicRmw(u8, &self.queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
824 _ = @atomicRmw(u8, &self.shared_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
825
826 // There might be a queue item. If we know the queue is empty, we can be done,
827 // because the other actor will try to obtain the lock.
828 // But if there's a queue item, we are the actor which must loop and attempt
829 // to grab the lock again.
830 if (@atomicLoad(u8, &self.queue_empty_bit, AtomicOrder.SeqCst) == 1) {
831 break;
832 } else {
833 continue;
834 }
835 }
836 unreachable;
837 }
838 }
839
840 // TODO this workaround to force my_tick_node to be in the coroutine frame should
841 // not be necessary
842 var trash1 = &my_tick_node;
843
844 return Held{ .lock = self };
845 }
846};
847
848/// Thread-safe async/await lock that protects one piece of data.
849/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and
850/// are resumed when the lock is released, in order.
851pub fn Locked(comptime T: type) type {
852 return struct {
853 lock: Lock,
854 private_data: T,
855
856 const Self = this;
857
858 pub const HeldLock = struct {
859 value: *T,
860 held: Lock.Held,
861
862 pub fn release(self: HeldLock) void {
863 self.held.release();
864 }
865 };
866
867 pub fn init(loop: *Loop, data: T) Self {
868 return Self{
869 .lock = Lock.init(loop),
870 .private_data = data,
871 };
872 }
873
874 pub fn deinit(self: *Self) void {
875 self.lock.deinit();
876 }
877
878 pub async fn acquire(self: *Self) HeldLock {
879 return HeldLock{
880 // TODO guaranteed allocation elision
881 .held = await (async self.lock.acquire() catch unreachable),
882 .value = &self.private_data,
883 };
884 }
885 };
886}
887
888test "std.event.Lock" {
889 var da = std.heap.DirectAllocator.init();
890 defer da.deinit();
891
892 const allocator = &da.allocator;
893
894 var loop: Loop = undefined;
895 try loop.initMultiThreaded(allocator);
896 defer loop.deinit();
897
898 var lock = Lock.init(&loop);
899 defer lock.deinit();
900
901 const handle = try async<allocator> testLock(&loop, &lock);
902 defer cancel handle;
903 loop.run();
904
905 assert(mem.eql(i32, shared_test_data, [1]i32{3 * 10} ** 10));
906}
907
908async fn testLock(loop: *Loop, lock: *Lock) void {
909 const handle1 = async lockRunner(lock) catch @panic("out of memory");
910 var tick_node1 = Loop.NextTickNode{
911 .next = undefined,
912 .data = handle1,
913 };
914 loop.onNextTick(&tick_node1);
915
916 const handle2 = async lockRunner(lock) catch @panic("out of memory");
917 var tick_node2 = Loop.NextTickNode{
918 .next = undefined,
919 .data = handle2,
920 };
921 loop.onNextTick(&tick_node2);
922
923 const handle3 = async lockRunner(lock) catch @panic("out of memory");
924 var tick_node3 = Loop.NextTickNode{
925 .next = undefined,
926 .data = handle3,
927 };
928 loop.onNextTick(&tick_node3);
929
930 await handle1;
931 await handle2;
932 await handle3;
933
934 // TODO this is to force tick node memory to be in the coro frame
935 // there should be a way to make it explicit where the memory is
936 var a = &tick_node1;
937 var b = &tick_node2;
938 var c = &tick_node3;
939}
940
941var shared_test_data = [1]i32{0} ** 10;
942var shared_test_index: usize = 0;
943
944async fn lockRunner(lock: *Lock) void {
945 suspend; // resumed by onNextTick
946
947 var i: usize = 0;
948 while (i < 10) : (i += 1) {
949 const handle = await (async lock.acquire() catch @panic("out of memory"));
950 defer handle.release();
951
952 shared_test_index = 0;
953 while (shared_test_index < shared_test_data.len) : (shared_test_index += 1) {
954 shared_test_data[shared_test_index] = shared_test_data[shared_test_index] + 1;
955 }
956 }
957}
std/heap.zig+15-15
...@@ -38,7 +38,7 @@ fn cFree(self: *Allocator, old_mem: []u8) void {...@@ -38,7 +38,7 @@ fn cFree(self: *Allocator, old_mem: []u8) void {
38}38}
3939
40/// This allocator makes a syscall directly for every allocation and free.40/// This allocator makes a syscall directly for every allocation and free.
41/// TODO make this thread-safe. The windows implementation will need some atomics.41/// Thread-safe and lock-free.
42pub const DirectAllocator = struct {42pub const DirectAllocator = struct {
43 allocator: Allocator,43 allocator: Allocator,
44 heap_handle: ?HeapHandle,44 heap_handle: ?HeapHandle,
...@@ -74,34 +74,34 @@ pub const DirectAllocator = struct {...@@ -74,34 +74,34 @@ pub const DirectAllocator = struct {
74 const alloc_size = if (alignment <= os.page_size) n else n + alignment;74 const alloc_size = if (alignment <= os.page_size) n else n + alignment;
75 const addr = p.mmap(null, alloc_size, p.PROT_READ | p.PROT_WRITE, p.MAP_PRIVATE | p.MAP_ANONYMOUS, -1, 0);75 const addr = p.mmap(null, alloc_size, p.PROT_READ | p.PROT_WRITE, p.MAP_PRIVATE | p.MAP_ANONYMOUS, -1, 0);
76 if (addr == p.MAP_FAILED) return error.OutOfMemory;76 if (addr == p.MAP_FAILED) return error.OutOfMemory;
77
78 if (alloc_size == n) return @intToPtr([*]u8, addr)[0..n];77 if (alloc_size == n) return @intToPtr([*]u8, addr)[0..n];
7978
80 var aligned_addr = addr & ~usize(alignment - 1);79 const aligned_addr = (addr & ~usize(alignment - 1)) + alignment;
81 aligned_addr += alignment;
8280
83 //We can unmap the unused portions of our mmap, but we must only81 // We can unmap the unused portions of our mmap, but we must only
84 // pass munmap bytes that exist outside our allocated pages or it82 // pass munmap bytes that exist outside our allocated pages or it
85 // will happily eat us too83 // will happily eat us too.
8684
87 //Since alignment > page_size, we are by definition on a page boundry85 // Since alignment > page_size, we are by definition on a page boundary.
88 const unused_start = addr;86 const unused_start = addr;
89 const unused_len = aligned_addr - 1 - unused_start;87 const unused_len = aligned_addr - 1 - unused_start;
9088
91 var err = p.munmap(unused_start, unused_len);89 const err = p.munmap(unused_start, unused_len);
92 debug.assert(p.getErrno(err) == 0);90 assert(p.getErrno(err) == 0);
9391
94 //It is impossible that there is an unoccupied page at the top of our92 // It is impossible that there is an unoccupied page at the top of our
95 // mmap.93 // mmap.
9694
97 return @intToPtr([*]u8, aligned_addr)[0..n];95 return @intToPtr([*]u8, aligned_addr)[0..n];
98 },96 },
99 Os.windows => {97 Os.windows => {
100 const amt = n + alignment + @sizeOf(usize);98 const amt = n + alignment + @sizeOf(usize);
101 const heap_handle = self.heap_handle orelse blk: {99 const optional_heap_handle = @atomicLoad(?HeapHandle, ?self.heap_handle, builtin.AtomicOrder.SeqCst);
100 const heap_handle = optional_heap_handle orelse blk: {
102 const hh = os.windows.HeapCreate(os.windows.HEAP_NO_SERIALIZE, amt, 0) orelse return error.OutOfMemory;101 const hh = os.windows.HeapCreate(os.windows.HEAP_NO_SERIALIZE, amt, 0) orelse return error.OutOfMemory;
103 self.heap_handle = hh;102 const other_hh = @cmpxchgStrong(?HeapHandle, &self.heap_handle, null, hh, builtin.AtomicOrder.SeqCst, builtin.AtomicOrder.SeqCst) orelse break :blk hh;
104 break :blk hh;103 _ = os.windows.HeapDestroy(hh);
104 break :blk other_hh;
105 };105 };
106 const ptr = os.windows.HeapAlloc(heap_handle, 0, amt) orelse return error.OutOfMemory;106 const ptr = os.windows.HeapAlloc(heap_handle, 0, amt) orelse return error.OutOfMemory;
107 const root_addr = @ptrToInt(ptr);107 const root_addr = @ptrToInt(ptr);
std/mem.zig+1-1
...@@ -6,7 +6,7 @@ const builtin = @import("builtin");...@@ -6,7 +6,7 @@ const builtin = @import("builtin");
6const mem = this;6const mem = this;
77
8pub const Allocator = struct {8pub const Allocator = struct {
9 const Error = error{OutOfMemory};9 pub const Error = error{OutOfMemory};
1010
11 /// Allocate byte_count bytes and return them in a slice, with the11 /// Allocate byte_count bytes and return them in a slice, with the
12 /// slice's pointer aligned at least to alignment bytes.12 /// slice's pointer aligned at least to alignment bytes.
std/os/index.zig+35-4
...@@ -2309,6 +2309,30 @@ pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usiz...@@ -2309,6 +2309,30 @@ pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usiz
2309 }2309 }
2310}2310}
23112311
2312pub const LinuxEventFdError = error{
2313 InvalidFlagValue,
2314 SystemResources,
2315 ProcessFdQuotaExceeded,
2316 SystemFdQuotaExceeded,
2317
2318 Unexpected,
2319};
2320
2321pub fn linuxEventFd(initval: u32, flags: u32) LinuxEventFdError!i32 {
2322 const rc = posix.eventfd(initval, flags);
2323 const err = posix.getErrno(rc);
2324 switch (err) {
2325 0 => return @intCast(i32, rc),
2326 else => return unexpectedErrorPosix(err),
2327
2328 posix.EINVAL => return LinuxEventFdError.InvalidFlagValue,
2329 posix.EMFILE => return LinuxEventFdError.ProcessFdQuotaExceeded,
2330 posix.ENFILE => return LinuxEventFdError.SystemFdQuotaExceeded,
2331 posix.ENODEV => return LinuxEventFdError.SystemResources,
2332 posix.ENOMEM => return LinuxEventFdError.SystemResources,
2333 }
2334}
2335
2312pub const PosixGetSockNameError = error{2336pub const PosixGetSockNameError = error{
2313 /// Insufficient resources were available in the system to perform the operation.2337 /// Insufficient resources were available in the system to perform the operation.
2314 SystemResources,2338 SystemResources,
...@@ -2605,10 +2629,17 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread...@@ -2605,10 +2629,17 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
26052629
2606 const MainFuncs = struct {2630 const MainFuncs = struct {
2607 extern fn linuxThreadMain(ctx_addr: usize) u8 {2631 extern fn linuxThreadMain(ctx_addr: usize) u8 {
2608 if (@sizeOf(Context) == 0) {2632 const arg = if (@sizeOf(Context) == 0) {} else @intToPtr(*const Context, ctx_addr).*;
2609 return startFn({});2633
2610 } else {2634 switch (@typeId(@typeOf(startFn).ReturnType)) {
2611 return startFn(@intToPtr(*const Context, ctx_addr).*);2635 builtin.TypeId.Int => {
2636 return startFn(arg);
2637 },
2638 builtin.TypeId.Void => {
2639 startFn(arg);
2640 return 0;
2641 },
2642 else => @compileError("expected return type of startFn to be 'u8', 'noreturn', 'void', or '!void'"),
2612 }2643 }
2613 }2644 }
2614 extern fn posixThreadMain(ctx: ?*c_void) ?*c_void {2645 extern fn posixThreadMain(ctx: ?*c_void) ?*c_void {
std/os/linux/index.zig+8
...@@ -523,6 +523,10 @@ pub const CLONE_NEWPID = 0x20000000;...@@ -523,6 +523,10 @@ pub const CLONE_NEWPID = 0x20000000;
523pub const CLONE_NEWNET = 0x40000000;523pub const CLONE_NEWNET = 0x40000000;
524pub const CLONE_IO = 0x80000000;524pub const CLONE_IO = 0x80000000;
525525
526pub const EFD_SEMAPHORE = 1;
527pub const EFD_CLOEXEC = O_CLOEXEC;
528pub const EFD_NONBLOCK = O_NONBLOCK;
529
526pub const MS_RDONLY = 1;530pub const MS_RDONLY = 1;
527pub const MS_NOSUID = 2;531pub const MS_NOSUID = 2;
528pub const MS_NODEV = 4;532pub const MS_NODEV = 4;
...@@ -1221,6 +1225,10 @@ pub fn epoll_wait(epoll_fd: i32, events: [*]epoll_event, maxevents: u32, timeout...@@ -1221,6 +1225,10 @@ pub fn epoll_wait(epoll_fd: i32, events: [*]epoll_event, maxevents: u32, timeout
1221 return syscall4(SYS_epoll_wait, @intCast(usize, epoll_fd), @ptrToInt(events), @intCast(usize, maxevents), @intCast(usize, timeout));1225 return syscall4(SYS_epoll_wait, @intCast(usize, epoll_fd), @ptrToInt(events), @intCast(usize, maxevents), @intCast(usize, timeout));
1222}1226}
12231227
1228pub fn eventfd(count: u32, flags: u32) usize {
1229 return syscall2(SYS_eventfd2, count, flags);
1230}
1231
1224pub fn timerfd_create(clockid: i32, flags: u32) usize {1232pub fn timerfd_create(clockid: i32, flags: u32) usize {
1225 return syscall2(SYS_timerfd_create, @intCast(usize, clockid), @intCast(usize, flags));1233 return syscall2(SYS_timerfd_create, @intCast(usize, clockid), @intCast(usize, flags));
1226}1234}