| ... | @@ -192,14 +192,22 @@ pub fn init(el: *EventLoop, gpa: Allocator) !void { | ... | @@ -192,14 +192,22 @@ pub fn init(el: *EventLoop, gpa: Allocator) !void { |
| 192 | }; | 192 | }; |
| 193 | const main_thread = &el.threads.allocated[0]; | 193 | const main_thread = &el.threads.allocated[0]; |
| 194 | Thread.self = main_thread; | 194 | Thread.self = main_thread; |
| 195 | const idle_stack_end: [*]usize = @alignCast(@ptrCast(allocated_slice[idle_stack_end_offset..].ptr)); | 195 | const idle_stack_end: [*]align(16) usize = @alignCast(@ptrCast(allocated_slice[idle_stack_end_offset..].ptr)); |
| 196 | (idle_stack_end - 1)[0..1].* = .{@intFromPtr(el)}; | 196 | (idle_stack_end - 1)[0..1].* = .{@intFromPtr(el)}; |
| 197 | main_thread.* = .{ | 197 | main_thread.* = .{ |
| 198 | .thread = undefined, | 198 | .thread = undefined, |
| 199 | .idle_context = .{ | 199 | .idle_context = switch (builtin.cpu.arch) { |
| 200 | .rsp = @intFromPtr(idle_stack_end - 1), | 200 | .aarch64 => .{ |
| 201 | .rbp = 0, | 201 | .sp = @intFromPtr(idle_stack_end), |
| 202 | .rip = @intFromPtr(&mainIdleEntry), | 202 | .fp = 0, |
| | 203 | .pc = @intFromPtr(&mainIdleEntry), |
| | 204 | }, |
| | 205 | .x86_64 => .{ |
| | 206 | .rsp = @intFromPtr(idle_stack_end - 1), |
| | 207 | .rbp = 0, |
| | 208 | .rip = @intFromPtr(&mainIdleEntry), |
| | 209 | }, |
| | 210 | else => @compileError("unimplemented architecture"), |
| 203 | }, | 211 | }, |
| 204 | .current_context = &main_fiber.context, | 212 | .current_context = &main_fiber.context, |
| 205 | .ready_queue = null, | 213 | .ready_queue = null, |
| ... | @@ -606,6 +614,11 @@ const SwitchMessage = struct { | ... | @@ -606,6 +614,11 @@ const SwitchMessage = struct { |
| 606 | }; | 614 | }; |
| 607 | | 615 | |
| 608 | const Context = switch (builtin.cpu.arch) { | 616 | const Context = switch (builtin.cpu.arch) { |
| | 617 | .aarch64 => extern struct { |
| | 618 | sp: u64, |
| | 619 | fp: u64, |
| | 620 | pc: u64, |
| | 621 | }, |
| 609 | .x86_64 => extern struct { | 622 | .x86_64 => extern struct { |
| 610 | rsp: u64, | 623 | rsp: u64, |
| 611 | rbp: u64, | 624 | rbp: u64, |
| ... | @@ -616,6 +629,102 @@ const Context = switch (builtin.cpu.arch) { | ... | @@ -616,6 +629,102 @@ const Context = switch (builtin.cpu.arch) { |
| 616 | | 629 | |
| 617 | inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage { | 630 | inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage { |
| 618 | return @fieldParentPtr("contexts", switch (builtin.cpu.arch) { | 631 | return @fieldParentPtr("contexts", switch (builtin.cpu.arch) { |
| | 632 | .aarch64 => asm volatile ( |
| | 633 | \\ ldp x0, x2, [x1] |
| | 634 | \\ ldr x3, [x2, #16] |
| | 635 | \\ mov x4, sp |
| | 636 | \\ stp x4, fp, [x0] |
| | 637 | \\ adr x5, 0f |
| | 638 | \\ ldp x4, fp, [x2] |
| | 639 | \\ str x5, [x0, #16] |
| | 640 | \\ mov sp, x4 |
| | 641 | \\ br x3 |
| | 642 | \\0: |
| | 643 | : [received_message] "={x1}" (-> *const @FieldType(SwitchMessage, "contexts")), |
| | 644 | : [message_to_send] "{x1}" (&message.contexts), |
| | 645 | : .{ |
| | 646 | .x1 = true, |
| | 647 | .x2 = true, |
| | 648 | .x3 = true, |
| | 649 | .x4 = true, |
| | 650 | .x5 = true, |
| | 651 | .x6 = true, |
| | 652 | .x7 = true, |
| | 653 | .x8 = true, |
| | 654 | .x9 = true, |
| | 655 | .x10 = true, |
| | 656 | .x11 = true, |
| | 657 | .x12 = true, |
| | 658 | .x13 = true, |
| | 659 | .x14 = true, |
| | 660 | .x15 = true, |
| | 661 | .x16 = true, |
| | 662 | .x17 = true, |
| | 663 | .x18 = true, |
| | 664 | .x19 = true, |
| | 665 | .x20 = true, |
| | 666 | .x21 = true, |
| | 667 | .x22 = true, |
| | 668 | .x23 = true, |
| | 669 | .x24 = true, |
| | 670 | .x25 = true, |
| | 671 | .x26 = true, |
| | 672 | .x27 = true, |
| | 673 | .x28 = true, |
| | 674 | .x30 = true, |
| | 675 | .z0 = true, |
| | 676 | .z1 = true, |
| | 677 | .z2 = true, |
| | 678 | .z3 = true, |
| | 679 | .z4 = true, |
| | 680 | .z5 = true, |
| | 681 | .z6 = true, |
| | 682 | .z7 = true, |
| | 683 | .z8 = true, |
| | 684 | .z9 = true, |
| | 685 | .z10 = true, |
| | 686 | .z11 = true, |
| | 687 | .z12 = true, |
| | 688 | .z13 = true, |
| | 689 | .z14 = true, |
| | 690 | .z15 = true, |
| | 691 | .z16 = true, |
| | 692 | .z17 = true, |
| | 693 | .z18 = true, |
| | 694 | .z19 = true, |
| | 695 | .z20 = true, |
| | 696 | .z21 = true, |
| | 697 | .z22 = true, |
| | 698 | .z23 = true, |
| | 699 | .z24 = true, |
| | 700 | .z25 = true, |
| | 701 | .z26 = true, |
| | 702 | .z27 = true, |
| | 703 | .z28 = true, |
| | 704 | .z29 = true, |
| | 705 | .z30 = true, |
| | 706 | .z31 = true, |
| | 707 | .p0 = true, |
| | 708 | .p1 = true, |
| | 709 | .p2 = true, |
| | 710 | .p3 = true, |
| | 711 | .p4 = true, |
| | 712 | .p5 = true, |
| | 713 | .p6 = true, |
| | 714 | .p7 = true, |
| | 715 | .p8 = true, |
| | 716 | .p9 = true, |
| | 717 | .p10 = true, |
| | 718 | .p11 = true, |
| | 719 | .p12 = true, |
| | 720 | .p13 = true, |
| | 721 | .p14 = true, |
| | 722 | .p15 = true, |
| | 723 | .fpcr = true, |
| | 724 | .fpsr = true, |
| | 725 | .ffr = true, |
| | 726 | .memory = true, |
| | 727 | }), |
| 619 | .x86_64 => asm volatile ( | 728 | .x86_64 => asm volatile ( |
| 620 | \\ movq 0(%%rsi), %%rax | 729 | \\ movq 0(%%rsi), %%rax |
| 621 | \\ movq 8(%%rsi), %%rcx | 730 | \\ movq 8(%%rsi), %%rcx |
| ... | @@ -629,15 +738,67 @@ inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage { | ... | @@ -629,15 +738,67 @@ inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage { |
| 629 | \\0: | 738 | \\0: |
| 630 | : [received_message] "={rsi}" (-> *const @FieldType(SwitchMessage, "contexts")), | 739 | : [received_message] "={rsi}" (-> *const @FieldType(SwitchMessage, "contexts")), |
| 631 | : [message_to_send] "{rsi}" (&message.contexts), | 740 | : [message_to_send] "{rsi}" (&message.contexts), |
| 632 | : "rax", "rcx", "rdx", "rbx", "rdi", // | 741 | : .{ |
| 633 | "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", // | 742 | .rax = true, |
| 634 | "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", // | 743 | .rcx = true, |
| 635 | "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5", "zmm6", "zmm7", // | 744 | .rdx = true, |
| 636 | "zmm8", "zmm9", "zmm10", "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", // | 745 | .rbx = true, |
| 637 | "zmm16", "zmm17", "zmm18", "zmm19", "zmm20", "zmm21", "zmm22", "zmm23", // | 746 | .rsi = true, |
| 638 | "zmm24", "zmm25", "zmm26", "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", // | 747 | .r8 = true, |
| 639 | "fpsr", "fpcr", "mxcsr", "rflags", "dirflag", "memory" | 748 | .r9 = true, |
| 640 | ), | 749 | .r10 = true, |
| | 750 | .r11 = true, |
| | 751 | .r12 = true, |
| | 752 | .r13 = true, |
| | 753 | .r14 = true, |
| | 754 | .r15 = true, |
| | 755 | .mm0 = true, |
| | 756 | .mm1 = true, |
| | 757 | .mm2 = true, |
| | 758 | .mm3 = true, |
| | 759 | .mm4 = true, |
| | 760 | .mm5 = true, |
| | 761 | .mm6 = true, |
| | 762 | .mm7 = true, |
| | 763 | .zmm0 = true, |
| | 764 | .zmm1 = true, |
| | 765 | .zmm2 = true, |
| | 766 | .zmm3 = true, |
| | 767 | .zmm4 = true, |
| | 768 | .zmm5 = true, |
| | 769 | .zmm6 = true, |
| | 770 | .zmm7 = true, |
| | 771 | .zmm8 = true, |
| | 772 | .zmm9 = true, |
| | 773 | .zmm10 = true, |
| | 774 | .zmm11 = true, |
| | 775 | .zmm12 = true, |
| | 776 | .zmm13 = true, |
| | 777 | .zmm14 = true, |
| | 778 | .zmm15 = true, |
| | 779 | .zmm16 = true, |
| | 780 | .zmm17 = true, |
| | 781 | .zmm18 = true, |
| | 782 | .zmm19 = true, |
| | 783 | .zmm20 = true, |
| | 784 | .zmm21 = true, |
| | 785 | .zmm22 = true, |
| | 786 | .zmm23 = true, |
| | 787 | .zmm24 = true, |
| | 788 | .zmm25 = true, |
| | 789 | .zmm26 = true, |
| | 790 | .zmm27 = true, |
| | 791 | .zmm28 = true, |
| | 792 | .zmm29 = true, |
| | 793 | .zmm30 = true, |
| | 794 | .zmm31 = true, |
| | 795 | .fpsr = true, |
| | 796 | .fpcr = true, |
| | 797 | .mxcsr = true, |
| | 798 | .rflags = true, |
| | 799 | .dirflag = true, |
| | 800 | .memory = true, |
| | 801 | }), |
| 641 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | 802 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), |
| 642 | }); | 803 | }); |
| 643 | } | 804 | } |
| ... | @@ -650,6 +811,12 @@ fn mainIdleEntry() callconv(.naked) void { | ... | @@ -650,6 +811,12 @@ fn mainIdleEntry() callconv(.naked) void { |
| 650 | : | 811 | : |
| 651 | : [mainIdle] "X" (&mainIdle), | 812 | : [mainIdle] "X" (&mainIdle), |
| 652 | ), | 813 | ), |
| | 814 | .aarch64 => asm volatile ( |
| | 815 | \\ ldr x0, [sp, #-8] |
| | 816 | \\ b %[mainIdle] |
| | 817 | : |
| | 818 | : [mainIdle] "X" (&mainIdle), |
| | 819 | ), |
| 653 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | 820 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), |
| 654 | } | 821 | } |
| 655 | } | 822 | } |
| ... | @@ -660,6 +827,11 @@ fn fiberEntry() callconv(.naked) void { | ... | @@ -660,6 +827,11 @@ fn fiberEntry() callconv(.naked) void { |
| 660 | \\ leaq 8(%%rsp), %%rdi | 827 | \\ leaq 8(%%rsp), %%rdi |
| 661 | \\ jmpq *(%%rsp) | 828 | \\ jmpq *(%%rsp) |
| 662 | ), | 829 | ), |
| | 830 | .aarch64 => asm volatile ( |
| | 831 | \\ mov x0, sp |
| | 832 | \\ ldr x2, [sp, #-8] |
| | 833 | \\ br x2 |
| | 834 | ), |
| 663 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | 835 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), |
| 664 | } | 836 | } |
| 665 | } | 837 | } |
| ... | @@ -718,7 +890,7 @@ fn async( | ... | @@ -718,7 +890,7 @@ fn async( |
| 718 | std.log.debug("allocated {*}", .{fiber}); | 890 | std.log.debug("allocated {*}", .{fiber}); |
| 719 | | 891 | |
| 720 | const closure: *AsyncClosure = .fromFiber(fiber); | 892 | const closure: *AsyncClosure = .fromFiber(fiber); |
| 721 | const stack_end: [*]usize = @alignCast(@ptrCast(closure)); | 893 | const stack_end: [*]align(16) usize = @alignCast(@ptrCast(closure)); |
| 722 | (stack_end - 1)[0..1].* = .{@intFromPtr(&AsyncClosure.call)}; | 894 | (stack_end - 1)[0..1].* = .{@intFromPtr(&AsyncClosure.call)}; |
| 723 | fiber.* = .{ | 895 | fiber.* = .{ |
| 724 | .required_align = {}, | 896 | .required_align = {}, |
| ... | @@ -728,6 +900,11 @@ fn async( | ... | @@ -728,6 +900,11 @@ fn async( |
| 728 | .rbp = 0, | 900 | .rbp = 0, |
| 729 | .rip = @intFromPtr(&fiberEntry), | 901 | .rip = @intFromPtr(&fiberEntry), |
| 730 | }, | 902 | }, |
| | 903 | .aarch64 => .{ |
| | 904 | .sp = @intFromPtr(stack_end), |
| | 905 | .fp = 0, |
| | 906 | .pc = @intFromPtr(&fiberEntry), |
| | 907 | }, |
| 731 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | 908 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), |
| 732 | }, | 909 | }, |
| 733 | .awaiter = null, | 910 | .awaiter = null, |
| ... | @@ -796,7 +973,7 @@ fn asyncDetached( | ... | @@ -796,7 +973,7 @@ fn asyncDetached( |
| 796 | const closure: *DetachedClosure = @ptrFromInt(Fiber.max_context_align.max(.of(DetachedClosure)).backward( | 973 | const closure: *DetachedClosure = @ptrFromInt(Fiber.max_context_align.max(.of(DetachedClosure)).backward( |
| 797 | @intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size, | 974 | @intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size, |
| 798 | ) - @sizeOf(DetachedClosure)); | 975 | ) - @sizeOf(DetachedClosure)); |
| 799 | const stack_end: [*]usize = @alignCast(@ptrCast(closure)); | 976 | const stack_end: [*]align(16) usize = @alignCast(@ptrCast(closure)); |
| 800 | (stack_end - 1)[0..1].* = .{@intFromPtr(&DetachedClosure.call)}; | 977 | (stack_end - 1)[0..1].* = .{@intFromPtr(&DetachedClosure.call)}; |
| 801 | fiber.* = .{ | 978 | fiber.* = .{ |
| 802 | .required_align = {}, | 979 | .required_align = {}, |
| ... | @@ -806,6 +983,11 @@ fn asyncDetached( | ... | @@ -806,6 +983,11 @@ fn asyncDetached( |
| 806 | .rbp = 0, | 983 | .rbp = 0, |
| 807 | .rip = @intFromPtr(&fiberEntry), | 984 | .rip = @intFromPtr(&fiberEntry), |
| 808 | }, | 985 | }, |
| | 986 | .aarch64 => .{ |
| | 987 | .sp = @intFromPtr(stack_end), |
| | 988 | .fp = 0, |
| | 989 | .pc = @intFromPtr(&fiberEntry), |
| | 990 | }, |
| 809 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | 991 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), |
| 810 | }, | 992 | }, |
| 811 | .awaiter = null, | 993 | .awaiter = null, |
Andrew Kelley <andrew@ziglang.org> 2025-07-11 17:51:55-07:00
lib