| ... | @@ -3097,20 +3097,47 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode | ... | @@ -3097,20 +3097,47 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode |
| 3097 | return return_inst; | 3097 | return return_inst; |
| 3098 | } | 3098 | } |
| 3099 | | 3099 | |
| 3100 | ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_field_ptr, return_value); | 3100 | IrBasicBlock *suspended_block = ir_create_basic_block(irb, scope, "Suspended"); |
| 3101 | IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node, | 3101 | IrBasicBlock *not_suspended_block = ir_create_basic_block(irb, scope, "NotSuspended"); |
| 3102 | get_optional_type(irb->codegen, irb->codegen->builtin_types.entry_promise)); | 3102 | IrBasicBlock *store_awaiter_block = ir_create_basic_block(irb, scope, "StoreAwaiter"); |
| 3103 | // TODO replace replacement_value with @intToPtr(?promise, 0x1) when it doesn't crash zig | 3103 | IrBasicBlock *check_canceled_block = ir_create_basic_block(irb, scope, "CheckCanceled"); |
| 3104 | IrInstruction *replacement_value = irb->exec->coro_handle; | 3104 | |
| 3105 | IrInstruction *maybe_await_handle = ir_build_atomic_rmw(irb, scope, node, | 3105 | IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, scope, node, 0x7); // 0b111 |
| 3106 | promise_type_val, irb->exec->coro_awaiter_field_ptr, nullptr, replacement_value, nullptr, | 3106 | IrInstruction *ptr_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000 |
| 3107 | AtomicRmwOp_xchg, AtomicOrderSeqCst); | 3107 | IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001 |
| 3108 | ir_build_store_ptr(irb, scope, node, irb->exec->await_handle_var_ptr, maybe_await_handle); | 3108 | IrInstruction *is_suspended_mask = ir_build_const_usize(irb, scope, node, 0x2); // 0b010 |
| 3109 | IrInstruction *is_non_null = ir_build_test_nonnull(irb, scope, node, maybe_await_handle); | 3109 | IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_promise); |
| 3110 | IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false); | 3110 | IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false); |
| 3111 | return ir_build_cond_br(irb, scope, node, is_non_null, irb->exec->coro_normal_final, irb->exec->coro_early_final, | 3111 | IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0); |
| 3112 | is_comptime); | 3112 | |
| 3113 | // the above blocks are rendered by ir_gen after the rest of codegen | 3113 | ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_field_ptr, return_value); |
| | 3114 | IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize); |
| | 3115 | IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| | 3116 | usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, ptr_mask, nullptr, |
| | 3117 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| | 3118 | |
| | 3119 | IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false); |
| | 3120 | IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false); |
| | 3121 | ir_build_cond_br(irb, scope, node, is_suspended_bool, suspended_block, not_suspended_block, is_comptime); |
| | 3122 | |
| | 3123 | ir_set_cursor_at_end_and_append_block(irb, suspended_block); |
| | 3124 | ir_build_unreachable(irb, scope, node); |
| | 3125 | |
| | 3126 | ir_set_cursor_at_end_and_append_block(irb, not_suspended_block); |
| | 3127 | IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false); |
| | 3128 | // if we ever add null checking safety to the ptrtoint instruction, it needs to be disabled here |
| | 3129 | IrInstruction *have_await_handle = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false); |
| | 3130 | ir_build_cond_br(irb, scope, node, have_await_handle, store_awaiter_block, check_canceled_block, is_comptime); |
| | 3131 | |
| | 3132 | ir_set_cursor_at_end_and_append_block(irb, store_awaiter_block); |
| | 3133 | IrInstruction *await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, await_handle_addr); |
| | 3134 | ir_build_store_ptr(irb, scope, node, irb->exec->await_handle_var_ptr, await_handle); |
| | 3135 | ir_build_br(irb, scope, node, irb->exec->coro_normal_final, is_comptime); |
| | 3136 | |
| | 3137 | ir_set_cursor_at_end_and_append_block(irb, check_canceled_block); |
| | 3138 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| | 3139 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| | 3140 | return ir_build_cond_br(irb, scope, node, is_canceled_bool, irb->exec->coro_final_cleanup_block, irb->exec->coro_early_final, is_comptime); |
| 3114 | } | 3141 | } |
| 3115 | | 3142 | |
| 3116 | static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, LVal lval) { | 3143 | static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, LVal lval) { |
| ... | @@ -6159,15 +6186,6 @@ static IrInstruction *ir_gen_return_from_block(IrBuilder *irb, Scope *break_scop | ... | @@ -6159,15 +6186,6 @@ static IrInstruction *ir_gen_return_from_block(IrBuilder *irb, Scope *break_scop |
| 6159 | return ir_build_br(irb, break_scope, node, dest_block, is_comptime); | 6186 | return ir_build_br(irb, break_scope, node, dest_block, is_comptime); |
| 6160 | } | 6187 | } |
| 6161 | | 6188 | |
| 6162 | static IrInstruction *ir_gen_break_from_suspend(IrBuilder *irb, Scope *break_scope, AstNode *node, ScopeSuspend *suspend_scope) { | | |
| 6163 | IrInstruction *is_comptime = ir_build_const_bool(irb, break_scope, node, false); | | |
| 6164 | | | |
| 6165 | IrBasicBlock *dest_block = suspend_scope->resume_block; | | |
| 6166 | ir_gen_defers_for_block(irb, break_scope, dest_block->scope, false); | | |
| 6167 | | | |
| 6168 | return ir_build_br(irb, break_scope, node, dest_block, is_comptime); | | |
| 6169 | } | | |
| 6170 | | | |
| 6171 | static IrInstruction *ir_gen_break(IrBuilder *irb, Scope *break_scope, AstNode *node) { | 6189 | static IrInstruction *ir_gen_break(IrBuilder *irb, Scope *break_scope, AstNode *node) { |
| 6172 | assert(node->type == NodeTypeBreak); | 6190 | assert(node->type == NodeTypeBreak); |
| 6173 | | 6191 | |
| ... | @@ -6208,12 +6226,8 @@ static IrInstruction *ir_gen_break(IrBuilder *irb, Scope *break_scope, AstNode * | ... | @@ -6208,12 +6226,8 @@ static IrInstruction *ir_gen_break(IrBuilder *irb, Scope *break_scope, AstNode * |
| 6208 | return ir_gen_return_from_block(irb, break_scope, node, this_block_scope); | 6226 | return ir_gen_return_from_block(irb, break_scope, node, this_block_scope); |
| 6209 | } | 6227 | } |
| 6210 | } else if (search_scope->id == ScopeIdSuspend) { | 6228 | } else if (search_scope->id == ScopeIdSuspend) { |
| 6211 | ScopeSuspend *this_suspend_scope = (ScopeSuspend *)search_scope; | 6229 | add_node_error(irb->codegen, node, buf_sprintf("cannot break out of suspend block")); |
| 6212 | if (node->data.break_expr.name != nullptr && | 6230 | return irb->codegen->invalid_instruction; |
| 6213 | (this_suspend_scope->name != nullptr && buf_eql_buf(node->data.break_expr.name, this_suspend_scope->name))) | | |
| 6214 | { | | |
| 6215 | return ir_gen_break_from_suspend(irb, break_scope, node, this_suspend_scope); | | |
| 6216 | } | | |
| 6217 | } | 6231 | } |
| 6218 | search_scope = search_scope->parent; | 6232 | search_scope = search_scope->parent; |
| 6219 | } | 6233 | } |
| ... | @@ -6649,30 +6663,150 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo | ... | @@ -6649,30 +6663,150 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo |
| 6649 | async_allocator_type_value, is_var_args); | 6663 | async_allocator_type_value, is_var_args); |
| 6650 | } | 6664 | } |
| 6651 | | 6665 | |
| 6652 | static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *parent_scope, AstNode *node) { | 6666 | static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode *node, |
| | 6667 | IrInstruction *target_inst, bool cancel_non_suspended, bool cancel_awaited) |
| | 6668 | { |
| | 6669 | IrBasicBlock *done_block = ir_create_basic_block(irb, scope, "CancelDone"); |
| | 6670 | IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled"); |
| | 6671 | IrBasicBlock *pre_return_block = ir_create_basic_block(irb, scope, "PreReturn"); |
| | 6672 | IrBasicBlock *post_return_block = ir_create_basic_block(irb, scope, "PostReturn"); |
| | 6673 | IrBasicBlock *do_cancel_block = ir_create_basic_block(irb, scope, "DoCancel"); |
| | 6674 | |
| | 6675 | IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0); |
| | 6676 | IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize); |
| | 6677 | IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false); |
| | 6678 | IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001 |
| | 6679 | IrInstruction *promise_T_type_val = ir_build_const_type(irb, scope, node, |
| | 6680 | get_promise_type(irb->codegen, irb->codegen->builtin_types.entry_void)); |
| | 6681 | IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, scope, node, 0x7); // 0b111 |
| | 6682 | IrInstruction *ptr_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000 |
| | 6683 | IrInstruction *await_mask = ir_build_const_usize(irb, scope, node, 0x4); // 0b100 |
| | 6684 | IrInstruction *is_suspended_mask = ir_build_const_usize(irb, scope, node, 0x2); // 0b010 |
| | 6685 | |
| | 6686 | // TODO relies on Zig not re-ordering fields |
| | 6687 | IrInstruction *casted_target_inst = ir_build_ptr_cast(irb, scope, node, promise_T_type_val, target_inst); |
| | 6688 | IrInstruction *coro_promise_ptr = ir_build_coro_promise(irb, scope, node, casted_target_inst); |
| | 6689 | Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME); |
| | 6690 | IrInstruction *atomic_state_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, |
| | 6691 | atomic_state_field_name); |
| | 6692 | |
| | 6693 | // set the is_canceled bit |
| | 6694 | IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| | 6695 | usize_type_val, atomic_state_ptr, nullptr, is_canceled_mask, nullptr, |
| | 6696 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| | 6697 | |
| | 6698 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| | 6699 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| | 6700 | ir_build_cond_br(irb, scope, node, is_canceled_bool, done_block, not_canceled_block, is_comptime); |
| | 6701 | |
| | 6702 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); |
| | 6703 | IrInstruction *awaiter_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false); |
| | 6704 | IrInstruction *is_returned_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpEq, awaiter_addr, ptr_mask, false); |
| | 6705 | ir_build_cond_br(irb, scope, node, is_returned_bool, post_return_block, pre_return_block, is_comptime); |
| | 6706 | |
| | 6707 | ir_set_cursor_at_end_and_append_block(irb, post_return_block); |
| | 6708 | if (cancel_awaited) { |
| | 6709 | ir_build_br(irb, scope, node, do_cancel_block, is_comptime); |
| | 6710 | } else { |
| | 6711 | IrInstruction *is_awaited_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, await_mask, false); |
| | 6712 | IrInstruction *is_awaited_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_awaited_value, zero, false); |
| | 6713 | ir_build_cond_br(irb, scope, node, is_awaited_bool, done_block, do_cancel_block, is_comptime); |
| | 6714 | } |
| | 6715 | |
| | 6716 | ir_set_cursor_at_end_and_append_block(irb, pre_return_block); |
| | 6717 | if (cancel_awaited) { |
| | 6718 | if (cancel_non_suspended) { |
| | 6719 | ir_build_br(irb, scope, node, do_cancel_block, is_comptime); |
| | 6720 | } else { |
| | 6721 | IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false); |
| | 6722 | IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false); |
| | 6723 | ir_build_cond_br(irb, scope, node, is_suspended_bool, do_cancel_block, done_block, is_comptime); |
| | 6724 | } |
| | 6725 | } else { |
| | 6726 | ir_build_br(irb, scope, node, done_block, is_comptime); |
| | 6727 | } |
| | 6728 | |
| | 6729 | ir_set_cursor_at_end_and_append_block(irb, do_cancel_block); |
| | 6730 | ir_build_cancel(irb, scope, node, target_inst); |
| | 6731 | ir_build_br(irb, scope, node, done_block, is_comptime); |
| | 6732 | |
| | 6733 | ir_set_cursor_at_end_and_append_block(irb, done_block); |
| | 6734 | return ir_build_const_void(irb, scope, node); |
| | 6735 | } |
| | 6736 | |
| | 6737 | static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *scope, AstNode *node) { |
| 6653 | assert(node->type == NodeTypeCancel); | 6738 | assert(node->type == NodeTypeCancel); |
| 6654 | | 6739 | |
| 6655 | IrInstruction *target_inst = ir_gen_node(irb, node->data.cancel_expr.expr, parent_scope); | 6740 | IrInstruction *target_inst = ir_gen_node(irb, node->data.cancel_expr.expr, scope); |
| 6656 | if (target_inst == irb->codegen->invalid_instruction) | 6741 | if (target_inst == irb->codegen->invalid_instruction) |
| 6657 | return irb->codegen->invalid_instruction; | 6742 | return irb->codegen->invalid_instruction; |
| 6658 | | 6743 | |
| 6659 | return ir_build_cancel(irb, parent_scope, node, target_inst); | 6744 | return ir_gen_cancel_target(irb, scope, node, target_inst, false, true); |
| | 6745 | } |
| | 6746 | |
| | 6747 | static IrInstruction *ir_gen_resume_target(IrBuilder *irb, Scope *scope, AstNode *node, |
| | 6748 | IrInstruction *target_inst) |
| | 6749 | { |
| | 6750 | IrBasicBlock *done_block = ir_create_basic_block(irb, scope, "ResumeDone"); |
| | 6751 | IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled"); |
| | 6752 | IrBasicBlock *suspended_block = ir_create_basic_block(irb, scope, "IsSuspended"); |
| | 6753 | IrBasicBlock *not_suspended_block = ir_create_basic_block(irb, scope, "IsNotSuspended"); |
| | 6754 | |
| | 6755 | IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0); |
| | 6756 | IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001 |
| | 6757 | IrInstruction *is_suspended_mask = ir_build_const_usize(irb, scope, node, 0x2); // 0b010 |
| | 6758 | IrInstruction *and_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, is_suspended_mask); |
| | 6759 | IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false); |
| | 6760 | IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize); |
| | 6761 | IrInstruction *promise_T_type_val = ir_build_const_type(irb, scope, node, |
| | 6762 | get_promise_type(irb->codegen, irb->codegen->builtin_types.entry_void)); |
| | 6763 | |
| | 6764 | // TODO relies on Zig not re-ordering fields |
| | 6765 | IrInstruction *casted_target_inst = ir_build_ptr_cast(irb, scope, node, promise_T_type_val, target_inst); |
| | 6766 | IrInstruction *coro_promise_ptr = ir_build_coro_promise(irb, scope, node, casted_target_inst); |
| | 6767 | Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME); |
| | 6768 | IrInstruction *atomic_state_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, |
| | 6769 | atomic_state_field_name); |
| | 6770 | |
| | 6771 | // clear the is_suspended bit |
| | 6772 | IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| | 6773 | usize_type_val, atomic_state_ptr, nullptr, and_mask, nullptr, |
| | 6774 | AtomicRmwOp_and, AtomicOrderSeqCst); |
| | 6775 | |
| | 6776 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| | 6777 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| | 6778 | ir_build_cond_br(irb, scope, node, is_canceled_bool, done_block, not_canceled_block, is_comptime); |
| | 6779 | |
| | 6780 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); |
| | 6781 | IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false); |
| | 6782 | IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false); |
| | 6783 | ir_build_cond_br(irb, scope, node, is_suspended_bool, suspended_block, not_suspended_block, is_comptime); |
| | 6784 | |
| | 6785 | ir_set_cursor_at_end_and_append_block(irb, not_suspended_block); |
| | 6786 | ir_build_unreachable(irb, scope, node); |
| | 6787 | |
| | 6788 | ir_set_cursor_at_end_and_append_block(irb, suspended_block); |
| | 6789 | ir_build_coro_resume(irb, scope, node, target_inst); |
| | 6790 | ir_build_br(irb, scope, node, done_block, is_comptime); |
| | 6791 | |
| | 6792 | ir_set_cursor_at_end_and_append_block(irb, done_block); |
| | 6793 | return ir_build_const_void(irb, scope, node); |
| 6660 | } | 6794 | } |
| 6661 | | 6795 | |
| 6662 | static IrInstruction *ir_gen_resume(IrBuilder *irb, Scope *parent_scope, AstNode *node) { | 6796 | static IrInstruction *ir_gen_resume(IrBuilder *irb, Scope *scope, AstNode *node) { |
| 6663 | assert(node->type == NodeTypeResume); | 6797 | assert(node->type == NodeTypeResume); |
| 6664 | | 6798 | |
| 6665 | IrInstruction *target_inst = ir_gen_node(irb, node->data.resume_expr.expr, parent_scope); | 6799 | IrInstruction *target_inst = ir_gen_node(irb, node->data.resume_expr.expr, scope); |
| 6666 | if (target_inst == irb->codegen->invalid_instruction) | 6800 | if (target_inst == irb->codegen->invalid_instruction) |
| 6667 | return irb->codegen->invalid_instruction; | 6801 | return irb->codegen->invalid_instruction; |
| 6668 | | 6802 | |
| 6669 | return ir_build_coro_resume(irb, parent_scope, node, target_inst); | 6803 | return ir_gen_resume_target(irb, scope, node, target_inst); |
| 6670 | } | 6804 | } |
| 6671 | | 6805 | |
| 6672 | static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, AstNode *node) { | 6806 | static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *node) { |
| 6673 | assert(node->type == NodeTypeAwaitExpr); | 6807 | assert(node->type == NodeTypeAwaitExpr); |
| 6674 | | 6808 | |
| 6675 | IrInstruction *target_inst = ir_gen_node(irb, node->data.await_expr.expr, parent_scope); | 6809 | IrInstruction *target_inst = ir_gen_node(irb, node->data.await_expr.expr, scope); |
| 6676 | if (target_inst == irb->codegen->invalid_instruction) | 6810 | if (target_inst == irb->codegen->invalid_instruction) |
| 6677 | return irb->codegen->invalid_instruction; | 6811 | return irb->codegen->invalid_instruction; |
| 6678 | | 6812 | |
| ... | @@ -6686,7 +6820,7 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast | ... | @@ -6686,7 +6820,7 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast |
| 6686 | return irb->codegen->invalid_instruction; | 6820 | return irb->codegen->invalid_instruction; |
| 6687 | } | 6821 | } |
| 6688 | | 6822 | |
| 6689 | ScopeDeferExpr *scope_defer_expr = get_scope_defer_expr(parent_scope); | 6823 | ScopeDeferExpr *scope_defer_expr = get_scope_defer_expr(scope); |
| 6690 | if (scope_defer_expr) { | 6824 | if (scope_defer_expr) { |
| 6691 | if (!scope_defer_expr->reported_err) { | 6825 | if (!scope_defer_expr->reported_err) { |
| 6692 | add_node_error(irb->codegen, node, buf_sprintf("cannot await inside defer expression")); | 6826 | add_node_error(irb->codegen, node, buf_sprintf("cannot await inside defer expression")); |
| ... | @@ -6697,81 +6831,157 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast | ... | @@ -6697,81 +6831,157 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast |
| 6697 | | 6831 | |
| 6698 | Scope *outer_scope = irb->exec->begin_scope; | 6832 | Scope *outer_scope = irb->exec->begin_scope; |
| 6699 | | 6833 | |
| 6700 | IrInstruction *coro_promise_ptr = ir_build_coro_promise(irb, parent_scope, node, target_inst); | 6834 | IrInstruction *coro_promise_ptr = ir_build_coro_promise(irb, scope, node, target_inst); |
| 6701 | Buf *result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); | 6835 | Buf *result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); |
| 6702 | IrInstruction *result_ptr_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_ptr_field_name); | 6836 | IrInstruction *result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name); |
| 6703 | | 6837 | |
| 6704 | if (irb->codegen->have_err_ret_tracing) { | 6838 | if (irb->codegen->have_err_ret_tracing) { |
| 6705 | IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, parent_scope, node, IrInstructionErrorReturnTrace::NonNull); | 6839 | IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::NonNull); |
| 6706 | Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); | 6840 | Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); |
| 6707 | IrInstruction *err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); | 6841 | IrInstruction *err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); |
| 6708 | ir_build_store_ptr(irb, parent_scope, node, err_ret_trace_ptr_field_ptr, err_ret_trace_ptr); | 6842 | ir_build_store_ptr(irb, scope, node, err_ret_trace_ptr_field_ptr, err_ret_trace_ptr); |
| 6709 | } | 6843 | } |
| 6710 | | 6844 | |
| 6711 | Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); | 6845 | IrBasicBlock *already_awaited_block = ir_create_basic_block(irb, scope, "AlreadyAwaited"); |
| 6712 | IrInstruction *awaiter_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, | 6846 | IrBasicBlock *not_awaited_block = ir_create_basic_block(irb, scope, "NotAwaited"); |
| 6713 | awaiter_handle_field_name); | 6847 | IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled"); |
| 6714 | | 6848 | IrBasicBlock *yes_suspend_block = ir_create_basic_block(irb, scope, "YesSuspend"); |
| 6715 | IrInstruction *const_bool_false = ir_build_const_bool(irb, parent_scope, node, false); | 6849 | IrBasicBlock *no_suspend_block = ir_create_basic_block(irb, scope, "NoSuspend"); |
| 6716 | VariableTableEntry *result_var = ir_create_var(irb, node, parent_scope, nullptr, | 6850 | IrBasicBlock *merge_block = ir_create_basic_block(irb, scope, "MergeSuspend"); |
| | 6851 | IrBasicBlock *cleanup_block = ir_create_basic_block(irb, scope, "SuspendCleanup"); |
| | 6852 | IrBasicBlock *resume_block = ir_create_basic_block(irb, scope, "SuspendResume"); |
| | 6853 | IrBasicBlock *cancel_target_block = ir_create_basic_block(irb, scope, "CancelTarget"); |
| | 6854 | IrBasicBlock *do_cancel_block = ir_create_basic_block(irb, scope, "DoCancel"); |
| | 6855 | IrBasicBlock *do_defers_block = ir_create_basic_block(irb, scope, "DoDefers"); |
| | 6856 | IrBasicBlock *destroy_block = ir_create_basic_block(irb, scope, "DestroyBlock"); |
| | 6857 | IrBasicBlock *my_suspended_block = ir_create_basic_block(irb, scope, "AlreadySuspended"); |
| | 6858 | IrBasicBlock *my_not_suspended_block = ir_create_basic_block(irb, scope, "NotAlreadySuspended"); |
| | 6859 | IrBasicBlock *do_suspend_block = ir_create_basic_block(irb, scope, "DoSuspend"); |
| | 6860 | |
| | 6861 | Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME); |
| | 6862 | IrInstruction *atomic_state_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, |
| | 6863 | atomic_state_field_name); |
| | 6864 | |
| | 6865 | IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_promise); |
| | 6866 | IrInstruction *const_bool_false = ir_build_const_bool(irb, scope, node, false); |
| | 6867 | IrInstruction *undefined_value = ir_build_const_undefined(irb, scope, node); |
| | 6868 | IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize); |
| | 6869 | IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0); |
| | 6870 | IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, scope, node, 0x7); // 0b111 |
| | 6871 | IrInstruction *ptr_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000 |
| | 6872 | IrInstruction *await_mask = ir_build_const_usize(irb, scope, node, 0x4); // 0b100 |
| | 6873 | IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001 |
| | 6874 | IrInstruction *is_suspended_mask = ir_build_const_usize(irb, scope, node, 0x2); // 0b010 |
| | 6875 | |
| | 6876 | VariableTableEntry *result_var = ir_create_var(irb, node, scope, nullptr, |
| 6717 | false, false, true, const_bool_false); | 6877 | false, false, true, const_bool_false); |
| 6718 | IrInstruction *undefined_value = ir_build_const_undefined(irb, parent_scope, node); | 6878 | IrInstruction *target_promise_type = ir_build_typeof(irb, scope, node, target_inst); |
| 6719 | IrInstruction *target_promise_type = ir_build_typeof(irb, parent_scope, node, target_inst); | 6879 | IrInstruction *promise_result_type = ir_build_promise_result_type(irb, scope, node, target_promise_type); |
| 6720 | IrInstruction *promise_result_type = ir_build_promise_result_type(irb, parent_scope, node, target_promise_type); | 6880 | ir_build_await_bookkeeping(irb, scope, node, promise_result_type); |
| 6721 | ir_build_await_bookkeeping(irb, parent_scope, node, promise_result_type); | 6881 | ir_build_var_decl(irb, scope, node, result_var, promise_result_type, nullptr, undefined_value); |
| 6722 | ir_build_var_decl(irb, parent_scope, node, result_var, promise_result_type, nullptr, undefined_value); | 6882 | IrInstruction *my_result_var_ptr = ir_build_var_ptr(irb, scope, node, result_var); |
| 6723 | IrInstruction *my_result_var_ptr = ir_build_var_ptr(irb, parent_scope, node, result_var); | 6883 | ir_build_store_ptr(irb, scope, node, result_ptr_field_ptr, my_result_var_ptr); |
| 6724 | ir_build_store_ptr(irb, parent_scope, node, result_ptr_field_ptr, my_result_var_ptr); | 6884 | IrInstruction *save_token = ir_build_coro_save(irb, scope, node, irb->exec->coro_handle); |
| 6725 | IrInstruction *save_token = ir_build_coro_save(irb, parent_scope, node, irb->exec->coro_handle); | 6885 | |
| 6726 | IrInstruction *promise_type_val = ir_build_const_type(irb, parent_scope, node, | 6886 | IrInstruction *coro_handle_addr = ir_build_ptr_to_int(irb, scope, node, irb->exec->coro_handle); |
| 6727 | get_optional_type(irb->codegen, irb->codegen->builtin_types.entry_promise)); | 6887 | IrInstruction *mask_bits = ir_build_bin_op(irb, scope, node, IrBinOpBinOr, coro_handle_addr, await_mask, false); |
| 6728 | IrInstruction *maybe_await_handle = ir_build_atomic_rmw(irb, parent_scope, node, | 6888 | IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| 6729 | promise_type_val, awaiter_field_ptr, nullptr, irb->exec->coro_handle, nullptr, | 6889 | usize_type_val, atomic_state_ptr, nullptr, mask_bits, nullptr, |
| 6730 | AtomicRmwOp_xchg, AtomicOrderSeqCst); | 6890 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| 6731 | IrInstruction *is_non_null = ir_build_test_nonnull(irb, parent_scope, node, maybe_await_handle); | 6891 | |
| 6732 | IrBasicBlock *yes_suspend_block = ir_create_basic_block(irb, parent_scope, "YesSuspend"); | 6892 | IrInstruction *is_awaited_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, await_mask, false); |
| 6733 | IrBasicBlock *no_suspend_block = ir_create_basic_block(irb, parent_scope, "NoSuspend"); | 6893 | IrInstruction *is_awaited_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_awaited_value, zero, false); |
| 6734 | IrBasicBlock *merge_block = ir_create_basic_block(irb, parent_scope, "MergeSuspend"); | 6894 | ir_build_cond_br(irb, scope, node, is_awaited_bool, already_awaited_block, not_awaited_block, const_bool_false); |
| 6735 | ir_build_cond_br(irb, parent_scope, node, is_non_null, no_suspend_block, yes_suspend_block, const_bool_false); | 6895 | |
| | 6896 | ir_set_cursor_at_end_and_append_block(irb, already_awaited_block); |
| | 6897 | ir_build_unreachable(irb, scope, node); |
| | 6898 | |
| | 6899 | ir_set_cursor_at_end_and_append_block(irb, not_awaited_block); |
| | 6900 | IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false); |
| | 6901 | IrInstruction *is_non_null = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false); |
| | 6902 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| | 6903 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| | 6904 | ir_build_cond_br(irb, scope, node, is_canceled_bool, cancel_target_block, not_canceled_block, const_bool_false); |
| | 6905 | |
| | 6906 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); |
| | 6907 | ir_build_cond_br(irb, scope, node, is_non_null, no_suspend_block, yes_suspend_block, const_bool_false); |
| | 6908 | |
| | 6909 | ir_set_cursor_at_end_and_append_block(irb, cancel_target_block); |
| | 6910 | ir_build_cancel(irb, scope, node, target_inst); |
| | 6911 | ir_mark_gen(ir_build_br(irb, scope, node, cleanup_block, const_bool_false)); |
| 6736 | | 6912 | |
| 6737 | ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); | 6913 | ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); |
| 6738 | if (irb->codegen->have_err_ret_tracing) { | 6914 | if (irb->codegen->have_err_ret_tracing) { |
| 6739 | Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); | 6915 | Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); |
| 6740 | IrInstruction *src_err_ret_trace_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_field_name); | 6916 | IrInstruction *src_err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name); |
| 6741 | IrInstruction *dest_err_ret_trace_ptr = ir_build_error_return_trace(irb, parent_scope, node, IrInstructionErrorReturnTrace::NonNull); | 6917 | IrInstruction *dest_err_ret_trace_ptr = ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::NonNull); |
| 6742 | ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, src_err_ret_trace_ptr, dest_err_ret_trace_ptr); | 6918 | ir_build_merge_err_ret_traces(irb, scope, node, coro_promise_ptr, src_err_ret_trace_ptr, dest_err_ret_trace_ptr); |
| 6743 | } | 6919 | } |
| 6744 | Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); | 6920 | Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); |
| 6745 | IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_field_name); | 6921 | IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); |
| 6746 | // If the type of the result handle_is_ptr then this does not actually perform a load. But we need it to, | 6922 | // If the type of the result handle_is_ptr then this does not actually perform a load. But we need it to, |
| 6747 | // because we're about to destroy the memory. So we store it into our result variable. | 6923 | // because we're about to destroy the memory. So we store it into our result variable. |
| 6748 | IrInstruction *no_suspend_result = ir_build_load_ptr(irb, parent_scope, node, promise_result_ptr); | 6924 | IrInstruction *no_suspend_result = ir_build_load_ptr(irb, scope, node, promise_result_ptr); |
| 6749 | ir_build_store_ptr(irb, parent_scope, node, my_result_var_ptr, no_suspend_result); | 6925 | ir_build_store_ptr(irb, scope, node, my_result_var_ptr, no_suspend_result); |
| 6750 | ir_build_cancel(irb, parent_scope, node, target_inst); | 6926 | ir_build_cancel(irb, scope, node, target_inst); |
| 6751 | ir_build_br(irb, parent_scope, node, merge_block, const_bool_false); | 6927 | ir_build_br(irb, scope, node, merge_block, const_bool_false); |
| | 6928 | |
| 6752 | | 6929 | |
| 6753 | ir_set_cursor_at_end_and_append_block(irb, yes_suspend_block); | 6930 | ir_set_cursor_at_end_and_append_block(irb, yes_suspend_block); |
| 6754 | IrInstruction *suspend_code = ir_build_coro_suspend(irb, parent_scope, node, save_token, const_bool_false); | 6931 | IrInstruction *my_prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| 6755 | IrBasicBlock *cleanup_block = ir_create_basic_block(irb, parent_scope, "SuspendCleanup"); | 6932 | usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, is_suspended_mask, nullptr, |
| 6756 | IrBasicBlock *resume_block = ir_create_basic_block(irb, parent_scope, "SuspendResume"); | 6933 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| | 6934 | IrInstruction *my_is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, my_prev_atomic_value, is_suspended_mask, false); |
| | 6935 | IrInstruction *my_is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, my_is_suspended_value, zero, false); |
| | 6936 | ir_build_cond_br(irb, scope, node, my_is_suspended_bool, my_suspended_block, my_not_suspended_block, const_bool_false); |
| | 6937 | |
| | 6938 | ir_set_cursor_at_end_and_append_block(irb, my_suspended_block); |
| | 6939 | ir_build_unreachable(irb, scope, node); |
| | 6940 | |
| | 6941 | ir_set_cursor_at_end_and_append_block(irb, my_not_suspended_block); |
| | 6942 | IrInstruction *my_is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, my_prev_atomic_value, is_canceled_mask, false); |
| | 6943 | IrInstruction *my_is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, my_is_canceled_value, zero, false); |
| | 6944 | ir_build_cond_br(irb, scope, node, my_is_canceled_bool, cleanup_block, do_suspend_block, const_bool_false); |
| | 6945 | |
| | 6946 | ir_set_cursor_at_end_and_append_block(irb, do_suspend_block); |
| | 6947 | IrInstruction *suspend_code = ir_build_coro_suspend(irb, scope, node, save_token, const_bool_false); |
| 6757 | | 6948 | |
| 6758 | IrInstructionSwitchBrCase *cases = allocate<IrInstructionSwitchBrCase>(2); | 6949 | IrInstructionSwitchBrCase *cases = allocate<IrInstructionSwitchBrCase>(2); |
| 6759 | cases[0].value = ir_build_const_u8(irb, parent_scope, node, 0); | 6950 | cases[0].value = ir_build_const_u8(irb, scope, node, 0); |
| 6760 | cases[0].block = resume_block; | 6951 | cases[0].block = resume_block; |
| 6761 | cases[1].value = ir_build_const_u8(irb, parent_scope, node, 1); | 6952 | cases[1].value = ir_build_const_u8(irb, scope, node, 1); |
| 6762 | cases[1].block = cleanup_block; | 6953 | cases[1].block = destroy_block; |
| 6763 | ir_build_switch_br(irb, parent_scope, node, suspend_code, irb->exec->coro_suspend_block, | 6954 | ir_build_switch_br(irb, scope, node, suspend_code, irb->exec->coro_suspend_block, |
| 6764 | 2, cases, const_bool_false, nullptr); | 6955 | 2, cases, const_bool_false, nullptr); |
| 6765 | | 6956 | |
| | 6957 | ir_set_cursor_at_end_and_append_block(irb, destroy_block); |
| | 6958 | ir_gen_cancel_target(irb, scope, node, target_inst, false, true); |
| | 6959 | ir_mark_gen(ir_build_br(irb, scope, node, cleanup_block, const_bool_false)); |
| | 6960 | |
| 6766 | ir_set_cursor_at_end_and_append_block(irb, cleanup_block); | 6961 | ir_set_cursor_at_end_and_append_block(irb, cleanup_block); |
| 6767 | ir_gen_defers_for_block(irb, parent_scope, outer_scope, true); | 6962 | IrInstruction *my_mask_bits = ir_build_bin_op(irb, scope, node, IrBinOpBinOr, ptr_mask, is_canceled_mask, false); |
| 6768 | ir_mark_gen(ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false)); | 6963 | IrInstruction *b_my_prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| | 6964 | usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, my_mask_bits, nullptr, |
| | 6965 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| | 6966 | IrInstruction *my_await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, b_my_prev_atomic_value, ptr_mask, false); |
| | 6967 | IrInstruction *dont_have_my_await_handle = ir_build_bin_op(irb, scope, node, IrBinOpCmpEq, my_await_handle_addr, zero, false); |
| | 6968 | IrInstruction *dont_destroy_ourselves = ir_build_bin_op(irb, scope, node, IrBinOpBoolAnd, dont_have_my_await_handle, is_canceled_bool, false); |
| | 6969 | ir_build_cond_br(irb, scope, node, dont_have_my_await_handle, do_defers_block, do_cancel_block, const_bool_false); |
| | 6970 | |
| | 6971 | ir_set_cursor_at_end_and_append_block(irb, do_cancel_block); |
| | 6972 | IrInstruction *my_await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, my_await_handle_addr); |
| | 6973 | ir_gen_cancel_target(irb, scope, node, my_await_handle, true, false); |
| | 6974 | ir_mark_gen(ir_build_br(irb, scope, node, do_defers_block, const_bool_false)); |
| | 6975 | |
| | 6976 | ir_set_cursor_at_end_and_append_block(irb, do_defers_block); |
| | 6977 | ir_gen_defers_for_block(irb, scope, outer_scope, true); |
| | 6978 | ir_mark_gen(ir_build_cond_br(irb, scope, node, dont_destroy_ourselves, irb->exec->coro_early_final, irb->exec->coro_final_cleanup_block, const_bool_false)); |
| 6769 | | 6979 | |
| 6770 | ir_set_cursor_at_end_and_append_block(irb, resume_block); | 6980 | ir_set_cursor_at_end_and_append_block(irb, resume_block); |
| 6771 | ir_build_br(irb, parent_scope, node, merge_block, const_bool_false); | 6981 | ir_build_br(irb, scope, node, merge_block, const_bool_false); |
| 6772 | | 6982 | |
| 6773 | ir_set_cursor_at_end_and_append_block(irb, merge_block); | 6983 | ir_set_cursor_at_end_and_append_block(irb, merge_block); |
| 6774 | return ir_build_load_ptr(irb, parent_scope, node, my_result_var_ptr); | 6984 | return ir_build_load_ptr(irb, scope, node, my_result_var_ptr); |
| 6775 | } | 6985 | } |
| 6776 | | 6986 | |
| 6777 | static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNode *node) { | 6987 | static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNode *node) { |
| ... | @@ -6810,9 +7020,52 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod | ... | @@ -6810,9 +7020,52 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod |
| 6810 | | 7020 | |
| 6811 | IrBasicBlock *cleanup_block = ir_create_basic_block(irb, parent_scope, "SuspendCleanup"); | 7021 | IrBasicBlock *cleanup_block = ir_create_basic_block(irb, parent_scope, "SuspendCleanup"); |
| 6812 | IrBasicBlock *resume_block = ir_create_basic_block(irb, parent_scope, "SuspendResume"); | 7022 | IrBasicBlock *resume_block = ir_create_basic_block(irb, parent_scope, "SuspendResume"); |
| 6813 | | 7023 | IrBasicBlock *suspended_block = ir_create_basic_block(irb, parent_scope, "AlreadySuspended"); |
| 6814 | IrInstruction *suspend_code; | 7024 | IrBasicBlock *canceled_block = ir_create_basic_block(irb, parent_scope, "IsCanceled"); |
| | 7025 | IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, parent_scope, "NotCanceled"); |
| | 7026 | IrBasicBlock *not_suspended_block = ir_create_basic_block(irb, parent_scope, "NotAlreadySuspended"); |
| | 7027 | IrBasicBlock *cancel_awaiter_block = ir_create_basic_block(irb, parent_scope, "CancelAwaiter"); |
| | 7028 | |
| | 7029 | IrInstruction *promise_type_val = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_promise); |
| | 7030 | IrInstruction *const_bool_true = ir_build_const_bool(irb, parent_scope, node, true); |
| 6815 | IrInstruction *const_bool_false = ir_build_const_bool(irb, parent_scope, node, false); | 7031 | IrInstruction *const_bool_false = ir_build_const_bool(irb, parent_scope, node, false); |
| | 7032 | IrInstruction *usize_type_val = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_usize); |
| | 7033 | IrInstruction *is_canceled_mask = ir_build_const_usize(irb, parent_scope, node, 0x1); // 0b001 |
| | 7034 | IrInstruction *is_suspended_mask = ir_build_const_usize(irb, parent_scope, node, 0x2); // 0b010 |
| | 7035 | IrInstruction *zero = ir_build_const_usize(irb, parent_scope, node, 0); |
| | 7036 | IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, parent_scope, node, 0x7); // 0b111 |
| | 7037 | IrInstruction *ptr_mask = ir_build_un_op(irb, parent_scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000 |
| | 7038 | |
| | 7039 | IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, parent_scope, node, |
| | 7040 | usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, is_suspended_mask, nullptr, |
| | 7041 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| | 7042 | |
| | 7043 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, parent_scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| | 7044 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| | 7045 | ir_build_cond_br(irb, parent_scope, node, is_canceled_bool, canceled_block, not_canceled_block, const_bool_false); |
| | 7046 | |
| | 7047 | ir_set_cursor_at_end_and_append_block(irb, canceled_block); |
| | 7048 | IrInstruction *await_handle_addr = ir_build_bin_op(irb, parent_scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false); |
| | 7049 | IrInstruction *have_await_handle = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false); |
| | 7050 | IrBasicBlock *post_canceled_block = irb->current_basic_block; |
| | 7051 | ir_build_cond_br(irb, parent_scope, node, have_await_handle, cancel_awaiter_block, cleanup_block, const_bool_false); |
| | 7052 | |
| | 7053 | ir_set_cursor_at_end_and_append_block(irb, cancel_awaiter_block); |
| | 7054 | IrInstruction *await_handle = ir_build_int_to_ptr(irb, parent_scope, node, promise_type_val, await_handle_addr); |
| | 7055 | ir_gen_cancel_target(irb, parent_scope, node, await_handle, true, false); |
| | 7056 | IrBasicBlock *post_cancel_awaiter_block = irb->current_basic_block; |
| | 7057 | ir_build_br(irb, parent_scope, node, cleanup_block, const_bool_false); |
| | 7058 | |
| | 7059 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); |
| | 7060 | IrInstruction *is_suspended_value = ir_build_bin_op(irb, parent_scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false); |
| | 7061 | IrInstruction *is_suspended_bool = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false); |
| | 7062 | ir_build_cond_br(irb, parent_scope, node, is_suspended_bool, suspended_block, not_suspended_block, const_bool_false); |
| | 7063 | |
| | 7064 | ir_set_cursor_at_end_and_append_block(irb, suspended_block); |
| | 7065 | ir_build_unreachable(irb, parent_scope, node); |
| | 7066 | |
| | 7067 | ir_set_cursor_at_end_and_append_block(irb, not_suspended_block); |
| | 7068 | IrInstruction *suspend_code; |
| 6816 | if (node->data.suspend.block == nullptr) { | 7069 | if (node->data.suspend.block == nullptr) { |
| 6817 | suspend_code = ir_build_coro_suspend(irb, parent_scope, node, nullptr, const_bool_false); | 7070 | suspend_code = ir_build_coro_suspend(irb, parent_scope, node, nullptr, const_bool_false); |
| 6818 | } else { | 7071 | } else { |
| ... | @@ -6840,13 +7093,20 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod | ... | @@ -6840,13 +7093,20 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod |
| 6840 | cases[0].value = ir_mark_gen(ir_build_const_u8(irb, parent_scope, node, 0)); | 7093 | cases[0].value = ir_mark_gen(ir_build_const_u8(irb, parent_scope, node, 0)); |
| 6841 | cases[0].block = resume_block; | 7094 | cases[0].block = resume_block; |
| 6842 | cases[1].value = ir_mark_gen(ir_build_const_u8(irb, parent_scope, node, 1)); | 7095 | cases[1].value = ir_mark_gen(ir_build_const_u8(irb, parent_scope, node, 1)); |
| 6843 | cases[1].block = cleanup_block; | 7096 | cases[1].block = canceled_block; |
| 6844 | ir_mark_gen(ir_build_switch_br(irb, parent_scope, node, suspend_code, irb->exec->coro_suspend_block, | 7097 | ir_mark_gen(ir_build_switch_br(irb, parent_scope, node, suspend_code, irb->exec->coro_suspend_block, |
| 6845 | 2, cases, const_bool_false, nullptr)); | 7098 | 2, cases, const_bool_false, nullptr)); |
| 6846 | | 7099 | |
| 6847 | ir_set_cursor_at_end_and_append_block(irb, cleanup_block); | 7100 | ir_set_cursor_at_end_and_append_block(irb, cleanup_block); |
| | 7101 | IrBasicBlock **incoming_blocks = allocate<IrBasicBlock *>(2); |
| | 7102 | IrInstruction **incoming_values = allocate<IrInstruction *>(2); |
| | 7103 | incoming_blocks[0] = post_canceled_block; |
| | 7104 | incoming_values[0] = const_bool_true; |
| | 7105 | incoming_blocks[1] = post_cancel_awaiter_block; |
| | 7106 | incoming_values[1] = const_bool_false; |
| | 7107 | IrInstruction *destroy_ourselves = ir_build_phi(irb, parent_scope, node, 2, incoming_blocks, incoming_values); |
| 6848 | ir_gen_defers_for_block(irb, parent_scope, outer_scope, true); | 7108 | ir_gen_defers_for_block(irb, parent_scope, outer_scope, true); |
| 6849 | ir_mark_gen(ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false)); | 7109 | ir_mark_gen(ir_build_cond_br(irb, parent_scope, node, destroy_ourselves, irb->exec->coro_final_cleanup_block, irb->exec->coro_early_final, const_bool_false)); |
| 6850 | | 7110 | |
| 6851 | ir_set_cursor_at_end_and_append_block(irb, resume_block); | 7111 | ir_set_cursor_at_end_and_append_block(irb, resume_block); |
| 6852 | return ir_mark_gen(ir_build_const_void(irb, parent_scope, node)); | 7112 | return ir_mark_gen(ir_build_const_void(irb, parent_scope, node)); |
| ... | @@ -7087,10 +7347,11 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec | ... | @@ -7087,10 +7347,11 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec |
| 7087 | IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, maybe_coro_mem_ptr); | 7347 | IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, maybe_coro_mem_ptr); |
| 7088 | irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr); | 7348 | irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr); |
| 7089 | | 7349 | |
| 7090 | Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); | 7350 | Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME); |
| 7091 | irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, | 7351 | irb->exec->atomic_state_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, |
| 7092 | awaiter_handle_field_name); | 7352 | atomic_state_field_name); |
| 7093 | ir_build_store_ptr(irb, scope, node, irb->exec->coro_awaiter_field_ptr, null_value); | 7353 | IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0); |
| | 7354 | ir_build_store_ptr(irb, scope, node, irb->exec->atomic_state_field_ptr, zero); |
| 7094 | Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); | 7355 | Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); |
| 7095 | irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); | 7356 | irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); |
| 7096 | result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); | 7357 | result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); |
| ... | @@ -7108,7 +7369,6 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec | ... | @@ -7108,7 +7369,6 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec |
| 7108 | // coordinate with builtin.zig | 7369 | // coordinate with builtin.zig |
| 7109 | Buf *index_name = buf_create_from_str("index"); | 7370 | Buf *index_name = buf_create_from_str("index"); |
| 7110 | IrInstruction *index_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, index_name); | 7371 | IrInstruction *index_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, index_name); |
| 7111 | IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0); | | |
| 7112 | ir_build_store_ptr(irb, scope, node, index_ptr, zero); | 7372 | ir_build_store_ptr(irb, scope, node, index_ptr, zero); |
| 7113 | | 7373 | |
| 7114 | Buf *instruction_addresses_name = buf_create_from_str("instruction_addresses"); | 7374 | Buf *instruction_addresses_name = buf_create_from_str("instruction_addresses"); |
| ... | @@ -7231,7 +7491,7 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec | ... | @@ -7231,7 +7491,7 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec |
| 7231 | ir_build_cond_br(irb, scope, node, resume_awaiter, resume_block, irb->exec->coro_suspend_block, const_bool_false); | 7491 | ir_build_cond_br(irb, scope, node, resume_awaiter, resume_block, irb->exec->coro_suspend_block, const_bool_false); |
| 7232 | | 7492 | |
| 7233 | ir_set_cursor_at_end_and_append_block(irb, resume_block); | 7493 | ir_set_cursor_at_end_and_append_block(irb, resume_block); |
| 7234 | ir_build_coro_resume(irb, scope, node, awaiter_handle); | 7494 | ir_gen_resume_target(irb, scope, node, awaiter_handle); |
| 7235 | ir_build_br(irb, scope, node, irb->exec->coro_suspend_block, const_bool_false); | 7495 | ir_build_br(irb, scope, node, irb->exec->coro_suspend_block, const_bool_false); |
| 7236 | } | 7496 | } |
| 7237 | | 7497 | |