| ... | ... | @@ -6663,7 +6663,9 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo |
| 6663 | 6663 | async_allocator_type_value, is_var_args); |
| 6664 | 6664 | } |
| 6665 | 6665 | |
| 6666 | | static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *target_inst) { |
| 6666 | static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode *node, |
| 6667 | IrInstruction *target_inst, bool cancel_non_suspended, bool cancel_awaited) |
| 6668 | { |
| 6667 | 6669 | IrBasicBlock *done_block = ir_create_basic_block(irb, scope, "CancelDone"); |
| 6668 | 6670 | IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled"); |
| 6669 | 6671 | IrBasicBlock *pre_return_block = ir_create_basic_block(irb, scope, "PreReturn"); |
| ... | ... | @@ -6691,7 +6693,7 @@ static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode |
| 6691 | 6693 | // set the is_canceled bit |
| 6692 | 6694 | IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| 6693 | 6695 | usize_type_val, atomic_state_ptr, nullptr, is_canceled_mask, nullptr, |
| 6694 | | AtomicRmwOp_and, AtomicOrderSeqCst); |
| 6696 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| 6695 | 6697 | |
| 6696 | 6698 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| 6697 | 6699 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| ... | ... | @@ -6703,14 +6705,26 @@ static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode |
| 6703 | 6705 | ir_build_cond_br(irb, scope, node, is_returned_bool, post_return_block, pre_return_block, is_comptime); |
| 6704 | 6706 | |
| 6705 | 6707 | ir_set_cursor_at_end_and_append_block(irb, post_return_block); |
| 6706 | | IrInstruction *is_awaited_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, await_mask, false); |
| 6707 | | IrInstruction *is_awaited_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_awaited_value, zero, false); |
| 6708 | | ir_build_cond_br(irb, scope, node, is_awaited_bool, done_block, do_cancel_block, is_comptime); |
| 6708 | if (cancel_awaited) { |
| 6709 | ir_build_br(irb, scope, node, do_cancel_block, is_comptime); |
| 6710 | } else { |
| 6711 | IrInstruction *is_awaited_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, await_mask, false); |
| 6712 | IrInstruction *is_awaited_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_awaited_value, zero, false); |
| 6713 | ir_build_cond_br(irb, scope, node, is_awaited_bool, done_block, do_cancel_block, is_comptime); |
| 6714 | } |
| 6709 | 6715 | |
| 6710 | 6716 | ir_set_cursor_at_end_and_append_block(irb, pre_return_block); |
| 6711 | | IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false); |
| 6712 | | IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false); |
| 6713 | | ir_build_cond_br(irb, scope, node, is_suspended_bool, do_cancel_block, done_block, is_comptime); |
| 6717 | if (cancel_awaited) { |
| 6718 | if (cancel_non_suspended) { |
| 6719 | ir_build_br(irb, scope, node, do_cancel_block, is_comptime); |
| 6720 | } else { |
| 6721 | IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false); |
| 6722 | IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false); |
| 6723 | ir_build_cond_br(irb, scope, node, is_suspended_bool, do_cancel_block, done_block, is_comptime); |
| 6724 | } |
| 6725 | } else { |
| 6726 | ir_build_br(irb, scope, node, done_block, is_comptime); |
| 6727 | } |
| 6714 | 6728 | |
| 6715 | 6729 | ir_set_cursor_at_end_and_append_block(irb, do_cancel_block); |
| 6716 | 6730 | ir_build_cancel(irb, scope, node, target_inst); |
| ... | ... | @@ -6727,7 +6741,7 @@ static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *scope, AstNode *node) |
| 6727 | 6741 | if (target_inst == irb->codegen->invalid_instruction) |
| 6728 | 6742 | return irb->codegen->invalid_instruction; |
| 6729 | 6743 | |
| 6730 | | return ir_gen_cancel_target(irb, scope, node, target_inst); |
| 6744 | return ir_gen_cancel_target(irb, scope, node, target_inst, false, true); |
| 6731 | 6745 | } |
| 6732 | 6746 | |
| 6733 | 6747 | static IrInstruction *ir_gen_resume(IrBuilder *irb, Scope *scope, AstNode *node) { |
| ... | ... | @@ -6872,15 +6886,19 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n |
| 6872 | 6886 | ir_build_unreachable(irb, scope, node); |
| 6873 | 6887 | |
| 6874 | 6888 | ir_set_cursor_at_end_and_append_block(irb, not_awaited_block); |
| 6889 | IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false); |
| 6890 | IrInstruction *is_non_null = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false); |
| 6875 | 6891 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| 6876 | 6892 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| 6877 | | ir_build_cond_br(irb, scope, node, is_canceled_bool, cleanup_block, not_canceled_block, const_bool_false); |
| 6893 | ir_build_cond_br(irb, scope, node, is_canceled_bool, cancel_target_block, not_canceled_block, const_bool_false); |
| 6878 | 6894 | |
| 6879 | 6895 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); |
| 6880 | | IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false); |
| 6881 | | IrInstruction *is_non_null = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false); |
| 6882 | 6896 | ir_build_cond_br(irb, scope, node, is_non_null, no_suspend_block, yes_suspend_block, const_bool_false); |
| 6883 | 6897 | |
| 6898 | ir_set_cursor_at_end_and_append_block(irb, cancel_target_block); |
| 6899 | ir_build_cancel(irb, scope, node, target_inst); |
| 6900 | ir_mark_gen(ir_build_br(irb, scope, node, cleanup_block, const_bool_false)); |
| 6901 | |
| 6884 | 6902 | ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); |
| 6885 | 6903 | if (irb->codegen->have_err_ret_tracing) { |
| 6886 | 6904 | Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); |
| ... | ... | @@ -6897,6 +6915,7 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n |
| 6897 | 6915 | ir_build_cancel(irb, scope, node, target_inst); |
| 6898 | 6916 | ir_build_br(irb, scope, node, merge_block, const_bool_false); |
| 6899 | 6917 | |
| 6918 | |
| 6900 | 6919 | ir_set_cursor_at_end_and_append_block(irb, yes_suspend_block); |
| 6901 | 6920 | IrInstruction *suspend_code = ir_build_coro_suspend(irb, scope, node, save_token, const_bool_false); |
| 6902 | 6921 | |
| ... | ... | @@ -6904,32 +6923,28 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n |
| 6904 | 6923 | cases[0].value = ir_build_const_u8(irb, scope, node, 0); |
| 6905 | 6924 | cases[0].block = resume_block; |
| 6906 | 6925 | cases[1].value = ir_build_const_u8(irb, scope, node, 1); |
| 6907 | | cases[1].block = cancel_target_block; |
| 6926 | cases[1].block = cleanup_block; |
| 6908 | 6927 | ir_build_switch_br(irb, scope, node, suspend_code, irb->exec->coro_suspend_block, |
| 6909 | 6928 | 2, cases, const_bool_false, nullptr); |
| 6910 | 6929 | |
| 6911 | | ir_set_cursor_at_end_and_append_block(irb, cancel_target_block); |
| 6912 | | IrInstruction *await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, await_handle_addr); |
| 6913 | | ir_gen_cancel_target(irb, scope, node, await_handle); |
| 6914 | | ir_mark_gen(ir_build_br(irb, scope, node, cleanup_block, const_bool_false)); |
| 6915 | | |
| 6916 | 6930 | ir_set_cursor_at_end_and_append_block(irb, cleanup_block); |
| 6917 | 6931 | IrInstruction *my_mask_bits = ir_build_bin_op(irb, scope, node, IrBinOpBinOr, ptr_mask, is_canceled_mask, false); |
| 6918 | 6932 | IrInstruction *my_prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| 6919 | 6933 | usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, my_mask_bits, nullptr, |
| 6920 | 6934 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| 6921 | 6935 | IrInstruction *my_await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, my_prev_atomic_value, ptr_mask, false); |
| 6922 | | IrInstruction *have_my_await_handle = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, my_await_handle_addr, zero, false); |
| 6923 | | ir_build_cond_br(irb, scope, node, have_my_await_handle, do_cancel_block, do_defers_block, const_bool_false); |
| 6936 | IrInstruction *dont_have_my_await_handle = ir_build_bin_op(irb, scope, node, IrBinOpCmpEq, my_await_handle_addr, zero, false); |
| 6937 | IrInstruction *dont_destroy_ourselves = ir_build_bin_op(irb, scope, node, IrBinOpBoolAnd, dont_have_my_await_handle, is_canceled_bool, false); |
| 6938 | ir_build_cond_br(irb, scope, node, dont_have_my_await_handle, do_defers_block, do_cancel_block, const_bool_false); |
| 6924 | 6939 | |
| 6925 | 6940 | ir_set_cursor_at_end_and_append_block(irb, do_cancel_block); |
| 6926 | 6941 | IrInstruction *my_await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, my_await_handle_addr); |
| 6927 | | ir_gen_cancel_target(irb, scope, node, my_await_handle); |
| 6942 | ir_gen_cancel_target(irb, scope, node, my_await_handle, true, false); |
| 6928 | 6943 | ir_mark_gen(ir_build_br(irb, scope, node, do_defers_block, const_bool_false)); |
| 6929 | 6944 | |
| 6930 | 6945 | ir_set_cursor_at_end_and_append_block(irb, do_defers_block); |
| 6931 | 6946 | ir_gen_defers_for_block(irb, scope, outer_scope, true); |
| 6932 | | ir_mark_gen(ir_build_br(irb, scope, node, irb->exec->coro_final_cleanup_block, const_bool_false)); |
| 6947 | ir_mark_gen(ir_build_cond_br(irb, scope, node, dont_destroy_ourselves, irb->exec->coro_early_final, irb->exec->coro_final_cleanup_block, const_bool_false)); |
| 6933 | 6948 | |
| 6934 | 6949 | ir_set_cursor_at_end_and_append_block(irb, resume_block); |
| 6935 | 6950 | ir_build_br(irb, scope, node, merge_block, const_bool_false); |
| ... | ... | @@ -7004,7 +7019,7 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod |
| 7004 | 7019 | |
| 7005 | 7020 | ir_set_cursor_at_end_and_append_block(irb, cancel_awaiter_block); |
| 7006 | 7021 | IrInstruction *await_handle = ir_build_int_to_ptr(irb, parent_scope, node, promise_type_val, await_handle_addr); |
| 7007 | | ir_gen_cancel_target(irb, parent_scope, node, await_handle); |
| 7022 | ir_gen_cancel_target(irb, parent_scope, node, await_handle, true, false); |
| 7008 | 7023 | ir_build_br(irb, parent_scope, node, cleanup_block, const_bool_false); |
| 7009 | 7024 | |
| 7010 | 7025 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); |