| ... | @@ -911,11 +911,13 @@ static Buf *panic_msg_buf(PanicMsgId msg_id) { | ... | @@ -911,11 +911,13 @@ static Buf *panic_msg_buf(PanicMsgId msg_id) { |
| 911 | case PanicMsgIdBadResume: | 911 | case PanicMsgIdBadResume: |
| 912 | return buf_create_from_str("resumed an async function which already returned"); | 912 | return buf_create_from_str("resumed an async function which already returned"); |
| 913 | case PanicMsgIdBadAwait: | 913 | case PanicMsgIdBadAwait: |
| 914 | return buf_create_from_str("async function awaited twice"); | 914 | return buf_create_from_str("async function awaited/canceled twice"); |
| 915 | case PanicMsgIdBadReturn: | 915 | case PanicMsgIdBadReturn: |
| 916 | return buf_create_from_str("async function returned twice"); | 916 | return buf_create_from_str("async function returned twice"); |
| 917 | case PanicMsgIdResumedAnAwaitingFn: | 917 | case PanicMsgIdResumedAnAwaitingFn: |
| 918 | return buf_create_from_str("awaiting function resumed"); | 918 | return buf_create_from_str("awaiting function resumed"); |
| | 919 | case PanicMsgIdResumedACancelingFn: |
| | 920 | return buf_create_from_str("canceling function resumed"); |
| 919 | case PanicMsgIdFrameTooSmall: | 921 | case PanicMsgIdFrameTooSmall: |
| 920 | return buf_create_from_str("frame too small"); | 922 | return buf_create_from_str("frame too small"); |
| 921 | case PanicMsgIdResumedFnPendingAwait: | 923 | case PanicMsgIdResumedFnPendingAwait: |
| ... | @@ -2189,12 +2191,12 @@ static void gen_assert_resume_id(CodeGen *g, IrInstruction *source_instr, Resume | ... | @@ -2189,12 +2191,12 @@ static void gen_assert_resume_id(CodeGen *g, IrInstruction *source_instr, Resume |
| 2189 | if (end_bb == nullptr) end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume"); | 2191 | if (end_bb == nullptr) end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume"); |
| 2190 | LLVMValueRef ok_bit; | 2192 | LLVMValueRef ok_bit; |
| 2191 | if (resume_id == ResumeIdAwaitEarlyReturn) { | 2193 | if (resume_id == ResumeIdAwaitEarlyReturn) { |
| 2192 | LLVMValueRef last_value = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref), | 2194 | LLVMValueRef last_value = LLVMConstSub(LLVMConstAllOnes(usize_type_ref), |
| 2193 | LLVMConstInt(usize_type_ref, ResumeIdAwaitEarlyReturn, false), ""); | 2195 | LLVMConstInt(usize_type_ref, ResumeIdAwaitEarlyReturn, false)); |
| 2194 | ok_bit = LLVMBuildICmp(g->builder, LLVMIntULT, LLVMGetParam(g->cur_fn_val, 1), last_value, ""); | 2196 | ok_bit = LLVMBuildICmp(g->builder, LLVMIntULT, LLVMGetParam(g->cur_fn_val, 1), last_value, ""); |
| 2195 | } else { | 2197 | } else { |
| 2196 | LLVMValueRef expected_value = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref), | 2198 | LLVMValueRef expected_value = LLVMConstSub(LLVMConstAllOnes(usize_type_ref), |
| 2197 | LLVMConstInt(usize_type_ref, resume_id, false), ""); | 2199 | LLVMConstInt(usize_type_ref, resume_id, false)); |
| 2198 | ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, LLVMGetParam(g->cur_fn_val, 1), expected_value, ""); | 2200 | ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, LLVMGetParam(g->cur_fn_val, 1), expected_value, ""); |
| 2199 | } | 2201 | } |
| 2200 | LLVMBuildCondBr(g->builder, ok_bit, end_bb, bad_resume_block); | 2202 | LLVMBuildCondBr(g->builder, ok_bit, end_bb, bad_resume_block); |
| ... | @@ -2210,11 +2212,13 @@ static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef tar | ... | @@ -2210,11 +2212,13 @@ static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef tar |
| 2210 | { | 2212 | { |
| 2211 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; | 2213 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| 2212 | if (fn_val == nullptr) { | 2214 | if (fn_val == nullptr) { |
| 2213 | if (g->anyframe_fn_type == nullptr) { | | |
| 2214 | (void)get_llvm_type(g, get_any_frame_type(g, nullptr)); | | |
| 2215 | } | | |
| 2216 | LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_fn_ptr_index, ""); | 2215 | LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_fn_ptr_index, ""); |
| 2217 | fn_val = LLVMBuildLoad(g->builder, fn_ptr_ptr, ""); | 2216 | LLVMValueRef fn_val_typed = LLVMBuildLoad(g->builder, fn_ptr_ptr, ""); |
| | 2217 | LLVMValueRef as_int = LLVMBuildPtrToInt(g->builder, fn_val_typed, usize_type_ref, ""); |
| | 2218 | LLVMValueRef one = LLVMConstInt(usize_type_ref, 1, false); |
| | 2219 | LLVMValueRef mask_val = LLVMConstNot(one); |
| | 2220 | LLVMValueRef as_int_masked = LLVMBuildAnd(g->builder, as_int, mask_val, ""); |
| | 2221 | fn_val = LLVMBuildIntToPtr(g->builder, as_int_masked, LLVMTypeOf(fn_val_typed), ""); |
| 2218 | } | 2222 | } |
| 2219 | if (arg_val == nullptr) { | 2223 | if (arg_val == nullptr) { |
| 2220 | arg_val = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref), | 2224 | arg_val = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref), |
| ... | @@ -2226,6 +2230,17 @@ static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef tar | ... | @@ -2226,6 +2230,17 @@ static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef tar |
| 2226 | return ZigLLVMBuildCall(g->builder, fn_val, args, 2, LLVMFastCallConv, ZigLLVM_FnInlineAuto, ""); | 2230 | return ZigLLVMBuildCall(g->builder, fn_val, args, 2, LLVMFastCallConv, ZigLLVM_FnInlineAuto, ""); |
| 2227 | } | 2231 | } |
| 2228 | | 2232 | |
| | 2233 | static LLVMBasicBlockRef gen_suspend_begin(CodeGen *g, const char *name_hint) { |
| | 2234 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| | 2235 | LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, name_hint); |
| | 2236 | size_t new_block_index = g->cur_resume_block_count; |
| | 2237 | g->cur_resume_block_count += 1; |
| | 2238 | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); |
| | 2239 | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb); |
| | 2240 | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); |
| | 2241 | return resume_bb; |
| | 2242 | } |
| | 2243 | |
| 2229 | static LLVMValueRef ir_render_return_begin(CodeGen *g, IrExecutable *executable, | 2244 | static LLVMValueRef ir_render_return_begin(CodeGen *g, IrExecutable *executable, |
| 2230 | IrInstructionReturnBegin *instruction) | 2245 | IrInstructionReturnBegin *instruction) |
| 2231 | { | 2246 | { |
| ... | @@ -2245,12 +2260,7 @@ static LLVMValueRef ir_render_return_begin(CodeGen *g, IrExecutable *executable, | ... | @@ -2245,12 +2260,7 @@ static LLVMValueRef ir_render_return_begin(CodeGen *g, IrExecutable *executable, |
| 2245 | } | 2260 | } |
| 2246 | | 2261 | |
| 2247 | // Prepare to be suspended. We might end up not having to suspend though. | 2262 | // Prepare to be suspended. We might end up not having to suspend though. |
| 2248 | LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, "ReturnResume"); | 2263 | LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "ReturnResume"); |
| 2249 | size_t new_block_index = g->cur_resume_block_count; | | |
| 2250 | g->cur_resume_block_count += 1; | | |
| 2251 | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); | | |
| 2252 | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb); | | |
| 2253 | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); | | |
| 2254 | | 2264 | |
| 2255 | LLVMValueRef zero = LLVMConstNull(usize_type_ref); | 2265 | LLVMValueRef zero = LLVMConstNull(usize_type_ref); |
| 2256 | LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref); | 2266 | LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref); |
| ... | @@ -2335,7 +2345,10 @@ static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrIns | ... | @@ -2335,7 +2345,10 @@ static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrIns |
| 2335 | | 2345 | |
| 2336 | // We need to resume the caller by tail calling them. | 2346 | // We need to resume the caller by tail calling them. |
| 2337 | ZigType *any_frame_type = get_any_frame_type(g, ret_type); | 2347 | ZigType *any_frame_type = get_any_frame_type(g, ret_type); |
| 2338 | LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, g->cur_async_prev_val, | 2348 | LLVMValueRef one = LLVMConstInt(usize_type_ref, 1, false); |
| | 2349 | LLVMValueRef mask_val = LLVMConstNot(one); |
| | 2350 | LLVMValueRef masked_prev_val = LLVMBuildAnd(g->builder, g->cur_async_prev_val, mask_val, ""); |
| | 2351 | LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, masked_prev_val, |
| 2339 | get_llvm_type(g, any_frame_type), ""); | 2352 | get_llvm_type(g, any_frame_type), ""); |
| 2340 | LLVMValueRef call_inst = gen_resume(g, nullptr, their_frame_ptr, ResumeIdReturn, nullptr); | 2353 | LLVMValueRef call_inst = gen_resume(g, nullptr, their_frame_ptr, ResumeIdReturn, nullptr); |
| 2341 | ZigLLVMSetTailCall(call_inst); | 2354 | ZigLLVMSetTailCall(call_inst); |
| ... | @@ -3945,13 +3958,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr | ... | @@ -3945,13 +3958,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr |
| 3945 | } else if (callee_is_async) { | 3958 | } else if (callee_is_async) { |
| 3946 | ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true); | 3959 | ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true); |
| 3947 | | 3960 | |
| 3948 | LLVMBasicBlockRef call_bb = LLVMAppendBasicBlock(g->cur_fn_val, "CallResume"); | 3961 | LLVMBasicBlockRef call_bb = gen_suspend_begin(g, "CallResume"); |
| 3949 | size_t new_block_index = g->cur_resume_block_count; | | |
| 3950 | g->cur_resume_block_count += 1; | | |
| 3951 | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); | | |
| 3952 | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, call_bb); | | |
| 3953 | | | |
| 3954 | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); | | |
| 3955 | | 3962 | |
| 3956 | LLVMValueRef call_inst = gen_resume(g, fn_val, frame_result_loc, ResumeIdCall, nullptr); | 3963 | LLVMValueRef call_inst = gen_resume(g, fn_val, frame_result_loc, ResumeIdCall, nullptr); |
| 3957 | ZigLLVMSetTailCall(call_inst); | 3964 | ZigLLVMSetTailCall(call_inst); |
| ... | @@ -4672,10 +4679,6 @@ static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *execu | ... | @@ -4672,10 +4679,6 @@ static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *execu |
| 4672 | return cur_err_ret_trace_val; | 4679 | return cur_err_ret_trace_val; |
| 4673 | } | 4680 | } |
| 4674 | | 4681 | |
| 4675 | static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) { | | |
| 4676 | zig_panic("TODO cancel"); | | |
| 4677 | } | | |
| 4678 | | | |
| 4679 | static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) { | 4682 | static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) { |
| 4680 | switch (atomic_order) { | 4683 | switch (atomic_order) { |
| 4681 | case AtomicOrderUnordered: return LLVMAtomicOrderingUnordered; | 4684 | case AtomicOrderUnordered: return LLVMAtomicOrderingUnordered; |
| ... | @@ -5416,13 +5419,7 @@ static LLVMValueRef ir_render_assert_non_null(CodeGen *g, IrExecutable *executab | ... | @@ -5416,13 +5419,7 @@ static LLVMValueRef ir_render_assert_non_null(CodeGen *g, IrExecutable *executab |
| 5416 | static LLVMValueRef ir_render_suspend_begin(CodeGen *g, IrExecutable *executable, | 5419 | static LLVMValueRef ir_render_suspend_begin(CodeGen *g, IrExecutable *executable, |
| 5417 | IrInstructionSuspendBegin *instruction) | 5420 | IrInstructionSuspendBegin *instruction) |
| 5418 | { | 5421 | { |
| 5419 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; | 5422 | instruction->resume_bb = gen_suspend_begin(g, "SuspendResume"); |
| 5420 | instruction->resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, "SuspendResume"); | | |
| 5421 | size_t new_block_index = g->cur_resume_block_count; | | |
| 5422 | g->cur_resume_block_count += 1; | | |
| 5423 | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); | | |
| 5424 | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, instruction->resume_bb); | | |
| 5425 | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); | | |
| 5426 | return nullptr; | 5423 | return nullptr; |
| 5427 | } | 5424 | } |
| 5428 | | 5425 | |
| ... | @@ -5436,6 +5433,43 @@ static LLVMValueRef ir_render_suspend_finish(CodeGen *g, IrExecutable *executabl | ... | @@ -5436,6 +5433,43 @@ static LLVMValueRef ir_render_suspend_finish(CodeGen *g, IrExecutable *executabl |
| 5436 | return nullptr; | 5433 | return nullptr; |
| 5437 | } | 5434 | } |
| 5438 | | 5435 | |
| | 5436 | static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) { |
| | 5437 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| | 5438 | LLVMValueRef zero = LLVMConstNull(usize_type_ref); |
| | 5439 | LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref); |
| | 5440 | LLVMValueRef one = LLVMConstInt(usize_type_ref, 1, false); |
| | 5441 | |
| | 5442 | LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame); |
| | 5443 | LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "CancelResume"); |
| | 5444 | |
| | 5445 | LLVMValueRef awaiter_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, ""); |
| | 5446 | LLVMValueRef awaiter_ored_val = LLVMBuildOr(g->builder, awaiter_val, one, ""); |
| | 5447 | LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_awaiter_index, ""); |
| | 5448 | |
| | 5449 | LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_ored_val, |
| | 5450 | LLVMAtomicOrderingRelease, g->is_single_threaded); |
| | 5451 | |
| | 5452 | LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CancelSuspend"); |
| | 5453 | LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn"); |
| | 5454 | |
| | 5455 | LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, resume_bb, 2); |
| | 5456 | LLVMAddCase(switch_instr, zero, complete_suspend_block); |
| | 5457 | LLVMAddCase(switch_instr, all_ones, early_return_block); |
| | 5458 | |
| | 5459 | LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block); |
| | 5460 | LLVMBuildRetVoid(g->builder); |
| | 5461 | |
| | 5462 | LLVMPositionBuilderAtEnd(g->builder, early_return_block); |
| | 5463 | LLVMValueRef call_inst = gen_resume(g, nullptr, target_frame_ptr, ResumeIdAwaitEarlyReturn, awaiter_ored_val); |
| | 5464 | ZigLLVMSetTailCall(call_inst); |
| | 5465 | LLVMBuildRetVoid(g->builder); |
| | 5466 | |
| | 5467 | LLVMPositionBuilderAtEnd(g->builder, resume_bb); |
| | 5468 | gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedACancelingFn, nullptr); |
| | 5469 | |
| | 5470 | return nullptr; |
| | 5471 | } |
| | 5472 | |
| 5439 | static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwaitGen *instruction) { | 5473 | static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwaitGen *instruction) { |
| 5440 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; | 5474 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| 5441 | LLVMValueRef zero = LLVMConstNull(usize_type_ref); | 5475 | LLVMValueRef zero = LLVMConstNull(usize_type_ref); |
| ... | @@ -5444,12 +5478,7 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst | ... | @@ -5444,12 +5478,7 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst |
| 5444 | ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true); | 5478 | ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true); |
| 5445 | | 5479 | |
| 5446 | // Prepare to be suspended | 5480 | // Prepare to be suspended |
| 5447 | LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, "AwaitResume"); | 5481 | LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "AwaitResume"); |
| 5448 | size_t new_block_index = g->cur_resume_block_count; | | |
| 5449 | g->cur_resume_block_count += 1; | | |
| 5450 | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); | | |
| 5451 | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb); | | |
| 5452 | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); | | |
| 5453 | | 5482 | |
| 5454 | // At this point resuming the function will do the correct thing. | 5483 | // At this point resuming the function will do the correct thing. |
| 5455 | // This code is as if it is running inside the suspend block. | 5484 | // This code is as if it is running inside the suspend block. |