| ... | ... | @@ -911,11 +911,13 @@ static Buf *panic_msg_buf(PanicMsgId msg_id) { |
| 911 | 911 | case PanicMsgIdBadResume: |
| 912 | 912 | return buf_create_from_str("resumed an async function which already returned"); |
| 913 | 913 | case PanicMsgIdBadAwait: |
| 914 | | return buf_create_from_str("async function awaited twice"); |
| 914 | return buf_create_from_str("async function awaited/canceled twice"); |
| 915 | 915 | case PanicMsgIdBadReturn: |
| 916 | 916 | return buf_create_from_str("async function returned twice"); |
| 917 | 917 | case PanicMsgIdResumedAnAwaitingFn: |
| 918 | 918 | return buf_create_from_str("awaiting function resumed"); |
| 919 | case PanicMsgIdResumedACancelingFn: |
| 920 | return buf_create_from_str("canceling function resumed"); |
| 919 | 921 | case PanicMsgIdFrameTooSmall: |
| 920 | 922 | return buf_create_from_str("frame too small"); |
| 921 | 923 | case PanicMsgIdResumedFnPendingAwait: |
| ... | ... | @@ -2189,12 +2191,12 @@ static void gen_assert_resume_id(CodeGen *g, IrInstruction *source_instr, Resume |
| 2189 | 2191 | if (end_bb == nullptr) end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume"); |
| 2190 | 2192 | LLVMValueRef ok_bit; |
| 2191 | 2193 | if (resume_id == ResumeIdAwaitEarlyReturn) { |
| 2192 | | LLVMValueRef last_value = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref), |
| 2193 | | LLVMConstInt(usize_type_ref, ResumeIdAwaitEarlyReturn, false), ""); |
| 2194 | LLVMValueRef last_value = LLVMConstSub(LLVMConstAllOnes(usize_type_ref), |
| 2195 | LLVMConstInt(usize_type_ref, ResumeIdAwaitEarlyReturn, false)); |
| 2194 | 2196 | ok_bit = LLVMBuildICmp(g->builder, LLVMIntULT, LLVMGetParam(g->cur_fn_val, 1), last_value, ""); |
| 2195 | 2197 | } else { |
| 2196 | | LLVMValueRef expected_value = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref), |
| 2197 | | LLVMConstInt(usize_type_ref, resume_id, false), ""); |
| 2198 | LLVMValueRef expected_value = LLVMConstSub(LLVMConstAllOnes(usize_type_ref), |
| 2199 | LLVMConstInt(usize_type_ref, resume_id, false)); |
| 2198 | 2200 | ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, LLVMGetParam(g->cur_fn_val, 1), expected_value, ""); |
| 2199 | 2201 | } |
| 2200 | 2202 | LLVMBuildCondBr(g->builder, ok_bit, end_bb, bad_resume_block); |
| ... | ... | @@ -2210,11 +2212,13 @@ static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef tar |
| 2210 | 2212 | { |
| 2211 | 2213 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| 2212 | 2214 | if (fn_val == nullptr) { |
| 2213 | | if (g->anyframe_fn_type == nullptr) { |
| 2214 | | (void)get_llvm_type(g, get_any_frame_type(g, nullptr)); |
| 2215 | | } |
| 2216 | 2215 | LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_fn_ptr_index, ""); |
| 2217 | | fn_val = LLVMBuildLoad(g->builder, fn_ptr_ptr, ""); |
| 2216 | LLVMValueRef fn_val_typed = LLVMBuildLoad(g->builder, fn_ptr_ptr, ""); |
| 2217 | LLVMValueRef as_int = LLVMBuildPtrToInt(g->builder, fn_val_typed, usize_type_ref, ""); |
| 2218 | LLVMValueRef one = LLVMConstInt(usize_type_ref, 1, false); |
| 2219 | LLVMValueRef mask_val = LLVMConstNot(one); |
| 2220 | LLVMValueRef as_int_masked = LLVMBuildAnd(g->builder, as_int, mask_val, ""); |
| 2221 | fn_val = LLVMBuildIntToPtr(g->builder, as_int_masked, LLVMTypeOf(fn_val_typed), ""); |
| 2218 | 2222 | } |
| 2219 | 2223 | if (arg_val == nullptr) { |
| 2220 | 2224 | arg_val = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref), |
| ... | ... | @@ -2226,6 +2230,17 @@ static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef tar |
| 2226 | 2230 | return ZigLLVMBuildCall(g->builder, fn_val, args, 2, LLVMFastCallConv, ZigLLVM_FnInlineAuto, ""); |
| 2227 | 2231 | } |
| 2228 | 2232 | |
| 2233 | static LLVMBasicBlockRef gen_suspend_begin(CodeGen *g, const char *name_hint) { |
| 2234 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| 2235 | LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, name_hint); |
| 2236 | size_t new_block_index = g->cur_resume_block_count; |
| 2237 | g->cur_resume_block_count += 1; |
| 2238 | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); |
| 2239 | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb); |
| 2240 | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); |
| 2241 | return resume_bb; |
| 2242 | } |
| 2243 | |
| 2229 | 2244 | static LLVMValueRef ir_render_return_begin(CodeGen *g, IrExecutable *executable, |
| 2230 | 2245 | IrInstructionReturnBegin *instruction) |
| 2231 | 2246 | { |
| ... | ... | @@ -2245,12 +2260,7 @@ static LLVMValueRef ir_render_return_begin(CodeGen *g, IrExecutable *executable, |
| 2245 | 2260 | } |
| 2246 | 2261 | |
| 2247 | 2262 | // Prepare to be suspended. We might end up not having to suspend though. |
| 2248 | | LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, "ReturnResume"); |
| 2249 | | size_t new_block_index = g->cur_resume_block_count; |
| 2250 | | g->cur_resume_block_count += 1; |
| 2251 | | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); |
| 2252 | | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb); |
| 2253 | | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); |
| 2263 | LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "ReturnResume"); |
| 2254 | 2264 | |
| 2255 | 2265 | LLVMValueRef zero = LLVMConstNull(usize_type_ref); |
| 2256 | 2266 | LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref); |
| ... | ... | @@ -2335,7 +2345,10 @@ static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrIns |
| 2335 | 2345 | |
| 2336 | 2346 | // We need to resume the caller by tail calling them. |
| 2337 | 2347 | ZigType *any_frame_type = get_any_frame_type(g, ret_type); |
| 2338 | | LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, g->cur_async_prev_val, |
| 2348 | LLVMValueRef one = LLVMConstInt(usize_type_ref, 1, false); |
| 2349 | LLVMValueRef mask_val = LLVMConstNot(one); |
| 2350 | LLVMValueRef masked_prev_val = LLVMBuildAnd(g->builder, g->cur_async_prev_val, mask_val, ""); |
| 2351 | LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, masked_prev_val, |
| 2339 | 2352 | get_llvm_type(g, any_frame_type), ""); |
| 2340 | 2353 | LLVMValueRef call_inst = gen_resume(g, nullptr, their_frame_ptr, ResumeIdReturn, nullptr); |
| 2341 | 2354 | ZigLLVMSetTailCall(call_inst); |
| ... | ... | @@ -3945,13 +3958,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr |
| 3945 | 3958 | } else if (callee_is_async) { |
| 3946 | 3959 | ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true); |
| 3947 | 3960 | |
| 3948 | | LLVMBasicBlockRef call_bb = LLVMAppendBasicBlock(g->cur_fn_val, "CallResume"); |
| 3949 | | size_t new_block_index = g->cur_resume_block_count; |
| 3950 | | g->cur_resume_block_count += 1; |
| 3951 | | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); |
| 3952 | | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, call_bb); |
| 3953 | | |
| 3954 | | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); |
| 3961 | LLVMBasicBlockRef call_bb = gen_suspend_begin(g, "CallResume"); |
| 3955 | 3962 | |
| 3956 | 3963 | LLVMValueRef call_inst = gen_resume(g, fn_val, frame_result_loc, ResumeIdCall, nullptr); |
| 3957 | 3964 | ZigLLVMSetTailCall(call_inst); |
| ... | ... | @@ -4672,10 +4679,6 @@ static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *execu |
| 4672 | 4679 | return cur_err_ret_trace_val; |
| 4673 | 4680 | } |
| 4674 | 4681 | |
| 4675 | | static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) { |
| 4676 | | zig_panic("TODO cancel"); |
| 4677 | | } |
| 4678 | | |
| 4679 | 4682 | static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) { |
| 4680 | 4683 | switch (atomic_order) { |
| 4681 | 4684 | case AtomicOrderUnordered: return LLVMAtomicOrderingUnordered; |
| ... | ... | @@ -5416,13 +5419,7 @@ static LLVMValueRef ir_render_assert_non_null(CodeGen *g, IrExecutable *executab |
| 5416 | 5419 | static LLVMValueRef ir_render_suspend_begin(CodeGen *g, IrExecutable *executable, |
| 5417 | 5420 | IrInstructionSuspendBegin *instruction) |
| 5418 | 5421 | { |
| 5419 | | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| 5420 | | instruction->resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, "SuspendResume"); |
| 5421 | | size_t new_block_index = g->cur_resume_block_count; |
| 5422 | | g->cur_resume_block_count += 1; |
| 5423 | | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); |
| 5424 | | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, instruction->resume_bb); |
| 5425 | | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); |
| 5422 | instruction->resume_bb = gen_suspend_begin(g, "SuspendResume"); |
| 5426 | 5423 | return nullptr; |
| 5427 | 5424 | } |
| 5428 | 5425 | |
| ... | ... | @@ -5436,6 +5433,43 @@ static LLVMValueRef ir_render_suspend_finish(CodeGen *g, IrExecutable *executabl |
| 5436 | 5433 | return nullptr; |
| 5437 | 5434 | } |
| 5438 | 5435 | |
| 5436 | static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) { |
| 5437 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| 5438 | LLVMValueRef zero = LLVMConstNull(usize_type_ref); |
| 5439 | LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref); |
| 5440 | LLVMValueRef one = LLVMConstInt(usize_type_ref, 1, false); |
| 5441 | |
| 5442 | LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame); |
| 5443 | LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "CancelResume"); |
| 5444 | |
| 5445 | LLVMValueRef awaiter_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, ""); |
| 5446 | LLVMValueRef awaiter_ored_val = LLVMBuildOr(g->builder, awaiter_val, one, ""); |
| 5447 | LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_awaiter_index, ""); |
| 5448 | |
| 5449 | LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_ored_val, |
| 5450 | LLVMAtomicOrderingRelease, g->is_single_threaded); |
| 5451 | |
| 5452 | LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CancelSuspend"); |
| 5453 | LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn"); |
| 5454 | |
| 5455 | LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, resume_bb, 2); |
| 5456 | LLVMAddCase(switch_instr, zero, complete_suspend_block); |
| 5457 | LLVMAddCase(switch_instr, all_ones, early_return_block); |
| 5458 | |
| 5459 | LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block); |
| 5460 | LLVMBuildRetVoid(g->builder); |
| 5461 | |
| 5462 | LLVMPositionBuilderAtEnd(g->builder, early_return_block); |
| 5463 | LLVMValueRef call_inst = gen_resume(g, nullptr, target_frame_ptr, ResumeIdAwaitEarlyReturn, awaiter_ored_val); |
| 5464 | ZigLLVMSetTailCall(call_inst); |
| 5465 | LLVMBuildRetVoid(g->builder); |
| 5466 | |
| 5467 | LLVMPositionBuilderAtEnd(g->builder, resume_bb); |
| 5468 | gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedACancelingFn, nullptr); |
| 5469 | |
| 5470 | return nullptr; |
| 5471 | } |
| 5472 | |
| 5439 | 5473 | static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwaitGen *instruction) { |
| 5440 | 5474 | LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; |
| 5441 | 5475 | LLVMValueRef zero = LLVMConstNull(usize_type_ref); |
| ... | ... | @@ -5444,12 +5478,7 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst |
| 5444 | 5478 | ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true); |
| 5445 | 5479 | |
| 5446 | 5480 | // Prepare to be suspended |
| 5447 | | LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, "AwaitResume"); |
| 5448 | | size_t new_block_index = g->cur_resume_block_count; |
| 5449 | | g->cur_resume_block_count += 1; |
| 5450 | | LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); |
| 5451 | | LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb); |
| 5452 | | LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); |
| 5481 | LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "AwaitResume"); |
| 5453 | 5482 | |
| 5454 | 5483 | // At this point resuming the function will do the correct thing. |
| 5455 | 5484 | // This code is as if it is running inside the suspend block. |