authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-08-04 18:24:10-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-08-04 18:24:10-04:00
logfa30ebfbe5949fc63aee9853d66932facfd1d168
tree170145ae5e992a7920d6288d76d8675c1b5af321
parent12924477a50d37bf055fd5fc1cdf0fd77684a472
signaturelock-open Commit is signed but in an unrecognized format.

suspension points inside branching control flow

6 files changed, 160 insertions(+), 82 deletions(-)

BRANCH_TODO-1
...@@ -1,4 +1,3 @@...@@ -1,4 +1,3 @@
1 * suspension points inside branching control flow
2 * go over the commented out tests1 * go over the commented out tests
3 * error return tracing2 * error return tracing
4 * compile error for error: expected anyframe->T, found 'anyframe'3 * compile error for error: expected anyframe->T, found 'anyframe'
src/all_types.hpp+11-4
...@@ -1716,6 +1716,9 @@ struct CodeGen {...@@ -1716,6 +1716,9 @@ struct CodeGen {
1716 ZigLLVMDIFile *dummy_di_file;1716 ZigLLVMDIFile *dummy_di_file;
1717 LLVMValueRef cur_ret_ptr;1717 LLVMValueRef cur_ret_ptr;
1718 LLVMValueRef cur_fn_val;1718 LLVMValueRef cur_fn_val;
1719 LLVMValueRef cur_async_switch_instr;
1720 LLVMValueRef cur_async_resume_index_ptr;
1721 LLVMValueRef cur_async_awaiter_ptr;
1719 LLVMValueRef cur_err_ret_trace_val_arg;1722 LLVMValueRef cur_err_ret_trace_val_arg;
1720 LLVMValueRef cur_err_ret_trace_val_stack;1723 LLVMValueRef cur_err_ret_trace_val_stack;
1721 LLVMValueRef memcpy_fn_val;1724 LLVMValueRef memcpy_fn_val;
...@@ -2166,8 +2169,8 @@ struct IrBasicBlock {...@@ -2166,8 +2169,8 @@ struct IrBasicBlock {
2166 size_t ref_count;2169 size_t ref_count;
2167 // index into the basic block list2170 // index into the basic block list
2168 size_t index;2171 size_t index;
2169 // for async functions, the split function which corresponds to this block2172 // for async functions, the resume index which corresponds to this block
2170 LLVMValueRef split_llvm_fn;2173 size_t resume_index;
2171 LLVMBasicBlockRef llvm_block;2174 LLVMBasicBlockRef llvm_block;
2172 LLVMBasicBlockRef llvm_exit_block;2175 LLVMBasicBlockRef llvm_exit_block;
2173 // The instruction that referenced this basic block and caused us to2176 // The instruction that referenced this basic block and caused us to
...@@ -3703,8 +3706,12 @@ static const size_t err_union_payload_index = 1;...@@ -3703,8 +3706,12 @@ static const size_t err_union_payload_index = 1;
37033706
3704// label (grep this): [coro_frame_struct_layout]3707// label (grep this): [coro_frame_struct_layout]
3705static const size_t coro_fn_ptr_index = 0;3708static const size_t coro_fn_ptr_index = 0;
3706static const size_t coro_awaiter_index = 1;3709static const size_t coro_resume_index = 1;
3707static const size_t coro_arg_start = 2;3710static const size_t coro_awaiter_index = 2;
3711static const size_t coro_arg_start = 3;
3712
3713// one for the Entry block, resume blocks are indexed after that.
3714static const size_t coro_extra_resume_block_count = 1;
37083715
3709// TODO call graph analysis to find out what this number needs to be for every function3716// TODO call graph analysis to find out what this number needs to be for every function
3710// MUST BE A POWER OF TWO.3717// MUST BE A POWER OF TWO.
src/analyze.cpp+27-8
...@@ -5215,6 +5215,9 @@ static Error resolve_coro_frame(CodeGen *g, ZigType *frame_type) {...@@ -5215,6 +5215,9 @@ static Error resolve_coro_frame(CodeGen *g, ZigType *frame_type) {
5215 field_names.append("fn_ptr");5215 field_names.append("fn_ptr");
5216 field_types.append(fn_type);5216 field_types.append(fn_type);
52175217
5218 field_names.append("resume_index");
5219 field_types.append(g->builtin_types.entry_usize);
5220
5218 field_names.append("awaiter");5221 field_names.append("awaiter");
5219 field_types.append(g->builtin_types.entry_usize);5222 field_types.append(g->builtin_types.entry_usize);
52205223
...@@ -7532,9 +7535,10 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re...@@ -7532,9 +7535,10 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
7532 // label (grep this): [coro_frame_struct_layout]7535 // label (grep this): [coro_frame_struct_layout]
7533 LLVMTypeRef field_types[] = {7536 LLVMTypeRef field_types[] = {
7534 ptr_result_type, // fn_ptr7537 ptr_result_type, // fn_ptr
7538 usize_type_ref, // resume_index
7535 usize_type_ref, // awaiter7539 usize_type_ref, // awaiter
7536 };7540 };
7537 LLVMStructSetBody(frame_header_type, field_types, 2, false);7541 LLVMStructSetBody(frame_header_type, field_types, 3, false);
75387542
7539 ZigLLVMDIType *di_element_types[] = {7543 ZigLLVMDIType *di_element_types[] = {
7540 ZigLLVMCreateDebugMemberType(g->dbuilder,7544 ZigLLVMCreateDebugMemberType(g->dbuilder,
...@@ -7545,12 +7549,19 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re...@@ -7545,12 +7549,19 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
7545 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 0),7549 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 0),
7546 ZigLLVM_DIFlags_Zero, usize_di_type),7550 ZigLLVM_DIFlags_Zero, usize_di_type),
7547 ZigLLVMCreateDebugMemberType(g->dbuilder,7551 ZigLLVMCreateDebugMemberType(g->dbuilder,
7548 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "awaiter",7552 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "resume_index",
7549 di_file, line,7553 di_file, line,
7550 8*LLVMABISizeOfType(g->target_data_ref, field_types[1]),7554 8*LLVMABISizeOfType(g->target_data_ref, field_types[1]),
7551 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types[1]),7555 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types[1]),
7552 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 1),7556 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 1),
7553 ZigLLVM_DIFlags_Zero, usize_di_type),7557 ZigLLVM_DIFlags_Zero, usize_di_type),
7558 ZigLLVMCreateDebugMemberType(g->dbuilder,
7559 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "awaiter",
7560 di_file, line,
7561 8*LLVMABISizeOfType(g->target_data_ref, field_types[2]),
7562 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types[2]),
7563 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 2),
7564 ZigLLVM_DIFlags_Zero, usize_di_type),
7554 };7565 };
7555 ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugStructType(g->dbuilder,7566 ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugStructType(g->dbuilder,
7556 compile_unit_scope, buf_ptr(name),7567 compile_unit_scope, buf_ptr(name),
...@@ -7558,7 +7569,7 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re...@@ -7558,7 +7569,7 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
7558 8*LLVMABISizeOfType(g->target_data_ref, frame_header_type),7569 8*LLVMABISizeOfType(g->target_data_ref, frame_header_type),
7559 8*LLVMABIAlignmentOfType(g->target_data_ref, frame_header_type),7570 8*LLVMABIAlignmentOfType(g->target_data_ref, frame_header_type),
7560 ZigLLVM_DIFlags_Zero,7571 ZigLLVM_DIFlags_Zero,
7561 nullptr, di_element_types, 2, 0, nullptr, "");7572 nullptr, di_element_types, 3, 0, nullptr, "");
75627573
7563 ZigLLVMReplaceTemporary(g->dbuilder, frame_header_di_type, replacement_di_type);7574 ZigLLVMReplaceTemporary(g->dbuilder, frame_header_di_type, replacement_di_type);
7564 } else {7575 } else {
...@@ -7566,11 +7577,12 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re...@@ -7566,11 +7577,12 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
7566 // label (grep this): [coro_frame_struct_layout]7577 // label (grep this): [coro_frame_struct_layout]
7567 LLVMTypeRef field_types[] = {7578 LLVMTypeRef field_types[] = {
7568 LLVMPointerType(fn_type, 0), // fn_ptr7579 LLVMPointerType(fn_type, 0), // fn_ptr
7580 usize_type_ref, // resume_index
7569 usize_type_ref, // awaiter7581 usize_type_ref, // awaiter
7570 get_llvm_type(g, ptr_result_type), // result_ptr7582 get_llvm_type(g, ptr_result_type), // result_ptr
7571 get_llvm_type(g, result_type), // result7583 get_llvm_type(g, result_type), // result
7572 };7584 };
7573 LLVMStructSetBody(frame_header_type, field_types, 4, false);7585 LLVMStructSetBody(frame_header_type, field_types, 5, false);
75747586
7575 ZigLLVMDIType *di_element_types[] = {7587 ZigLLVMDIType *di_element_types[] = {
7576 ZigLLVMCreateDebugMemberType(g->dbuilder,7588 ZigLLVMCreateDebugMemberType(g->dbuilder,
...@@ -7588,18 +7600,25 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re...@@ -7588,18 +7600,25 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
7588 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 1),7600 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 1),
7589 ZigLLVM_DIFlags_Zero, usize_di_type),7601 ZigLLVM_DIFlags_Zero, usize_di_type),
7590 ZigLLVMCreateDebugMemberType(g->dbuilder,7602 ZigLLVMCreateDebugMemberType(g->dbuilder,
7591 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result_ptr",7603 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "awaiter",
7592 di_file, line,7604 di_file, line,
7593 8*LLVMABISizeOfType(g->target_data_ref, field_types[2]),7605 8*LLVMABISizeOfType(g->target_data_ref, field_types[2]),
7594 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types[2]),7606 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types[2]),
7595 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 2),7607 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 2),
7596 ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, ptr_result_type)),7608 ZigLLVM_DIFlags_Zero, usize_di_type),
7597 ZigLLVMCreateDebugMemberType(g->dbuilder,7609 ZigLLVMCreateDebugMemberType(g->dbuilder,
7598 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result",7610 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result_ptr",
7599 di_file, line,7611 di_file, line,
7600 8*LLVMABISizeOfType(g->target_data_ref, field_types[3]),7612 8*LLVMABISizeOfType(g->target_data_ref, field_types[3]),
7601 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types[3]),7613 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types[3]),
7602 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 3),7614 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 3),
7615 ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, ptr_result_type)),
7616 ZigLLVMCreateDebugMemberType(g->dbuilder,
7617 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result",
7618 di_file, line,
7619 8*LLVMABISizeOfType(g->target_data_ref, field_types[4]),
7620 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types[4]),
7621 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, 4),
7603 ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, result_type)),7622 ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, result_type)),
7604 };7623 };
7605 ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugStructType(g->dbuilder,7624 ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugStructType(g->dbuilder,
...@@ -7608,7 +7627,7 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re...@@ -7608,7 +7627,7 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
7608 8*LLVMABISizeOfType(g->target_data_ref, frame_header_type),7627 8*LLVMABISizeOfType(g->target_data_ref, frame_header_type),
7609 8*LLVMABIAlignmentOfType(g->target_data_ref, frame_header_type),7628 8*LLVMABIAlignmentOfType(g->target_data_ref, frame_header_type),
7610 ZigLLVM_DIFlags_Zero,7629 ZigLLVM_DIFlags_Zero,
7611 nullptr, di_element_types, 2, 0, nullptr, "");7630 nullptr, di_element_types, 5, 0, nullptr, "");
76127631
7613 ZigLLVMReplaceTemporary(g->dbuilder, frame_header_di_type, replacement_di_type);7632 ZigLLVMReplaceTemporary(g->dbuilder, frame_header_di_type, replacement_di_type);
7614 }7633 }
src/codegen.cpp+95-68
...@@ -1997,7 +1997,9 @@ static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *execut...@@ -1997,7 +1997,9 @@ static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *execut
1997 return call_instruction;1997 return call_instruction;
1998}1998}
19991999
2000static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrInstructionReturn *return_instruction) {2000static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable,
2001 IrInstructionReturn *return_instruction)
2002{
2001 if (fn_is_async(g->cur_fn)) {2003 if (fn_is_async(g->cur_fn)) {
2002 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;2004 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2003 LLVMValueRef locals_ptr = g->cur_ret_ptr;2005 LLVMValueRef locals_ptr = g->cur_ret_ptr;
...@@ -2006,12 +2008,10 @@ static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrIns...@@ -2006,12 +2008,10 @@ static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrIns
2006 ZigType *ret_type = ret_type_has_bits ? return_instruction->value->value.type : nullptr;2008 ZigType *ret_type = ret_type_has_bits ? return_instruction->value->value.type : nullptr;
20072009
2008 if (ir_want_runtime_safety(g, &return_instruction->base)) {2010 if (ir_want_runtime_safety(g, &return_instruction->base)) {
2009 LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, locals_ptr, coro_fn_ptr_index, "");2011 LLVMValueRef new_resume_index = LLVMConstAllOnes(usize_type_ref);
2010 LLVMValueRef new_resume_fn = g->cur_fn->resume_blocks.last()->split_llvm_fn;2012 LLVMBuildStore(g->builder, new_resume_index, g->cur_async_resume_index_ptr);
2011 LLVMBuildStore(g->builder, new_resume_fn, resume_index_ptr);
2012 }2013 }
20132014
2014 LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, locals_ptr, coro_awaiter_index, "");
2015 LLVMValueRef result_ptr_as_usize;2015 LLVMValueRef result_ptr_as_usize;
2016 if (ret_type_has_bits) {2016 if (ret_type_has_bits) {
2017 LLVMValueRef result_ptr_ptr = LLVMBuildStructGEP(g->builder, locals_ptr, coro_arg_start, "");2017 LLVMValueRef result_ptr_ptr = LLVMBuildStructGEP(g->builder, locals_ptr, coro_arg_start, "");
...@@ -2029,8 +2029,8 @@ static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrIns...@@ -2029,8 +2029,8 @@ static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrIns
2029 }2029 }
2030 LLVMValueRef zero = LLVMConstNull(usize_type_ref);2030 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
2031 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);2031 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
2032 LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, awaiter_ptr,2032 LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, g->cur_async_awaiter_ptr,
2033 all_ones, LLVMAtomicOrderingSequentiallyConsistent, g->is_single_threaded);2033 all_ones, LLVMAtomicOrderingMonotonic, g->is_single_threaded);
20342034
2035 LLVMBasicBlockRef bad_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadReturn");2035 LLVMBasicBlockRef bad_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadReturn");
2036 LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");2036 LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
...@@ -3453,7 +3453,6 @@ static void render_async_spills(CodeGen *g) {...@@ -3453,7 +3453,6 @@ static void render_async_spills(CodeGen *g) {
3453}3453}
34543454
3455static void render_async_var_decls(CodeGen *g, Scope *scope) {3455static void render_async_var_decls(CodeGen *g, Scope *scope) {
3456 render_async_spills(g);
3457 for (;;) {3456 for (;;) {
3458 switch (scope->id) {3457 switch (scope->id) {
3459 case ScopeIdCImport:3458 case ScopeIdCImport:
...@@ -3573,6 +3572,14 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr...@@ -3573,6 +3572,14 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr
3573 if (instruction->is_async || callee_is_async) {3572 if (instruction->is_async || callee_is_async) {
3574 assert(frame_result_loc != nullptr);3573 assert(frame_result_loc != nullptr);
35753574
3575 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, coro_fn_ptr_index, "");
3576 LLVMValueRef bitcasted_fn_val = LLVMBuildBitCast(g->builder, fn_val,
3577 LLVMGetElementType(LLVMTypeOf(fn_ptr_ptr)), "");
3578 LLVMBuildStore(g->builder, bitcasted_fn_val, fn_ptr_ptr);
3579
3580 LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, coro_resume_index, "");
3581 LLVMBuildStore(g->builder, zero, resume_index_ptr);
3582
3576 if (prefix_arg_err_ret_stack) {3583 if (prefix_arg_err_ret_stack) {
3577 zig_panic("TODO");3584 zig_panic("TODO");
3578 }3585 }
...@@ -3652,23 +3659,24 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr...@@ -3652,23 +3659,24 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr
3652 return nullptr;3659 return nullptr;
3653 } else if (callee_is_async) {3660 } else if (callee_is_async) {
3654 ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true);3661 ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true);
3655 LLVMValueRef split_llvm_fn = make_fn_llvm_value(g, g->cur_fn);3662
3656 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_fn_ptr_index, "");3663 LLVMBasicBlockRef call_bb = LLVMAppendBasicBlock(g->cur_fn_val, "CallResume");
3657 LLVMBuildStore(g->builder, split_llvm_fn, fn_ptr_ptr);3664 size_t new_block_index = g->cur_fn->resume_blocks.length + coro_extra_resume_block_count;
3665 g->cur_fn->resume_blocks.append(nullptr);
3666 LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false);
3667 LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, call_bb);
3668
3669 LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr);
3658 LLVMValueRef args[] = {frame_result_loc, LLVMGetUndef(usize_type_ref)};3670 LLVMValueRef args[] = {frame_result_loc, LLVMGetUndef(usize_type_ref)};
3659 LLVMValueRef call_inst = ZigLLVMBuildCall(g->builder, fn_val, args, 2, llvm_cc, fn_inline, "");3671 LLVMValueRef call_inst = ZigLLVMBuildCall(g->builder, fn_val, args, 2, llvm_cc, fn_inline, "");
3660 ZigLLVMSetTailCall(call_inst);3672 ZigLLVMSetTailCall(call_inst);
3661 LLVMBuildRetVoid(g->builder);3673 LLVMBuildRetVoid(g->builder);
36623674
3663 g->cur_fn_val = split_llvm_fn;
3664 g->cur_ret_ptr = LLVMGetParam(split_llvm_fn, 0);
3665 LLVMBasicBlockRef call_bb = LLVMAppendBasicBlock(split_llvm_fn, "CallResume");
3666 LLVMPositionBuilderAtEnd(g->builder, call_bb);3675 LLVMPositionBuilderAtEnd(g->builder, call_bb);
3667
3668 if (ir_want_runtime_safety(g, &instruction->base)) {3676 if (ir_want_runtime_safety(g, &instruction->base)) {
3669 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(split_llvm_fn, "BadResume");3677 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
3670 LLVMBasicBlockRef ok_resume_block = LLVMAppendBasicBlock(split_llvm_fn, "OkResume");3678 LLVMBasicBlockRef ok_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume");
3671 LLVMValueRef arg_val = LLVMGetParam(split_llvm_fn, 1);3679 LLVMValueRef arg_val = LLVMGetParam(g->cur_fn_val, 1);
3672 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);3680 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
3673 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntNE, arg_val, all_ones, "");3681 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntNE, arg_val, all_ones, "");
3674 LLVMBuildCondBr(g->builder, ok_bit, ok_resume_block, bad_resume_block);3682 LLVMBuildCondBr(g->builder, ok_bit, ok_resume_block, bad_resume_block);
...@@ -5144,10 +5152,9 @@ static LLVMValueRef ir_render_assert_non_null(CodeGen *g, IrExecutable *executab...@@ -5144,10 +5152,9 @@ static LLVMValueRef ir_render_assert_non_null(CodeGen *g, IrExecutable *executab
5144static LLVMValueRef ir_render_suspend_begin(CodeGen *g, IrExecutable *executable,5152static LLVMValueRef ir_render_suspend_begin(CodeGen *g, IrExecutable *executable,
5145 IrInstructionSuspendBegin *instruction)5153 IrInstructionSuspendBegin *instruction)
5146{5154{
5147 LLVMValueRef locals_ptr = g->cur_ret_ptr;5155 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
5148 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, locals_ptr, coro_fn_ptr_index, "");5156 LLVMValueRef new_resume_index = LLVMConstInt(usize_type_ref, instruction->resume_block->resume_index, false);
5149 LLVMValueRef new_fn_ptr = instruction->resume_block->split_llvm_fn;5157 LLVMBuildStore(g->builder, new_resume_index, g->cur_async_resume_index_ptr);
5150 LLVMBuildStore(g->builder, new_fn_ptr, fn_ptr_ptr);
5151 return nullptr;5158 return nullptr;
5152}5159}
51535160
...@@ -5159,19 +5166,22 @@ static LLVMValueRef ir_render_suspend_br(CodeGen *g, IrExecutable *executable,...@@ -5159,19 +5166,22 @@ static LLVMValueRef ir_render_suspend_br(CodeGen *g, IrExecutable *executable,
5159}5166}
51605167
5161static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwait *instruction) {5168static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwait *instruction) {
5169 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
5162 LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame);5170 LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame);
5163 ZigType *result_type = instruction->base.value.type;5171 ZigType *result_type = instruction->base.value.type;
5164 ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true);5172 ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true);
51655173
5166 // Prepare to be suspended5174 // Prepare to be suspended
5167 LLVMValueRef split_llvm_fn = make_fn_llvm_value(g, g->cur_fn);5175 LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, "AwaitResume");
5168 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_fn_ptr_index, "");5176 size_t new_block_index = g->cur_fn->resume_blocks.length + coro_extra_resume_block_count;
5169 LLVMBuildStore(g->builder, split_llvm_fn, fn_ptr_ptr);5177 g->cur_fn->resume_blocks.append(nullptr);
5178 LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false);
5179 LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb);
5180 LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr);
51705181
5171 // At this point resuming the function will do the correct thing.5182 // At this point resuming the function will do the correct thing.
5172 // This code is as if it is running inside the suspend block.5183 // This code is as if it is running inside the suspend block.
51735184
5174 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
5175 // caller's own frame pointer5185 // caller's own frame pointer
5176 LLVMValueRef awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_ret_ptr, usize_type_ref, "");5186 LLVMValueRef awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_ret_ptr, usize_type_ref, "");
5177 LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_awaiter_index, "");5187 LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_awaiter_index, "");
...@@ -5184,18 +5194,20 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst...@@ -5184,18 +5194,20 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst
5184 result_ptr_as_usize = LLVMGetUndef(usize_type_ref);5194 result_ptr_as_usize = LLVMGetUndef(usize_type_ref);
5185 }5195 }
5186 LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_init_val,5196 LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_init_val,
5187 LLVMAtomicOrderingSequentiallyConsistent, g->is_single_threaded);5197 LLVMAtomicOrderingMonotonic, g->is_single_threaded);
51885198
5189 LLVMBasicBlockRef bad_await_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadAwait");5199 LLVMBasicBlockRef bad_await_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadAwait");
5190 LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CompleteSuspend");5200 LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CompleteSuspend");
5191 LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
51925201
5193 LLVMValueRef zero = LLVMConstNull(usize_type_ref);5202 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
5194 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);5203 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
5195 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, bad_await_block, 2);5204 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, bad_await_block, 2);
5205 LLVMBasicBlockRef predecessor_bb = LLVMGetInsertBlock(g->builder);
51965206
5197 LLVMAddCase(switch_instr, zero, complete_suspend_block);5207 LLVMAddCase(switch_instr, zero, complete_suspend_block);
5198 LLVMAddCase(switch_instr, all_ones, early_return_block);5208
5209 // Early return: The async function has already completed. No need to suspend.
5210 LLVMAddCase(switch_instr, all_ones, resume_bb);
51995211
5200 // We discovered that another awaiter was already here.5212 // We discovered that another awaiter was already here.
5201 LLVMPositionBuilderAtEnd(g->builder, bad_await_block);5213 LLVMPositionBuilderAtEnd(g->builder, bad_await_block);
...@@ -5205,25 +5217,18 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst...@@ -5205,25 +5217,18 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst
5205 LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block);5217 LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block);
5206 LLVMBuildRetVoid(g->builder);5218 LLVMBuildRetVoid(g->builder);
52075219
5208 // The async function has already completed. So we use a tail call to resume ourselves.5220 LLVMPositionBuilderAtEnd(g->builder, resume_bb);
5209 LLVMPositionBuilderAtEnd(g->builder, early_return_block);5221 // We either got here from Entry (function call) or from the switch above
5210 LLVMValueRef args[] = {g->cur_ret_ptr, result_ptr_as_usize};5222 LLVMValueRef spilled_result_ptr = LLVMBuildPhi(g->builder, usize_type_ref, "");
5211 LLVMValueRef call_inst = ZigLLVMBuildCall(g->builder, split_llvm_fn, args, 2, LLVMFastCallConv,5223 LLVMValueRef incoming_values[] = { LLVMGetParam(g->cur_fn_val, 1), result_ptr_as_usize };
5212 ZigLLVM_FnInlineAuto, "");5224 LLVMBasicBlockRef incoming_blocks[] = { g->cur_fn->preamble_llvm_block, predecessor_bb };
5213 ZigLLVMSetTailCall(call_inst);5225 LLVMAddIncoming(spilled_result_ptr, incoming_values, incoming_blocks, 2);
5214 LLVMBuildRetVoid(g->builder);
5215
5216 g->cur_fn_val = split_llvm_fn;
5217 g->cur_ret_ptr = LLVMGetParam(split_llvm_fn, 0);
5218 LLVMBasicBlockRef call_bb = LLVMAppendBasicBlock(split_llvm_fn, "AwaitResume");
5219 LLVMPositionBuilderAtEnd(g->builder, call_bb);
52205226
5221 if (ir_want_runtime_safety(g, &instruction->base)) {5227 if (ir_want_runtime_safety(g, &instruction->base)) {
5222 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(split_llvm_fn, "BadResume");5228 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
5223 LLVMBasicBlockRef ok_resume_block = LLVMAppendBasicBlock(split_llvm_fn, "OkResume");5229 LLVMBasicBlockRef ok_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume");
5224 LLVMValueRef arg_val = LLVMGetParam(split_llvm_fn, 1);
5225 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);5230 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
5226 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntNE, arg_val, all_ones, "");5231 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntNE, spilled_result_ptr, all_ones, "");
5227 LLVMBuildCondBr(g->builder, ok_bit, ok_resume_block, bad_resume_block);5232 LLVMBuildCondBr(g->builder, ok_bit, ok_resume_block, bad_resume_block);
52285233
5229 LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);5234 LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
...@@ -5235,7 +5240,6 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst...@@ -5235,7 +5240,6 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst
5235 render_async_var_decls(g, instruction->base.scope);5240 render_async_var_decls(g, instruction->base.scope);
52365241
5237 if (type_has_bits(result_type)) {5242 if (type_has_bits(result_type)) {
5238 LLVMValueRef spilled_result_ptr = LLVMGetParam(g->cur_fn_val, 1);
5239 LLVMValueRef casted_spilled_result_ptr = LLVMBuildIntToPtr(g->builder, spilled_result_ptr,5243 LLVMValueRef casted_spilled_result_ptr = LLVMBuildIntToPtr(g->builder, spilled_result_ptr,
5240 get_llvm_type(g, ptr_result_type), "");5244 get_llvm_type(g, ptr_result_type), "");
5241 return get_handle_value(g, casted_spilled_result_ptr, result_type, ptr_result_type);5245 return get_handle_value(g, casted_spilled_result_ptr, result_type, ptr_result_type);
...@@ -5547,13 +5551,18 @@ static void ir_render(CodeGen *g, ZigFn *fn_entry) {...@@ -5547,13 +5551,18 @@ static void ir_render(CodeGen *g, ZigFn *fn_entry) {
55475551
5548 IrExecutable *executable = &fn_entry->analyzed_executable;5552 IrExecutable *executable = &fn_entry->analyzed_executable;
5549 assert(executable->basic_block_list.length > 0);5553 assert(executable->basic_block_list.length > 0);
5554
5555 if (fn_is_async(fn_entry)) {
5556 IrBasicBlock *entry_block = executable->basic_block_list.at(0);
5557 LLVMPositionBuilderAtEnd(g->builder, entry_block->llvm_block);
5558 render_async_var_decls(g, entry_block->instruction_list.at(0)->scope);
5559 }
5560
5550 for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {5561 for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {
5551 IrBasicBlock *current_block = executable->basic_block_list.at(block_i);5562 IrBasicBlock *current_block = executable->basic_block_list.at(block_i);
5552 assert(current_block->llvm_block);5563 assert(current_block->llvm_block);
5553 LLVMPositionBuilderAtEnd(g->builder, current_block->llvm_block);5564 LLVMPositionBuilderAtEnd(g->builder, current_block->llvm_block);
5554 if (current_block->split_llvm_fn != nullptr) {5565 if (current_block->resume_index != 0) {
5555 g->cur_fn_val = current_block->split_llvm_fn;
5556 g->cur_ret_ptr = LLVMGetParam(g->cur_fn_val, 0);
5557 render_async_var_decls(g, current_block->instruction_list.at(0)->scope);5566 render_async_var_decls(g, current_block->instruction_list.at(0)->scope);
5558 }5567 }
5559 for (size_t instr_i = 0; instr_i < current_block->instruction_list.length; instr_i += 1) {5568 for (size_t instr_i = 0; instr_i < current_block->instruction_list.length; instr_i += 1) {
...@@ -6416,17 +6425,19 @@ static void build_all_basic_blocks(CodeGen *g, ZigFn *fn) {...@@ -6416,17 +6425,19 @@ static void build_all_basic_blocks(CodeGen *g, ZigFn *fn) {
6416 IrExecutable *executable = &fn->analyzed_executable;6425 IrExecutable *executable = &fn->analyzed_executable;
6417 assert(executable->basic_block_list.length > 0);6426 assert(executable->basic_block_list.length > 0);
6418 LLVMValueRef fn_val = fn_llvm_value(g, fn);6427 LLVMValueRef fn_val = fn_llvm_value(g, fn);
6428 LLVMBasicBlockRef first_bb = nullptr;
6429 if (fn_is_async(fn)) {
6430 first_bb = LLVMAppendBasicBlock(fn_val, "AsyncSwitch");
6431 fn->preamble_llvm_block = first_bb;
6432 }
6419 for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {6433 for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {
6420 IrBasicBlock *bb = executable->basic_block_list.at(block_i);6434 IrBasicBlock *bb = executable->basic_block_list.at(block_i);
6421 if (bb->split_llvm_fn != nullptr) {
6422 assert(bb->split_llvm_fn == reinterpret_cast<LLVMValueRef>(0x1));
6423 fn_val = make_fn_llvm_value(g, fn);
6424 bb->split_llvm_fn = fn_val;
6425 }
6426 bb->llvm_block = LLVMAppendBasicBlock(fn_val, bb->name_hint);6435 bb->llvm_block = LLVMAppendBasicBlock(fn_val, bb->name_hint);
6427 }6436 }
6428 IrBasicBlock *entry_bb = executable->basic_block_list.at(0);6437 if (first_bb == nullptr) {
6429 LLVMPositionBuilderAtEnd(g->builder, entry_bb->llvm_block);6438 first_bb = executable->basic_block_list.at(0)->llvm_block;
6439 }
6440 LLVMPositionBuilderAtEnd(g->builder, first_bb);
6430}6441}
64316442
6432static void gen_global_var(CodeGen *g, ZigVar *var, LLVMValueRef init_val,6443static void gen_global_var(CodeGen *g, ZigVar *var, LLVMValueRef init_val,
...@@ -6636,9 +6647,7 @@ static void do_code_gen(CodeGen *g) {...@@ -6636,9 +6647,7 @@ static void do_code_gen(CodeGen *g) {
6636 g->cur_err_ret_trace_val_stack = nullptr;6647 g->cur_err_ret_trace_val_stack = nullptr;
6637 }6648 }
66386649
6639 if (is_async) {6650 if (!is_async) {
6640 render_async_spills(g);
6641 } else {
6642 // allocate temporary stack data6651 // allocate temporary stack data
6643 for (size_t alloca_i = 0; alloca_i < fn_table_entry->alloca_gen_list.length; alloca_i += 1) {6652 for (size_t alloca_i = 0; alloca_i < fn_table_entry->alloca_gen_list.length; alloca_i += 1) {
6644 IrInstructionAllocaGen *instruction = fn_table_entry->alloca_gen_list.at(alloca_i);6653 IrInstructionAllocaGen *instruction = fn_table_entry->alloca_gen_list.at(alloca_i);
...@@ -6752,17 +6761,35 @@ static void do_code_gen(CodeGen *g) {...@@ -6752,17 +6761,35 @@ static void do_code_gen(CodeGen *g) {
6752 LLVMValueRef size_val = LLVMConstInt(usize_type_ref, fn_table_entry->frame_type->abi_size, false);6761 LLVMValueRef size_val = LLVMConstInt(usize_type_ref, fn_table_entry->frame_type->abi_size, false);
6753 ZigLLVMFunctionSetPrefixData(fn_table_entry->llvm_value, size_val);6762 ZigLLVMFunctionSetPrefixData(fn_table_entry->llvm_value, size_val);
67546763
6755 if (ir_want_runtime_safety_scope(g, fn_table_entry->child_scope)) {6764 if (!g->strip_debug_symbols) {
6756 IrBasicBlock *bad_resume_block = allocate<IrBasicBlock>(1);6765 AstNode *source_node = fn_table_entry->proto_node;
6757 bad_resume_block->name_hint = "BadResume";6766 ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1,
6758 bad_resume_block->split_llvm_fn = make_fn_llvm_value(g, fn_table_entry);6767 (int)source_node->column + 1, get_di_scope(g, fn_table_entry->child_scope));
67596768 }
6760 LLVMBasicBlockRef llvm_block = LLVMAppendBasicBlock(bad_resume_block->split_llvm_fn, "BadResume");6769 IrExecutable *executable = &fn_table_entry->analyzed_executable;
6761 LLVMPositionBuilderAtEnd(g->builder, llvm_block);6770 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
6762 gen_safety_crash(g, PanicMsgIdBadResume);6771 LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
6772 gen_assertion_scope(g, PanicMsgIdBadResume, fn_table_entry->child_scope);
67636773
6764 fn_table_entry->resume_blocks.append(bad_resume_block);6774 LLVMPositionBuilderAtEnd(g->builder, fn_table_entry->preamble_llvm_block);
6775 render_async_spills(g);
6776 g->cur_async_awaiter_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_awaiter_index, "");
6777 LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_resume_index, "");
6778 g->cur_async_resume_index_ptr = resume_index_ptr;
6779 LLVMValueRef resume_index = LLVMBuildLoad(g->builder, resume_index_ptr, "");
6780 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, resume_index, bad_resume_block,
6781 fn_table_entry->resume_blocks.length + coro_extra_resume_block_count);
6782 g->cur_async_switch_instr = switch_instr;
6783
6784 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
6785 LLVMAddCase(switch_instr, zero, executable->basic_block_list.at(0)->llvm_block);
6786
6787 for (size_t resume_i = 0; resume_i < fn_table_entry->resume_blocks.length; resume_i += 1) {
6788 IrBasicBlock *resume_block = fn_table_entry->resume_blocks.at(resume_i);
6789 LLVMValueRef case_value = LLVMConstInt(usize_type_ref, resume_block->resume_index, false);
6790 LLVMAddCase(switch_instr, case_value, resume_block->llvm_block);
6765 }6791 }
6792
6766 } else {6793 } else {
6767 // create debug variable declarations for parameters6794 // create debug variable declarations for parameters
6768 // rely on the first variables in the variable_list being parameters.6795 // rely on the first variables in the variable_list being parameters.
src/ir.cpp+1-1
...@@ -24474,7 +24474,7 @@ static IrInstruction *ir_analyze_instruction_suspend_br(IrAnalyze *ira, IrInstru...@@ -24474,7 +24474,7 @@ static IrInstruction *ir_analyze_instruction_suspend_br(IrAnalyze *ira, IrInstru
24474 ZigFn *fn_entry = exec_fn_entry(ira->new_irb.exec);24474 ZigFn *fn_entry = exec_fn_entry(ira->new_irb.exec);
24475 ir_assert(fn_entry != nullptr, &instruction->base);24475 ir_assert(fn_entry != nullptr, &instruction->base);
2447624476
24477 new_bb->split_llvm_fn = reinterpret_cast<LLVMValueRef>(0x1);24477 new_bb->resume_index = fn_entry->resume_blocks.length + coro_extra_resume_block_count;
2447824478
24479 fn_entry->resume_blocks.append(new_bb);24479 fn_entry->resume_blocks.append(new_bb);
24480 if (fn_entry->inferred_async_node == nullptr) {24480 if (fn_entry->inferred_async_node == nullptr) {
test/stage1/behavior/coroutines.zig+26
...@@ -419,3 +419,29 @@ test "async function call return value" {...@@ -419,3 +419,29 @@ test "async function call return value" {
419 };419 };
420 S.doTheTest();420 S.doTheTest();
421}421}
422
423test "suspension points inside branching control flow" {
424 const S = struct {
425 var global_result: i32 = 10;
426
427 fn doTheTest() void {
428 expect(10 == global_result);
429 var frame = async func(true);
430 expect(10 == global_result);
431 resume frame;
432 expect(11 == global_result);
433 resume frame;
434 expect(12 == global_result);
435 resume frame;
436 expect(13 == global_result);
437 }
438
439 fn func(b: bool) void {
440 while (b) {
441 suspend;
442 global_result += 1;
443 }
444 }
445 };
446 S.doTheTest();
447}