authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-30 19:19:05-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-12-30 19:19:05-05:00
log73e535e1125d76bcd4e85123defea8b76412ab09
treee347bc78a7ea39a2009b490e3be97074d05f9fcb
parentcbfd66f68a5a390abcf99e8cc6923d056ee1e4fa
parent2252951066f772ac6b77ab183ad3bc074ae8f09f
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3683 from Vexu/atomic-float

Support floats with some atomic operations

7 files changed, 145 insertions(+), 17 deletions(-)

doc/langref.html.in+3-3
...@@ -6699,7 +6699,7 @@ async fn func(y: *i32) void {...@@ -6699,7 +6699,7 @@ async fn func(y: *i32) void {
6699 This builtin function atomically dereferences a pointer and returns the value.6699 This builtin function atomically dereferences a pointer and returns the value.
6700 </p>6700 </p>
6701 <p>6701 <p>
6702 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}6702 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}, a float,
6703 an integer whose bit count meets these requirements:6703 an integer whose bit count meets these requirements:
6704 </p>6704 </p>
6705 <ul>6705 <ul>
...@@ -6734,7 +6734,7 @@ async fn func(y: *i32) void {...@@ -6734,7 +6734,7 @@ async fn func(y: *i32) void {
6734 Supported operations:6734 Supported operations:
6735 </p>6735 </p>
6736 <ul>6736 <ul>
6737 <li>{#syntax#}.Xchg{#endsyntax#} - stores the operand unmodified.</li>6737 <li>{#syntax#}.Xchg{#endsyntax#} - stores the operand unmodified. Supports enums, integers and floats.</li>
6738 <li>{#syntax#}.Add{#endsyntax#} - for integers, twos complement wraparound addition.6738 <li>{#syntax#}.Add{#endsyntax#} - for integers, twos complement wraparound addition.
6739 Also supports {#link|Floats#}.</li>6739 Also supports {#link|Floats#}.</li>
6740 <li>{#syntax#}.Sub{#endsyntax#} - for integers, twos complement wraparound subtraction.6740 <li>{#syntax#}.Sub{#endsyntax#} - for integers, twos complement wraparound subtraction.
...@@ -6753,7 +6753,7 @@ async fn func(y: *i32) void {...@@ -6753,7 +6753,7 @@ async fn func(y: *i32) void {
6753 This builtin function atomically stores a value.6753 This builtin function atomically stores a value.
6754 </p>6754 </p>
6755 <p>6755 <p>
6756 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}6756 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}, a float,
6757 an integer whose bit count meets these requirements:6757 an integer whose bit count meets these requirements:
6758 </p>6758 </p>
6759 <ul>6759 <ul>
src/codegen.cpp+16-13
...@@ -5132,19 +5132,21 @@ static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) {...@@ -5132,19 +5132,21 @@ static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) {
5132 zig_unreachable();5132 zig_unreachable();
5133}5133}
51345134
5135static LLVMAtomicRMWBinOp to_LLVMAtomicRMWBinOp(AtomicRmwOp op, bool is_signed) {5135static enum ZigLLVM_AtomicRMWBinOp to_ZigLLVMAtomicRMWBinOp(AtomicRmwOp op, bool is_signed, bool is_float) {
5136 switch (op) {5136 switch (op) {
5137 case AtomicRmwOp_xchg: return LLVMAtomicRMWBinOpXchg;5137 case AtomicRmwOp_xchg: return ZigLLVMAtomicRMWBinOpXchg;
5138 case AtomicRmwOp_add: return LLVMAtomicRMWBinOpAdd;5138 case AtomicRmwOp_add:
5139 case AtomicRmwOp_sub: return LLVMAtomicRMWBinOpSub;5139 return is_float ? ZigLLVMAtomicRMWBinOpFAdd : ZigLLVMAtomicRMWBinOpAdd;
5140 case AtomicRmwOp_and: return LLVMAtomicRMWBinOpAnd;5140 case AtomicRmwOp_sub:
5141 case AtomicRmwOp_nand: return LLVMAtomicRMWBinOpNand;5141 return is_float ? ZigLLVMAtomicRMWBinOpFSub : ZigLLVMAtomicRMWBinOpSub;
5142 case AtomicRmwOp_or: return LLVMAtomicRMWBinOpOr;5142 case AtomicRmwOp_and: return ZigLLVMAtomicRMWBinOpAnd;
5143 case AtomicRmwOp_xor: return LLVMAtomicRMWBinOpXor;5143 case AtomicRmwOp_nand: return ZigLLVMAtomicRMWBinOpNand;
5144 case AtomicRmwOp_or: return ZigLLVMAtomicRMWBinOpOr;
5145 case AtomicRmwOp_xor: return ZigLLVMAtomicRMWBinOpXor;
5144 case AtomicRmwOp_max:5146 case AtomicRmwOp_max:
5145 return is_signed ? LLVMAtomicRMWBinOpMax : LLVMAtomicRMWBinOpUMax;5147 return is_signed ? ZigLLVMAtomicRMWBinOpMax : ZigLLVMAtomicRMWBinOpUMax;
5146 case AtomicRmwOp_min:5148 case AtomicRmwOp_min:
5147 return is_signed ? LLVMAtomicRMWBinOpMin : LLVMAtomicRMWBinOpUMin;5149 return is_signed ? ZigLLVMAtomicRMWBinOpMin : ZigLLVMAtomicRMWBinOpUMin;
5148 }5150 }
5149 zig_unreachable();5151 zig_unreachable();
5150}5152}
...@@ -5738,25 +5740,26 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable,...@@ -5738,25 +5740,26 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable,
5738{5740{
5739 bool is_signed;5741 bool is_signed;
5740 ZigType *operand_type = instruction->operand->value->type;5742 ZigType *operand_type = instruction->operand->value->type;
5743 bool is_float = operand_type->id == ZigTypeIdFloat;
5741 if (operand_type->id == ZigTypeIdInt) {5744 if (operand_type->id == ZigTypeIdInt) {
5742 is_signed = operand_type->data.integral.is_signed;5745 is_signed = operand_type->data.integral.is_signed;
5743 } else {5746 } else {
5744 is_signed = false;5747 is_signed = false;
5745 }5748 }
5746 LLVMAtomicRMWBinOp op = to_LLVMAtomicRMWBinOp(instruction->resolved_op, is_signed);5749 enum ZigLLVM_AtomicRMWBinOp op = to_ZigLLVMAtomicRMWBinOp(instruction->resolved_op, is_signed, is_float);
5747 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->resolved_ordering);5750 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->resolved_ordering);
5748 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);5751 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
5749 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);5752 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
57505753
5751 if (get_codegen_ptr_type(operand_type) == nullptr) {5754 if (get_codegen_ptr_type(operand_type) == nullptr) {
5752 return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);5755 return ZigLLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);
5753 }5756 }
57545757
5755 // it's a pointer but we need to treat it as an int5758 // it's a pointer but we need to treat it as an int
5756 LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,5759 LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,
5757 LLVMPointerType(g->builtin_types.entry_usize->llvm_type, 0), "");5760 LLVMPointerType(g->builtin_types.entry_usize->llvm_type, 0), "");
5758 LLVMValueRef casted_operand = LLVMBuildPtrToInt(g->builder, operand, g->builtin_types.entry_usize->llvm_type, "");5761 LLVMValueRef casted_operand = LLVMBuildPtrToInt(g->builder, operand, g->builtin_types.entry_usize->llvm_type, "");
5759 LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,5762 LLVMValueRef uncasted_result = ZigLLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
5760 g->is_single_threaded);5763 g->is_single_threaded);
5761 return LLVMBuildIntToPtr(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");5764 return LLVMBuildIntToPtr(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
5762}5765}
src/ir.cpp+19-1
...@@ -23959,6 +23959,12 @@ static IrInstruction *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstructi...@@ -23959,6 +23959,12 @@ static IrInstruction *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstructi
23959 if (type_is_invalid(operand_type))23959 if (type_is_invalid(operand_type))
23960 return ira->codegen->invalid_instruction;23960 return ira->codegen->invalid_instruction;
2396123961
23962 if (operand_type->id == ZigTypeIdFloat) {
23963 ir_add_error(ira, instruction->type_value->child,
23964 buf_sprintf("expected integer, enum or pointer type, found '%s'", buf_ptr(&operand_type->name)));
23965 return ira->codegen->invalid_instruction;
23966 }
23967
23962 IrInstruction *ptr = instruction->ptr->child;23968 IrInstruction *ptr = instruction->ptr->child;
23963 if (type_is_invalid(ptr->value->type))23969 if (type_is_invalid(ptr->value->type))
23964 return ira->codegen->invalid_instruction;23970 return ira->codegen->invalid_instruction;
...@@ -27440,9 +27446,17 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstruction *op...@@ -27440,9 +27446,17 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstruction *op
27440 buf_sprintf("%" PRIu32 "-bit enum tag type is not a power of 2", int_type->data.integral.bit_count));27446 buf_sprintf("%" PRIu32 "-bit enum tag type is not a power of 2", int_type->data.integral.bit_count));
27441 return ira->codegen->builtin_types.entry_invalid;27447 return ira->codegen->builtin_types.entry_invalid;
27442 }27448 }
27449 } else if (operand_type->id == ZigTypeIdFloat) {
27450 uint32_t max_atomic_bits = target_arch_largest_atomic_bits(ira->codegen->zig_target->arch);
27451 if (operand_type->data.floating.bit_count > max_atomic_bits) {
27452 ir_add_error(ira, op,
27453 buf_sprintf("expected %" PRIu32 "-bit float or smaller, found %" PRIu32 "-bit float",
27454 max_atomic_bits, (uint32_t) operand_type->data.floating.bit_count));
27455 return ira->codegen->builtin_types.entry_invalid;
27456 }
27443 } else if (get_codegen_ptr_type(operand_type) == nullptr) {27457 } else if (get_codegen_ptr_type(operand_type) == nullptr) {
27444 ir_add_error(ira, op,27458 ir_add_error(ira, op,
27445 buf_sprintf("expected integer, enum or pointer type, found '%s'", buf_ptr(&operand_type->name)));27459 buf_sprintf("expected integer, float, enum or pointer type, found '%s'", buf_ptr(&operand_type->name)));
27446 return ira->codegen->builtin_types.entry_invalid;27460 return ira->codegen->builtin_types.entry_invalid;
27447 }27461 }
2744827462
...@@ -27477,6 +27491,10 @@ static IrInstruction *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstru...@@ -27477,6 +27491,10 @@ static IrInstruction *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstru
27477 ir_add_error(ira, instruction->op,27491 ir_add_error(ira, instruction->op,
27478 buf_sprintf("@atomicRmw on enum only works with .Xchg"));27492 buf_sprintf("@atomicRmw on enum only works with .Xchg"));
27479 return ira->codegen->invalid_instruction;27493 return ira->codegen->invalid_instruction;
27494 } else if (operand_type->id == ZigTypeIdFloat && op > AtomicRmwOp_sub) {
27495 ir_add_error(ira, instruction->op,
27496 buf_sprintf("@atomicRmw with float only works with .Xchg, .Add and .Sub"));
27497 return ira->codegen->invalid_instruction;
27480 }27498 }
2748127499
27482 IrInstruction *operand = instruction->operand->child;27500 IrInstruction *operand = instruction->operand->child;
src/zig_llvm.cpp+50
...@@ -1096,6 +1096,56 @@ bool ZigLLDLink(ZigLLVM_ObjectFormatType oformat, const char **args, size_t arg_...@@ -1096,6 +1096,56 @@ bool ZigLLDLink(ZigLLVM_ObjectFormatType oformat, const char **args, size_t arg_
1096 abort();1096 abort();
1097}1097}
10981098
1099static AtomicRMWInst::BinOp toLLVMRMWBinOp(enum ZigLLVM_AtomicRMWBinOp BinOp) {
1100 switch (BinOp) {
1101 default:
1102 case ZigLLVMAtomicRMWBinOpXchg: return AtomicRMWInst::Xchg;
1103 case ZigLLVMAtomicRMWBinOpAdd: return AtomicRMWInst::Add;
1104 case ZigLLVMAtomicRMWBinOpSub: return AtomicRMWInst::Sub;
1105 case ZigLLVMAtomicRMWBinOpAnd: return AtomicRMWInst::And;
1106 case ZigLLVMAtomicRMWBinOpNand: return AtomicRMWInst::Nand;
1107 case ZigLLVMAtomicRMWBinOpOr: return AtomicRMWInst::Or;
1108 case ZigLLVMAtomicRMWBinOpXor: return AtomicRMWInst::Xor;
1109 case ZigLLVMAtomicRMWBinOpMax: return AtomicRMWInst::Max;
1110 case ZigLLVMAtomicRMWBinOpMin: return AtomicRMWInst::Min;
1111 case ZigLLVMAtomicRMWBinOpUMax: return AtomicRMWInst::UMax;
1112 case ZigLLVMAtomicRMWBinOpUMin: return AtomicRMWInst::UMin;
1113 case ZigLLVMAtomicRMWBinOpFAdd: return AtomicRMWInst::FAdd;
1114 case ZigLLVMAtomicRMWBinOpFSub: return AtomicRMWInst::FSub;
1115 }
1116}
1117
1118static AtomicOrdering toLLVMOrdering(LLVMAtomicOrdering Ordering) {
1119 switch (Ordering) {
1120 default:
1121 case LLVMAtomicOrderingNotAtomic: return AtomicOrdering::NotAtomic;
1122 case LLVMAtomicOrderingUnordered: return AtomicOrdering::Unordered;
1123 case LLVMAtomicOrderingMonotonic: return AtomicOrdering::Monotonic;
1124 case LLVMAtomicOrderingAcquire: return AtomicOrdering::Acquire;
1125 case LLVMAtomicOrderingRelease: return AtomicOrdering::Release;
1126 case LLVMAtomicOrderingAcquireRelease: return AtomicOrdering::AcquireRelease;
1127 case LLVMAtomicOrderingSequentiallyConsistent: return AtomicOrdering::SequentiallyConsistent;
1128 }
1129}
1130
1131inline LLVMAttributeRef wrap(Attribute Attr) {
1132 return reinterpret_cast<LLVMAttributeRef>(Attr.getRawPointer());
1133}
1134
1135inline Attribute unwrap(LLVMAttributeRef Attr) {
1136 return Attribute::fromRawPointer(Attr);
1137}
1138
1139LLVMValueRef ZigLLVMBuildAtomicRMW(LLVMBuilderRef B, enum ZigLLVM_AtomicRMWBinOp op,
1140 LLVMValueRef PTR, LLVMValueRef Val,
1141 LLVMAtomicOrdering ordering, LLVMBool singleThread)
1142{
1143 AtomicRMWInst::BinOp intop = toLLVMRMWBinOp(op);
1144 return wrap(unwrap(B)->CreateAtomicRMW(intop, unwrap(PTR),
1145 unwrap(Val), toLLVMOrdering(ordering),
1146 singleThread ? SyncScope::SingleThread : SyncScope::System));
1147}
1148
1099static_assert((Triple::ArchType)ZigLLVM_UnknownArch == Triple::UnknownArch, "");1149static_assert((Triple::ArchType)ZigLLVM_UnknownArch == Triple::UnknownArch, "");
1100static_assert((Triple::ArchType)ZigLLVM_arm == Triple::arm, "");1150static_assert((Triple::ArchType)ZigLLVM_arm == Triple::arm, "");
1101static_assert((Triple::ArchType)ZigLLVM_armeb == Triple::armeb, "");1151static_assert((Triple::ArchType)ZigLLVM_armeb == Triple::armeb, "");
src/zig_llvm.h+20
...@@ -422,6 +422,26 @@ enum ZigLLVM_ObjectFormatType {...@@ -422,6 +422,26 @@ enum ZigLLVM_ObjectFormatType {
422 ZigLLVM_XCOFF,422 ZigLLVM_XCOFF,
423};423};
424424
425enum ZigLLVM_AtomicRMWBinOp {
426 ZigLLVMAtomicRMWBinOpXchg,
427 ZigLLVMAtomicRMWBinOpAdd,
428 ZigLLVMAtomicRMWBinOpSub,
429 ZigLLVMAtomicRMWBinOpAnd,
430 ZigLLVMAtomicRMWBinOpNand,
431 ZigLLVMAtomicRMWBinOpOr,
432 ZigLLVMAtomicRMWBinOpXor,
433 ZigLLVMAtomicRMWBinOpMax,
434 ZigLLVMAtomicRMWBinOpMin,
435 ZigLLVMAtomicRMWBinOpUMax,
436 ZigLLVMAtomicRMWBinOpUMin,
437 ZigLLVMAtomicRMWBinOpFAdd,
438 ZigLLVMAtomicRMWBinOpFSub,
439};
440
441LLVMValueRef ZigLLVMBuildAtomicRMW(LLVMBuilderRef B, enum ZigLLVM_AtomicRMWBinOp op,
442 LLVMValueRef PTR, LLVMValueRef Val,
443 LLVMAtomicOrdering ordering, LLVMBool singleThread);
444
425#define ZigLLVM_DIFlags_Zero 0U445#define ZigLLVM_DIFlags_Zero 0U
426#define ZigLLVM_DIFlags_Private 1U446#define ZigLLVM_DIFlags_Private 1U
427#define ZigLLVM_DIFlags_Protected 2U447#define ZigLLVM_DIFlags_Protected 2U
test/compile_errors.zig+20
...@@ -31,6 +31,26 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {...@@ -31,6 +31,26 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
31 "tmp.zig:3:12: note: destination pointer requires a terminating '0' sentinel",31 "tmp.zig:3:12: note: destination pointer requires a terminating '0' sentinel",
32 });32 });
3333
34 cases.add(
35 "cmpxchg with float",
36 \\export fn entry() void {
37 \\ var x: f32 = 0;
38 \\ _ = @cmpxchgWeak(f32, &x, 1, 2, .SeqCst, .SeqCst);
39 \\}
40 , &[_][]const u8{
41 "tmp.zig:3:22: error: expected integer, enum or pointer type, found 'f32'",
42 });
43
44 cases.add(
45 "atomicrmw with float op not .Xchg, .Add or .Sub",
46 \\export fn entry() void {
47 \\ var x: f32 = 0;
48 \\ _ = @atomicRmw(f32, &x, .And, 2, .SeqCst);
49 \\}
50 , &[_][]const u8{
51 "tmp.zig:3:29: error: @atomicRmw with float only works with .Xchg, .Add and .Sub",
52 });
53
34 cases.add("intToPtr with misaligned address",54 cases.add("intToPtr with misaligned address",
35 \\pub fn main() void {55 \\pub fn main() void {
36 \\ var y = @intToPtr([*]align(4) u8, 5);56 \\ var y = @intToPtr([*]align(4) u8, 5);
test/stage1/behavior/atomics.zig+17
...@@ -144,3 +144,20 @@ fn testAtomicStore() void {...@@ -144,3 +144,20 @@ fn testAtomicStore() void {
144 @atomicStore(u32, &x, 12345678, .SeqCst);144 @atomicStore(u32, &x, 12345678, .SeqCst);
145 expect(@atomicLoad(u32, &x, .SeqCst) == 12345678);145 expect(@atomicLoad(u32, &x, .SeqCst) == 12345678);
146}146}
147
148test "atomicrmw with floats" {
149 if (builtin.arch == .aarch64 or builtin.arch == .arm)
150 return;
151 testAtomicRmwFloat();
152}
153
154fn testAtomicRmwFloat() void {
155 var x: f32 = 0;
156 expect(x == 0);
157 _ = @atomicRmw(f32, &x, .Xchg, 1, .SeqCst);
158 expect(x == 1);
159 _ = @atomicRmw(f32, &x, .Add, 5, .SeqCst);
160 expect(x == 6);
161 _ = @atomicRmw(f32, &x, .Sub, 2, .SeqCst);
162 expect(x == 4);
163}