authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-30 19:19:05-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-12-30 19:19:05-05:00
log73e535e1125d76bcd4e85123defea8b76412ab09
treee347bc78a7ea39a2009b490e3be97074d05f9fcb
parentcbfd66f68a5a390abcf99e8cc6923d056ee1e4fa
parent2252951066f772ac6b77ab183ad3bc074ae8f09f
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3683 from Vexu/atomic-float

Support floats with some atomic operations

7 files changed, 145 insertions(+), 17 deletions(-)

doc/langref.html.in+3-3
......@@ -6699,7 +6699,7 @@ async fn func(y: *i32) void {
66996699 This builtin function atomically dereferences a pointer and returns the value.
67006700 </p>
67016701 <p>
6702 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}
6702 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}, a float,
67036703 an integer whose bit count meets these requirements:
67046704 </p>
67056705 <ul>
......@@ -6734,7 +6734,7 @@ async fn func(y: *i32) void {
67346734 Supported operations:
67356735 </p>
67366736 <ul>
6737 <li>{#syntax#}.Xchg{#endsyntax#} - stores the operand unmodified.</li>
6737 <li>{#syntax#}.Xchg{#endsyntax#} - stores the operand unmodified. Supports enums, integers and floats.</li>
67386738 <li>{#syntax#}.Add{#endsyntax#} - for integers, twos complement wraparound addition.
67396739 Also supports {#link|Floats#}.</li>
67406740 <li>{#syntax#}.Sub{#endsyntax#} - for integers, twos complement wraparound subtraction.
......@@ -6753,7 +6753,7 @@ async fn func(y: *i32) void {
67536753 This builtin function atomically stores a value.
67546754 </p>
67556755 <p>
6756 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}
6756 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}, a float,
67576757 an integer whose bit count meets these requirements:
67586758 </p>
67596759 <ul>
src/codegen.cpp+16-13
......@@ -5132,19 +5132,21 @@ static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) {
51325132 zig_unreachable();
51335133}
51345134
5135static LLVMAtomicRMWBinOp to_LLVMAtomicRMWBinOp(AtomicRmwOp op, bool is_signed) {
5135static enum ZigLLVM_AtomicRMWBinOp to_ZigLLVMAtomicRMWBinOp(AtomicRmwOp op, bool is_signed, bool is_float) {
51365136 switch (op) {
5137 case AtomicRmwOp_xchg: return LLVMAtomicRMWBinOpXchg;
5138 case AtomicRmwOp_add: return LLVMAtomicRMWBinOpAdd;
5139 case AtomicRmwOp_sub: return LLVMAtomicRMWBinOpSub;
5140 case AtomicRmwOp_and: return LLVMAtomicRMWBinOpAnd;
5141 case AtomicRmwOp_nand: return LLVMAtomicRMWBinOpNand;
5142 case AtomicRmwOp_or: return LLVMAtomicRMWBinOpOr;
5143 case AtomicRmwOp_xor: return LLVMAtomicRMWBinOpXor;
5137 case AtomicRmwOp_xchg: return ZigLLVMAtomicRMWBinOpXchg;
5138 case AtomicRmwOp_add:
5139 return is_float ? ZigLLVMAtomicRMWBinOpFAdd : ZigLLVMAtomicRMWBinOpAdd;
5140 case AtomicRmwOp_sub:
5141 return is_float ? ZigLLVMAtomicRMWBinOpFSub : ZigLLVMAtomicRMWBinOpSub;
5142 case AtomicRmwOp_and: return ZigLLVMAtomicRMWBinOpAnd;
5143 case AtomicRmwOp_nand: return ZigLLVMAtomicRMWBinOpNand;
5144 case AtomicRmwOp_or: return ZigLLVMAtomicRMWBinOpOr;
5145 case AtomicRmwOp_xor: return ZigLLVMAtomicRMWBinOpXor;
51445146 case AtomicRmwOp_max:
5145 return is_signed ? LLVMAtomicRMWBinOpMax : LLVMAtomicRMWBinOpUMax;
5147 return is_signed ? ZigLLVMAtomicRMWBinOpMax : ZigLLVMAtomicRMWBinOpUMax;
51465148 case AtomicRmwOp_min:
5147 return is_signed ? LLVMAtomicRMWBinOpMin : LLVMAtomicRMWBinOpUMin;
5149 return is_signed ? ZigLLVMAtomicRMWBinOpMin : ZigLLVMAtomicRMWBinOpUMin;
51485150 }
51495151 zig_unreachable();
51505152}
......@@ -5738,25 +5740,26 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable,
57385740{
57395741 bool is_signed;
57405742 ZigType *operand_type = instruction->operand->value->type;
5743 bool is_float = operand_type->id == ZigTypeIdFloat;
57415744 if (operand_type->id == ZigTypeIdInt) {
57425745 is_signed = operand_type->data.integral.is_signed;
57435746 } else {
57445747 is_signed = false;
57455748 }
5746 LLVMAtomicRMWBinOp op = to_LLVMAtomicRMWBinOp(instruction->resolved_op, is_signed);
5749 enum ZigLLVM_AtomicRMWBinOp op = to_ZigLLVMAtomicRMWBinOp(instruction->resolved_op, is_signed, is_float);
57475750 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->resolved_ordering);
57485751 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
57495752 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
57505753
57515754 if (get_codegen_ptr_type(operand_type) == nullptr) {
5752 return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);
5755 return ZigLLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);
57535756 }
57545757
57555758 // it's a pointer but we need to treat it as an int
57565759 LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,
57575760 LLVMPointerType(g->builtin_types.entry_usize->llvm_type, 0), "");
57585761 LLVMValueRef casted_operand = LLVMBuildPtrToInt(g->builder, operand, g->builtin_types.entry_usize->llvm_type, "");
5759 LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
5762 LLVMValueRef uncasted_result = ZigLLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
57605763 g->is_single_threaded);
57615764 return LLVMBuildIntToPtr(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
57625765}
src/ir.cpp+19-1
......@@ -23959,6 +23959,12 @@ static IrInstruction *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstructi
2395923959 if (type_is_invalid(operand_type))
2396023960 return ira->codegen->invalid_instruction;
2396123961
23962 if (operand_type->id == ZigTypeIdFloat) {
23963 ir_add_error(ira, instruction->type_value->child,
23964 buf_sprintf("expected integer, enum or pointer type, found '%s'", buf_ptr(&operand_type->name)));
23965 return ira->codegen->invalid_instruction;
23966 }
23967
2396223968 IrInstruction *ptr = instruction->ptr->child;
2396323969 if (type_is_invalid(ptr->value->type))
2396423970 return ira->codegen->invalid_instruction;
......@@ -27440,9 +27446,17 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstruction *op
2744027446 buf_sprintf("%" PRIu32 "-bit enum tag type is not a power of 2", int_type->data.integral.bit_count));
2744127447 return ira->codegen->builtin_types.entry_invalid;
2744227448 }
27449 } else if (operand_type->id == ZigTypeIdFloat) {
27450 uint32_t max_atomic_bits = target_arch_largest_atomic_bits(ira->codegen->zig_target->arch);
27451 if (operand_type->data.floating.bit_count > max_atomic_bits) {
27452 ir_add_error(ira, op,
27453 buf_sprintf("expected %" PRIu32 "-bit float or smaller, found %" PRIu32 "-bit float",
27454 max_atomic_bits, (uint32_t) operand_type->data.floating.bit_count));
27455 return ira->codegen->builtin_types.entry_invalid;
27456 }
2744327457 } else if (get_codegen_ptr_type(operand_type) == nullptr) {
2744427458 ir_add_error(ira, op,
27445 buf_sprintf("expected integer, enum or pointer type, found '%s'", buf_ptr(&operand_type->name)));
27459 buf_sprintf("expected integer, float, enum or pointer type, found '%s'", buf_ptr(&operand_type->name)));
2744627460 return ira->codegen->builtin_types.entry_invalid;
2744727461 }
2744827462
......@@ -27477,6 +27491,10 @@ static IrInstruction *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstru
2747727491 ir_add_error(ira, instruction->op,
2747827492 buf_sprintf("@atomicRmw on enum only works with .Xchg"));
2747927493 return ira->codegen->invalid_instruction;
27494 } else if (operand_type->id == ZigTypeIdFloat && op > AtomicRmwOp_sub) {
27495 ir_add_error(ira, instruction->op,
27496 buf_sprintf("@atomicRmw with float only works with .Xchg, .Add and .Sub"));
27497 return ira->codegen->invalid_instruction;
2748027498 }
2748127499
2748227500 IrInstruction *operand = instruction->operand->child;
src/zig_llvm.cpp+50
......@@ -1096,6 +1096,56 @@ bool ZigLLDLink(ZigLLVM_ObjectFormatType oformat, const char **args, size_t arg_
10961096 abort();
10971097}
10981098
1099static AtomicRMWInst::BinOp toLLVMRMWBinOp(enum ZigLLVM_AtomicRMWBinOp BinOp) {
1100 switch (BinOp) {
1101 default:
1102 case ZigLLVMAtomicRMWBinOpXchg: return AtomicRMWInst::Xchg;
1103 case ZigLLVMAtomicRMWBinOpAdd: return AtomicRMWInst::Add;
1104 case ZigLLVMAtomicRMWBinOpSub: return AtomicRMWInst::Sub;
1105 case ZigLLVMAtomicRMWBinOpAnd: return AtomicRMWInst::And;
1106 case ZigLLVMAtomicRMWBinOpNand: return AtomicRMWInst::Nand;
1107 case ZigLLVMAtomicRMWBinOpOr: return AtomicRMWInst::Or;
1108 case ZigLLVMAtomicRMWBinOpXor: return AtomicRMWInst::Xor;
1109 case ZigLLVMAtomicRMWBinOpMax: return AtomicRMWInst::Max;
1110 case ZigLLVMAtomicRMWBinOpMin: return AtomicRMWInst::Min;
1111 case ZigLLVMAtomicRMWBinOpUMax: return AtomicRMWInst::UMax;
1112 case ZigLLVMAtomicRMWBinOpUMin: return AtomicRMWInst::UMin;
1113 case ZigLLVMAtomicRMWBinOpFAdd: return AtomicRMWInst::FAdd;
1114 case ZigLLVMAtomicRMWBinOpFSub: return AtomicRMWInst::FSub;
1115 }
1116}
1117
1118static AtomicOrdering toLLVMOrdering(LLVMAtomicOrdering Ordering) {
1119 switch (Ordering) {
1120 default:
1121 case LLVMAtomicOrderingNotAtomic: return AtomicOrdering::NotAtomic;
1122 case LLVMAtomicOrderingUnordered: return AtomicOrdering::Unordered;
1123 case LLVMAtomicOrderingMonotonic: return AtomicOrdering::Monotonic;
1124 case LLVMAtomicOrderingAcquire: return AtomicOrdering::Acquire;
1125 case LLVMAtomicOrderingRelease: return AtomicOrdering::Release;
1126 case LLVMAtomicOrderingAcquireRelease: return AtomicOrdering::AcquireRelease;
1127 case LLVMAtomicOrderingSequentiallyConsistent: return AtomicOrdering::SequentiallyConsistent;
1128 }
1129}
1130
1131inline LLVMAttributeRef wrap(Attribute Attr) {
1132 return reinterpret_cast<LLVMAttributeRef>(Attr.getRawPointer());
1133}
1134
1135inline Attribute unwrap(LLVMAttributeRef Attr) {
1136 return Attribute::fromRawPointer(Attr);
1137}
1138
1139LLVMValueRef ZigLLVMBuildAtomicRMW(LLVMBuilderRef B, enum ZigLLVM_AtomicRMWBinOp op,
1140 LLVMValueRef PTR, LLVMValueRef Val,
1141 LLVMAtomicOrdering ordering, LLVMBool singleThread)
1142{
1143 AtomicRMWInst::BinOp intop = toLLVMRMWBinOp(op);
1144 return wrap(unwrap(B)->CreateAtomicRMW(intop, unwrap(PTR),
1145 unwrap(Val), toLLVMOrdering(ordering),
1146 singleThread ? SyncScope::SingleThread : SyncScope::System));
1147}
1148
10991149static_assert((Triple::ArchType)ZigLLVM_UnknownArch == Triple::UnknownArch, "");
11001150static_assert((Triple::ArchType)ZigLLVM_arm == Triple::arm, "");
11011151static_assert((Triple::ArchType)ZigLLVM_armeb == Triple::armeb, "");
src/zig_llvm.h+20
......@@ -422,6 +422,26 @@ enum ZigLLVM_ObjectFormatType {
422422 ZigLLVM_XCOFF,
423423};
424424
425enum ZigLLVM_AtomicRMWBinOp {
426 ZigLLVMAtomicRMWBinOpXchg,
427 ZigLLVMAtomicRMWBinOpAdd,
428 ZigLLVMAtomicRMWBinOpSub,
429 ZigLLVMAtomicRMWBinOpAnd,
430 ZigLLVMAtomicRMWBinOpNand,
431 ZigLLVMAtomicRMWBinOpOr,
432 ZigLLVMAtomicRMWBinOpXor,
433 ZigLLVMAtomicRMWBinOpMax,
434 ZigLLVMAtomicRMWBinOpMin,
435 ZigLLVMAtomicRMWBinOpUMax,
436 ZigLLVMAtomicRMWBinOpUMin,
437 ZigLLVMAtomicRMWBinOpFAdd,
438 ZigLLVMAtomicRMWBinOpFSub,
439};
440
441LLVMValueRef ZigLLVMBuildAtomicRMW(LLVMBuilderRef B, enum ZigLLVM_AtomicRMWBinOp op,
442 LLVMValueRef PTR, LLVMValueRef Val,
443 LLVMAtomicOrdering ordering, LLVMBool singleThread);
444
425445#define ZigLLVM_DIFlags_Zero 0U
426446#define ZigLLVM_DIFlags_Private 1U
427447#define ZigLLVM_DIFlags_Protected 2U
test/compile_errors.zig+20
......@@ -31,6 +31,26 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
3131 "tmp.zig:3:12: note: destination pointer requires a terminating '0' sentinel",
3232 });
3333
34 cases.add(
35 "cmpxchg with float",
36 \\export fn entry() void {
37 \\ var x: f32 = 0;
38 \\ _ = @cmpxchgWeak(f32, &x, 1, 2, .SeqCst, .SeqCst);
39 \\}
40 , &[_][]const u8{
41 "tmp.zig:3:22: error: expected integer, enum or pointer type, found 'f32'",
42 });
43
44 cases.add(
45 "atomicrmw with float op not .Xchg, .Add or .Sub",
46 \\export fn entry() void {
47 \\ var x: f32 = 0;
48 \\ _ = @atomicRmw(f32, &x, .And, 2, .SeqCst);
49 \\}
50 , &[_][]const u8{
51 "tmp.zig:3:29: error: @atomicRmw with float only works with .Xchg, .Add and .Sub",
52 });
53
3454 cases.add("intToPtr with misaligned address",
3555 \\pub fn main() void {
3656 \\ var y = @intToPtr([*]align(4) u8, 5);
test/stage1/behavior/atomics.zig+17
......@@ -144,3 +144,20 @@ fn testAtomicStore() void {
144144 @atomicStore(u32, &x, 12345678, .SeqCst);
145145 expect(@atomicLoad(u32, &x, .SeqCst) == 12345678);
146146}
147
148test "atomicrmw with floats" {
149 if (builtin.arch == .aarch64 or builtin.arch == .arm)
150 return;
151 testAtomicRmwFloat();
152}
153
154fn testAtomicRmwFloat() void {
155 var x: f32 = 0;
156 expect(x == 0);
157 _ = @atomicRmw(f32, &x, .Xchg, 1, .SeqCst);
158 expect(x == 1);
159 _ = @atomicRmw(f32, &x, .Add, 5, .SeqCst);
160 expect(x == 6);
161 _ = @atomicRmw(f32, &x, .Sub, 2, .SeqCst);
162 expect(x == 4);
163}