authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2022-12-02 19:54:00-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2022-12-02 22:21:24-05:00
logfdedd62365b5e3f3fa908ae063df86d11a3c02bb
tree46f6b65879ff774baec072d6ad745106f7ff3032
parentaf4361f57af388238a075b0c8ef5b34e75b73787

cbe: use memcpy for underaligned loads and stores

8 files changed, 22 insertions(+), 29 deletions(-)

src/codegen/c.zig+22-15
...@@ -2999,30 +2999,31 @@ fn airArg(f: *Function) CValue {...@@ -2999,30 +2999,31 @@ fn airArg(f: *Function) CValue {
2999fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {2999fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
3000 const ty_op = f.air.instructions.items(.data)[inst].ty_op;3000 const ty_op = f.air.instructions.items(.data)[inst].ty_op;
3001 const ptr_info = f.air.typeOf(ty_op.operand).ptrInfo().data;3001 const ptr_info = f.air.typeOf(ty_op.operand).ptrInfo().data;
3002 const src_ty = ptr_info.pointee_type;
30023003
3003 const inst_ty = f.air.typeOfIndex(inst);3004 if (!src_ty.hasRuntimeBitsIgnoreComptime() or
3004 if (!inst_ty.hasRuntimeBitsIgnoreComptime() or
3005 !ptr_info.@"volatile" and f.liveness.isUnused(inst))3005 !ptr_info.@"volatile" and f.liveness.isUnused(inst))
3006 return CValue.none;3006 return CValue.none;
30073007
3008 const target = f.object.dg.module.getTarget();3008 const target = f.object.dg.module.getTarget();
3009 const is_array = lowersToArray(inst_ty, target);3009 const is_aligned = ptr_info.@"align" == 0 or ptr_info.@"align" >= src_ty.abiAlignment(target);
3010 const is_array = lowersToArray(src_ty, target);
3011 const need_memcpy = !is_aligned or is_array;
3010 const operand = try f.resolveInst(ty_op.operand);3012 const operand = try f.resolveInst(ty_op.operand);
3011 const writer = f.object.writer();3013 const writer = f.object.writer();
30123014
3013 // We need to separately initialize arrays with a memcpy so they must be mutable.3015 // We need to initialize arrays and unaligned loads with a memcpy so they must be mutable.
3014 const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const);3016 const local = try f.allocLocal(src_ty, if (need_memcpy) .Mut else .Const);
30153017
3016 if (is_array) {3018 if (need_memcpy) {
3017 // Insert a memcpy to initialize this array. The source operand is always a pointer
3018 // and thus we only need to know size/type information from the local type/dest.
3019 try writer.writeAll(";\n");3019 try writer.writeAll(";\n");
3020 try writer.writeAll("memcpy(");3020 try writer.writeAll("memcpy(");
3021 if (!is_array) try writer.writeByte('&');
3021 try f.writeCValue(writer, local, .FunctionArgument);3022 try f.writeCValue(writer, local, .FunctionArgument);
3022 try writer.writeAll(", ");3023 try writer.writeAll(", (const char *)");
3023 try f.writeCValue(writer, operand, .FunctionArgument);3024 try f.writeCValue(writer, operand, .Other);
3024 try writer.writeAll(", sizeof(");3025 try writer.writeAll(", sizeof(");
3025 try f.renderTypecast(writer, inst_ty);3026 try f.renderTypecast(writer, src_ty);
3026 try writer.writeAll("))");3027 try writer.writeAll("))");
3027 } else if (ptr_info.host_size != 0) {3028 } else if (ptr_info.host_size != 0) {
3028 var host_pl = Type.Payload.Bits{3029 var host_pl = Type.Payload.Bits{
...@@ -3045,12 +3046,12 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3045,12 +3046,12 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
30453046
3046 var field_pl = Type.Payload.Bits{3047 var field_pl = Type.Payload.Bits{
3047 .base = .{ .tag = .int_unsigned },3048 .base = .{ .tag = .int_unsigned },
3048 .data = @intCast(u16, inst_ty.bitSize(target)),3049 .data = @intCast(u16, src_ty.bitSize(target)),
3049 };3050 };
3050 const field_ty = Type.initPayload(&field_pl.base);3051 const field_ty = Type.initPayload(&field_pl.base);
30513052
3052 try writer.writeAll(" = (");3053 try writer.writeAll(" = (");
3053 try f.renderTypecast(writer, inst_ty);3054 try f.renderTypecast(writer, src_ty);
3054 try writer.writeAll(")zig_wrap_");3055 try writer.writeAll(")zig_wrap_");
3055 try f.object.dg.renderTypeForBuiltinFnName(writer, field_ty);3056 try f.object.dg.renderTypeForBuiltinFnName(writer, field_ty);
3056 try writer.writeAll("((");3057 try writer.writeAll("((");
...@@ -3226,8 +3227,13 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3226,8 +3227,13 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue {
3226 return try airStoreUndefined(f, ptr_info.pointee_type, ptr_val);3227 return try airStoreUndefined(f, ptr_info.pointee_type, ptr_val);
32273228
3228 const target = f.object.dg.module.getTarget();3229 const target = f.object.dg.module.getTarget();
3230 const is_aligned = ptr_info.@"align" == 0 or
3231 ptr_info.@"align" >= ptr_info.pointee_type.abiAlignment(target);
3232 const is_array = lowersToArray(ptr_info.pointee_type, target);
3233 const need_memcpy = !is_aligned or is_array;
3229 const writer = f.object.writer();3234 const writer = f.object.writer();
3230 if (lowersToArray(ptr_info.pointee_type, target)) {3235
3236 if (need_memcpy) {
3231 // For this memcpy to safely work we need the rhs to have the same3237 // For this memcpy to safely work we need the rhs to have the same
3232 // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).3238 // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).
3233 assert(src_ty.eql(ptr_info.pointee_type, f.object.dg.module));3239 assert(src_ty.eql(ptr_info.pointee_type, f.object.dg.module));
...@@ -3244,9 +3250,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3244,9 +3250,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue {
3244 break :blk new_local;3250 break :blk new_local;
3245 } else src_val;3251 } else src_val;
32463252
3247 try writer.writeAll("memcpy(");3253 try writer.writeAll("memcpy((char *)");
3248 try f.writeCValue(writer, ptr_val, .FunctionArgument);3254 try f.writeCValue(writer, ptr_val, .FunctionArgument);
3249 try writer.writeAll(", ");3255 try writer.writeAll(", ");
3256 if (!is_array) try writer.writeByte('&');
3250 try f.writeCValue(writer, array_src, .FunctionArgument);3257 try f.writeCValue(writer, array_src, .FunctionArgument);
3251 try writer.writeAll(", sizeof(");3258 try writer.writeAll(", sizeof(");
3252 try f.renderTypecast(writer, src_ty);3259 try f.renderTypecast(writer, src_ty);
test/behavior/basic.zig-2
...@@ -754,7 +754,6 @@ fn maybe(x: bool) anyerror!?u32 {...@@ -754,7 +754,6 @@ fn maybe(x: bool) anyerror!?u32 {
754}754}
755755
756test "pointer to thread local array" {756test "pointer to thread local array" {
757 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
758 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO757 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
759 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO758 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
760 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO759 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
...@@ -770,7 +769,6 @@ threadlocal var buffer: [11]u8 = undefined;...@@ -770,7 +769,6 @@ threadlocal var buffer: [11]u8 = undefined;
770769
771test "auto created variables have correct alignment" {770test "auto created variables have correct alignment" {
772 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO771 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
773 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
774772
775 const S = struct {773 const S = struct {
776 fn foo(str: [*]const u8) u32 {774 fn foo(str: [*]const u8) u32 {
test/behavior/bitreverse.zig-1
...@@ -155,7 +155,6 @@ fn vector0() !void {...@@ -155,7 +155,6 @@ fn vector0() !void {
155test "bitReverse vectors u0" {155test "bitReverse vectors u0" {
156 if (builtin.zig_backend == .stage1) return error.SkipZigTest;156 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
157 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;157 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
158 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
159 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;158 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
160159
161 comptime try vector0();160 comptime try vector0();
test/behavior/byteswap.zig-1
...@@ -121,7 +121,6 @@ fn vector0() !void {...@@ -121,7 +121,6 @@ fn vector0() !void {
121test "@byteSwap vectors u0" {121test "@byteSwap vectors u0" {
122 if (builtin.zig_backend == .stage1) return error.SkipZigTest;122 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
123 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;123 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
124 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
125 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;124 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
126125
127 comptime try vector0();126 comptime try vector0();
test/behavior/math.zig-1
...@@ -1518,7 +1518,6 @@ fn testRound(comptime T: type, x: T) !void {...@@ -1518,7 +1518,6 @@ fn testRound(comptime T: type, x: T) !void {
1518}1518}
15191519
1520test "vector integer addition" {1520test "vector integer addition" {
1521 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
1522 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1521 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1523 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1522 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1524 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1523 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
test/behavior/ptrcast.zig-1
...@@ -78,7 +78,6 @@ fn testReinterpretBytesAsExternStruct() !void {...@@ -78,7 +78,6 @@ fn testReinterpretBytesAsExternStruct() !void {
7878
79test "reinterpret bytes of an extern struct (with under-aligned fields) into another" {79test "reinterpret bytes of an extern struct (with under-aligned fields) into another" {
80 if (builtin.zig_backend == .stage1) return error.SkipZigTest;80 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
81 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO: Under-aligned fields are not yet supported in the CBE
8281
83 try testReinterpretExternStructAsExternStruct();82 try testReinterpretExternStructAsExternStruct();
84 comptime try testReinterpretExternStructAsExternStruct();83 comptime try testReinterpretExternStructAsExternStruct();
test/behavior/struct.zig-1
...@@ -641,7 +641,6 @@ test "default struct initialization fields" {...@@ -641,7 +641,6 @@ test "default struct initialization fields" {
641641
642test "packed array 24bits" {642test "packed array 24bits" {
643 if (builtin.zig_backend == .stage1) return error.SkipZigTest;643 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
644 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
645 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;644 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
646 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;645 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
647 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;646 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
test/behavior/vector.zig-7
...@@ -75,7 +75,6 @@ test "vector bin compares with mem.eql" {...@@ -75,7 +75,6 @@ test "vector bin compares with mem.eql" {
7575
76test "vector int operators" {76test "vector int operators" {
77 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO77 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
78 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
79 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO78 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
80 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO79 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
81 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO80 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...@@ -139,7 +138,6 @@ test "vector bit operators" {...@@ -139,7 +138,6 @@ test "vector bit operators" {
139138
140test "implicit cast vector to array" {139test "implicit cast vector to array" {
141 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO140 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
142 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
143 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO141 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
144 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO142 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
145 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO143 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...@@ -158,7 +156,6 @@ test "implicit cast vector to array" {...@@ -158,7 +156,6 @@ test "implicit cast vector to array" {
158156
159test "array to vector" {157test "array to vector" {
160 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO158 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
161 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
162 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO159 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
163 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO160 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
164 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO161 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...@@ -177,7 +174,6 @@ test "array to vector" {...@@ -177,7 +174,6 @@ test "array to vector" {
177174
178test "tuple to vector" {175test "tuple to vector" {
179 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO176 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
180 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
181 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO177 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
182 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO178 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
183 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO179 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...@@ -210,7 +206,6 @@ test "tuple to vector" {...@@ -210,7 +206,6 @@ test "tuple to vector" {
210206
211test "vector casts of sizes not divisible by 8" {207test "vector casts of sizes not divisible by 8" {
212 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO208 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
213 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
214 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO209 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
215 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO210 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
216 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO211 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...@@ -945,7 +940,6 @@ test "multiplication-assignment operator with an array operand" {...@@ -945,7 +940,6 @@ test "multiplication-assignment operator with an array operand" {
945 return error.SkipZigTest;940 return error.SkipZigTest;
946 }941 }
947 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO942 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
948 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
949 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO943 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
950 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO944 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
951 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO945 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...@@ -1194,7 +1188,6 @@ test "zero divisor" {...@@ -1194,7 +1188,6 @@ test "zero divisor" {
11941188
1195test "zero multiplicand" {1189test "zero multiplicand" {
1196 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1190 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1197 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
1198 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO1191 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1199 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1192 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1200 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1193 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO