authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-11-28 19:21:21+01:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-11-30 17:56:02+01:00
log6924f21bbd81683b0889994ce86aa0ae22e5b317
treea78d5cf5efbe86a6f21856ab9cdbb1a5113b5c49
parenta7ad1212cb1b127754c7e48ead8d80d66f0f6623
signaturelock-open Commit is signed but in an unrecognized format.

wasm: support non-natural alignment in load/store

This implements support for loading and storing where the lhs is of pointer type with host_size != 0. e.g. when loading a specific field from a packed struct with a non-byte alignment such as (0:1:3).

1 files changed, 144 insertions(+), 78 deletions(-)

src/arch/wasm/CodeGen.zig+144-78
......@@ -1492,12 +1492,15 @@ fn memcpy(func: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
14921492 // when the length is comptime-known, rather than a runtime value, we can optimize the generated code by having
14931493 // the loop during codegen, rather than inserting a runtime loop into the binary.
14941494 switch (len) {
1495 .imm32, .imm64 => {
1495 .imm32, .imm64 => blk: {
14961496 const length = switch (len) {
14971497 .imm32 => |val| val,
14981498 .imm64 => |val| val,
14991499 else => unreachable,
15001500 };
1501 // if the size (length) is more than 1024 bytes, we use a runtime loop instead to prevent
1502 // binary size bloat.
1503 if (length > 1024) break :blk;
15011504 var offset: u32 = 0;
15021505 const lhs_base = dst.offset();
15031506 const rhs_base = src.offset();
......@@ -1518,80 +1521,81 @@ fn memcpy(func: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
15181521 else => unreachable,
15191522 }
15201523 }
1524 return;
15211525 },
1522 else => {
1523 // TODO: We should probably lower this to a call to compiler_rt
1524 // But for now, we implement it manually
1525 var offset = try func.ensureAllocLocal(Type.usize); // local for counter
1526 defer offset.free(func);
1526 else => {},
1527 }
15271528
1528 // outer block to jump to when loop is done
1529 try func.startBlock(.block, wasm.block_empty);
1530 try func.startBlock(.loop, wasm.block_empty);
1529 // TODO: We should probably lower this to a call to compiler_rt
1530 // But for now, we implement it manually
1531 var offset = try func.ensureAllocLocal(Type.usize); // local for counter
1532 defer offset.free(func);
15311533
1532 // loop condition (offset == length -> break)
1533 {
1534 try func.emitWValue(offset);
1535 try func.emitWValue(len);
1536 switch (func.arch()) {
1537 .wasm32 => try func.addTag(.i32_eq),
1538 .wasm64 => try func.addTag(.i64_eq),
1539 else => unreachable,
1540 }
1541 try func.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
1542 }
1534 // outer block to jump to when loop is done
1535 try func.startBlock(.block, wasm.block_empty);
1536 try func.startBlock(.loop, wasm.block_empty);
15431537
1544 // get dst ptr
1545 {
1546 try func.emitWValue(dst);
1547 try func.emitWValue(offset);
1548 switch (func.arch()) {
1549 .wasm32 => try func.addTag(.i32_add),
1550 .wasm64 => try func.addTag(.i64_add),
1551 else => unreachable,
1552 }
1553 }
1538 // loop condition (offset == length -> break)
1539 {
1540 try func.emitWValue(offset);
1541 try func.emitWValue(len);
1542 switch (func.arch()) {
1543 .wasm32 => try func.addTag(.i32_eq),
1544 .wasm64 => try func.addTag(.i64_eq),
1545 else => unreachable,
1546 }
1547 try func.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
1548 }
15541549
1555 // get src value and also store in dst
1556 {
1557 try func.emitWValue(src);
1558 try func.emitWValue(offset);
1559 switch (func.arch()) {
1560 .wasm32 => {
1561 try func.addTag(.i32_add);
1562 try func.addMemArg(.i32_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1563 try func.addMemArg(.i32_store8, .{ .offset = dst.offset(), .alignment = 1 });
1564 },
1565 .wasm64 => {
1566 try func.addTag(.i64_add);
1567 try func.addMemArg(.i64_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1568 try func.addMemArg(.i64_store8, .{ .offset = dst.offset(), .alignment = 1 });
1569 },
1570 else => unreachable,
1571 }
1572 }
1550 // get dst ptr
1551 {
1552 try func.emitWValue(dst);
1553 try func.emitWValue(offset);
1554 switch (func.arch()) {
1555 .wasm32 => try func.addTag(.i32_add),
1556 .wasm64 => try func.addTag(.i64_add),
1557 else => unreachable,
1558 }
1559 }
15731560
1574 // increment loop counter
1575 {
1576 try func.emitWValue(offset);
1577 switch (func.arch()) {
1578 .wasm32 => {
1579 try func.addImm32(1);
1580 try func.addTag(.i32_add);
1581 },
1582 .wasm64 => {
1583 try func.addImm64(1);
1584 try func.addTag(.i64_add);
1585 },
1586 else => unreachable,
1587 }
1588 try func.addLabel(.local_set, offset.local.value);
1589 try func.addLabel(.br, 0); // jump to start of loop
1590 }
1591 try func.endBlock(); // close off loop block
1592 try func.endBlock(); // close off outer block
1593 },
1561 // get src value and also store in dst
1562 {
1563 try func.emitWValue(src);
1564 try func.emitWValue(offset);
1565 switch (func.arch()) {
1566 .wasm32 => {
1567 try func.addTag(.i32_add);
1568 try func.addMemArg(.i32_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1569 try func.addMemArg(.i32_store8, .{ .offset = dst.offset(), .alignment = 1 });
1570 },
1571 .wasm64 => {
1572 try func.addTag(.i64_add);
1573 try func.addMemArg(.i64_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1574 try func.addMemArg(.i64_store8, .{ .offset = dst.offset(), .alignment = 1 });
1575 },
1576 else => unreachable,
1577 }
15941578 }
1579
1580 // increment loop counter
1581 {
1582 try func.emitWValue(offset);
1583 switch (func.arch()) {
1584 .wasm32 => {
1585 try func.addImm32(1);
1586 try func.addTag(.i32_add);
1587 },
1588 .wasm64 => {
1589 try func.addImm64(1);
1590 try func.addTag(.i64_add);
1591 },
1592 else => unreachable,
1593 }
1594 try func.addLabel(.local_set, offset.local.value);
1595 try func.addLabel(.br, 0); // jump to start of loop
1596 }
1597 try func.endBlock(); // close off loop block
1598 try func.endBlock(); // close off outer block
15951599}
15961600
15971601fn ptrSize(func: *const CodeGen) u16 {
......@@ -2128,9 +2132,45 @@ fn airStore(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
21282132
21292133 const lhs = try func.resolveInst(bin_op.lhs);
21302134 const rhs = try func.resolveInst(bin_op.rhs);
2131 const ty = func.air.typeOf(bin_op.lhs).childType();
2135 const ptr_ty = func.air.typeOf(bin_op.lhs);
2136 const ptr_info = ptr_ty.ptrInfo().data;
2137 const ty = ptr_ty.childType();
2138 if (ptr_info.host_size == 0) {
2139 try func.store(lhs, rhs, ty, 0);
2140 } else {
2141 // at this point we have a non-natural alignment, we must
2142 // load the value, and then shift+or the rhs into the result location.
2143 var int_ty_payload: Type.Payload.Bits = .{
2144 .base = .{ .tag = .int_unsigned },
2145 .data = ptr_info.host_size * 8,
2146 };
2147 const int_elem_ty = Type.initPayload(&int_ty_payload.base);
2148
2149 var mask = @intCast(u64, (@as(u65, 1) << @intCast(u7, ty.bitSize(func.target))) - 1);
2150 mask <<= @intCast(u6, ptr_info.bit_offset);
2151 mask ^= ~@as(u64, 0);
2152 const shift_val = if (ptr_info.host_size <= 4)
2153 WValue{ .imm32 = ptr_info.bit_offset }
2154 else
2155 WValue{ .imm64 = ptr_info.bit_offset };
2156 const mask_val = if (ptr_info.host_size <= 4)
2157 WValue{ .imm32 = @truncate(u32, mask) }
2158 else
2159 WValue{ .imm64 = mask };
2160
2161 try func.emitWValue(lhs);
2162 const loaded = try func.load(lhs, int_elem_ty, 0);
2163 const anded = try func.binOp(loaded, mask_val, int_elem_ty, .@"and");
2164 const extended_value = try func.intcast(rhs, ty, int_elem_ty);
2165 const shifted_value = if (ptr_info.bit_offset > 0) shifted: {
2166 break :shifted try func.binOp(extended_value, shift_val, int_elem_ty, .shl);
2167 } else extended_value;
2168 const result = try func.binOp(anded, shifted_value, int_elem_ty, .@"or");
2169 std.debug.print("Host: {} ty {} ty {}\n", .{ ptr_info.host_size, int_elem_ty.fmtDebug(), ty.fmtDebug() });
2170 // lhs is still on the stack
2171 try func.store(.stack, result, int_elem_ty, 0);
2172 }
21322173
2133 try func.store(lhs, rhs, ty, 0);
21342174 func.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
21352175}
21362176
......@@ -2218,6 +2258,8 @@ fn airLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
22182258 const ty_op = func.air.instructions.items(.data)[inst].ty_op;
22192259 const operand = try func.resolveInst(ty_op.operand);
22202260 const ty = func.air.getRefType(ty_op.ty);
2261 const ptr_ty = func.air.typeOf(ty_op.operand);
2262 const ptr_info = ptr_ty.ptrInfo().data;
22212263
22222264 if (!ty.hasRuntimeBitsIgnoreComptime()) return func.finishAir(inst, .none, &.{ty_op.operand});
22232265
......@@ -2228,8 +2270,28 @@ fn airLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
22282270 break :result new_local;
22292271 }
22302272
2231 const stack_loaded = try func.load(operand, ty, 0);
2232 break :result try stack_loaded.toLocal(func, ty);
2273 if (ptr_info.host_size == 0) {
2274 const stack_loaded = try func.load(operand, ty, 0);
2275 break :result try stack_loaded.toLocal(func, ty);
2276 }
2277
2278 // at this point we have a non-natural alignment, we must
2279 // shift the value to obtain the correct bit.
2280 var int_ty_payload: Type.Payload.Bits = .{
2281 .base = .{ .tag = .int_unsigned },
2282 .data = ptr_info.host_size * 8,
2283 };
2284 const int_elem_ty = Type.initPayload(&int_ty_payload.base);
2285 const shift_val = if (ptr_info.host_size <= 4)
2286 WValue{ .imm32 = ptr_info.bit_offset }
2287 else
2288 WValue{ .imm64 = ptr_info.bit_offset };
2289
2290 const stack_loaded = try func.load(operand, int_elem_ty, 0);
2291 const shifted = try func.binOp(stack_loaded, shift_val, int_elem_ty, .shr);
2292 const result = try func.trunc(shifted, ty, int_elem_ty);
2293 // const wrapped = try func.wrapOperand(shifted, ty);
2294 break :result try result.toLocal(func, ty);
22332295 };
22342296 func.finishAir(inst, result, &.{ty_op.operand});
22352297}
......@@ -3151,7 +3213,7 @@ fn airStructFieldPtr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
31513213
31523214 const struct_ptr = try func.resolveInst(extra.data.struct_operand);
31533215 const struct_ty = func.air.typeOf(extra.data.struct_operand).childType();
3154 const result = try func.structFieldPtr(struct_ptr, struct_ty, extra.data.field_index);
3216 const result = try func.structFieldPtr(extra.data.struct_operand, struct_ptr, struct_ty, extra.data.field_index);
31553217 func.finishAir(inst, result, &.{extra.data.struct_operand});
31563218}
31573219
......@@ -3161,11 +3223,11 @@ fn airStructFieldPtrIndex(func: *CodeGen, inst: Air.Inst.Index, index: u32) Inne
31613223 const struct_ptr = try func.resolveInst(ty_op.operand);
31623224 const struct_ty = func.air.typeOf(ty_op.operand).childType();
31633225
3164 const result = try func.structFieldPtr(struct_ptr, struct_ty, index);
3226 const result = try func.structFieldPtr(ty_op.operand, struct_ptr, struct_ty, index);
31653227 func.finishAir(inst, result, &.{ty_op.operand});
31663228}
31673229
3168fn structFieldPtr(func: *CodeGen, struct_ptr: WValue, struct_ty: Type, index: u32) InnerError!WValue {
3230fn structFieldPtr(func: *CodeGen, ref: Air.Inst.Ref, struct_ptr: WValue, struct_ty: Type, index: u32) InnerError!WValue {
31693231 const offset = switch (struct_ty.containerLayout()) {
31703232 .Packed => switch (struct_ty.zigTypeTag()) {
31713233 .Struct => struct_ty.packedStructFieldByteOffset(index, func.target),
......@@ -3174,6 +3236,10 @@ fn structFieldPtr(func: *CodeGen, struct_ptr: WValue, struct_ty: Type, index: u3
31743236 },
31753237 else => struct_ty.structFieldOffset(index, func.target),
31763238 };
3239 // save a load and store when we can simply reuse the operand
3240 if (offset == 0) {
3241 return func.reuseOperand(ref, struct_ptr);
3242 }
31773243 switch (struct_ptr) {
31783244 .stack_offset => |stack_offset| {
31793245 return WValue{ .stack_offset = .{ .value = stack_offset.value + @intCast(u32, offset), .references = 1 } };
......@@ -3893,9 +3959,9 @@ fn airTrunc(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
38933959/// Truncates a given operand to a given type, discarding any overflown bits.
38943960/// NOTE: Resulting value is left on the stack.
38953961fn trunc(func: *CodeGen, operand: WValue, wanted_ty: Type, given_ty: Type) InnerError!WValue {
3896 const int_info = given_ty.intInfo(func.target);
3897 if (toWasmBits(int_info.bits) == null) {
3898 return func.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{int_info.bits});
3962 const given_bits = @intCast(u16, given_ty.bitSize(func.target));
3963 if (toWasmBits(given_bits) == null) {
3964 return func.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{given_bits});
38993965 }
39003966
39013967 var result = try func.intcast(operand, given_ty, wanted_ty);