| ... | ... | @@ -759,6 +759,16 @@ fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue { |
| 759 | 759 | return result; |
| 760 | 760 | } |
| 761 | 761 | |
| 762 | fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue { |
| 763 | const zcu = cg.pt.zcu; |
| 764 | const ty = val.typeOf(zcu); |
| 765 | |
| 766 | return if (isByRef(ty, zcu, cg.target)) |
| 767 | .{ .uav_ref = .{ .ip_index = val.toIntern() } } |
| 768 | else |
| 769 | try cg.lowerConstant(val, ty); |
| 770 | } |
| 771 | |
| 762 | 772 | /// NOTE: if result == .stack, it will be stored in .local |
| 763 | 773 | fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void { |
| 764 | 774 | assert(operands.len <= Liveness.bpi - 1); |
| ... | ... | @@ -1588,12 +1598,18 @@ fn toWasmBits(bits: u16) ?u16 { |
| 1588 | 1598 | /// Performs a copy of bytes for a given type. Copying all bytes |
| 1589 | 1599 | /// from rhs to lhs. |
| 1590 | 1600 | fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { |
| 1601 | const len_known_neq_0 = switch (len) { |
| 1602 | .imm32 => |val| if (val != 0) true else return, |
| 1603 | .imm64 => |val| if (val != 0) true else return, |
| 1604 | else => false, |
| 1605 | }; |
| 1591 | 1606 | // When bulk_memory is enabled, we lower it to wasm's memcpy instruction. |
| 1592 | 1607 | // If not, we lower it ourselves manually |
| 1593 | 1608 | if (std.Target.wasm.featureSetHas(cg.target.cpu.features, .bulk_memory)) { |
| 1594 | 1609 | const len0_ok = std.Target.wasm.featureSetHas(cg.target.cpu.features, .nontrapping_bulk_memory_len0); |
| 1610 | const emit_check = !(len0_ok or len_known_neq_0); |
| 1595 | 1611 | |
| 1596 | | if (!len0_ok) { |
| 1612 | if (emit_check) { |
| 1597 | 1613 | try cg.startBlock(.block, .empty); |
| 1598 | 1614 | |
| 1599 | 1615 | // Even if `len` is zero, the spec requires an implementation to trap if `src + len` or |
| ... | ... | @@ -1616,7 +1632,7 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { |
| 1616 | 1632 | try cg.emitWValue(len); |
| 1617 | 1633 | try cg.addExtended(.memory_copy); |
| 1618 | 1634 | |
| 1619 | | if (!len0_ok) { |
| 1635 | if (emit_check) { |
| 1620 | 1636 | try cg.endBlock(); |
| 1621 | 1637 | } |
| 1622 | 1638 | |
| ... | ... | @@ -2296,11 +2312,6 @@ fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 2296 | 2312 | fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { |
| 2297 | 2313 | const pt = cg.pt; |
| 2298 | 2314 | const zcu = pt.zcu; |
| 2299 | | if (safety) { |
| 2300 | | // TODO if the value is undef, write 0xaa bytes to dest |
| 2301 | | } else { |
| 2302 | | // TODO if the value is undef, don't lower this instruction |
| 2303 | | } |
| 2304 | 2315 | const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 2305 | 2316 | |
| 2306 | 2317 | const lhs = try cg.resolveInst(bin_op.lhs); |
| ... | ... | @@ -2309,44 +2320,65 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { |
| 2309 | 2320 | const ptr_info = ptr_ty.ptrInfo(zcu); |
| 2310 | 2321 | const ty = ptr_ty.childType(zcu); |
| 2311 | 2322 | |
| 2323 | if (!safety and bin_op.rhs == .undef) { |
| 2324 | return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); |
| 2325 | } |
| 2326 | |
| 2312 | 2327 | if (ptr_info.packed_offset.host_size == 0) { |
| 2313 | 2328 | try cg.store(lhs, rhs, ty, 0); |
| 2314 | 2329 | } else { |
| 2315 | 2330 | // at this point we have a non-natural alignment, we must |
| 2316 | 2331 | // load the value, and then shift+or the rhs into the result location. |
| 2317 | | const int_elem_ty = try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8); |
| 2332 | const host_size = ptr_info.packed_offset.host_size * 8; |
| 2333 | const host_ty = try pt.intType(.unsigned, host_size); |
| 2334 | const bit_size: u16 = @intCast(ty.bitSize(zcu)); |
| 2335 | const bit_offset = ptr_info.packed_offset.bit_offset; |
| 2336 | |
| 2337 | const mask_val = try cg.resolveValue(val: { |
| 2338 | const limbs = try cg.gpa.alloc( |
| 2339 | std.math.big.Limb, |
| 2340 | std.math.big.int.calcTwosCompLimbCount(host_size) + 1, |
| 2341 | ); |
| 2342 | defer cg.gpa.free(limbs); |
| 2318 | 2343 | |
| 2319 | | if (isByRef(int_elem_ty, zcu, cg.target)) { |
| 2320 | | return cg.fail("TODO: airStore for pointers to bitfields with backing type larger than 64bits", .{}); |
| 2321 | | } |
| 2344 | var mask_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; |
| 2345 | mask_bigint.setTwosCompIntLimit(.max, .unsigned, host_size); |
| 2346 | |
| 2347 | if (bit_size != host_size) { |
| 2348 | mask_bigint.shiftRight(mask_bigint.toConst(), host_size - bit_size); |
| 2349 | } |
| 2350 | if (bit_offset != 0) { |
| 2351 | mask_bigint.shiftLeft(mask_bigint.toConst(), bit_offset); |
| 2352 | } |
| 2353 | mask_bigint.bitNotWrap(mask_bigint.toConst(), .unsigned, host_size); |
| 2354 | |
| 2355 | break :val try pt.intValue_big(host_ty, mask_bigint.toConst()); |
| 2356 | }); |
| 2322 | 2357 | |
| 2323 | | var mask = @as(u64, @intCast((@as(u65, 1) << @as(u7, @intCast(ty.bitSize(zcu)))) - 1)); |
| 2324 | | mask <<= @as(u6, @intCast(ptr_info.packed_offset.bit_offset)); |
| 2325 | | mask ^= ~@as(u64, 0); |
| 2326 | | const shift_val: WValue = if (ptr_info.packed_offset.host_size <= 4) |
| 2327 | | .{ .imm32 = ptr_info.packed_offset.bit_offset } |
| 2358 | const shift_val: WValue = if (33 <= host_size and host_size <= 64) |
| 2359 | .{ .imm64 = bit_offset } |
| 2328 | 2360 | else |
| 2329 | | .{ .imm64 = ptr_info.packed_offset.bit_offset }; |
| 2330 | | const mask_val: WValue = if (ptr_info.packed_offset.host_size <= 4) |
| 2331 | | .{ .imm32 = @as(u32, @truncate(mask)) } |
| 2361 | .{ .imm32 = bit_offset }; |
| 2362 | |
| 2363 | if (host_size <= 64) { |
| 2364 | try cg.emitWValue(lhs); |
| 2365 | } |
| 2366 | const loaded = if (host_size <= 64) |
| 2367 | try cg.load(lhs, host_ty, 0) |
| 2332 | 2368 | else |
| 2333 | | .{ .imm64 = mask }; |
| 2334 | | const wrap_mask_val: WValue = if (ptr_info.packed_offset.host_size <= 4) |
| 2335 | | .{ .imm32 = @truncate(~@as(u64, 0) >> @intCast(64 - ty.bitSize(zcu))) } |
| 2369 | lhs; |
| 2370 | const anded = try cg.binOp(loaded, mask_val, host_ty, .@"and"); |
| 2371 | const extended_value = try cg.intcast(rhs, ty, host_ty); |
| 2372 | const shifted_value = if (bit_offset > 0) |
| 2373 | try cg.binOp(extended_value, shift_val, host_ty, .shl) |
| 2336 | 2374 | else |
| 2337 | | .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - ty.bitSize(zcu)) }; |
| 2338 | | |
| 2339 | | try cg.emitWValue(lhs); |
| 2340 | | const loaded = try cg.load(lhs, int_elem_ty, 0); |
| 2341 | | const anded = try cg.binOp(loaded, mask_val, int_elem_ty, .@"and"); |
| 2342 | | const extended_value = try cg.intcast(rhs, ty, int_elem_ty); |
| 2343 | | const masked_value = try cg.binOp(extended_value, wrap_mask_val, int_elem_ty, .@"and"); |
| 2344 | | const shifted_value = if (ptr_info.packed_offset.bit_offset > 0) shifted: { |
| 2345 | | break :shifted try cg.binOp(masked_value, shift_val, int_elem_ty, .shl); |
| 2346 | | } else masked_value; |
| 2347 | | const result = try cg.binOp(anded, shifted_value, int_elem_ty, .@"or"); |
| 2348 | | // lhs is still on the stack |
| 2349 | | try cg.store(.stack, result, int_elem_ty, lhs.offset()); |
| 2375 | extended_value; |
| 2376 | const result = try cg.binOp(anded, shifted_value, host_ty, .@"or"); |
| 2377 | if (host_size <= 64) { |
| 2378 | try cg.store(.stack, result, host_ty, lhs.offset()); |
| 2379 | } else { |
| 2380 | try cg.store(lhs, result, host_ty, lhs.offset()); |
| 2381 | } |
| 2350 | 2382 | } |
| 2351 | 2383 | |
| 2352 | 2384 | return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); |
| ... | ... | @@ -2357,33 +2389,39 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr |
| 2357 | 2389 | const pt = cg.pt; |
| 2358 | 2390 | const zcu = pt.zcu; |
| 2359 | 2391 | const abi_size = ty.abiSize(zcu); |
| 2392 | |
| 2393 | if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) return; |
| 2394 | |
| 2360 | 2395 | switch (ty.zigTypeTag(zcu)) { |
| 2361 | 2396 | .error_union => { |
| 2362 | 2397 | const pl_ty = ty.errorUnionPayload(zcu); |
| 2363 | 2398 | if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 2364 | | return cg.store(lhs, rhs, Type.anyerror, 0); |
| 2399 | return cg.store(lhs, rhs, Type.anyerror, offset); |
| 2365 | 2400 | } |
| 2366 | 2401 | |
| 2367 | 2402 | const len = @as(u32, @intCast(abi_size)); |
| 2403 | assert(offset == 0); |
| 2368 | 2404 | return cg.memcpy(lhs, rhs, .{ .imm32 = len }); |
| 2369 | 2405 | }, |
| 2370 | 2406 | .optional => { |
| 2371 | 2407 | if (ty.isPtrLikeOptional(zcu)) { |
| 2372 | | return cg.store(lhs, rhs, Type.usize, 0); |
| 2408 | return cg.store(lhs, rhs, Type.usize, offset); |
| 2373 | 2409 | } |
| 2374 | 2410 | const pl_ty = ty.optionalChild(zcu); |
| 2375 | 2411 | if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 2376 | | return cg.store(lhs, rhs, Type.u8, 0); |
| 2412 | return cg.store(lhs, rhs, Type.u8, offset); |
| 2377 | 2413 | } |
| 2378 | 2414 | if (pl_ty.zigTypeTag(zcu) == .error_set) { |
| 2379 | | return cg.store(lhs, rhs, Type.anyerror, 0); |
| 2415 | return cg.store(lhs, rhs, Type.anyerror, offset); |
| 2380 | 2416 | } |
| 2381 | 2417 | |
| 2382 | 2418 | const len = @as(u32, @intCast(abi_size)); |
| 2419 | assert(offset == 0); |
| 2383 | 2420 | return cg.memcpy(lhs, rhs, .{ .imm32 = len }); |
| 2384 | 2421 | }, |
| 2385 | 2422 | .@"struct", .array, .@"union" => if (isByRef(ty, zcu, cg.target)) { |
| 2386 | 2423 | const len = @as(u32, @intCast(abi_size)); |
| 2424 | assert(offset == 0); |
| 2387 | 2425 | return cg.memcpy(lhs, rhs, .{ .imm32 = len }); |
| 2388 | 2426 | }, |
| 2389 | 2427 | .vector => switch (determineSimdStoreStrategy(ty, zcu, cg.target)) { |
| ... | ... | @@ -2407,6 +2445,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr |
| 2407 | 2445 | }, |
| 2408 | 2446 | .pointer => { |
| 2409 | 2447 | if (ty.isSlice(zcu)) { |
| 2448 | assert(offset == 0); |
| 2410 | 2449 | // store pointer first |
| 2411 | 2450 | // lower it to the stack so we do not have to store rhs into a local first |
| 2412 | 2451 | try cg.emitWValue(lhs); |
| ... | ... | @@ -2421,6 +2460,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr |
| 2421 | 2460 | } |
| 2422 | 2461 | }, |
| 2423 | 2462 | .int, .@"enum", .float => if (abi_size > 8 and abi_size <= 16) { |
| 2463 | assert(offset == 0); |
| 2424 | 2464 | try cg.emitWValue(lhs); |
| 2425 | 2465 | const lsb = try cg.load(rhs, Type.u64, 0); |
| 2426 | 2466 | try cg.store(.stack, lsb, Type.u64, 0 + lhs.offset()); |
| ... | ... | @@ -2430,6 +2470,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr |
| 2430 | 2470 | try cg.store(.stack, msb, Type.u64, 8 + lhs.offset()); |
| 2431 | 2471 | return; |
| 2432 | 2472 | } else if (abi_size > 16) { |
| 2473 | assert(offset == 0); |
| 2433 | 2474 | try cg.memcpy(lhs, rhs, .{ .imm32 = @as(u32, @intCast(ty.abiSize(zcu))) }); |
| 2434 | 2475 | }, |
| 2435 | 2476 | else => if (abi_size > 8) { |
| ... | ... | @@ -2480,22 +2521,30 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 2480 | 2521 | } |
| 2481 | 2522 | |
| 2482 | 2523 | if (ptr_info.packed_offset.host_size == 0) { |
| 2483 | | break :result try cg.load(operand, ty, 0); |
| 2484 | | } |
| 2485 | | |
| 2486 | | // at this point we have a non-natural alignment, we must |
| 2487 | | // shift the value to obtain the correct bit. |
| 2488 | | const int_elem_ty = try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8); |
| 2489 | | const shift_val: WValue = if (ptr_info.packed_offset.host_size <= 4) |
| 2490 | | .{ .imm32 = ptr_info.packed_offset.bit_offset } |
| 2491 | | else if (ptr_info.packed_offset.host_size <= 8) |
| 2492 | | .{ .imm64 = ptr_info.packed_offset.bit_offset } |
| 2493 | | else |
| 2494 | | return cg.fail("TODO: airLoad where ptr to bitfield exceeds 64 bits", .{}); |
| 2524 | const loaded = try cg.load(operand, ty, 0); |
| 2525 | const ty_size = ty.abiSize(zcu); |
| 2526 | if (ty.isAbiInt(zcu) and ty_size * 8 > ty.bitSize(zcu)) { |
| 2527 | const int_elem_ty = try pt.intType(.unsigned, @intCast(ty_size * 8)); |
| 2528 | break :result try cg.trunc(loaded, ty, int_elem_ty); |
| 2529 | } else { |
| 2530 | break :result loaded; |
| 2531 | } |
| 2532 | } else { |
| 2533 | const int_elem_ty = try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8); |
| 2534 | const shift_val: WValue = if (ptr_info.packed_offset.host_size <= 4) |
| 2535 | .{ .imm32 = ptr_info.packed_offset.bit_offset } |
| 2536 | else if (ptr_info.packed_offset.host_size <= 8) |
| 2537 | .{ .imm64 = ptr_info.packed_offset.bit_offset } |
| 2538 | else |
| 2539 | .{ .imm32 = ptr_info.packed_offset.bit_offset }; |
| 2495 | 2540 | |
| 2496 | | const stack_loaded = try cg.load(operand, int_elem_ty, 0); |
| 2497 | | const shifted = try cg.binOp(stack_loaded, shift_val, int_elem_ty, .shr); |
| 2498 | | break :result try cg.trunc(shifted, ty, int_elem_ty); |
| 2541 | const stack_loaded = if (ptr_info.packed_offset.host_size <= 8) |
| 2542 | try cg.load(operand, int_elem_ty, 0) |
| 2543 | else |
| 2544 | operand; |
| 2545 | const shifted = try cg.binOp(stack_loaded, shift_val, int_elem_ty, .shr); |
| 2546 | break :result try cg.trunc(shifted, ty, int_elem_ty); |
| 2547 | } |
| 2499 | 2548 | }; |
| 2500 | 2549 | return cg.finishAir(inst, result, &.{ty_op.operand}); |
| 2501 | 2550 | } |
| ... | ... | @@ -3118,7 +3167,13 @@ fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerErro |
| 3118 | 3167 | .nav => |nav| return .{ .nav_ref = .{ .nav_index = nav, .offset = @intCast(offset) } }, |
| 3119 | 3168 | .uav => |uav| return .{ .uav_ref = .{ .ip_index = uav.val, .offset = @intCast(offset), .orig_ptr_ty = uav.orig_ty } }, |
| 3120 | 3169 | .int => return cg.lowerConstant(try pt.intValue(Type.usize, offset), Type.usize), |
| 3121 | | .eu_payload => return cg.fail("Wasm TODO: lower error union payload pointer", .{}), |
| 3170 | .eu_payload => |eu_ptr| try cg.lowerPtr( |
| 3171 | eu_ptr, |
| 3172 | offset + codegen.errUnionPayloadOffset( |
| 3173 | Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu), |
| 3174 | zcu, |
| 3175 | ), |
| 3176 | ), |
| 3122 | 3177 | .opt_payload => |opt_ptr| return cg.lowerPtr(opt_ptr, offset), |
| 3123 | 3178 | .field => |field| { |
| 3124 | 3179 | const base_ptr = Value.fromInterned(field.base); |
| ... | ... | @@ -3137,19 +3192,7 @@ fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerErro |
| 3137 | 3192 | .@"extern", .@"packed" => unreachable, |
| 3138 | 3193 | }, |
| 3139 | 3194 | .@"union" => switch (base_ty.containerLayout(zcu)) { |
| 3140 | | .auto => off: { |
| 3141 | | // Keep in sync with the `un` case of `generateSymbol`. |
| 3142 | | const layout = base_ty.unionGetLayout(zcu); |
| 3143 | | if (layout.payload_size == 0) break :off 0; |
| 3144 | | if (layout.tag_size == 0) break :off 0; |
| 3145 | | if (layout.tag_align.compare(.gte, layout.payload_align)) { |
| 3146 | | // Tag first. |
| 3147 | | break :off layout.tag_size; |
| 3148 | | } else { |
| 3149 | | // Payload first. |
| 3150 | | break :off 0; |
| 3151 | | } |
| 3152 | | }, |
| 3195 | .auto => base_ty.structFieldOffset(@intCast(field.index), zcu), |
| 3153 | 3196 | .@"extern", .@"packed" => unreachable, |
| 3154 | 3197 | }, |
| 3155 | 3198 | else => unreachable, |
| ... | ... | @@ -3295,16 +3338,16 @@ fn lowerConstant(cg: *CodeGen, val: Value, ty: Type) InnerError!WValue { |
| 3295 | 3338 | }, |
| 3296 | 3339 | else => unreachable, |
| 3297 | 3340 | }, |
| 3298 | | .un => |un| { |
| 3299 | | // in this case we have a packed union which will not be passed by reference. |
| 3300 | | const constant_ty = if (un.tag == .none) |
| 3301 | | try ty.unionBackingType(pt) |
| 3302 | | else field_ty: { |
| 3303 | | const union_obj = zcu.typeToUnion(ty).?; |
| 3304 | | const field_index = zcu.unionTagFieldIndex(union_obj, Value.fromInterned(un.tag)).?; |
| 3305 | | break :field_ty Type.fromInterned(union_obj.field_types.get(ip)[field_index]); |
| 3306 | | }; |
| 3307 | | return cg.lowerConstant(Value.fromInterned(un.val), constant_ty); |
| 3341 | .un => { |
| 3342 | const int_type = try pt.intType(.unsigned, @intCast(ty.bitSize(zcu))); |
| 3343 | |
| 3344 | var buf: [8]u8 = .{0} ** 8; // zero the buffer so we do not read 0xaa as integer |
| 3345 | val.writeToPackedMemory(ty, pt, &buf, 0) catch unreachable; |
| 3346 | const int_val = try pt.intValue( |
| 3347 | int_type, |
| 3348 | mem.readInt(u64, &buf, .little), |
| 3349 | ); |
| 3350 | return cg.lowerConstant(int_val, int_type); |
| 3308 | 3351 | }, |
| 3309 | 3352 | .memoized_call => unreachable, |
| 3310 | 3353 | } |
| ... | ... | @@ -3352,6 +3395,14 @@ fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue { |
| 3352 | 3395 | const packed_struct = zcu.typeToPackedStruct(ty).?; |
| 3353 | 3396 | return cg.emitUndefined(Type.fromInterned(packed_struct.backingIntTypeUnordered(ip))); |
| 3354 | 3397 | }, |
| 3398 | .@"union" => switch (ty.containerLayout(zcu)) { |
| 3399 | .@"packed" => switch (ty.bitSize(zcu)) { |
| 3400 | 0...32 => return .{ .imm32 = 0xaaaaaaaa }, |
| 3401 | 33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa }, |
| 3402 | else => unreachable, |
| 3403 | }, |
| 3404 | else => unreachable, |
| 3405 | }, |
| 3355 | 3406 | else => return cg.fail("Wasm TODO: emitUndefined for type: {}\n", .{ty.zigTypeTag(zcu)}), |
| 3356 | 3407 | } |
| 3357 | 3408 | } |
| ... | ... | @@ -3841,15 +3892,12 @@ fn airStructFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 3841 | 3892 | const packed_struct = zcu.typeToPackedStruct(struct_ty).?; |
| 3842 | 3893 | const offset = pt.structPackedFieldBitOffset(packed_struct, field_index); |
| 3843 | 3894 | const backing_ty = Type.fromInterned(packed_struct.backingIntTypeUnordered(ip)); |
| 3844 | | const wasm_bits = toWasmBits(backing_ty.intInfo(zcu).bits) orelse { |
| 3845 | | return cg.fail("TODO: airStructFieldVal for packed structs larger than 128 bits", .{}); |
| 3846 | | }; |
| 3847 | | const const_wvalue: WValue = if (wasm_bits == 32) |
| 3848 | | .{ .imm32 = offset } |
| 3849 | | else if (wasm_bits == 64) |
| 3895 | const host_bits = backing_ty.intInfo(zcu).bits; |
| 3896 | |
| 3897 | const const_wvalue: WValue = if (33 <= host_bits and host_bits <= 64) |
| 3850 | 3898 | .{ .imm64 = offset } |
| 3851 | 3899 | else |
| 3852 | | return cg.fail("TODO: airStructFieldVal for packed structs larger than 64 bits", .{}); |
| 3900 | .{ .imm32 = offset }; |
| 3853 | 3901 | |
| 3854 | 3902 | // for first field we don't require any shifting |
| 3855 | 3903 | const shifted_value = if (offset == 0) |
| ... | ... | @@ -4027,7 +4075,7 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner |
| 4027 | 4075 | if (use_br_table) { |
| 4028 | 4076 | const width = width_maybe.?; |
| 4029 | 4077 | |
| 4030 | | const br_value_original = try cg.binOp(target, try cg.resolveInst(Air.internedToRef(min.?.toIntern())), target_ty, .sub); |
| 4078 | const br_value_original = try cg.binOp(target, try cg.resolveValue(min.?), target_ty, .sub); |
| 4031 | 4079 | _ = try cg.intcast(br_value_original, target_ty, Type.u32); |
| 4032 | 4080 | |
| 4033 | 4081 | const jump_table: Mir.JumpTable = .{ .length = width + 1 }; |
| ... | ... | @@ -4167,52 +4215,61 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode) InnerEr |
| 4167 | 4215 | return cg.finishAir(inst, result, &.{un_op}); |
| 4168 | 4216 | } |
| 4169 | 4217 | |
| 4218 | /// E!T -> T op_is_ptr == false |
| 4219 | /// *(E!T) -> *T op_is_prt == true |
| 4170 | 4220 | fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void { |
| 4171 | 4221 | const zcu = cg.pt.zcu; |
| 4172 | 4222 | const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 4173 | 4223 | |
| 4174 | 4224 | const operand = try cg.resolveInst(ty_op.operand); |
| 4175 | 4225 | const op_ty = cg.typeOf(ty_op.operand); |
| 4176 | | const err_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty; |
| 4177 | | const payload_ty = err_ty.errorUnionPayload(zcu); |
| 4226 | const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty; |
| 4227 | const payload_ty = eu_ty.errorUnionPayload(zcu); |
| 4178 | 4228 | |
| 4179 | 4229 | const result: WValue = result: { |
| 4180 | 4230 | if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 4181 | 4231 | if (op_is_ptr) { |
| 4182 | 4232 | break :result cg.reuseOperand(ty_op.operand, operand); |
| 4233 | } else { |
| 4234 | break :result .none; |
| 4183 | 4235 | } |
| 4184 | | break :result .none; |
| 4185 | 4236 | } |
| 4186 | 4237 | |
| 4187 | | const pl_offset = @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))); |
| 4238 | const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); |
| 4188 | 4239 | if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) { |
| 4189 | 4240 | break :result try cg.buildPointerOffset(operand, pl_offset, .new); |
| 4241 | } else { |
| 4242 | assert(isByRef(eu_ty, zcu, cg.target)); |
| 4243 | break :result try cg.load(operand, payload_ty, pl_offset); |
| 4190 | 4244 | } |
| 4191 | | |
| 4192 | | break :result try cg.load(operand, payload_ty, pl_offset); |
| 4193 | 4245 | }; |
| 4194 | 4246 | return cg.finishAir(inst, result, &.{ty_op.operand}); |
| 4195 | 4247 | } |
| 4196 | 4248 | |
| 4249 | /// E!T -> E op_is_ptr == false |
| 4250 | /// *(E!T) -> E op_is_prt == true |
| 4251 | /// NOTE: op_is_ptr will not change return type |
| 4197 | 4252 | fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void { |
| 4198 | 4253 | const zcu = cg.pt.zcu; |
| 4199 | 4254 | const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 4200 | 4255 | |
| 4201 | 4256 | const operand = try cg.resolveInst(ty_op.operand); |
| 4202 | 4257 | const op_ty = cg.typeOf(ty_op.operand); |
| 4203 | | const err_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty; |
| 4204 | | const payload_ty = err_ty.errorUnionPayload(zcu); |
| 4258 | const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty; |
| 4259 | const payload_ty = eu_ty.errorUnionPayload(zcu); |
| 4205 | 4260 | |
| 4206 | 4261 | const result: WValue = result: { |
| 4207 | | if (err_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { |
| 4262 | if (eu_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { |
| 4208 | 4263 | break :result .{ .imm32 = 0 }; |
| 4209 | 4264 | } |
| 4210 | 4265 | |
| 4211 | | if (op_is_ptr or !payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 4266 | const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu)); |
| 4267 | if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) { |
| 4268 | break :result try cg.load(operand, Type.anyerror, err_offset); |
| 4269 | } else { |
| 4270 | assert(!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)); |
| 4212 | 4271 | break :result cg.reuseOperand(ty_op.operand, operand); |
| 4213 | 4272 | } |
| 4214 | | |
| 4215 | | break :result try cg.load(operand, Type.anyerror, @intCast(errUnionErrorOffset(payload_ty, zcu))); |
| 4216 | 4273 | }; |
| 4217 | 4274 | return cg.finishAir(inst, result, &.{ty_op.operand}); |
| 4218 | 4275 | } |
| ... | ... | @@ -4438,9 +4495,6 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void |
| 4438 | 4495 | const operand = try cg.resolveInst(ty_op.operand); |
| 4439 | 4496 | const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 4440 | 4497 | const payload_ty = opt_ty.optionalChild(zcu); |
| 4441 | | if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 4442 | | return cg.fail("TODO: Implement OptionalPayloadPtrSet for optional with zero-sized type {}", .{payload_ty.fmtDebug()}); |
| 4443 | | } |
| 4444 | 4498 | |
| 4445 | 4499 | if (opt_ty.optionalReprIsPayload(zcu)) { |
| 4446 | 4500 | return cg.finishAir(inst, operand, &.{ty_op.operand}); |
| ... | ... | @@ -4731,11 +4785,6 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { |
| 4731 | 4785 | |
| 4732 | 4786 | fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { |
| 4733 | 4787 | const zcu = cg.pt.zcu; |
| 4734 | | if (safety) { |
| 4735 | | // TODO if the value is undef, write 0xaa bytes to dest |
| 4736 | | } else { |
| 4737 | | // TODO if the value is undef, don't lower this instruction |
| 4738 | | } |
| 4739 | 4788 | const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 4740 | 4789 | |
| 4741 | 4790 | const ptr = try cg.resolveInst(bin_op.lhs); |
| ... | ... | @@ -4752,6 +4801,10 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { |
| 4752 | 4801 | else |
| 4753 | 4802 | ptr_ty.childType(zcu); |
| 4754 | 4803 | |
| 4804 | if (!safety and bin_op.rhs == .undef) { |
| 4805 | return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); |
| 4806 | } |
| 4807 | |
| 4755 | 4808 | const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty); |
| 4756 | 4809 | try cg.memset(elem_ty, dst_ptr, len, value); |
| 4757 | 4810 | |
| ... | ... | @@ -5193,9 +5246,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 5193 | 5246 | const result = try cg.allocStack(result_ty); |
| 5194 | 5247 | const elem_ty = result_ty.childType(zcu); |
| 5195 | 5248 | const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); |
| 5196 | | const sentinel = if (result_ty.sentinel(zcu)) |sent| blk: { |
| 5197 | | break :blk try cg.lowerConstant(sent, elem_ty); |
| 5198 | | } else null; |
| 5249 | const sentinel = result_ty.sentinel(zcu); |
| 5199 | 5250 | |
| 5200 | 5251 | // When the element type is by reference, we must copy the entire |
| 5201 | 5252 | // value. It is therefore safer to move the offset pointer and store |
| ... | ... | @@ -5208,12 +5259,13 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 5208 | 5259 | const elem_val = try cg.resolveInst(elem); |
| 5209 | 5260 | try cg.store(offset, elem_val, elem_ty, 0); |
| 5210 | 5261 | |
| 5211 | | if (elem_index < elements.len - 1 and sentinel == null) { |
| 5262 | if (elem_index < elements.len - 1 or sentinel != null) { |
| 5212 | 5263 | _ = try cg.buildPointerOffset(offset, elem_size, .modify); |
| 5213 | 5264 | } |
| 5214 | 5265 | } |
| 5215 | | if (sentinel) |sent| { |
| 5216 | | try cg.store(offset, sent, elem_ty, 0); |
| 5266 | if (sentinel) |s| { |
| 5267 | const val = try cg.resolveValue(s); |
| 5268 | try cg.store(offset, val, elem_ty, 0); |
| 5217 | 5269 | } |
| 5218 | 5270 | } else { |
| 5219 | 5271 | var offset: u32 = 0; |
| ... | ... | @@ -5222,8 +5274,9 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 5222 | 5274 | try cg.store(result, elem_val, elem_ty, offset); |
| 5223 | 5275 | offset += elem_size; |
| 5224 | 5276 | } |
| 5225 | | if (sentinel) |sent| { |
| 5226 | | try cg.store(result, sent, elem_ty, offset); |
| 5277 | if (sentinel) |s| { |
| 5278 | const val = try cg.resolveValue(s); |
| 5279 | try cg.store(result, val, elem_ty, offset); |
| 5227 | 5280 | } |
| 5228 | 5281 | } |
| 5229 | 5282 | break :result_value result; |
| ... | ... | @@ -5407,31 +5460,40 @@ fn cmpOptionals(cg: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: st |
| 5407 | 5460 | assert(operand_ty.hasRuntimeBitsIgnoreComptime(zcu)); |
| 5408 | 5461 | assert(op == .eq or op == .neq); |
| 5409 | 5462 | const payload_ty = operand_ty.optionalChild(zcu); |
| 5463 | assert(!isByRef(payload_ty, zcu, cg.target)); |
| 5410 | 5464 | |
| 5411 | | // We store the final result in here that will be validated |
| 5412 | | // if the optional is truly equal. |
| 5413 | | var result = try cg.ensureAllocLocal(Type.i32); |
| 5465 | var result = try cg.allocLocal(Type.i32); |
| 5414 | 5466 | defer result.free(cg); |
| 5415 | 5467 | |
| 5468 | var lhs_null = try cg.allocLocal(Type.i32); |
| 5469 | defer lhs_null.free(cg); |
| 5470 | |
| 5416 | 5471 | try cg.startBlock(.block, .empty); |
| 5472 | |
| 5473 | try cg.addImm32(if (op == .eq) 0 else 1); |
| 5474 | try cg.addLocal(.local_set, result.local.value); |
| 5475 | |
| 5417 | 5476 | _ = try cg.isNull(lhs, operand_ty, .i32_eq); |
| 5477 | try cg.addLocal(.local_tee, lhs_null.local.value); |
| 5418 | 5478 | _ = try cg.isNull(rhs, operand_ty, .i32_eq); |
| 5419 | | try cg.addTag(.i32_ne); // inverse so we can exit early |
| 5420 | | try cg.addLabel(.br_if, 0); |
| 5479 | try cg.addTag(.i32_ne); |
| 5480 | try cg.addLabel(.br_if, 0); // only one is null |
| 5481 | |
| 5482 | try cg.addImm32(if (op == .eq) 1 else 0); |
| 5483 | try cg.addLocal(.local_set, result.local.value); |
| 5484 | |
| 5485 | try cg.addLocal(.local_get, lhs_null.local.value); |
| 5486 | try cg.addLabel(.br_if, 0); // both are null |
| 5421 | 5487 | |
| 5422 | 5488 | _ = try cg.load(lhs, payload_ty, 0); |
| 5423 | 5489 | _ = try cg.load(rhs, payload_ty, 0); |
| 5424 | | const opcode = buildOpcode(.{ .op = .ne, .valtype1 = typeToValtype(payload_ty, zcu, cg.target) }); |
| 5425 | | try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode)); |
| 5426 | | try cg.addLabel(.br_if, 0); |
| 5427 | | |
| 5428 | | try cg.addImm32(1); |
| 5490 | _ = try cg.cmp(.stack, .stack, payload_ty, op); |
| 5429 | 5491 | try cg.addLocal(.local_set, result.local.value); |
| 5492 | |
| 5430 | 5493 | try cg.endBlock(); |
| 5431 | 5494 | |
| 5432 | | try cg.emitWValue(result); |
| 5433 | | try cg.addImm32(0); |
| 5434 | | try cg.addTag(if (op == .eq) .i32_ne else .i32_eq); |
| 5495 | try cg.addLocal(.local_get, result.local.value); |
| 5496 | |
| 5435 | 5497 | return .stack; |
| 5436 | 5498 | } |
| 5437 | 5499 | |
| ... | ... | @@ -5659,13 +5721,25 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void |
| 5659 | 5721 | } |
| 5660 | 5722 | |
| 5661 | 5723 | fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 5662 | | const zcu = cg.pt.zcu; |
| 5724 | const pt = cg.pt; |
| 5725 | const zcu = pt.zcu; |
| 5663 | 5726 | const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 5664 | 5727 | const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; |
| 5665 | 5728 | |
| 5666 | 5729 | const field_ptr = try cg.resolveInst(extra.field_ptr); |
| 5667 | | const parent_ty = ty_pl.ty.toType().childType(zcu); |
| 5668 | | const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu); |
| 5730 | const parent_ptr_ty = cg.typeOfIndex(inst); |
| 5731 | const parent_ty = parent_ptr_ty.childType(zcu); |
| 5732 | const field_ptr_ty = cg.typeOf(extra.field_ptr); |
| 5733 | const field_index = extra.field_index; |
| 5734 | const field_offset = switch (parent_ty.containerLayout(zcu)) { |
| 5735 | .auto, .@"extern" => parent_ty.structFieldOffset(field_index, zcu), |
| 5736 | .@"packed" => offset: { |
| 5737 | const parent_ptr_offset = parent_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset; |
| 5738 | const field_offset = if (zcu.typeToStruct(parent_ty)) |loaded_struct| pt.structPackedFieldBitOffset(loaded_struct, field_index) else 0; |
| 5739 | const field_ptr_offset = field_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset; |
| 5740 | break :offset @divExact(parent_ptr_offset + field_offset - field_ptr_offset, 8); |
| 5741 | }, |
| 5742 | }; |
| 5669 | 5743 | |
| 5670 | 5744 | const result = if (field_offset != 0) result: { |
| 5671 | 5745 | const base = try cg.buildPointerOffset(field_ptr, 0, .new); |
| ... | ... | @@ -6260,11 +6334,21 @@ fn airClz(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 6260 | 6334 | |
| 6261 | 6335 | switch (wasm_bits) { |
| 6262 | 6336 | 32 => { |
| 6263 | | try cg.emitWValue(operand); |
| 6337 | if (int_info.signedness == .signed) { |
| 6338 | const mask = ~@as(u32, 0) >> @intCast(32 - int_info.bits); |
| 6339 | _ = try cg.binOp(operand, .{ .imm32 = mask }, ty, .@"and"); |
| 6340 | } else { |
| 6341 | try cg.emitWValue(operand); |
| 6342 | } |
| 6264 | 6343 | try cg.addTag(.i32_clz); |
| 6265 | 6344 | }, |
| 6266 | 6345 | 64 => { |
| 6267 | | try cg.emitWValue(operand); |
| 6346 | if (int_info.signedness == .signed) { |
| 6347 | const mask = ~@as(u64, 0) >> @intCast(64 - int_info.bits); |
| 6348 | _ = try cg.binOp(operand, .{ .imm64 = mask }, ty, .@"and"); |
| 6349 | } else { |
| 6350 | try cg.emitWValue(operand); |
| 6351 | } |
| 6268 | 6352 | try cg.addTag(.i64_clz); |
| 6269 | 6353 | try cg.addTag(.i32_wrap_i64); |
| 6270 | 6354 | }, |
Andrew Kelley <andrew@ziglang.org> 2025-04-11 15:56:35-04:00
GitHub <noreply@github.com> 2025-04-11 15:56:35-04:00
Signed by PGP key B5690EEEBB952194
src