| ... | ... | @@ -983,7 +983,14 @@ pub const Value = struct { |
| 983 | 983 | while (part_it.next()) |part_vi| { |
| 984 | 984 | const part_offset, const part_size = part_vi.positionInParent(isel); |
| 985 | 985 | const part_mat = try part_vi.mat(isel, .{}); |
| 986 | | try isel.moveLoc(def_loc, part_offset, part_mat.loc(), 0, part_size, .preserved); |
| 986 | try isel.moveLoc( |
| 987 | def_loc, |
| 988 | part_offset, |
| 989 | part_mat.loc(), |
| 990 | 0, |
| 991 | part_size, |
| 992 | .{ .dst_prot = .preserved }, |
| 993 | ); |
| 987 | 994 | try part_mat.finish(isel); |
| 988 | 995 | if (reextend_parts) |
| 989 | 996 | try part_vi.reextend(isel, def_ext); |
| ... | ... | @@ -1003,7 +1010,7 @@ pub const Value = struct { |
| 1003 | 1010 | const def_loc = def_vi.takeLocationMarkWritten(isel) orelse return; |
| 1004 | 1011 | const def_offset, const def_size = def_vi.positionInParent(isel); |
| 1005 | 1012 | const parent_mat = try parent_vi.mat(isel, .{}); |
| 1006 | | try isel.moveLoc(def_loc, 0, parent_mat.loc(), def_offset, def_size, .none); |
| 1013 | try isel.moveLoc(def_loc, 0, parent_mat.loc(), def_offset, def_size, .{}); |
| 1007 | 1014 | try parent_mat.finish(isel); |
| 1008 | 1015 | } |
| 1009 | 1016 | |
| ... | ... | @@ -1115,7 +1122,7 @@ pub const Value = struct { |
| 1115 | 1122 | .{ .stack_slot = .{ .base = base_reg, .offset = 0 } }, |
| 1116 | 1123 | offset, |
| 1117 | 1124 | vi.size(isel), |
| 1118 | | .none, |
| 1125 | .{ .atomic = opts.atomic }, |
| 1119 | 1126 | ); |
| 1120 | 1127 | return true; |
| 1121 | 1128 | } |
| ... | ... | @@ -1167,7 +1174,7 @@ pub const Value = struct { |
| 1167 | 1174 | const src_loc = src_mat.loc(); |
| 1168 | 1175 | if (!std.meta.eql(loc, src_loc)) { |
| 1169 | 1176 | loc.markRegWritten(isel); |
| 1170 | | try isel.moveLoc(loc, 0, src_mat.loc(), 0, copy_size, .none); |
| 1177 | try isel.moveLoc(loc, 0, src_mat.loc(), 0, copy_size, .{}); |
| 1171 | 1178 | } |
| 1172 | 1179 | try src_mat.finish(isel); |
| 1173 | 1180 | } |
| ... | ... | @@ -1271,7 +1278,14 @@ pub const Value = struct { |
| 1271 | 1278 | _ = try isel.fillReg(src_reg); |
| 1272 | 1279 | } |
| 1273 | 1280 | // TODO: replace reextending def_part_vi to .zero_ext with moveLoc .wipe when applicable |
| 1274 | | try isel.moveLoc(def_part_loc, dst_offset, src_loc, src_offset, mat_size, .preserved); |
| 1281 | try isel.moveLoc( |
| 1282 | def_part_loc, |
| 1283 | dst_offset, |
| 1284 | src_loc, |
| 1285 | src_offset, |
| 1286 | mat_size, |
| 1287 | .{ .dst_prot = .preserved }, |
| 1288 | ); |
| 1275 | 1289 | } |
| 1276 | 1290 | } |
| 1277 | 1291 | if (maybe_def_addr_mat) |def_addr_mat| try def_addr_mat.finish(isel); |
| ... | ... | @@ -1279,6 +1293,8 @@ pub const Value = struct { |
| 1279 | 1293 | |
| 1280 | 1294 | const MemoryAccessOptions = struct { |
| 1281 | 1295 | @"volatile": bool = false, |
| 1296 | // Atomicity. This is unrelated to ordering and visibility. |
| 1297 | atomic: bool = false, |
| 1282 | 1298 | }; |
| 1283 | 1299 | |
| 1284 | 1300 | const MatOptions = struct { |
| ... | ... | @@ -1478,7 +1494,6 @@ pub const Value = struct { |
| 1478 | 1494 | opts: MemoryAccessOptions, |
| 1479 | 1495 | ) !void { |
| 1480 | 1496 | wip_mir_log.debug(" | # store {f} -> [${t}, #{d}]", .{ vi, base_reg, offset }); |
| 1481 | | _ = opts; |
| 1482 | 1497 | |
| 1483 | 1498 | const hint_stack: Indirect = if (std.math.cast(@FieldType(Indirect, "offset"), offset)) |stack_off| |
| 1484 | 1499 | .{ .base = base_reg, .offset = stack_off } |
| ... | ... | @@ -1491,7 +1506,7 @@ pub const Value = struct { |
| 1491 | 1506 | value_mat.loc(), |
| 1492 | 1507 | 0, |
| 1493 | 1508 | vi.size(isel), |
| 1494 | | .none, |
| 1509 | .{ .atomic = opts.atomic }, |
| 1495 | 1510 | ); |
| 1496 | 1511 | try value_mat.finish(isel); |
| 1497 | 1512 | } |
| ... | ... | @@ -1543,7 +1558,7 @@ pub const Value = struct { |
| 1543 | 1558 | .size = @intCast(part_size), |
| 1544 | 1559 | .extension = part_vi.extension(isel), |
| 1545 | 1560 | }); |
| 1546 | | try isel.moveLoc(.{ .register = part_ra }, 0, value_mat.loc(), 0, part_size, .none); |
| 1561 | try isel.moveLoc(.{ .register = part_ra }, 0, value_mat.loc(), 0, part_size, .{}); |
| 1547 | 1562 | try value_mat.finish(isel); |
| 1548 | 1563 | } |
| 1549 | 1564 | |
| ... | ... | @@ -1558,7 +1573,7 @@ pub const Value = struct { |
| 1558 | 1573 | .size = @intCast(part_size), |
| 1559 | 1574 | .extension = part_vi.extension(isel), |
| 1560 | 1575 | }); |
| 1561 | | try isel.moveLoc(.{ .stack_slot = layout_part_stack }, 0, value_mat.loc(), 0, part_size, .none); |
| 1576 | try isel.moveLoc(.{ .stack_slot = layout_part_stack }, 0, value_mat.loc(), 0, part_size, .{}); |
| 1562 | 1577 | try value_mat.finish(isel); |
| 1563 | 1578 | } |
| 1564 | 1579 | } |
| ... | ... | @@ -1569,7 +1584,7 @@ pub const Value = struct { |
| 1569 | 1584 | if (src_loc.asRegister()) |src_reg| _ = try isel.fillReg(src_reg); |
| 1570 | 1585 | tracking_log.debug("{f} -> {f} (move to)", .{ vi, src_loc }); |
| 1571 | 1586 | if (vi.takeLocationMarkWritten(isel)) |dst_loc| |
| 1572 | | try isel.moveLoc(dst_loc, 0, src_loc, 0, vi.size(isel), .none); |
| 1587 | try isel.moveLoc(dst_loc, 0, src_loc, 0, vi.size(isel), .{}); |
| 1573 | 1588 | if (vi.isSmall(isel)) { |
| 1574 | 1589 | vi.setSmallLocation(isel, src_loc); |
| 1575 | 1590 | if (src_loc.asRegister()) |src_reg| { |
| ... | ... | @@ -2130,7 +2145,7 @@ pub const Value = struct { |
| 2130 | 2145 | vi_loc, |
| 2131 | 2146 | mat.offset, |
| 2132 | 2147 | mat.size, |
| 2133 | | .preserved, |
| 2148 | .{ .dst_prot = .preserved }, |
| 2134 | 2149 | ); |
| 2135 | 2150 | if (maybe_loc_reg) |loc_reg| { |
| 2136 | 2151 | const loc_live = isel.live_registers.getPtr(loc_reg); |
| ... | ... | @@ -2190,7 +2205,7 @@ pub const Value = struct { |
| 2190 | 2205 | .{ .stack_slot = .{ .base = addr_mat.reg(), .offset = 0 } }, |
| 2191 | 2206 | offset_from_root + mat.offset, |
| 2192 | 2207 | mat.size, |
| 2193 | | .none, |
| 2208 | .{}, |
| 2194 | 2209 | ); |
| 2195 | 2210 | try addr_mat.finish(isel); |
| 2196 | 2211 | }, |
| ... | ... | @@ -3415,7 +3430,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void |
| 3415 | 3430 | output_loc, |
| 3416 | 3431 | 0, |
| 3417 | 3432 | output_vi.value.size(isel), |
| 3418 | | .none, |
| 3433 | .{}, |
| 3419 | 3434 | ); |
| 3420 | 3435 | } |
| 3421 | 3436 | break :output_reg output_reg; |
| ... | ... | @@ -3544,7 +3559,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void |
| 3544 | 3559 | const dst_loc: Value.Location = .{ .register = input_ra }; |
| 3545 | 3560 | if (!std.meta.eql(input_val_loc, dst_loc)) { |
| 3546 | 3561 | dst_loc.markRegWritten(isel); |
| 3547 | | try isel.moveLoc(dst_loc, 0, input_val_loc, 0, input_ra.mod.byteSize(isel.target), .none); |
| 3562 | try isel.moveLoc(dst_loc, 0, input_val_loc, 0, input_ra.mod.byteSize(isel.target), .{}); |
| 3548 | 3563 | } |
| 3549 | 3564 | try input_val_mat.finish(isel); |
| 3550 | 3565 | }, |
| ... | ... | @@ -3962,6 +3977,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void |
| 3962 | 3977 | const ptr_mat = try ptr_vi.matReg(isel); |
| 3963 | 3978 | try src_vi.matStore(isel, ptr_mat.reg(), 0, .{ |
| 3964 | 3979 | .@"volatile" = ptr_info.flags.is_volatile, |
| 3980 | .atomic = (air_tag == .atomic_store_unordered), |
| 3965 | 3981 | }); |
| 3966 | 3982 | try ptr_mat.finish(isel); |
| 3967 | 3983 | }, |
| ... | ... | @@ -3986,6 +4002,29 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void |
| 3986 | 4002 | try ptr_mat.finish(isel); |
| 3987 | 4003 | } |
| 3988 | 4004 | }, |
| 4005 | .atomic_load => { |
| 4006 | const atomic_load = air.data(air.inst_index).atomic_load; |
| 4007 | const ptr_ty = isel.air.typeOf(atomic_load.ptr, ip); |
| 4008 | const ptr_info = ptr_ty.ptrInfo(zcu); |
| 4009 | if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed atomic load", .{}); |
| 4010 | |
| 4011 | if (ptr_info.flags.is_volatile) |
| 4012 | _ = try isel.use(air.inst_index.toRef()); |
| 4013 | if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| { |
| 4014 | defer dst_vi.value.deref(isel); |
| 4015 | |
| 4016 | const ptr_vi = try isel.use(atomic_load.ptr); |
| 4017 | const ptr_mat = try ptr_vi.matIntRegZeroExt(isel); |
| 4018 | _ = try dst_vi.value.defLoad(isel, ptr_mat.reg(), 0, .{ |
| 4019 | .@"volatile" = ptr_info.flags.is_volatile, |
| 4020 | .atomic = true, |
| 4021 | }); |
| 4022 | try ptr_mat.finish(isel); |
| 4023 | |
| 4024 | if (atomic_load.order != .unordered) |
| 4025 | try isel.failUnimplemented("ordered atomic load", .{}); |
| 4026 | } |
| 4027 | }, |
| 3989 | 4028 | .int_cast => if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| { |
| 3990 | 4029 | defer dst_vi.value.deref(isel); |
| 3991 | 4030 | |
| ... | ... | @@ -6206,6 +6245,11 @@ fn elemPtr( |
| 6206 | 6245 | try index_mat.finish(isel); |
| 6207 | 6246 | } |
| 6208 | 6247 | |
| 6248 | const MoveLocOptions = struct { |
| 6249 | dst_prot: DestProtection = .none, |
| 6250 | atomic: bool = false, |
| 6251 | }; |
| 6252 | |
| 6209 | 6253 | fn moveLoc( |
| 6210 | 6254 | isel: *Select, |
| 6211 | 6255 | dst_loc: Value.Location, |
| ... | ... | @@ -6213,22 +6257,23 @@ fn moveLoc( |
| 6213 | 6257 | src_loc: Value.Location, |
| 6214 | 6258 | src_off: u64, |
| 6215 | 6259 | size: u64, |
| 6216 | | dst_prot: DestProtection, |
| 6260 | opts: MoveLocOptions, |
| 6217 | 6261 | ) !void { |
| 6218 | 6262 | if (dst_loc.isUnallocated()) return; |
| 6219 | 6263 | if (std.meta.eql(dst_loc, src_loc) and dst_off == src_off) return; |
| 6220 | 6264 | if (size == 0) return; |
| 6221 | 6265 | assert(!src_loc.isUnallocated()); |
| 6222 | | wip_mir_log.debug(" | # move {f}[{d}] <- {f}[{d}], {d}B, dst prot={t}", .{ |
| 6266 | wip_mir_log.debug(" | # move {f}[{d}] <- {f}[{d}], {d}B, dst prot={t}, atomic={}", .{ |
| 6223 | 6267 | dst_loc, |
| 6224 | 6268 | dst_off, |
| 6225 | 6269 | src_loc, |
| 6226 | 6270 | src_off, |
| 6227 | 6271 | size, |
| 6228 | | dst_prot, |
| 6272 | opts.dst_prot, |
| 6273 | opts.atomic, |
| 6229 | 6274 | }); |
| 6230 | 6275 | |
| 6231 | | const dst_lock: RegLock = if (dst_prot != .preserved) .empty else dst_loc.tryLock(isel); |
| 6276 | const dst_lock: RegLock = if (opts.dst_prot != .preserved) .empty else dst_loc.tryLock(isel); |
| 6232 | 6277 | defer dst_lock.unlock(isel); |
| 6233 | 6278 | const src_lock = src_loc.tryLock(isel); |
| 6234 | 6279 | defer src_lock.unlock(isel); |
| ... | ... | @@ -6241,7 +6286,7 @@ fn moveLoc( |
| 6241 | 6286 | src_ra, |
| 6242 | 6287 | @intCast(src_off * 8), |
| 6243 | 6288 | @intCast(size * 8), |
| 6244 | | dst_prot, |
| 6289 | opts.dst_prot, |
| 6245 | 6290 | ), |
| 6246 | 6291 | .stack_slot => |src_stack| { |
| 6247 | 6292 | const tmp_reg = if (dst_ra.mod == .integer and dst_off == 0) |
| ... | ... | @@ -6255,7 +6300,7 @@ fn moveLoc( |
| 6255 | 6300 | .{ .reg = tmp_reg, .mod = .integer }, |
| 6256 | 6301 | 0, |
| 6257 | 6302 | @intCast(size * 8), |
| 6258 | | dst_prot, |
| 6303 | opts.dst_prot, |
| 6259 | 6304 | ); |
| 6260 | 6305 | try isel.loadReg( |
| 6261 | 6306 | tmp_reg, |
| ... | ... | @@ -6267,6 +6312,8 @@ fn moveLoc( |
| 6267 | 6312 | }, |
| 6268 | 6313 | }, |
| 6269 | 6314 | .stack_slot => |dst_stack| { |
| 6315 | if (size > isel.gprSize() and opts.atomic) |
| 6316 | return isel.fail("Memory store of {d} bytes cannot be atomic", .{size}); |
| 6270 | 6317 | if (size > isel.gprSize()) { |
| 6271 | 6318 | // large memory copies, src must be stack_slot |
| 6272 | 6319 | const src_stack = src_loc.stack_slot; |
| ... | ... | @@ -6294,7 +6341,7 @@ fn moveLoc( |
| 6294 | 6341 | if (gen_low_to_high) { |
| 6295 | 6342 | var off: u64 = 0; |
| 6296 | 6343 | for (0..@intCast(steps)) |_| { |
| 6297 | | try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, dst_prot); |
| 6344 | try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, opts); |
| 6298 | 6345 | off += gpr_size; |
| 6299 | 6346 | } |
| 6300 | 6347 | } |
| ... | ... | @@ -6302,7 +6349,7 @@ fn moveLoc( |
| 6302 | 6349 | var off: u64 = steps * gpr_size; |
| 6303 | 6350 | for (0..@intCast(steps)) |_| { |
| 6304 | 6351 | off -= gpr_size; |
| 6305 | | try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, dst_prot); |
| 6352 | try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, opts); |
| 6306 | 6353 | } |
| 6307 | 6354 | } |
| 6308 | 6355 | |
| ... | ... | @@ -6313,7 +6360,9 @@ fn moveLoc( |
| 6313 | 6360 | // If size is not direct mem op size and !kill_dst, old values have to |
| 6314 | 6361 | // be loaded first. |
| 6315 | 6362 | const memop_size = memOpSizeFitting(size); |
| 6316 | | const need_load = memop_size != size; |
| 6363 | const need_load = memop_size != size and opts.dst_prot != .none; |
| 6364 | if (opts.atomic and need_load) |
| 6365 | return isel.fail("Memory store of {d} bytes cannot be atomic without killing other bytes", .{size}); |
| 6317 | 6366 | const tmp_reg, const tmp_allocated = tmp_reg: { |
| 6318 | 6367 | if (src_off == 0 and !need_load) { |
| 6319 | 6368 | switch (src_loc) { |
| ... | ... | @@ -6335,7 +6384,7 @@ fn moveLoc( |
| 6335 | 6384 | src_loc, |
| 6336 | 6385 | src_off, |
| 6337 | 6386 | size, |
| 6338 | | if (need_load) .preserved else .none, |
| 6387 | opts, |
| 6339 | 6388 | ); |
| 6340 | 6389 | if (need_load) |
| 6341 | 6390 | try isel.loadReg(tmp_reg, memop_size, .unsigned, dst_stack.base, dst_stack_off); |
| ... | ... | @@ -6463,7 +6512,7 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini |
| 6463 | 6512 | const rd = dst_ra.reg; |
| 6464 | 6513 | defer if (use_tmp_reg) isel.freeReg(rd); |
| 6465 | 6514 | |
| 6466 | | if (use_tmp_reg) try isel.moveLoc(dst, 0, .{ .register = dst_ra }, 0, isel.gprSize(), .none); |
| 6515 | if (use_tmp_reg) try isel.moveLoc(dst, 0, .{ .register = dst_ra }, 0, isel.gprSize(), .{}); |
| 6467 | 6516 | return switch (ptr.base_addr) { |
| 6468 | 6517 | .nav => |nav| if (ZigType.fromInterned(ip.getNav(nav).resolved.?.type).isRuntimeFnOrHasRuntimeBits(zcu)) { |
| 6469 | 6518 | // TODO code model |
| ... | ... | @@ -6724,7 +6773,14 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini |
| 6724 | 6773 | ), |
| 6725 | 6774 | }; |
| 6726 | 6775 | |
| 6727 | | try isel.moveLoc(dst, part_offset, .{ .register = tmp_ra }, 0, part_size, .preserved); |
| 6776 | try isel.moveLoc( |
| 6777 | dst, |
| 6778 | part_offset, |
| 6779 | .{ .register = tmp_ra }, |
| 6780 | 0, |
| 6781 | part_size, |
| 6782 | .{ .dst_prot = .preserved }, |
| 6783 | ); |
| 6728 | 6784 | try isel.moveIntImm(tmp_reg, @bitCast(part_value)); |
| 6729 | 6785 | |
| 6730 | 6786 | if (part_offset == 0) break else part_offset -= gpr_size; |
| ... | ... | @@ -6755,7 +6811,7 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini |
| 6755 | 6811 | .{ .stack_slot = .{ .base = tmp_reg, .offset = 0 } }, |
| 6756 | 6812 | offset, |
| 6757 | 6813 | size, |
| 6758 | | .none, |
| 6814 | .{}, |
| 6759 | 6815 | ); |
| 6760 | 6816 | |
| 6761 | 6817 | // load constant pointer |
| ... | ... | @@ -7237,7 +7293,7 @@ const call = struct { |
| 7237 | 7293 | .{ .register = .{ .mod = .integer, .reg = tmp_reg } }, |
| 7238 | 7294 | 0, |
| 7239 | 7295 | isel.gprSize(), |
| 7240 | | .preserved, |
| 7296 | .{ .dst_prot = .preserved }, |
| 7241 | 7297 | ); |
| 7242 | 7298 | } else unreachable; |
| 7243 | 7299 | } |
xtex <xtex@astrafall.org> 2026-09-30 19:51:59+08:00