authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-10-01 06:50:50+02:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-10-01 06:50:50+02:00
logc29a215217b50989d431ba4d0ee97a323cdedc7b
tree8d487a860b43ebc31d1cebf79378cbd087d4e333
parentf0b51497baa5d6fe1fd5f5d5ece19364f71489bf
parentacc6b7a7d7823ff1b806152763a9adc224c32ed5

Merge pull request 'loongarch: TLS LE; unordered atomic load/store' (#37022) from AstraFall/zig:loongarch/pr into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/37022 Reviewed-by: Alex Rønne Petersen <alex@alexrp.com>

1 files changed, 125 insertions(+), 36 deletions(-)

src/codegen/loongarch/Select.zig+125-36
......@@ -983,7 +983,14 @@ pub const Value = struct {
983983 while (part_it.next()) |part_vi| {
984984 const part_offset, const part_size = part_vi.positionInParent(isel);
985985 const part_mat = try part_vi.mat(isel, .{});
986 try isel.moveLoc(def_loc, part_offset, part_mat.loc(), 0, part_size, .preserved);
986 try isel.moveLoc(
987 def_loc,
988 part_offset,
989 part_mat.loc(),
990 0,
991 part_size,
992 .{ .dst_prot = .preserved },
993 );
987994 try part_mat.finish(isel);
988995 if (reextend_parts)
989996 try part_vi.reextend(isel, def_ext);
......@@ -1003,7 +1010,7 @@ pub const Value = struct {
10031010 const def_loc = def_vi.takeLocationMarkWritten(isel) orelse return;
10041011 const def_offset, const def_size = def_vi.positionInParent(isel);
10051012 const parent_mat = try parent_vi.mat(isel, .{});
1006 try isel.moveLoc(def_loc, 0, parent_mat.loc(), def_offset, def_size, .none);
1013 try isel.moveLoc(def_loc, 0, parent_mat.loc(), def_offset, def_size, .{});
10071014 try parent_mat.finish(isel);
10081015 }
10091016
......@@ -1104,9 +1111,10 @@ pub const Value = struct {
11041111 ) !bool {
11051112 try vi.reextendToPcs(isel);
11061113 try vi.collectDefs(isel);
1114 if (opts.@"volatile")
1115 try (try vi.mat(isel, .{})).finish(isel);
11071116 const loc = vi.takeLocationMarkWritten(isel) orelse return false;
11081117 wip_mir_log.debug(" | # load {f} <- [${t}, #{d}] ({d}B)", .{ vi, base_reg, offset, vi.size(isel) });
1109 _ = opts;
11101118
11111119 try isel.moveLoc(
11121120 loc,
......@@ -1114,7 +1122,7 @@ pub const Value = struct {
11141122 .{ .stack_slot = .{ .base = base_reg, .offset = 0 } },
11151123 offset,
11161124 vi.size(isel),
1117 .none,
1125 .{ .atomic = opts.atomic },
11181126 );
11191127 return true;
11201128 }
......@@ -1166,7 +1174,7 @@ pub const Value = struct {
11661174 const src_loc = src_mat.loc();
11671175 if (!std.meta.eql(loc, src_loc)) {
11681176 loc.markRegWritten(isel);
1169 try isel.moveLoc(loc, 0, src_mat.loc(), 0, copy_size, .none);
1177 try isel.moveLoc(loc, 0, src_mat.loc(), 0, copy_size, .{});
11701178 }
11711179 try src_mat.finish(isel);
11721180 }
......@@ -1270,15 +1278,23 @@ pub const Value = struct {
12701278 _ = try isel.fillReg(src_reg);
12711279 }
12721280 // TODO: replace reextending def_part_vi to .zero_ext with moveLoc .wipe when applicable
1273 try isel.moveLoc(def_part_loc, dst_offset, src_loc, src_offset, mat_size, .preserved);
1281 try isel.moveLoc(
1282 def_part_loc,
1283 dst_offset,
1284 src_loc,
1285 src_offset,
1286 mat_size,
1287 .{ .dst_prot = .preserved },
1288 );
12741289 }
12751290 }
12761291 if (maybe_def_addr_mat) |def_addr_mat| try def_addr_mat.finish(isel);
12771292 }
12781293
12791294 const MemoryAccessOptions = struct {
1280 // TODO unimplemented, remove?
12811295 @"volatile": bool = false,
1296 // Atomicity. This is unrelated to ordering and visibility.
1297 atomic: bool = false,
12821298 };
12831299
12841300 const MatOptions = struct {
......@@ -1308,7 +1324,7 @@ pub const Value = struct {
13081324 };
13091325
13101326 /// Materializes a value
1311 fn mat(vi: Value.Index, isel: *Select, opts: MatOptions) Mat.Error!Mat {
1327 fn mat(vi: Value.Index, isel: *Select, opts: MatOptions) codegen.Error!Mat {
13121328 // try vi.split(isel, true);
13131329 const mat_size = @min(opts.size, @as(u32, @intCast(vi.size(isel) - opts.offset)));
13141330 const loc_pref = if (opts.extension == .garbage)
......@@ -1478,7 +1494,6 @@ pub const Value = struct {
14781494 opts: MemoryAccessOptions,
14791495 ) !void {
14801496 wip_mir_log.debug(" | # store {f} -> [${t}, #{d}]", .{ vi, base_reg, offset });
1481 _ = opts;
14821497
14831498 const hint_stack: Indirect = if (std.math.cast(@FieldType(Indirect, "offset"), offset)) |stack_off|
14841499 .{ .base = base_reg, .offset = stack_off }
......@@ -1491,7 +1506,7 @@ pub const Value = struct {
14911506 value_mat.loc(),
14921507 0,
14931508 vi.size(isel),
1494 .none,
1509 .{ .atomic = opts.atomic },
14951510 );
14961511 try value_mat.finish(isel);
14971512 }
......@@ -1543,7 +1558,7 @@ pub const Value = struct {
15431558 .size = @intCast(part_size),
15441559 .extension = part_vi.extension(isel),
15451560 });
1546 try isel.moveLoc(.{ .register = part_ra }, 0, value_mat.loc(), 0, part_size, .none);
1561 try isel.moveLoc(.{ .register = part_ra }, 0, value_mat.loc(), 0, part_size, .{});
15471562 try value_mat.finish(isel);
15481563 }
15491564
......@@ -1558,7 +1573,7 @@ pub const Value = struct {
15581573 .size = @intCast(part_size),
15591574 .extension = part_vi.extension(isel),
15601575 });
1561 try isel.moveLoc(.{ .stack_slot = layout_part_stack }, 0, value_mat.loc(), 0, part_size, .none);
1576 try isel.moveLoc(.{ .stack_slot = layout_part_stack }, 0, value_mat.loc(), 0, part_size, .{});
15621577 try value_mat.finish(isel);
15631578 }
15641579 }
......@@ -1569,7 +1584,7 @@ pub const Value = struct {
15691584 if (src_loc.asRegister()) |src_reg| _ = try isel.fillReg(src_reg);
15701585 tracking_log.debug("{f} -> {f} (move to)", .{ vi, src_loc });
15711586 if (vi.takeLocationMarkWritten(isel)) |dst_loc|
1572 try isel.moveLoc(dst_loc, 0, src_loc, 0, vi.size(isel), .none);
1587 try isel.moveLoc(dst_loc, 0, src_loc, 0, vi.size(isel), .{});
15731588 if (vi.isSmall(isel)) {
15741589 vi.setSmallLocation(isel, src_loc);
15751590 if (src_loc.asRegister()) |src_reg| {
......@@ -2046,8 +2061,6 @@ pub const Value = struct {
20462061 if (!std.debug.runtime_safety) assert(@sizeOf(Mat) <= 32);
20472062 }
20482063
2049 const Error = codegen.Error;
2050
20512064 pub fn ra(mat: Value.Mat) Register.Alias {
20522065 return mat.location.register;
20532066 }
......@@ -2066,7 +2079,7 @@ pub const Value = struct {
20662079 };
20672080 }
20682081
2069 fn finish(mat: Value.Mat, isel: *Select) Mat.Error!void {
2082 fn finish(mat: Value.Mat, isel: *Select) codegen.Error!void {
20702083 const vi = mat.vi;
20712084 const value = vi.get(isel);
20722085 tracking_log.debug("{f}[{d}..{d}] <- {f} (mat finish)", .{ vi, mat.offset, mat.offset + mat.size - 1, mat.loc() });
......@@ -2132,7 +2145,7 @@ pub const Value = struct {
21322145 vi_loc,
21332146 mat.offset,
21342147 mat.size,
2135 .preserved,
2148 .{ .dst_prot = .preserved },
21362149 );
21372150 if (maybe_loc_reg) |loc_reg| {
21382151 const loc_live = isel.live_registers.getPtr(loc_reg);
......@@ -2192,7 +2205,7 @@ pub const Value = struct {
21922205 .{ .stack_slot = .{ .base = addr_mat.reg(), .offset = 0 } },
21932206 offset_from_root + mat.offset,
21942207 mat.size,
2195 .none,
2208 .{},
21962209 );
21972210 try addr_mat.finish(isel);
21982211 },
......@@ -2315,7 +2328,7 @@ fn failUnimplemented(isel: *Select, comptime format: []const u8, args: anytype)
23152328 } else return isel.fail(format, args);
23162329}
23172330
2318fn moveDebugString(isel: *Select, reg: Register, msg: [:0]const u8) error{ OutOfMemory, AlreadyReported }!void {
2331fn moveDebugString(isel: *Select, reg: Register, msg: [:0]const u8) codegen.Error!void {
23192332 @branchHint(.cold);
23202333 assert(debug_trap_unimplemented_code);
23212334
......@@ -3417,7 +3430,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
34173430 output_loc,
34183431 0,
34193432 output_vi.value.size(isel),
3420 .none,
3433 .{},
34213434 );
34223435 }
34233436 break :output_reg output_reg;
......@@ -3546,7 +3559,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
35463559 const dst_loc: Value.Location = .{ .register = input_ra };
35473560 if (!std.meta.eql(input_val_loc, dst_loc)) {
35483561 dst_loc.markRegWritten(isel);
3549 try isel.moveLoc(dst_loc, 0, input_val_loc, 0, input_ra.mod.byteSize(isel.target), .none);
3562 try isel.moveLoc(dst_loc, 0, input_val_loc, 0, input_ra.mod.byteSize(isel.target), .{});
35503563 }
35513564 try input_val_mat.finish(isel);
35523565 },
......@@ -3964,6 +3977,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
39643977 const ptr_mat = try ptr_vi.matReg(isel);
39653978 try src_vi.matStore(isel, ptr_mat.reg(), 0, .{
39663979 .@"volatile" = ptr_info.flags.is_volatile,
3980 .atomic = (air_tag == .atomic_store_unordered),
39673981 });
39683982 try ptr_mat.finish(isel);
39693983 },
......@@ -3973,7 +3987,8 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
39733987 const ptr_info = ptr_ty.ptrInfo(zcu);
39743988 if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed load", .{});
39753989
3976 if (ptr_info.flags.is_volatile) _ = try isel.use(air.inst_index.toRef());
3990 if (ptr_info.flags.is_volatile)
3991 _ = try isel.use(air.inst_index.toRef());
39773992 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
39783993 defer dst_vi.value.deref(isel);
39793994
......@@ -3987,6 +4002,29 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
39874002 try ptr_mat.finish(isel);
39884003 }
39894004 },
4005 .atomic_load => {
4006 const atomic_load = air.data(air.inst_index).atomic_load;
4007 const ptr_ty = isel.air.typeOf(atomic_load.ptr, ip);
4008 const ptr_info = ptr_ty.ptrInfo(zcu);
4009 if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed atomic load", .{});
4010
4011 if (ptr_info.flags.is_volatile)
4012 _ = try isel.use(air.inst_index.toRef());
4013 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
4014 defer dst_vi.value.deref(isel);
4015
4016 const ptr_vi = try isel.use(atomic_load.ptr);
4017 const ptr_mat = try ptr_vi.matIntRegZeroExt(isel);
4018 _ = try dst_vi.value.defLoad(isel, ptr_mat.reg(), 0, .{
4019 .@"volatile" = ptr_info.flags.is_volatile,
4020 .atomic = true,
4021 });
4022 try ptr_mat.finish(isel);
4023
4024 if (atomic_load.order != .unordered)
4025 try isel.failUnimplemented("ordered atomic load", .{});
4026 }
4027 },
39904028 .int_cast => if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
39914029 defer dst_vi.value.deref(isel);
39924030
......@@ -4709,6 +4747,40 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
47094747 try error_union_ptr_mat.finish(isel);
47104748 }
47114749 },
4750 .runtime_nav_ptr => if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| unused: {
4751 defer ptr_vi.value.deref(isel);
4752 const ptr_reg = try ptr_vi.value.defRegMod(isel, .integer) orelse break :unused;
4753 const ty_nav = air.data(air.inst_index).ty_nav;
4754 const target_nav = ip.getNav(ty_nav.nav);
4755 const target_nav_resolved = target_nav.resolved.?;
4756
4757 if (target_nav_resolved.@"threadlocal") {
4758 // TODO: implement TLS models other than LE
4759 try isel.emit(switch (isel.gprBits()) {
4760 32 => .@"add.w"(ptr_reg, ptr_reg, .tp),
4761 64 => .@"add.d"(ptr_reg, ptr_reg, .tp),
4762 else => unreachable,
4763 });
4764 try isel.nav_relocs.append(zcu.gpa, .{
4765 .nav = ty_nav.nav,
4766 .reloc = .{
4767 .label = @intCast(isel.instructions.items.len),
4768 .addend = 0,
4769 .type = .TLS_LE_LO12,
4770 },
4771 });
4772 try isel.emit(.ori(ptr_reg, ptr_reg, 0));
4773 try isel.nav_relocs.append(zcu.gpa, .{
4774 .nav = ty_nav.nav,
4775 .reloc = .{
4776 .label = @intCast(isel.instructions.items.len),
4777 .addend = 0,
4778 .type = .TLS_LE_HI20,
4779 },
4780 });
4781 try isel.emit(.@"lu12i.w"(ptr_reg, 0));
4782 } else try isel.failUnimplemented("unimplemented runtime_nav_ptr", .{});
4783 },
47124784 }
47134785 if (air_tag != .arg) {
47144786 var live_reg_it = isel.live_registers.iterator();
......@@ -6207,6 +6279,11 @@ fn elemPtr(
62076279 try index_mat.finish(isel);
62086280}
62096281
6282const MoveLocOptions = struct {
6283 dst_prot: DestProtection = .none,
6284 atomic: bool = false,
6285};
6286
62106287fn moveLoc(
62116288 isel: *Select,
62126289 dst_loc: Value.Location,
......@@ -6214,22 +6291,23 @@ fn moveLoc(
62146291 src_loc: Value.Location,
62156292 src_off: u64,
62166293 size: u64,
6217 dst_prot: DestProtection,
6294 opts: MoveLocOptions,
62186295) !void {
62196296 if (dst_loc.isUnallocated()) return;
62206297 if (std.meta.eql(dst_loc, src_loc) and dst_off == src_off) return;
62216298 if (size == 0) return;
62226299 assert(!src_loc.isUnallocated());
6223 wip_mir_log.debug(" | # move {f}[{d}] <- {f}[{d}], {d}B, dst prot={t}", .{
6300 wip_mir_log.debug(" | # move {f}[{d}] <- {f}[{d}], {d}B, dst prot={t}, atomic={}", .{
62246301 dst_loc,
62256302 dst_off,
62266303 src_loc,
62276304 src_off,
62286305 size,
6229 dst_prot,
6306 opts.dst_prot,
6307 opts.atomic,
62306308 });
62316309
6232 const dst_lock: RegLock = if (dst_prot != .preserved) .empty else dst_loc.tryLock(isel);
6310 const dst_lock: RegLock = if (opts.dst_prot != .preserved) .empty else dst_loc.tryLock(isel);
62336311 defer dst_lock.unlock(isel);
62346312 const src_lock = src_loc.tryLock(isel);
62356313 defer src_lock.unlock(isel);
......@@ -6242,7 +6320,7 @@ fn moveLoc(
62426320 src_ra,
62436321 @intCast(src_off * 8),
62446322 @intCast(size * 8),
6245 dst_prot,
6323 opts.dst_prot,
62466324 ),
62476325 .stack_slot => |src_stack| {
62486326 const tmp_reg = if (dst_ra.mod == .integer and dst_off == 0)
......@@ -6256,7 +6334,7 @@ fn moveLoc(
62566334 .{ .reg = tmp_reg, .mod = .integer },
62576335 0,
62586336 @intCast(size * 8),
6259 dst_prot,
6337 opts.dst_prot,
62606338 );
62616339 try isel.loadReg(
62626340 tmp_reg,
......@@ -6268,6 +6346,8 @@ fn moveLoc(
62686346 },
62696347 },
62706348 .stack_slot => |dst_stack| {
6349 if (size > isel.gprSize() and opts.atomic)
6350 return isel.fail("Memory store of {d} bytes cannot be atomic", .{size});
62716351 if (size > isel.gprSize()) {
62726352 // large memory copies, src must be stack_slot
62736353 const src_stack = src_loc.stack_slot;
......@@ -6295,7 +6375,7 @@ fn moveLoc(
62956375 if (gen_low_to_high) {
62966376 var off: u64 = 0;
62976377 for (0..@intCast(steps)) |_| {
6298 try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, dst_prot);
6378 try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, opts);
62996379 off += gpr_size;
63006380 }
63016381 }
......@@ -6303,7 +6383,7 @@ fn moveLoc(
63036383 var off: u64 = steps * gpr_size;
63046384 for (0..@intCast(steps)) |_| {
63056385 off -= gpr_size;
6306 try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, dst_prot);
6386 try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, opts);
63076387 }
63086388 }
63096389
......@@ -6314,7 +6394,9 @@ fn moveLoc(
63146394 // If size is not direct mem op size and !kill_dst, old values have to
63156395 // be loaded first.
63166396 const memop_size = memOpSizeFitting(size);
6317 const need_load = memop_size != size;
6397 const need_load = memop_size != size and opts.dst_prot != .none;
6398 if (opts.atomic and need_load)
6399 return isel.fail("Memory store of {d} bytes cannot be atomic without killing other bytes", .{size});
63186400 const tmp_reg, const tmp_allocated = tmp_reg: {
63196401 if (src_off == 0 and !need_load) {
63206402 switch (src_loc) {
......@@ -6336,7 +6418,7 @@ fn moveLoc(
63366418 src_loc,
63376419 src_off,
63386420 size,
6339 if (need_load) .preserved else .none,
6421 opts,
63406422 );
63416423 if (need_load)
63426424 try isel.loadReg(tmp_reg, memop_size, .unsigned, dst_stack.base, dst_stack_off);
......@@ -6464,7 +6546,7 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini
64646546 const rd = dst_ra.reg;
64656547 defer if (use_tmp_reg) isel.freeReg(rd);
64666548
6467 if (use_tmp_reg) try isel.moveLoc(dst, 0, .{ .register = dst_ra }, 0, isel.gprSize(), .none);
6549 if (use_tmp_reg) try isel.moveLoc(dst, 0, .{ .register = dst_ra }, 0, isel.gprSize(), .{});
64686550 return switch (ptr.base_addr) {
64696551 .nav => |nav| if (ZigType.fromInterned(ip.getNav(nav).resolved.?.type).isRuntimeFnOrHasRuntimeBits(zcu)) {
64706552 // TODO code model
......@@ -6725,7 +6807,14 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini
67256807 ),
67266808 };
67276809
6728 try isel.moveLoc(dst, part_offset, .{ .register = tmp_ra }, 0, part_size, .preserved);
6810 try isel.moveLoc(
6811 dst,
6812 part_offset,
6813 .{ .register = tmp_ra },
6814 0,
6815 part_size,
6816 .{ .dst_prot = .preserved },
6817 );
67296818 try isel.moveIntImm(tmp_reg, @bitCast(part_value));
67306819
67316820 if (part_offset == 0) break else part_offset -= gpr_size;
......@@ -6756,7 +6845,7 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini
67566845 .{ .stack_slot = .{ .base = tmp_reg, .offset = 0 } },
67576846 offset,
67586847 size,
6759 .none,
6848 .{},
67606849 );
67616850
67626851 // load constant pointer
......@@ -7238,7 +7327,7 @@ const call = struct {
72387327 .{ .register = .{ .mod = .integer, .reg = tmp_reg } },
72397328 0,
72407329 isel.gprSize(),
7241 .preserved,
7330 .{ .dst_prot = .preserved },
72427331 );
72437332 } else unreachable;
72447333 }