Commit bfaaeab734e6cef3cd2a1186e14a19ba8481ec6e | zig

author	xtex <xtex@astrafall.org> 2026-09-30 19:51:59+08:00
committer	xtex <xtex@astrafall.org> 2026-10-01 08:39:15+08:00
log	bfaaeab734e6cef3cd2a1186e14a19ba8481ec6e
tree	e666248bb6a99bb3b632b429199879cefc9c6d86
parent	e2d0773388ae25b3a55787968a5a652efdae07ea
signature	Signed by SSH key SHA256:IEYEjkZlkUTr5U9GiDAmZU/4eZus2t2RsxusyhQqwao

loongarch: support unordered load/store atomicity

1 files changed, 84 insertions(+), 28 deletions(-)

src
- codegen
  - loongarch
    Select.zig

src/codegen/loongarch/Select.zig+84-28

 @@ -983,7 +983,14 @@ pub const Value = struct {
             while (part_it.next()) |part_vi| {
                 const part_offset, const part_size = part_vi.positionInParent(isel);
                 const part_mat = try part_vi.mat(isel, .{});
-                try isel.moveLoc(def_loc, part_offset, part_mat.loc(), 0, part_size, .preserved);
+                try isel.moveLoc(
+                    def_loc,
+                    part_offset,
+                    part_mat.loc(),
+                    0,
+                    part_size,
+                    .{ .dst_prot = .preserved },
+                );
                 try part_mat.finish(isel);
                 if (reextend_parts)
                     try part_vi.reextend(isel, def_ext);
 @@ -1003,7 +1010,7 @@ pub const Value = struct {
             const def_loc = def_vi.takeLocationMarkWritten(isel) orelse return;
             const def_offset, const def_size = def_vi.positionInParent(isel);
             const parent_mat = try parent_vi.mat(isel, .{});
-            try isel.moveLoc(def_loc, 0, parent_mat.loc(), def_offset, def_size, .none);
+            try isel.moveLoc(def_loc, 0, parent_mat.loc(), def_offset, def_size, .{});
             try parent_mat.finish(isel);
         }
 @@ -1115,7 +1122,7 @@ pub const Value = struct {
                 .{ .stack_slot = .{ .base = base_reg, .offset = 0 } },
                 offset,
                 vi.size(isel),
-                .none,
+                .{ .atomic = opts.atomic },
             );
             return true;
         }
 @@ -1167,7 +1174,7 @@ pub const Value = struct {
                 const src_loc = src_mat.loc();
                 if (!std.meta.eql(loc, src_loc)) {
                     loc.markRegWritten(isel);
-                    try isel.moveLoc(loc, 0, src_mat.loc(), 0, copy_size, .none);
+                    try isel.moveLoc(loc, 0, src_mat.loc(), 0, copy_size, .{});
                 }
                 try src_mat.finish(isel);
             }
 @@ -1271,7 +1278,14 @@ pub const Value = struct {
                             _ = try isel.fillReg(src_reg);
                     }
                     // TODO: replace reextending def_part_vi to .zero_ext with moveLoc .wipe when applicable
-                    try isel.moveLoc(def_part_loc, dst_offset, src_loc, src_offset, mat_size, .preserved);
+                    try isel.moveLoc(
+                        def_part_loc,
+                        dst_offset,
+                        src_loc,
+                        src_offset,
+                        mat_size,
+                        .{ .dst_prot = .preserved },
+                    );
                 }
             }
             if (maybe_def_addr_mat) |def_addr_mat| try def_addr_mat.finish(isel);
 @@ -1279,6 +1293,8 @@ pub const Value = struct {
         const MemoryAccessOptions = struct {
             @"volatile": bool = false,
+            // Atomicity. This is unrelated to ordering and visibility.
+            atomic: bool = false,
         };
         const MatOptions = struct {
 @@ -1478,7 +1494,6 @@ pub const Value = struct {
             opts: MemoryAccessOptions,
         ) !void {
             wip_mir_log.debug("  | # store {f} -> [${t}, #{d}]", .{ vi, base_reg, offset });
-            _ = opts;
             const hint_stack: Indirect = if (std.math.cast(@FieldType(Indirect, "offset"), offset)) |stack_off|
                 .{ .base = base_reg, .offset = stack_off }
 @@ -1491,7 +1506,7 @@ pub const Value = struct {
                 value_mat.loc(),
                 0,
                 vi.size(isel),
-                .none,
+                .{ .atomic = opts.atomic },
             );
             try value_mat.finish(isel);
         }
 @@ -1543,7 +1558,7 @@ pub const Value = struct {
                         .size = @intCast(part_size),
                         .extension = part_vi.extension(isel),
                     });
-                    try isel.moveLoc(.{ .register = part_ra }, 0, value_mat.loc(), 0, part_size, .none);
+                    try isel.moveLoc(.{ .register = part_ra }, 0, value_mat.loc(), 0, part_size, .{});
                     try value_mat.finish(isel);
                 }
 @@ -1558,7 +1573,7 @@ pub const Value = struct {
                         .size = @intCast(part_size),
                         .extension = part_vi.extension(isel),
                     });
-                    try isel.moveLoc(.{ .stack_slot = layout_part_stack }, 0, value_mat.loc(), 0, part_size, .none);
+                    try isel.moveLoc(.{ .stack_slot = layout_part_stack }, 0, value_mat.loc(), 0, part_size, .{});
                     try value_mat.finish(isel);
                 }
             }
 @@ -1569,7 +1584,7 @@ pub const Value = struct {
             if (src_loc.asRegister()) |src_reg| _ = try isel.fillReg(src_reg);
             tracking_log.debug("{f} -> {f} (move to)", .{ vi, src_loc });
             if (vi.takeLocationMarkWritten(isel)) |dst_loc|
-                try isel.moveLoc(dst_loc, 0, src_loc, 0, vi.size(isel), .none);
+                try isel.moveLoc(dst_loc, 0, src_loc, 0, vi.size(isel), .{});
             if (vi.isSmall(isel)) {
                 vi.setSmallLocation(isel, src_loc);
                 if (src_loc.asRegister()) |src_reg| {
 @@ -2130,7 +2145,7 @@ pub const Value = struct {
                         vi_loc,
                         mat.offset,
                         mat.size,
-                        .preserved,
+                        .{ .dst_prot = .preserved },
                     );
                     if (maybe_loc_reg) |loc_reg| {
                         const loc_live = isel.live_registers.getPtr(loc_reg);
 @@ -2190,7 +2205,7 @@ pub const Value = struct {
                         .{ .stack_slot = .{ .base = addr_mat.reg(), .offset = 0 } },
                         offset_from_root + mat.offset,
                         mat.size,
-                        .none,
+                        .{},
                     );
                     try addr_mat.finish(isel);
                 },
 @@ -3415,7 +3430,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
                                                 output_loc,
                                                 0,
                                                 output_vi.value.size(isel),
-                                                .none,
+                                                .{},
                                             );
                                     }
                                     break :output_reg output_reg;
 @@ -3544,7 +3559,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
                             const dst_loc: Value.Location = .{ .register = input_ra };
                             if (!std.meta.eql(input_val_loc, dst_loc)) {
                                 dst_loc.markRegWritten(isel);
-                                try isel.moveLoc(dst_loc, 0, input_val_loc, 0, input_ra.mod.byteSize(isel.target), .none);
+                                try isel.moveLoc(dst_loc, 0, input_val_loc, 0, input_ra.mod.byteSize(isel.target), .{});
                             }
                             try input_val_mat.finish(isel);
                         },
 @@ -3962,6 +3977,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
                 const ptr_mat = try ptr_vi.matReg(isel);
                 try src_vi.matStore(isel, ptr_mat.reg(), 0, .{
                     .@"volatile" = ptr_info.flags.is_volatile,
+                    .atomic = (air_tag == .atomic_store_unordered),
                 });
                 try ptr_mat.finish(isel);
             },
 @@ -3986,6 +4002,29 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void
                     try ptr_mat.finish(isel);
                 }
             },
+            .atomic_load => {
+                const atomic_load = air.data(air.inst_index).atomic_load;
+                const ptr_ty = isel.air.typeOf(atomic_load.ptr, ip);
+                const ptr_info = ptr_ty.ptrInfo(zcu);
+                if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed atomic load", .{});
+                if (ptr_info.flags.is_volatile)
+                    _ = try isel.use(air.inst_index.toRef());
+                if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
+                    defer dst_vi.value.deref(isel);
+                    const ptr_vi = try isel.use(atomic_load.ptr);
+                    const ptr_mat = try ptr_vi.matIntRegZeroExt(isel);
+                    _ = try dst_vi.value.defLoad(isel, ptr_mat.reg(), 0, .{
+                        .@"volatile" = ptr_info.flags.is_volatile,
+                        .atomic = true,
+                    });
+                    try ptr_mat.finish(isel);
+                    if (atomic_load.order != .unordered)
+                        try isel.failUnimplemented("ordered atomic load", .{});
+                }
+            },
             .int_cast => if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
                 defer dst_vi.value.deref(isel);
 @@ -6206,6 +6245,11 @@ fn elemPtr(
     try index_mat.finish(isel);
 }
+const MoveLocOptions = struct {
+    dst_prot: DestProtection = .none,
+    atomic: bool = false,
+};
 fn moveLoc(
     isel: *Select,
     dst_loc: Value.Location,
 @@ -6213,22 +6257,23 @@ fn moveLoc(
     src_loc: Value.Location,
     src_off: u64,
     size: u64,
-    dst_prot: DestProtection,
+    opts: MoveLocOptions,
 ) !void {
     if (dst_loc.isUnallocated()) return;
     if (std.meta.eql(dst_loc, src_loc) and dst_off == src_off) return;
     if (size == 0) return;
     assert(!src_loc.isUnallocated());
-    wip_mir_log.debug("  | # move {f}[{d}] <- {f}[{d}], {d}B, dst prot={t}", .{
+    wip_mir_log.debug("  | # move {f}[{d}] <- {f}[{d}], {d}B, dst prot={t}, atomic={}", .{
         dst_loc,
         dst_off,
         src_loc,
         src_off,
         size,
-        dst_prot,
+        opts.dst_prot,
+        opts.atomic,
     });
-    const dst_lock: RegLock = if (dst_prot != .preserved) .empty else dst_loc.tryLock(isel);
+    const dst_lock: RegLock = if (opts.dst_prot != .preserved) .empty else dst_loc.tryLock(isel);
     defer dst_lock.unlock(isel);
     const src_lock = src_loc.tryLock(isel);
     defer src_lock.unlock(isel);
 @@ -6241,7 +6286,7 @@ fn moveLoc(
                 src_ra,
                 @intCast(src_off * 8),
                 @intCast(size * 8),
-                dst_prot,
+                opts.dst_prot,
             ),
             .stack_slot => |src_stack| {
                 const tmp_reg = if (dst_ra.mod == .integer and dst_off == 0)
 @@ -6255,7 +6300,7 @@ fn moveLoc(
                     .{ .reg = tmp_reg, .mod = .integer },
                     0,
                     @intCast(size * 8),
-                    dst_prot,
+                    opts.dst_prot,
                 );
                 try isel.loadReg(
                     tmp_reg,
 @@ -6267,6 +6312,8 @@ fn moveLoc(
             },
         },
         .stack_slot => |dst_stack| {
+            if (size > isel.gprSize() and opts.atomic)
+                return isel.fail("Memory store of {d} bytes cannot be atomic", .{size});
             if (size > isel.gprSize()) {
                 // large memory copies, src must be stack_slot
                 const src_stack = src_loc.stack_slot;
 @@ -6294,7 +6341,7 @@ fn moveLoc(
                 if (gen_low_to_high) {
                     var off: u64 = 0;
                     for (0..@intCast(steps)) |_| {
-                        try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, dst_prot);
+                        try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, opts);
                         off += gpr_size;
                     }
                 }
 @@ -6302,7 +6349,7 @@ fn moveLoc(
                     var off: u64 = steps * gpr_size;
                     for (0..@intCast(steps)) |_| {
                         off -= gpr_size;
-                        try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, dst_prot);
+                        try isel.moveLoc(dst_loc, dst_off + off, src_loc, src_off + off, gpr_size, opts);
                     }
                 }
 @@ -6313,7 +6360,9 @@ fn moveLoc(
             // If size is not direct mem op size and !kill_dst, old values have to
             // be loaded first.
             const memop_size = memOpSizeFitting(size);
-            const need_load = memop_size != size;
+            const need_load = memop_size != size and opts.dst_prot != .none;
+            if (opts.atomic and need_load)
+                return isel.fail("Memory store of {d} bytes cannot be atomic without killing other bytes", .{size});
             const tmp_reg, const tmp_allocated = tmp_reg: {
                 if (src_off == 0 and !need_load) {
                     switch (src_loc) {
 @@ -6335,7 +6384,7 @@ fn moveLoc(
                     src_loc,
                     src_off,
                     size,
-                    if (need_load) .preserved else .none,
+                    opts,
                 );
             if (need_load)
                 try isel.loadReg(tmp_reg, memop_size, .unsigned, dst_stack.base, dst_stack_off);
 @@ -6463,7 +6512,7 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini
                 const rd = dst_ra.reg;
                 defer if (use_tmp_reg) isel.freeReg(rd);
-                if (use_tmp_reg) try isel.moveLoc(dst, 0, .{ .register = dst_ra }, 0, isel.gprSize(), .none);
+                if (use_tmp_reg) try isel.moveLoc(dst, 0, .{ .register = dst_ra }, 0, isel.gprSize(), .{});
                 return switch (ptr.base_addr) {
                     .nav => |nav| if (ZigType.fromInterned(ip.getNav(nav).resolved.?.type).isRuntimeFnOrHasRuntimeBits(zcu)) {
                         // TODO code model
 @@ -6724,7 +6773,14 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini
                     ),
                 };
-                try isel.moveLoc(dst, part_offset, .{ .register = tmp_ra }, 0, part_size, .preserved);
+                try isel.moveLoc(
+                    dst,
+                    part_offset,
+                    .{ .register = tmp_ra },
+                    0,
+                    part_size,
+                    .{ .dst_prot = .preserved },
+                );
                 try isel.moveIntImm(tmp_reg, @bitCast(part_value));
                 if (part_offset == 0) break else part_offset -= gpr_size;
 @@ -6755,7 +6811,7 @@ fn moveConstant(isel: *Select, dst: Value.Location, init_constant: Constant, ini
                 .{ .stack_slot = .{ .base = tmp_reg, .offset = 0 } },
                 offset,
                 size,
-                .none,
+                .{},
             );
             // load constant pointer
 @@ -7237,7 +7293,7 @@ const call = struct {
                 .{ .register = .{ .mod = .integer, .reg = tmp_reg } },
                 0,
                 isel.gprSize(),
-                .preserved,
+                .{ .dst_prot = .preserved },
             );
         } else unreachable;
     }