authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-18 23:25:44-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-01-18 23:25:44-05:00
logbeb7495e19d23b4814e16772888e80688ad10e47
treec202d5e2367cf81596e1e7dca1ee83044d55b5d8
parent8375b71f75c8b8567bc7d97867716c4911dace65
parentde5908548c65944ce21c05bae274c03bb1188aa1
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10624 from ziglang/prefetch

stage2: implement `@prefetch`

15 files changed, 300 insertions(+), 45 deletions(-)

lib/std/builtin.zig+2-2
...@@ -643,12 +643,12 @@ pub const PrefetchOptions = struct {...@@ -643,12 +643,12 @@ pub const PrefetchOptions = struct {
643 /// The cache that the prefetch should be preformed on.643 /// The cache that the prefetch should be preformed on.
644 cache: Cache = .data,644 cache: Cache = .data,
645645
646 pub const Rw = enum {646 pub const Rw = enum(u1) {
647 read,647 read,
648 write,648 write,
649 };649 };
650650
651 pub const Cache = enum {651 pub const Cache = enum(u1) {
652 instruction,652 instruction,
653 data,653 data,
654 };654 };
src/Air.zig+12
...@@ -515,6 +515,11 @@ pub const Inst = struct {...@@ -515,6 +515,11 @@ pub const Inst = struct {
515 /// is a `Ref`. Length of the array is given by the vector type.515 /// is a `Ref`. Length of the array is given by the vector type.
516 vector_init,516 vector_init,
517517
518 /// Communicates an intent to load memory.
519 /// Result is always unused.
520 /// Uses the `prefetch` field.
521 prefetch,
522
518 pub fn fromCmpOp(op: std.math.CompareOperator) Tag {523 pub fn fromCmpOp(op: std.math.CompareOperator) Tag {
519 return switch (op) {524 return switch (op) {
520 .lt => .cmp_lt,525 .lt => .cmp_lt,
...@@ -586,6 +591,12 @@ pub const Inst = struct {...@@ -586,6 +591,12 @@ pub const Inst = struct {
586 ptr: Ref,591 ptr: Ref,
587 order: std.builtin.AtomicOrder,592 order: std.builtin.AtomicOrder,
588 },593 },
594 prefetch: struct {
595 ptr: Ref,
596 rw: std.builtin.PrefetchOptions.Rw,
597 locality: u2,
598 cache: std.builtin.PrefetchOptions.Cache,
599 },
589600
590 // Make sure we don't accidentally add a field to make this union601 // Make sure we don't accidentally add a field to make this union
591 // bigger than expected. Note that in Debug builds, Zig is allowed602 // bigger than expected. Note that in Debug builds, Zig is allowed
...@@ -823,6 +834,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {...@@ -823,6 +834,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
823 .memset,834 .memset,
824 .memcpy,835 .memcpy,
825 .set_union_tag,836 .set_union_tag,
837 .prefetch,
826 => return Type.initTag(.void),838 => return Type.initTag(.void),
827839
828 .ptrtoint,840 .ptrtoint,
src/Liveness.zig+5
...@@ -342,6 +342,11 @@ fn analyzeInst(...@@ -342,6 +342,11 @@ fn analyzeInst(
342 return trackOperands(a, new_set, inst, main_tomb, .{ operand, .none, .none });342 return trackOperands(a, new_set, inst, main_tomb, .{ operand, .none, .none });
343 },343 },
344344
345 .prefetch => {
346 const prefetch = inst_datas[inst].prefetch;
347 return trackOperands(a, new_set, inst, main_tomb, .{ prefetch.ptr, .none, .none });
348 },
349
345 .call => {350 .call => {
346 const inst_data = inst_datas[inst].pl_op;351 const inst_data = inst_datas[inst].pl_op;
347 const callee = inst_data.operand;352 const callee = inst_data.operand;
src/Sema.zig+82-38
...@@ -10397,49 +10397,53 @@ fn zirStructInitEmpty(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileE...@@ -10397,49 +10397,53 @@ fn zirStructInitEmpty(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileE
10397 const inst_data = sema.code.instructions.items(.data)[inst].un_node;10397 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
10398 const src = inst_data.src();10398 const src = inst_data.src();
10399 const obj_ty = try sema.resolveType(block, src, inst_data.operand);10399 const obj_ty = try sema.resolveType(block, src, inst_data.operand);
10400 const gpa = sema.gpa;
1040110400
10402 switch (obj_ty.zigTypeTag()) {10401 switch (obj_ty.zigTypeTag()) {
10403 .Struct => {10402 .Struct => return structInitEmpty(sema, block, obj_ty, src, src),
10404 // This logic must be synchronized with that in `zirStructInit`.10403 .Array => return arrayInitEmpty(sema, obj_ty),
10405 const struct_ty = try sema.resolveTypeFields(block, src, obj_ty);
10406 const struct_obj = struct_ty.castTag(.@"struct").?.data;
10407
10408 // The init values to use for the struct instance.
10409 const field_inits = try gpa.alloc(Air.Inst.Ref, struct_obj.fields.count());
10410 defer gpa.free(field_inits);
10411
10412 var root_msg: ?*Module.ErrorMsg = null;
10413
10414 for (struct_obj.fields.values()) |field, i| {
10415 if (field.default_val.tag() == .unreachable_value) {
10416 const field_name = struct_obj.fields.keys()[i];
10417 const template = "missing struct field: {s}";
10418 const args = .{field_name};
10419 if (root_msg) |msg| {
10420 try sema.errNote(block, src, msg, template, args);
10421 } else {
10422 root_msg = try sema.errMsg(block, src, template, args);
10423 }
10424 } else {
10425 field_inits[i] = try sema.addConstant(field.ty, field.default_val);
10426 }
10427 }
10428 return sema.finishStructInit(block, src, field_inits, root_msg, struct_obj, struct_ty, false);
10429 },
10430 .Array => {
10431 if (obj_ty.sentinel()) |sentinel| {
10432 const val = try Value.Tag.empty_array_sentinel.create(sema.arena, sentinel);
10433 return sema.addConstant(obj_ty, val);
10434 } else {
10435 return sema.addConstant(obj_ty, Value.initTag(.empty_array));
10436 }
10437 },
10438 .Void => return sema.addConstant(obj_ty, Value.void),10404 .Void => return sema.addConstant(obj_ty, Value.void),
10439 else => unreachable,10405 else => unreachable,
10440 }10406 }
10441}10407}
1044210408
10409fn structInitEmpty(sema: *Sema, block: *Block, obj_ty: Type, dest_src: LazySrcLoc, init_src: LazySrcLoc) CompileError!Air.Inst.Ref {
10410 const gpa = sema.gpa;
10411 // This logic must be synchronized with that in `zirStructInit`.
10412 const struct_ty = try sema.resolveTypeFields(block, dest_src, obj_ty);
10413 const struct_obj = struct_ty.castTag(.@"struct").?.data;
10414
10415 // The init values to use for the struct instance.
10416 const field_inits = try gpa.alloc(Air.Inst.Ref, struct_obj.fields.count());
10417 defer gpa.free(field_inits);
10418
10419 var root_msg: ?*Module.ErrorMsg = null;
10420
10421 for (struct_obj.fields.values()) |field, i| {
10422 if (field.default_val.tag() == .unreachable_value) {
10423 const field_name = struct_obj.fields.keys()[i];
10424 const template = "missing struct field: {s}";
10425 const args = .{field_name};
10426 if (root_msg) |msg| {
10427 try sema.errNote(block, init_src, msg, template, args);
10428 } else {
10429 root_msg = try sema.errMsg(block, init_src, template, args);
10430 }
10431 } else {
10432 field_inits[i] = try sema.addConstant(field.ty, field.default_val);
10433 }
10434 }
10435 return sema.finishStructInit(block, dest_src, field_inits, root_msg, struct_obj, struct_ty, false);
10436}
10437
10438fn arrayInitEmpty(sema: *Sema, obj_ty: Type) CompileError!Air.Inst.Ref {
10439 if (obj_ty.sentinel()) |sentinel| {
10440 const val = try Value.Tag.empty_array_sentinel.create(sema.arena, sentinel);
10441 return sema.addConstant(obj_ty, val);
10442 } else {
10443 return sema.addConstant(obj_ty, Value.initTag(.empty_array));
10444 }
10445}
10446
10443fn zirUnionInitPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {10447fn zirUnionInitPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
10444 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;10448 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
10445 const src = inst_data.src();10449 const src = inst_data.src();
...@@ -12407,8 +12411,38 @@ fn zirPrefetch(...@@ -12407,8 +12411,38 @@ fn zirPrefetch(
12407 extended: Zir.Inst.Extended.InstData,12411 extended: Zir.Inst.Extended.InstData,
12408) CompileError!Air.Inst.Ref {12412) CompileError!Air.Inst.Ref {
12409 const extra = sema.code.extraData(Zir.Inst.BinNode, extended.operand).data;12413 const extra = sema.code.extraData(Zir.Inst.BinNode, extended.operand).data;
12410 const src: LazySrcLoc = .{ .node_offset = extra.node };12414 const ptr_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = extra.node };
12411 return sema.fail(block, src, "TODO: implement Sema.zirPrefetch", .{});12415 const opts_src: LazySrcLoc = .{ .node_offset_builtin_call_arg1 = extra.node };
12416 const options_ty = try sema.getBuiltinType(block, opts_src, "PrefetchOptions");
12417 const ptr = sema.resolveInst(extra.lhs);
12418 try sema.checkPtrType(block, ptr_src, sema.typeOf(ptr));
12419 const options = try sema.coerce(block, options_ty, sema.resolveInst(extra.rhs), opts_src);
12420
12421 const rw = try sema.fieldVal(block, opts_src, options, "rw", opts_src);
12422 const rw_val = try sema.resolveConstValue(block, opts_src, rw);
12423 const rw_tag = rw_val.toEnum(std.builtin.PrefetchOptions.Rw);
12424
12425 const locality = try sema.fieldVal(block, opts_src, options, "locality", opts_src);
12426 const locality_val = try sema.resolveConstValue(block, opts_src, locality);
12427 const locality_int = @intCast(u2, locality_val.toUnsignedInt());
12428
12429 const cache = try sema.fieldVal(block, opts_src, options, "cache", opts_src);
12430 const cache_val = try sema.resolveConstValue(block, opts_src, cache);
12431 const cache_tag = cache_val.toEnum(std.builtin.PrefetchOptions.Cache);
12432
12433 if (!block.is_comptime) {
12434 _ = try block.addInst(.{
12435 .tag = .prefetch,
12436 .data = .{ .prefetch = .{
12437 .ptr = ptr,
12438 .rw = rw_tag,
12439 .locality = locality_int,
12440 .cache = cache_tag,
12441 } },
12442 });
12443 }
12444
12445 return Air.Inst.Ref.void_value;
12412}12446}
1241312447
12414fn zirBuiltinExtern(12448fn zirBuiltinExtern(
...@@ -13868,6 +13902,11 @@ fn coerce(...@@ -13868,6 +13902,11 @@ fn coerce(
13868 },13902 },
13869 .Array => switch (inst_ty.zigTypeTag()) {13903 .Array => switch (inst_ty.zigTypeTag()) {
13870 .Vector => return sema.coerceVectorInMemory(block, dest_ty, dest_ty_src, inst, inst_src),13904 .Vector => return sema.coerceVectorInMemory(block, dest_ty, dest_ty_src, inst, inst_src),
13905 .Struct => {
13906 if (inst == .empty_struct) {
13907 return arrayInitEmpty(sema, dest_ty);
13908 }
13909 },
13871 else => {},13910 else => {},
13872 },13911 },
13873 .Vector => switch (inst_ty.zigTypeTag()) {13912 .Vector => switch (inst_ty.zigTypeTag()) {
...@@ -13875,6 +13914,11 @@ fn coerce(...@@ -13875,6 +13914,11 @@ fn coerce(
13875 .Vector => return sema.coerceVectors(block, dest_ty, dest_ty_src, inst, inst_src),13914 .Vector => return sema.coerceVectors(block, dest_ty, dest_ty_src, inst, inst_src),
13876 else => {},13915 else => {},
13877 },13916 },
13917 .Struct => {
13918 if (inst == .empty_struct) {
13919 return structInitEmpty(sema, block, dest_ty, dest_ty_src, inst_src);
13920 }
13921 },
13878 else => {},13922 else => {},
13879 }13923 }
1388013924
src/arch/aarch64/CodeGen.zig+6
...@@ -595,6 +595,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -595,6 +595,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
595 .error_name => try self.airErrorName(inst),595 .error_name => try self.airErrorName(inst),
596 .splat => try self.airSplat(inst),596 .splat => try self.airSplat(inst),
597 .vector_init => try self.airVectorInit(inst),597 .vector_init => try self.airVectorInit(inst),
598 .prefetch => try self.airPrefetch(inst),
598599
599 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),600 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
600 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),601 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -2597,6 +2598,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -2597,6 +2598,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
2597 return bt.finishAir(result);2598 return bt.finishAir(result);
2598}2599}
25992600
2601fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
2602 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
2603 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
2604}
2605
2600fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {2606fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
2601 // First section of indexes correspond to a set number of constant values.2607 // First section of indexes correspond to a set number of constant values.
2602 const ref_int = @enumToInt(inst);2608 const ref_int = @enumToInt(inst);
src/arch/arm/CodeGen.zig+6
...@@ -586,6 +586,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -586,6 +586,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
586 .error_name => try self.airErrorName(inst),586 .error_name => try self.airErrorName(inst),
587 .splat => try self.airSplat(inst),587 .splat => try self.airSplat(inst),
588 .vector_init => try self.airVectorInit(inst),588 .vector_init => try self.airVectorInit(inst),
589 .prefetch => try self.airPrefetch(inst),
589590
590 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),591 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
591 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),592 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -3719,6 +3720,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -3719,6 +3720,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
3719 return bt.finishAir(result);3720 return bt.finishAir(result);
3720}3721}
37213722
3723fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
3724 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
3725 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
3726}
3727
3722fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {3728fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
3723 // First section of indexes correspond to a set number of constant values.3729 // First section of indexes correspond to a set number of constant values.
3724 const ref_int = @enumToInt(inst);3730 const ref_int = @enumToInt(inst);
src/arch/riscv64/CodeGen.zig+6
...@@ -574,6 +574,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -574,6 +574,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
574 .error_name => try self.airErrorName(inst),574 .error_name => try self.airErrorName(inst),
575 .splat => try self.airSplat(inst),575 .splat => try self.airSplat(inst),
576 .vector_init => try self.airVectorInit(inst),576 .vector_init => try self.airVectorInit(inst),
577 .prefetch => try self.airPrefetch(inst),
577578
578 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),579 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
579 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),580 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -2096,6 +2097,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -2096,6 +2097,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
2096 return bt.finishAir(result);2097 return bt.finishAir(result);
2097}2098}
20982099
2100fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
2101 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
2102 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
2103}
2104
2099fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {2105fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
2100 // First section of indexes correspond to a set number of constant values.2106 // First section of indexes correspond to a set number of constant values.
2101 const ref_int = @enumToInt(inst);2107 const ref_int = @enumToInt(inst);
src/arch/wasm/CodeGen.zig+63-4
...@@ -1297,6 +1297,9 @@ fn copyLocal(self: *Self, value: WValue, ty: Type) InnerError!WValue {...@@ -1297,6 +1297,9 @@ fn copyLocal(self: *Self, value: WValue, ty: Type) InnerError!WValue {
1297fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {1297fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1298 const air_tags = self.air.instructions.items(.tag);1298 const air_tags = self.air.instructions.items(.tag);
1299 return switch (air_tags[inst]) {1299 return switch (air_tags[inst]) {
1300 .constant => unreachable,
1301 .const_ty => unreachable,
1302
1300 .add => self.airBinOp(inst, .add),1303 .add => self.airBinOp(inst, .add),
1301 .addwrap => self.airWrapBinOp(inst, .add),1304 .addwrap => self.airWrapBinOp(inst, .add),
1302 .sub => self.airBinOp(inst, .sub),1305 .sub => self.airBinOp(inst, .sub),
...@@ -1330,7 +1333,6 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1330,7 +1333,6 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1330 .bool_to_int => self.airBoolToInt(inst),1333 .bool_to_int => self.airBoolToInt(inst),
1331 .call => self.airCall(inst),1334 .call => self.airCall(inst),
1332 .cond_br => self.airCondBr(inst),1335 .cond_br => self.airCondBr(inst),
1333 .constant => unreachable,
1334 .dbg_stmt => WValue.none,1336 .dbg_stmt => WValue.none,
1335 .intcast => self.airIntcast(inst),1337 .intcast => self.airIntcast(inst),
1336 .float_to_int => self.airFloatToInt(inst),1338 .float_to_int => self.airFloatToInt(inst),
...@@ -1358,6 +1360,9 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1358,6 +1360,9 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1358 .ret => self.airRet(inst),1360 .ret => self.airRet(inst),
1359 .ret_ptr => self.airRetPtr(inst),1361 .ret_ptr => self.airRetPtr(inst),
1360 .ret_load => self.airRetLoad(inst),1362 .ret_load => self.airRetLoad(inst),
1363 .splat => self.airSplat(inst),
1364 .vector_init => self.airVectorInit(inst),
1365 .prefetch => self.airPrefetch(inst),
13611366
1362 .slice => self.airSlice(inst),1367 .slice => self.airSlice(inst),
1363 .slice_len => self.airSliceLen(inst),1368 .slice_len => self.airSliceLen(inst),
...@@ -1382,7 +1387,55 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1382,7 +1387,55 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1382 .unwrap_errunion_err => self.airUnwrapErrUnionError(inst),1387 .unwrap_errunion_err => self.airUnwrapErrUnionError(inst),
1383 .wrap_errunion_payload => self.airWrapErrUnionPayload(inst),1388 .wrap_errunion_payload => self.airWrapErrUnionPayload(inst),
1384 .wrap_errunion_err => self.airWrapErrUnionErr(inst),1389 .wrap_errunion_err => self.airWrapErrUnionErr(inst),
1385 else => |tag| self.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),1390
1391 .add_sat,
1392 .sub_sat,
1393 .mul_sat,
1394 .div_float,
1395 .div_floor,
1396 .div_exact,
1397 .rem,
1398 .mod,
1399 .max,
1400 .min,
1401 .assembly,
1402 .shl_exact,
1403 .shl_sat,
1404 .ret_addr,
1405 .clz,
1406 .ctz,
1407 .popcount,
1408 .is_err_ptr,
1409 .is_non_err_ptr,
1410 .fptrunc,
1411 .fpext,
1412 .unwrap_errunion_payload_ptr,
1413 .unwrap_errunion_err_ptr,
1414 .set_union_tag,
1415 .get_union_tag,
1416 .ptr_slice_len_ptr,
1417 .ptr_slice_ptr_ptr,
1418 .int_to_float,
1419 .memcpy,
1420 .cmpxchg_weak,
1421 .cmpxchg_strong,
1422 .fence,
1423 .atomic_load,
1424 .atomic_store_unordered,
1425 .atomic_store_monotonic,
1426 .atomic_store_release,
1427 .atomic_store_seq_cst,
1428 .atomic_rmw,
1429 .tag_name,
1430 .error_name,
1431
1432 // For these 4, probably best to wait until https://github.com/ziglang/zig/issues/10248
1433 // is implemented in the frontend before implementing them here in the wasm backend.
1434 .add_with_overflow,
1435 .sub_with_overflow,
1436 .mul_with_overflow,
1437 .shl_with_overflow,
1438 => |tag| self.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),
1386 };1439 };
1387}1440}
13881441
...@@ -3211,7 +3264,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3211,7 +3264,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3211 return result;3264 return result;
3212}3265}
32133266
3214fn airSplat(self: *Self, inst: Air.Inst.Index) !void {3267fn airSplat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3215 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };3268 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };
32163269
3217 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3270 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
...@@ -3222,7 +3275,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -3222,7 +3275,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
3222 return self.fail("TODO: Implement wasm airSplat", .{});3275 return self.fail("TODO: Implement wasm airSplat", .{});
3223}3276}
32243277
3225fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {3278fn airVectorInit(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3226 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };3279 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };
32273280
3228 const vector_ty = self.air.typeOfIndex(inst);3281 const vector_ty = self.air.typeOfIndex(inst);
...@@ -3234,6 +3287,12 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -3234,6 +3287,12 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
3234 return self.fail("TODO: Wasm backend: implement airVectorInit", .{});3287 return self.fail("TODO: Wasm backend: implement airVectorInit", .{});
3235}3288}
32363289
3290fn airPrefetch(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3291 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
3292 _ = prefetch;
3293 return WValue{ .none = {} };
3294}
3295
3237fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {3296fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {
3238 assert(operand_ty.hasCodeGenBits());3297 assert(operand_ty.hasCodeGenBits());
3239 assert(op == .eq or op == .neq);3298 assert(op == .eq or op == .neq);
src/arch/x86_64/CodeGen.zig+6
...@@ -638,6 +638,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -638,6 +638,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
638 .error_name => try self.airErrorName(inst),638 .error_name => try self.airErrorName(inst),
639 .splat => try self.airSplat(inst),639 .splat => try self.airSplat(inst),
640 .vector_init => try self.airVectorInit(inst),640 .vector_init => try self.airVectorInit(inst),
641 .prefetch => try self.airPrefetch(inst),
641642
642 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),643 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
643 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),644 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -3893,6 +3894,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -3893,6 +3894,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
3893 return bt.finishAir(result);3894 return bt.finishAir(result);
3894}3895}
38953896
3897fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
3898 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
3899 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
3900}
3901
3896fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {3902fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
3897 // First section of indexes correspond to a set number of constant values.3903 // First section of indexes correspond to a set number of constant values.
3898 const ref_int = @enumToInt(inst);3904 const ref_int = @enumToInt(inst);
src/codegen/c.zig+20
...@@ -1278,6 +1278,7 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO...@@ -1278,6 +1278,7 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO
1278 .error_name => try airErrorName(f, inst),1278 .error_name => try airErrorName(f, inst),
1279 .splat => try airSplat(f, inst),1279 .splat => try airSplat(f, inst),
1280 .vector_init => try airVectorInit(f, inst),1280 .vector_init => try airVectorInit(f, inst),
1281 .prefetch => try airPrefetch(f, inst),
12811282
1282 .int_to_float,1283 .int_to_float,
1283 .float_to_int,1284 .float_to_int,
...@@ -3089,6 +3090,25 @@ fn airVectorInit(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3089,6 +3090,25 @@ fn airVectorInit(f: *Function, inst: Air.Inst.Index) !CValue {
3089 return f.fail("TODO: C backend: implement airVectorInit", .{});3090 return f.fail("TODO: C backend: implement airVectorInit", .{});
3090}3091}
30913092
3093fn airPrefetch(f: *Function, inst: Air.Inst.Index) !CValue {
3094 const prefetch = f.air.instructions.items(.data)[inst].prefetch;
3095 switch (prefetch.cache) {
3096 .data => {},
3097 // The available prefetch intrinsics do not accept a cache argument; only
3098 // address, rw, and locality. So unless the cache is data, we do not lower
3099 // this instruction.
3100 .instruction => return CValue.none,
3101 }
3102 const ptr = try f.resolveInst(prefetch.ptr);
3103 const writer = f.object.writer();
3104 try writer.writeAll("zig_prefetch(");
3105 try f.writeCValue(writer, ptr);
3106 try writer.print(", {d}, {d});\n", .{
3107 @enumToInt(prefetch.rw), prefetch.locality,
3108 });
3109 return CValue.none;
3110}
3111
3092fn toMemoryOrder(order: std.builtin.AtomicOrder) [:0]const u8 {3112fn toMemoryOrder(order: std.builtin.AtomicOrder) [:0]const u8 {
3093 return switch (order) {3113 return switch (order) {
3094 .Unordered => "memory_order_relaxed",3114 .Unordered => "memory_order_relaxed",
src/codegen/llvm.zig+62
...@@ -2073,6 +2073,7 @@ pub const FuncGen = struct {...@@ -2073,6 +2073,7 @@ pub const FuncGen = struct {
2073 .error_name => try self.airErrorName(inst),2073 .error_name => try self.airErrorName(inst),
2074 .splat => try self.airSplat(inst),2074 .splat => try self.airSplat(inst),
2075 .vector_init => try self.airVectorInit(inst),2075 .vector_init => try self.airVectorInit(inst),
2076 .prefetch => try self.airPrefetch(inst),
20762077
2077 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),2078 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
2078 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),2079 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -4384,6 +4385,67 @@ pub const FuncGen = struct {...@@ -4384,6 +4385,67 @@ pub const FuncGen = struct {
4384 return vector;4385 return vector;
4385 }4386 }
43864387
4388 fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
4389 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
4390
4391 comptime assert(@enumToInt(std.builtin.PrefetchOptions.Rw.read) == 0);
4392 comptime assert(@enumToInt(std.builtin.PrefetchOptions.Rw.write) == 1);
4393
4394 // TODO these two asserts should be able to be comptime because the type is a u2
4395 assert(prefetch.locality >= 0);
4396 assert(prefetch.locality <= 3);
4397
4398 comptime assert(@enumToInt(std.builtin.PrefetchOptions.Cache.instruction) == 0);
4399 comptime assert(@enumToInt(std.builtin.PrefetchOptions.Cache.data) == 1);
4400
4401 // LLVM fails during codegen of instruction cache prefetchs for these architectures.
4402 // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported
4403 // by the target.
4404 // To work around this, don't emit llvm.prefetch in this case.
4405 // See https://bugs.llvm.org/show_bug.cgi?id=21037
4406 const target = self.dg.module.getTarget();
4407 switch (prefetch.cache) {
4408 .instruction => switch (target.cpu.arch) {
4409 .x86_64, .i386 => return null,
4410 .arm, .armeb, .thumb, .thumbeb => {
4411 switch (prefetch.rw) {
4412 .write => return null,
4413 else => {},
4414 }
4415 },
4416 else => {},
4417 },
4418 .data => {},
4419 }
4420
4421 const llvm_u8 = self.context.intType(8);
4422 const llvm_ptr_u8 = llvm_u8.pointerType(0);
4423 const llvm_u32 = self.context.intType(32);
4424
4425 const llvm_fn_name = "llvm.prefetch.p0i8";
4426 const fn_val = self.dg.object.llvm_module.getNamedFunction(llvm_fn_name) orelse blk: {
4427 // declare void @llvm.prefetch(i8*, i32, i32, i32)
4428 const llvm_void = self.context.voidType();
4429 const param_types = [_]*const llvm.Type{
4430 llvm_ptr_u8, llvm_u32, llvm_u32, llvm_u32,
4431 };
4432 const fn_type = llvm.functionType(llvm_void, &param_types, param_types.len, .False);
4433 break :blk self.dg.object.llvm_module.addFunction(llvm_fn_name, fn_type);
4434 };
4435
4436 const ptr = try self.resolveInst(prefetch.ptr);
4437 const ptr_u8 = self.builder.buildBitCast(ptr, llvm_ptr_u8, "");
4438
4439 const params = [_]*const llvm.Value{
4440 ptr_u8,
4441 llvm_u32.constInt(@enumToInt(prefetch.rw), .False),
4442 llvm_u32.constInt(prefetch.locality, .False),
4443 llvm_u32.constInt(@enumToInt(prefetch.cache), .False),
4444 };
4445 _ = self.builder.buildCall(fn_val, &params, params.len, .C, .Auto, "");
4446 return null;
4447 }
4448
4387 fn getErrorNameTable(self: *FuncGen) !*const llvm.Value {4449 fn getErrorNameTable(self: *FuncGen) !*const llvm.Value {
4388 if (self.dg.object.error_name_table) |table| {4450 if (self.dg.object.error_name_table) |table| {
4389 return table;4451 return table;
src/link/C/zig.h+6
...@@ -74,6 +74,12 @@...@@ -74,6 +74,12 @@
74#define zig_frame_address() 074#define zig_frame_address() 0
75#endif75#endif
7676
77#if defined(__GNUC__)
78#define zig_prefetch(addr, rw, locality) __builtin_prefetch(addr, rw, locality)
79#else
80#define zig_prefetch(addr, rw, locality)
81#endif
82
77#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)83#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
78#include <stdatomic.h>84#include <stdatomic.h>
79#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)85#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)
src/print_air.zig+10
...@@ -226,6 +226,7 @@ const Writer = struct {...@@ -226,6 +226,7 @@ const Writer = struct {
226 .cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),226 .cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),
227 .fence => try w.writeFence(s, inst),227 .fence => try w.writeFence(s, inst),
228 .atomic_load => try w.writeAtomicLoad(s, inst),228 .atomic_load => try w.writeAtomicLoad(s, inst),
229 .prefetch => try w.writePrefetch(s, inst),
229 .atomic_store_unordered => try w.writeAtomicStore(s, inst, .Unordered),230 .atomic_store_unordered => try w.writeAtomicStore(s, inst, .Unordered),
230 .atomic_store_monotonic => try w.writeAtomicStore(s, inst, .Monotonic),231 .atomic_store_monotonic => try w.writeAtomicStore(s, inst, .Monotonic),
231 .atomic_store_release => try w.writeAtomicStore(s, inst, .Release),232 .atomic_store_release => try w.writeAtomicStore(s, inst, .Release),
...@@ -350,6 +351,15 @@ const Writer = struct {...@@ -350,6 +351,15 @@ const Writer = struct {
350 try s.print(", {s}", .{@tagName(atomic_load.order)});351 try s.print(", {s}", .{@tagName(atomic_load.order)});
351 }352 }
352353
354 fn writePrefetch(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
355 const prefetch = w.air.instructions.items(.data)[inst].prefetch;
356
357 try w.writeOperand(s, inst, 0, prefetch.ptr);
358 try s.print(", {s}, {d}, {s}", .{
359 @tagName(prefetch.rw), prefetch.locality, @tagName(prefetch.cache),
360 });
361 }
362
353 fn writeAtomicStore(363 fn writeAtomicStore(
354 w: *Writer,364 w: *Writer,
355 s: anytype,365 s: anytype,
test/behavior.zig+1-1
...@@ -10,6 +10,7 @@ test {...@@ -10,6 +10,7 @@ test {
10 _ = @import("behavior/fn_in_struct_in_comptime.zig");10 _ = @import("behavior/fn_in_struct_in_comptime.zig");
11 _ = @import("behavior/hasdecl.zig");11 _ = @import("behavior/hasdecl.zig");
12 _ = @import("behavior/hasfield.zig");12 _ = @import("behavior/hasfield.zig");
13 _ = @import("behavior/prefetch.zig");
13 _ = @import("behavior/pub_enum.zig");14 _ = @import("behavior/pub_enum.zig");
14 _ = @import("behavior/type.zig");15 _ = @import("behavior/type.zig");
15 _ = @import("behavior/bugs/655.zig");16 _ = @import("behavior/bugs/655.zig");
...@@ -178,7 +179,6 @@ test {...@@ -178,7 +179,6 @@ test {
178 _ = @import("behavior/optional_stage1.zig");179 _ = @import("behavior/optional_stage1.zig");
179 _ = @import("behavior/pointers_stage1.zig");180 _ = @import("behavior/pointers_stage1.zig");
180 _ = @import("behavior/popcount_stage1.zig");181 _ = @import("behavior/popcount_stage1.zig");
181 _ = @import("behavior/prefetch.zig");
182 _ = @import("behavior/ptrcast_stage1.zig");182 _ = @import("behavior/ptrcast_stage1.zig");
183 _ = @import("behavior/reflection.zig");183 _ = @import("behavior/reflection.zig");
184 _ = @import("behavior/saturating_arithmetic_stage1.zig");184 _ = @import("behavior/saturating_arithmetic_stage1.zig");
test/behavior/atomics.zig+13
...@@ -115,6 +115,13 @@ fn test_u128_cmpxchg() !void {...@@ -115,6 +115,13 @@ fn test_u128_cmpxchg() !void {
115var a_global_variable = @as(u32, 1234);115var a_global_variable = @as(u32, 1234);
116116
117test "cmpxchg on a global variable" {117test "cmpxchg on a global variable" {
118 if ((builtin.zig_backend == .stage1 or builtin.zig_backend == .stage2_llvm) and
119 builtin.cpu.arch == .aarch64)
120 {
121 // https://github.com/ziglang/zig/issues/10627
122 return error.SkipZigTest;
123 }
124
118 _ = @cmpxchgWeak(u32, &a_global_variable, 1234, 42, .Acquire, .Monotonic);125 _ = @cmpxchgWeak(u32, &a_global_variable, 1234, 42, .Acquire, .Monotonic);
119 try expect(a_global_variable == 42);126 try expect(a_global_variable == 42);
120}127}
...@@ -153,6 +160,12 @@ fn testAtomicStore() !void {...@@ -153,6 +160,12 @@ fn testAtomicStore() !void {
153}160}
154161
155test "atomicrmw with floats" {162test "atomicrmw with floats" {
163 if ((builtin.zig_backend == .stage1 or builtin.zig_backend == .stage2_llvm) and
164 builtin.cpu.arch == .aarch64)
165 {
166 // https://github.com/ziglang/zig/issues/10627
167 return error.SkipZigTest;
168 }
156 try testAtomicRmwFloat();169 try testAtomicRmwFloat();
157 comptime try testAtomicRmwFloat();170 comptime try testAtomicRmwFloat();
158}171}