authorgravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2024-02-17 14:01:07+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-02-17 14:01:07+01:00
logdd4d320eb9663c7a0ef8dbe3aca220a64795d683
tree9fb2dfc90b33b5efecf31a7d7ee286ddb4ef5f1b
parent50330ec22a79646d65ad0562ae0e580669a8c9cc
parent66e6d0d3142cfc32ea238a84d630732191c1dc1f
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #18948 from alichraghi/vector

spirv: use extended instructions whenever possible

5 files changed, 161 insertions(+), 93 deletions(-)

src/codegen/spirv.zig+101-76
...@@ -632,11 +632,15 @@ const DeclGen = struct {...@@ -632,11 +632,15 @@ const DeclGen = struct {
632 /// Checks whether the type can be directly translated to SPIR-V vectors632 /// Checks whether the type can be directly translated to SPIR-V vectors
633 fn isVector(self: *DeclGen, ty: Type) bool {633 fn isVector(self: *DeclGen, ty: Type) bool {
634 const mod = self.module;634 const mod = self.module;
635 const target = self.getTarget();
635 if (ty.zigTypeTag(mod) != .Vector) return false;636 if (ty.zigTypeTag(mod) != .Vector) return false;
636 const elem_ty = ty.childType(mod);637 const elem_ty = ty.childType(mod);
638
637 const len = ty.vectorLen(mod);639 const len = ty.vectorLen(mod);
638 const is_scalar = elem_ty.isNumeric(mod) or elem_ty.toIntern() == .bool_type;640 const is_scalar = elem_ty.isNumeric(mod) or elem_ty.toIntern() == .bool_type;
639 return is_scalar and len > 1 and len <= 4;641 const spirv_len = len > 1 and len <= 4;
642 const opencl_len = if (target.os.tag == .opencl) (len == 8 or len == 16) else false;
643 return is_scalar and (spirv_len or opencl_len);
640 }644 }
641645
642 fn arithmeticTypeInfo(self: *DeclGen, ty: Type) ArithmeticTypeInfo {646 fn arithmeticTypeInfo(self: *DeclGen, ty: Type) ArithmeticTypeInfo {
...@@ -1968,7 +1972,10 @@ const DeclGen = struct {...@@ -1968,7 +1972,10 @@ const DeclGen = struct {
1968 try self.func.prologue.emit(self.spv.gpa, .OpFunction, .{1972 try self.func.prologue.emit(self.spv.gpa, .OpFunction, .{
1969 .id_result_type = self.typeId(return_ty_ref),1973 .id_result_type = self.typeId(return_ty_ref),
1970 .id_result = decl_id,1974 .id_result = decl_id,
1971 .function_control = .{}, // TODO: We can set inline here if the type requires it.1975 .function_control = switch (fn_info.cc) {
1976 .Inline => .{ .Inline = true },
1977 else => .{},
1978 },
1972 .function_type = prototype_id,1979 .function_type = prototype_id,
1973 });1980 });
19741981
...@@ -2437,48 +2444,71 @@ const DeclGen = struct {...@@ -2437,48 +2444,71 @@ const DeclGen = struct {
24372444
2438 fn minMax(self: *DeclGen, result_ty: Type, op: std.math.CompareOperator, lhs_id: IdRef, rhs_id: IdRef) !IdRef {2445 fn minMax(self: *DeclGen, result_ty: Type, op: std.math.CompareOperator, lhs_id: IdRef, rhs_id: IdRef) !IdRef {
2439 const info = self.arithmeticTypeInfo(result_ty);2446 const info = self.arithmeticTypeInfo(result_ty);
2447 const target = self.getTarget();
24402448
2441 var wip = try self.elementWise(result_ty, true);2449 const use_backup_codegen = target.os.tag == .opencl and info.class != .float;
2450 var wip = try self.elementWise(result_ty, use_backup_codegen);
2442 defer wip.deinit();2451 defer wip.deinit();
2452
2443 for (wip.results, 0..) |*result_id, i| {2453 for (wip.results, 0..) |*result_id, i| {
2444 const lhs_elem_id = try wip.elementAt(result_ty, lhs_id, i);2454 const lhs_elem_id = try wip.elementAt(result_ty, lhs_id, i);
2445 const rhs_elem_id = try wip.elementAt(result_ty, rhs_id, i);2455 const rhs_elem_id = try wip.elementAt(result_ty, rhs_id, i);
24462456
2447 // TODO: Use fmin for OpenCL2457 if (use_backup_codegen) {
2448 const cmp_id = try self.cmp(op, Type.bool, wip.ty, lhs_elem_id, rhs_elem_id);2458 const cmp_id = try self.cmp(op, Type.bool, wip.ty, lhs_elem_id, rhs_elem_id);
2449 const selection_id = switch (info.class) {2459 result_id.* = self.spv.allocId();
2450 .float => blk: {2460 try self.func.body.emit(self.spv.gpa, .OpSelect, .{
2451 // cmp uses OpFOrd. When we have 0 [<>] nan this returns false,2461 .id_result_type = wip.ty_id,
2452 // but we want it to pick lhs. Therefore we also have to check if2462 .id_result = result_id.*,
2453 // rhs is nan. We don't need to care about the result when both2463 .condition = cmp_id,
2454 // are nan.2464 .object_1 = lhs_elem_id,
2455 const rhs_is_nan_id = self.spv.allocId();2465 .object_2 = rhs_elem_id,
2456 const bool_ty_ref = try self.resolveType(Type.bool, .direct);2466 });
2457 try self.func.body.emit(self.spv.gpa, .OpIsNan, .{2467 } else {
2458 .id_result_type = self.typeId(bool_ty_ref),2468 const ext_inst: Word = switch (target.os.tag) {
2459 .id_result = rhs_is_nan_id,2469 .opencl => switch (op) {
2460 .x = rhs_elem_id,2470 .lt => 28, // fmin
2461 });2471 .gt => 27, // fmax
2462 const float_cmp_id = self.spv.allocId();2472 else => unreachable,
2463 try self.func.body.emit(self.spv.gpa, .OpLogicalOr, .{2473 },
2464 .id_result_type = self.typeId(bool_ty_ref),2474 .vulkan => switch (info.class) {
2465 .id_result = float_cmp_id,2475 .float => switch (op) {
2466 .operand_1 = cmp_id,2476 .lt => 37, // FMin
2467 .operand_2 = rhs_is_nan_id,2477 .gt => 40, // FMax
2468 });2478 else => unreachable,
2469 break :blk float_cmp_id;2479 },
2470 },2480 .integer, .strange_integer => switch (info.signedness) {
2471 else => cmp_id,2481 .signed => switch (op) {
2472 };2482 .lt => 39, // SMin
2483 .gt => 42, // SMax
2484 else => unreachable,
2485 },
2486 .unsigned => switch (op) {
2487 .lt => 38, // UMin
2488 .gt => 41, // UMax
2489 else => unreachable,
2490 },
2491 },
2492 .composite_integer => unreachable, // TODO
2493 .bool => unreachable,
2494 },
2495 else => unreachable,
2496 };
2497 const set_id = switch (target.os.tag) {
2498 .opencl => try self.spv.importInstructionSet(.opencl),
2499 .vulkan => try self.spv.importInstructionSet(.glsl),
2500 else => unreachable,
2501 };
24732502
2474 result_id.* = self.spv.allocId();2503 result_id.* = self.spv.allocId();
2475 try self.func.body.emit(self.spv.gpa, .OpSelect, .{2504 try self.func.body.emit(self.spv.gpa, .OpExtInst, .{
2476 .id_result_type = wip.ty_id,2505 .id_result_type = wip.ty_id,
2477 .id_result = result_id.*,2506 .id_result = result_id.*,
2478 .condition = selection_id,2507 .set = set_id,
2479 .object_1 = lhs_elem_id,2508 .instruction = .{ .inst = ext_inst },
2480 .object_2 = rhs_elem_id,2509 .id_ref_4 = &.{ lhs_elem_id, rhs_elem_id },
2481 });2510 });
2511 }
2482 }2512 }
2483 return wip.finalize();2513 return wip.finalize();
2484 }2514 }
...@@ -2607,57 +2637,52 @@ const DeclGen = struct {...@@ -2607,57 +2637,52 @@ const DeclGen = struct {
2607 }2637 }
26082638
2609 fn airAbs(self: *DeclGen, inst: Air.Inst.Index) !?IdRef {2639 fn airAbs(self: *DeclGen, inst: Air.Inst.Index) !?IdRef {
2610 const mod = self.module;2640 const target = self.getTarget();
2611 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2641 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2612 const operand_id = try self.resolve(ty_op.operand);2642 const operand_id = try self.resolve(ty_op.operand);
2613 // Note: operand_ty may be signed, while ty is always unsigned!2643 // Note: operand_ty may be signed, while ty is always unsigned!
2614 const operand_ty = self.typeOf(ty_op.operand);2644 const operand_ty = self.typeOf(ty_op.operand);
2615 const result_ty = self.typeOfIndex(inst);2645 const result_ty = self.typeOfIndex(inst);
2616 const info = self.arithmeticTypeInfo(result_ty);2646 const operand_info = self.arithmeticTypeInfo(operand_ty);
2617 const operand_scalar_ty = operand_ty.scalarType(mod);
2618 const operand_scalar_ty_ref = try self.resolveType(operand_scalar_ty, .direct);
26192647
2620 var wip = try self.elementWise(result_ty, true);2648 var wip = try self.elementWise(result_ty, false);
2621 defer wip.deinit();2649 defer wip.deinit();
26222650
2623 const zero_id = switch (info.class) {
2624 .float => try self.constFloat(operand_scalar_ty_ref, 0),
2625 .integer, .strange_integer => try self.constInt(operand_scalar_ty_ref, 0),
2626 .composite_integer => unreachable, // TODO
2627 .bool => unreachable,
2628 };
2629 for (wip.results, 0..) |*result_id, i| {2651 for (wip.results, 0..) |*result_id, i| {
2630 const elem_id = try wip.elementAt(operand_ty, operand_id, i);2652 const elem_id = try wip.elementAt(operand_ty, operand_id, i);
2631 // Idk why spir-v doesn't have a dedicated abs() instruction in the base2653
2632 // instruction set. For now we're just going to negate and check to avoid2654 const ext_inst: Word = switch (target.os.tag) {
2633 // importing the extinst.2655 .opencl => switch (operand_info.class) {
2634 // TODO: Make this a call to compiler rt / ext inst2656 .float => 23, // fabs
2635 const neg_id = self.spv.allocId();2657 .integer, .strange_integer => switch (operand_info.signedness) {
2636 const args = .{2658 .signed => 141, // s_abs
2637 .id_result_type = self.typeId(operand_scalar_ty_ref),2659 .unsigned => 201, // u_abs
2638 .id_result = neg_id,2660 },
2639 .operand_1 = zero_id,2661 .composite_integer => unreachable, // TODO
2640 .operand_2 = elem_id,2662 .bool => unreachable,
2663 },
2664 .vulkan => switch (operand_info.class) {
2665 .float => 4, // FAbs
2666 .integer, .strange_integer => 5, // SAbs
2667 .composite_integer => unreachable, // TODO
2668 .bool => unreachable,
2669 },
2670 else => unreachable,
2641 };2671 };
2642 switch (info.class) {2672 const set_id = switch (target.os.tag) {
2643 .float => try self.func.body.emit(self.spv.gpa, .OpFSub, args),2673 .opencl => try self.spv.importInstructionSet(.opencl),
2644 .integer, .strange_integer => try self.func.body.emit(self.spv.gpa, .OpISub, args),2674 .vulkan => try self.spv.importInstructionSet(.glsl),
2645 .composite_integer => unreachable, // TODO2675 else => unreachable,
2646 .bool => unreachable,2676 };
2647 }2677
2648 const neg_norm_id = try self.normalize(wip.ty_ref, neg_id, info);2678 result_id.* = self.spv.allocId();
26492679 try self.func.body.emit(self.spv.gpa, .OpExtInst, .{
2650 const gt_zero_id = try self.cmp(.gt, Type.bool, operand_scalar_ty, elem_id, zero_id);2680 .id_result_type = wip.ty_id,
2651 const abs_id = self.spv.allocId();2681 .id_result = result_id.*,
2652 try self.func.body.emit(self.spv.gpa, .OpSelect, .{2682 .set = set_id,
2653 .id_result_type = self.typeId(operand_scalar_ty_ref),2683 .instruction = .{ .inst = ext_inst },
2654 .id_result = abs_id,2684 .id_ref_4 = &.{elem_id},
2655 .condition = gt_zero_id,
2656 .object_1 = elem_id,
2657 .object_2 = neg_norm_id,
2658 });2685 });
2659 // For Shader, we may need to cast from signed to unsigned here.
2660 result_id.* = try self.bitCast(wip.ty, operand_scalar_ty, abs_id);
2661 }2686 }
2662 return try wip.finalize();2687 return try wip.finalize();
2663 }2688 }
src/codegen/spirv/Module.zig+56-14
...@@ -8,6 +8,7 @@...@@ -8,6 +8,7 @@
8const Module = @This();8const Module = @This();
99
10const std = @import("std");10const std = @import("std");
11const builtin = @import("builtin");
11const Allocator = std.mem.Allocator;12const Allocator = std.mem.Allocator;
12const assert = std.debug.assert;13const assert = std.debug.assert;
1314
...@@ -114,8 +115,10 @@ sections: struct {...@@ -114,8 +115,10 @@ sections: struct {
114 capabilities: Section = .{},115 capabilities: Section = .{},
115 /// OpExtension instructions116 /// OpExtension instructions
116 extensions: Section = .{},117 extensions: Section = .{},
117 // OpExtInstImport instructions - skip for now.118 /// OpExtInstImport
118 // memory model defined by target, not required here.119 extended_instruction_set: Section = .{},
120 /// memory model defined by target
121 memory_model: Section = .{},
119 /// OpEntryPoint instructions - Handled by `self.entry_points`.122 /// OpEntryPoint instructions - Handled by `self.entry_points`.
120 /// OpExecutionMode and OpExecutionModeId instructions.123 /// OpExecutionMode and OpExecutionModeId instructions.
121 execution_modes: Section = .{},124 execution_modes: Section = .{},
...@@ -172,6 +175,9 @@ globals: struct {...@@ -172,6 +175,9 @@ globals: struct {
172 section: Section = .{},175 section: Section = .{},
173} = .{},176} = .{},
174177
178/// The list of extended instruction sets that should be imported.
179extended_instruction_set: std.AutoHashMapUnmanaged(ExtendedInstructionSet, IdRef) = .{},
180
175pub fn init(gpa: Allocator) Module {181pub fn init(gpa: Allocator) Module {
176 return .{182 return .{
177 .gpa = gpa,183 .gpa = gpa,
...@@ -182,6 +188,8 @@ pub fn init(gpa: Allocator) Module {...@@ -182,6 +188,8 @@ pub fn init(gpa: Allocator) Module {
182pub fn deinit(self: *Module) void {188pub fn deinit(self: *Module) void {
183 self.sections.capabilities.deinit(self.gpa);189 self.sections.capabilities.deinit(self.gpa);
184 self.sections.extensions.deinit(self.gpa);190 self.sections.extensions.deinit(self.gpa);
191 self.sections.extended_instruction_set.deinit(self.gpa);
192 self.sections.memory_model.deinit(self.gpa);
185 self.sections.execution_modes.deinit(self.gpa);193 self.sections.execution_modes.deinit(self.gpa);
186 self.sections.debug_strings.deinit(self.gpa);194 self.sections.debug_strings.deinit(self.gpa);
187 self.sections.debug_names.deinit(self.gpa);195 self.sections.debug_names.deinit(self.gpa);
...@@ -200,6 +208,8 @@ pub fn deinit(self: *Module) void {...@@ -200,6 +208,8 @@ pub fn deinit(self: *Module) void {
200 self.globals.globals.deinit(self.gpa);208 self.globals.globals.deinit(self.gpa);
201 self.globals.section.deinit(self.gpa);209 self.globals.section.deinit(self.gpa);
202210
211 self.extended_instruction_set.deinit(self.gpa);
212
203 self.* = undefined;213 self.* = undefined;
204}214}
205215
...@@ -448,6 +458,8 @@ pub fn flush(self: *Module, file: std.fs.File, target: std.Target) !void {...@@ -448,6 +458,8 @@ pub fn flush(self: *Module, file: std.fs.File, target: std.Target) !void {
448 &header,458 &header,
449 self.sections.capabilities.toWords(),459 self.sections.capabilities.toWords(),
450 self.sections.extensions.toWords(),460 self.sections.extensions.toWords(),
461 self.sections.extended_instruction_set.toWords(),
462 self.sections.memory_model.toWords(),
451 entry_points.toWords(),463 entry_points.toWords(),
452 self.sections.execution_modes.toWords(),464 self.sections.execution_modes.toWords(),
453 source.toWords(),465 source.toWords(),
...@@ -460,19 +472,26 @@ pub fn flush(self: *Module, file: std.fs.File, target: std.Target) !void {...@@ -460,19 +472,26 @@ pub fn flush(self: *Module, file: std.fs.File, target: std.Target) !void {
460 self.sections.functions.toWords(),472 self.sections.functions.toWords(),
461 };473 };
462474
463 var iovc_buffers: [buffers.len]std.os.iovec_const = undefined;475 if (builtin.zig_backend == .stage2_x86_64) {
464 var file_size: u64 = 0;476 for (buffers) |buf| {
465 for (&iovc_buffers, 0..) |*iovc, i| {477 try file.writeAll(std.mem.sliceAsBytes(buf));
466 // Note, since spir-v supports both little and big endian we can ignore byte order here and478 }
467 // just treat the words as a sequence of bytes.479 } else {
468 const bytes = std.mem.sliceAsBytes(buffers[i]);480 // miscompiles with x86_64 backend
469 iovc.* = .{ .iov_base = bytes.ptr, .iov_len = bytes.len };481 var iovc_buffers: [buffers.len]std.os.iovec_const = undefined;
470 file_size += bytes.len;482 var file_size: u64 = 0;
471 }483 for (&iovc_buffers, 0..) |*iovc, i| {
484 // Note, since spir-v supports both little and big endian we can ignore byte order here and
485 // just treat the words as a sequence of bytes.
486 const bytes = std.mem.sliceAsBytes(buffers[i]);
487 iovc.* = .{ .iov_base = bytes.ptr, .iov_len = bytes.len };
488 file_size += bytes.len;
489 }
472490
473 try file.seekTo(0);491 try file.seekTo(0);
474 try file.setEndPos(file_size);492 try file.setEndPos(file_size);
475 try file.pwritevAll(&iovc_buffers, 0);493 try file.pwritevAll(&iovc_buffers, 0);
494 }
476}495}
477496
478/// Merge the sections making up a function declaration into this module.497/// Merge the sections making up a function declaration into this module.
...@@ -482,6 +501,29 @@ pub fn addFunction(self: *Module, decl_index: Decl.Index, func: Fn) !void {...@@ -482,6 +501,29 @@ pub fn addFunction(self: *Module, decl_index: Decl.Index, func: Fn) !void {
482 try self.declareDeclDeps(decl_index, func.decl_deps.keys());501 try self.declareDeclDeps(decl_index, func.decl_deps.keys());
483}502}
484503
504pub const ExtendedInstructionSet = enum {
505 glsl,
506 opencl,
507};
508
509/// Imports or returns the existing id of an extended instruction set
510pub fn importInstructionSet(self: *Module, set: ExtendedInstructionSet) !IdRef {
511 const gop = try self.extended_instruction_set.getOrPut(self.gpa, set);
512 if (gop.found_existing) return gop.value_ptr.*;
513
514 const result_id = self.allocId();
515 try self.sections.extended_instruction_set.emit(self.gpa, .OpExtInstImport, .{
516 .id_result = result_id,
517 .name = switch (set) {
518 .glsl => "GLSL.std.450",
519 .opencl => "OpenCL.std",
520 },
521 });
522 gop.value_ptr.* = result_id;
523
524 return result_id;
525}
526
485/// Fetch the result-id of an OpString instruction that encodes the path of the source527/// Fetch the result-id of an OpString instruction that encodes the path of the source
486/// file of the decl. This function may also emit an OpSource with source-level information regarding528/// file of the decl. This function may also emit an OpSource with source-level information regarding
487/// the decl.529/// the decl.
src/link/SpirV.zig+2-3
...@@ -246,7 +246,7 @@ fn writeCapabilities(spv: *SpvModule, target: std.Target) !void {...@@ -246,7 +246,7 @@ fn writeCapabilities(spv: *SpvModule, target: std.Target) !void {
246 const gpa = spv.gpa;246 const gpa = spv.gpa;
247 // TODO: Integrate with a hypothetical feature system247 // TODO: Integrate with a hypothetical feature system
248 const caps: []const spec.Capability = switch (target.os.tag) {248 const caps: []const spec.Capability = switch (target.os.tag) {
249 .opencl => &.{ .Kernel, .Addresses, .Int8, .Int16, .Int64, .Float64, .Float16, .GenericPointer },249 .opencl => &.{ .Kernel, .Addresses, .Int8, .Int16, .Int64, .Float64, .Float16, .Vector16, .GenericPointer },
250 .glsl450 => &.{.Shader},250 .glsl450 => &.{.Shader},
251 .vulkan => &.{ .Shader, .VariablePointersStorageBuffer, .Int8, .Int16, .Int64, .Float64, .Float16 },251 .vulkan => &.{ .Shader, .VariablePointersStorageBuffer, .Int8, .Int16, .Int64, .Float64, .Float16 },
252 else => unreachable, // TODO252 else => unreachable, // TODO
...@@ -279,8 +279,7 @@ fn writeMemoryModel(spv: *SpvModule, target: std.Target) !void {...@@ -279,8 +279,7 @@ fn writeMemoryModel(spv: *SpvModule, target: std.Target) !void {
279 else => unreachable,279 else => unreachable,
280 };280 };
281281
282 // TODO: Put this in a proper section.282 try spv.sections.memory_model.emit(gpa, .OpMemoryModel, .{
283 try spv.sections.extensions.emit(gpa, .OpMemoryModel, .{
284 .addressing_model = addressing_model,283 .addressing_model = addressing_model,
285 .memory_model = memory_model,284 .memory_model = memory_model,
286 });285 });
test/behavior/math.zig+1
...@@ -1062,6 +1062,7 @@ test "@mulWithOverflow bitsize > 32" {...@@ -1062,6 +1062,7 @@ test "@mulWithOverflow bitsize > 32" {
1062test "@mulWithOverflow u256" {1062test "@mulWithOverflow u256" {
1063 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;1063 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1064 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;1064 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
1065 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
10651066
1066 {1067 {
1067 const const_lhs: u256 = 8035709466408580321693645878924206181189;1068 const const_lhs: u256 = 8035709466408580321693645878924206181189;
test/behavior/struct.zig+1
...@@ -2117,6 +2117,7 @@ test "initiate global variable with runtime value" {...@@ -2117,6 +2117,7 @@ test "initiate global variable with runtime value" {
21172117
2118test "struct containing optional pointer to array of @This()" {2118test "struct containing optional pointer to array of @This()" {
2119 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;2119 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
2120 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
21202121
2121 const S = struct {2122 const S = struct {
2122 x: ?*const [1]@This(),2123 x: ?*const [1]@This(),