diff --git a/lib/std/Target.zig b/lib/std/Target.zig index f4c7e29f5f63d13464e48be7c66130d4d058a574..76d8122a561ad747d8c7c05badd529388d22ec77 100644 --- a/lib/std/Target.zig +++ b/lib/std/Target.zig @@ -2392,6 +2392,7 @@ pub fn supportsAddressSpace( .lut => arch == .propeller and std.Target.propeller.featureSetHas(target.cpu.features, .p2), .global, .local, .shared => is_gpu, + .private => is_spirv, .constant => (is_gpu and (context == null or context == .constant)) or (is_spirv and (context == null or context == .constant or context == .pointer)), .param => is_nvptx, diff --git a/lib/std/lang.zig b/lib/std/lang.zig index b96acc3c5efa47b6092f7cc82c8592200ad4e662..d0fd96a912a1b8dde4a6be70aea5983d225f1204 100644 --- a/lib/std/lang.zig +++ b/lib/std/lang.zig @@ -591,6 +591,7 @@ pub const AddressSpace = enum(u5) { param, shared, local, + private, input, output, uniform, diff --git a/src/Sema.zig b/src/Sema.zig index a3533e99233d4e8f8e0d61184d65733906d1ad86..3485075948e23534b447c15588bad8b32a9ca116 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -20914,13 +20914,8 @@ fn zirReifySpirvType( const depth = try sema.interpretStdLangType(block, operand_src, depth_val, std.lang.Type.Spirv.Image.Depth); const access = try sema.interpretStdLangType(block, operand_src, access_val, std.lang.Type.Spirv.Image.Access); - switch (target.os.tag) { - .opencl => if (access == .unknown) { - return sema.fail(block, operand_src, "'access' field must be specified under the 'opencl' os", .{}); - }, - else => if (access != .unknown) { - return sema.fail(block, operand_src, "access qualifier '.{t}' is only valid under the 'opencl' os", .{access}); - }, + if (target.os.tag == .opencl and access == .unknown) { + return sema.fail(block, operand_src, "'access' field must be specified under the 'opencl' OS", .{}); } const arrayed = try sema.interpretStdLangType(block, operand_src, arrayed_val, bool); @@ -20930,31 +20925,24 @@ fn zirReifySpirvType( const usage_tag = try sema.interpretStdLangType(block, operand_src, usage_tag_val, @typeInfo(std.lang.Type.Spirv.Image.Usage).@"union".tag_type.?); switch (target.os.tag) { - .vulkan => { - if (usage_tag == .unknown) { - return sema.fail( - block, - operand_src, - "'usage' must be '.sampled' or '.storage' under the 'vulkan' os (Sampled == 0 is forbidden)", - .{}, - ); - } + .vulkan => if (usage_tag == .unknown) { + return sema.fail(block, operand_src, "'usage' must be '.sampled' or '.storage' under the 'vulkan' OS", .{}); }, .opencl => { if (usage_tag != .unknown) { - return sema.fail(block, operand_src, "'usage' must be '.unknown' under the 'opencl' os", .{}); + return sema.fail(block, operand_src, "'usage' must be '.unknown' under the 'opencl' OS", .{}); } if (multisampled) { - return sema.fail(block, operand_src, "'multisampled' must be 'false' under the 'opencl' os", .{}); + return sema.fail(block, operand_src, "'multisampled' must be 'false' under the 'opencl' OS", .{}); } if (format != .unknown) { - return sema.fail(block, operand_src, "'format' must be '.unknown' under the 'opencl' os", .{}); + return sema.fail(block, operand_src, "'format' must be '.unknown' under the 'opencl' OS", .{}); } if (dim == .cube) { - return sema.fail(block, operand_src, "'dim' '.cube' is not allowed under the 'opencl' os", .{}); + return sema.fail(block, operand_src, "'dim' '.cube' is not allowed under the 'opencl' OS", .{}); } if (arrayed and dim != .@"1d" and dim != .@"2d") { - return sema.fail(block, operand_src, "'arrayed' may only be 'true' when 'dim' is '.1d' or '.2d' under the 'opencl' os", .{}); + return sema.fail(block, operand_src, "'arrayed' may only be 'true' when 'dim' is '.1d' or '.2d' under the 'opencl' OS", .{}); } }, else => {}, @@ -20965,10 +20953,10 @@ fn zirReifySpirvType( const sampled_type = usage_val.unionPayload(zcu).toType(); if (target.os.tag != .opencl and sampled_type.toIntern() == .void_type) { - return sema.fail(block, operand_src, "'void' type for '{t}' field is only valid under the 'opencl' os", .{usage_tag}); + return sema.fail(block, operand_src, "'void' type for '{t}' field is only valid under the 'opencl' OS", .{usage_tag}); } if (target.os.tag == .opencl and sampled_type.toIntern() != .void_type) { - return sema.fail(block, operand_src, "'{t}' field type must be 'void' under the 'opencl' os", .{usage_tag}); + return sema.fail(block, operand_src, "'{t}' field type must be 'void' under the 'opencl' OS", .{usage_tag}); } if (sampled_type.toIntern() != .void_type and @@ -20984,7 +20972,7 @@ fn zirReifySpirvType( return sema.fail( block, operand_src, - "'{t}' field value must be a 32-bit int, 64-bit int or 32-bit float under the 'vulkan' os", + "'{t}' field value must be a 32-bit int, 64-bit int or 32-bit float under the 'vulkan' OS", .{usage_tag}, ); } @@ -21005,7 +20993,7 @@ fn zirReifySpirvType( return sema.fail( block, operand_src, - "image 'format' '.{t}' does not match '{t}' type '{f}' under the 'vulkan' os", + "image 'format' '.{t}' does not match '{t}' type '{f}' under the 'vulkan' OS", .{ format, usage_tag, sampled_type.fmt(pt) }, ); } @@ -21058,7 +21046,7 @@ fn zirReifySpirvType( elem_ty.zigTypeTag(zcu) == .spirv and ip.loadSpirvType(elem_ty.toIntern()).flags.tag == .runtime_array) { - return sema.fail(block, operand_src, "'runtime_array' of 'runtime_array' is not allowed under the 'vulkan' os", .{}); + return sema.fail(block, operand_src, "'runtime_array' of 'runtime_array' is not allowed under the 'vulkan' OS", .{}); } break :blk .{ .ty = union_val.val, @@ -25315,11 +25303,14 @@ fn zirBuiltinExtern( } if (options.decoration) |decoration| switch (decoration) { - .flat => switch (ptr_info.flags.address_space) { + .location, .flat => switch (ptr_info.flags.address_space) { .input, .output => {}, - else => return sema.fail(block, options_src, "\"flat\" decoration requires \"input\" or \"output\" address space", .{}), + else => return sema.fail(block, options_src, "'{t}' decoration requires 'input' or 'output' address space", .{decoration}), + }, + .descriptor => switch (ptr_info.flags.address_space) { + .global, .storage_buffer, .uniform, .constant => {}, + else => return sema.fail(block, options_src, "'descriptor' decoration requires 'storage_buffer', 'uniform', 'constant' or 'global' address space", .{}), }, - .location, .descriptor => {}, }; const target = zcu.getTarget(); @@ -34913,6 +34904,7 @@ pub fn resolveNavPtrModifiers( const gpa = comp.gpa; const io = comp.io; const ip = &zcu.intern_pool; + const target = zcu.getTarget(); const align_src = block.src(.{ .node_offset_var_decl_align = .zero }); const section_src = block.src(.{ .node_offset_var_decl_section = .zero }); @@ -34944,7 +34936,6 @@ pub fn resolveNavPtrModifiers( else => .constant, }, }; - const target = zcu.getTarget(); const addrspace_body = zir_decl.addrspace_body orelse { if (zir_decl.linkage == .@"extern" and target.cpu.arch.isSpirV() and @@ -34968,6 +34959,21 @@ pub fn resolveNavPtrModifiers( break :as try sema.analyzeAsAddressSpace(block, addrspace_src, addrspace_ref, addrspace_ctx); }; + if (target.cpu.arch.isSpirV()) { + if (zir_decl.kind == .@"var" and zir_decl.linkage != .@"extern" and !zir_decl.is_threadlocal) { + return sema.failWithOwnedErrorMsg(block, msg: { + const decl_src = block.nodeOffset(.zero); + const msg = try sema.errMsg(decl_src, "SPIR-V target does not support global variables", .{}); + errdefer msg.destroy(gpa); + try sema.errNote(decl_src, msg, "consider using 'threadlocal'", .{}); + break :msg msg; + }); + } + if (zir_decl.is_threadlocal and @"addrspace" != .private and target.os.tag == .vulkan) { + return sema.fail(block, addrspace_src, "threadlocal variables with address space '{t}' are not supported on {t}", .{ @"addrspace", target.os.tag }); + } + } + return .{ .@"align" = @"align", .@"linksection" = @"linksection", diff --git a/src/codegen/spirv/Assembler.zig b/src/codegen/spirv/Assembler.zig index 6a96b5e3a00445176a67e085a0ca27cbd2f88dd3..ec9f608e38675d47a258bb16dada49c6262c70d5 100644 --- a/src/codegen/spirv/Assembler.zig +++ b/src/codegen/spirv/Assembler.zig @@ -1,16 +1,11 @@ const std = @import("std"); -const Allocator = std.mem.Allocator; const assert = std.debug.assert; - const CodeGen = @import("CodeGen.zig"); -const Decl = @import("CodeGen.zig").Decl; - const spec = @import("spec.zig"); const Opcode = spec.Opcode; const Word = spec.Word; const Id = spec.Id; const StorageClass = spec.StorageClass; - const Assembler = @This(); cg: *CodeGen, @@ -20,7 +15,10 @@ src: []const u8 = undefined, tokens: std.ArrayList(Token) = .empty, current_token: u32 = 0, /// The instruction that is currently being parsed or has just been parsed. -inst: struct { +inst: Instruction = .{}, +value_map: std.array_hash_map.String(AsmValue) = .empty, + +const Instruction = struct { opcode: Opcode = undefined, operands: std.ArrayList(Operand) = .empty, string_bytes: std.ArrayList(u8) = .empty, @@ -35,9 +33,7 @@ inst: struct { } return null; } -} = .{}, -value_map: std.array_hash_map.String(AsmValue) = .empty, -inst_map: std.array_hash_map.String(void) = .empty, +}; const Operand = union(enum) { /// Any 'simple' 32-bit value. This could be a mask or @@ -68,27 +64,14 @@ pub fn deinit(ass: *Assembler) void { ass.inst.operands.deinit(gpa); ass.inst.string_bytes.deinit(gpa); ass.value_map.deinit(gpa); - ass.inst_map.deinit(gpa); } const Error = error{ AssembleFail, OutOfMemory }; pub fn assemble(ass: *Assembler, src: []const u8) Error!void { - const gpa = ass.cg.gpa; - ass.src = src; ass.errors.clearRetainingCapacity(); - // Populate the opcode map if it isn't already - if (ass.inst_map.count() == 0) { - const instructions = spec.InstructionSet.core.instructions(); - try ass.inst_map.ensureUnusedCapacity(gpa, @intCast(instructions.len)); - for (instructions, 0..) |inst, i| { - const entry = try ass.inst_map.getOrPut(gpa, inst.name); - assert(entry.index == i); - } - } - try ass.tokenize(); while (!ass.testToken(.eof)) { try ass.parseInstruction(); @@ -144,18 +127,20 @@ const AsmValue = union(enum) { /// zero-extended from the float's width to 64 bits. constant: u64, /// A vector "c" input expanded by `processSpecConstVector`. - constant_composite: struct { + constant_composite: ConstantComposite, + string: []const u8, + + const ConstantComposite = struct { child: Id, child_kind: std.lang.TypeId, child_bit_width: u16, values: []u64, - }, - string: []const u8, + }; /// Retrieve the result-id of this AsmValue. Asserts that this AsmValue /// is of a variant that allows the result to be obtained (not an unresolved /// forward declaration, not in the process of being declared, etc). - pub fn resultId(value: AsmValue) Id { + fn resultId(value: AsmValue) Id { return switch (value) { .just_declared, .unresolved_forward_reference, @@ -221,7 +206,7 @@ fn processTypeInstruction(ass: *Assembler) !AsmValue { const cg = ass.cg; const gpa = cg.gpa; const operands = ass.inst.operands.items; - const section = &cg.sections.globals; + const section = &cg.module; const id = switch (ass.inst.opcode) { .OpTypeVoid => try cg.voidType(), .OpTypeBool => try cg.boolType(), @@ -339,26 +324,16 @@ fn processTypeInstruction(ass: *Assembler) !AsmValue { /// - Target section is determined from instruction type. fn processGenericInstruction(ass: *Assembler) !?AsmValue { const cg = ass.cg; - const target = cg.zcu.getTarget(); const operands = ass.inst.operands.items; - var maybe_spv_decl_index: ?Decl.Index = null; const section = switch (ass.inst.opcode.class()) { - .constant_creation => &cg.sections.globals, - .annotation => &cg.sections.annotations, + .constant_creation => &cg.module, + .annotation => &cg.module, .type_declaration => unreachable, // Handled elsewhere. else => switch (ass.inst.opcode) { - .OpEntryPoint => unreachable, - .OpExecutionMode, .OpExecutionModeId => &cg.sections.execution_modes, .OpVariable => section: { const storage_class: spec.StorageClass = @fromBackingInt(@intCast(operands[2].value)); if (storage_class == .function) break :section &ass.cg.prologue; - maybe_spv_decl_index = try cg.allocDecl(.global); - if (!target.cpu.has(.spirv, .v1_4) and storage_class != .input and storage_class != .output) { - // Before version 1.4, the interface’s storage classes are limited to the Input and Output - break :section &cg.sections.globals; - } - try ass.cg.decl_deps.append(cg.gpa, maybe_spv_decl_index.?); - break :section &cg.sections.globals; + break :section &cg.module; }, else => &ass.cg.body, }, @@ -388,10 +363,7 @@ fn processGenericInstruction(ass: *Assembler) !?AsmValue { .value, .literal32 => |word| section.writeWord(word), .literal64 => |dword| section.writeDoubleWord(dword), .result_id => { - maybe_result_id = if (maybe_spv_decl_index) |spv_decl_index| - cg.declPtr(spv_decl_index).result_id - else - cg.allocId(); + maybe_result_id = cg.allocId(); section.writeOperand(Id, maybe_result_id.?); }, .ref_id => |index| { @@ -406,7 +378,8 @@ fn processGenericInstruction(ass: *Assembler) !?AsmValue { } const actual_word_count = section.instructions.items.len - first_word; - section.instructions.items[first_word] |= @as(u32, @as(u16, @intCast(actual_word_count))) << 16 | @backingInt(ass.inst.opcode); + const header: spec.InstructionHeader = .{ .opcode = ass.inst.opcode, .word_count = @intCast(actual_word_count) }; + section.instructions.items[first_word] = @bitCast(header); switch (ass.inst.opcode) { .OpKill, @@ -454,8 +427,8 @@ fn processSpecConstVector(ass: *Assembler) !?AsmValue { else => return ass.fail(ass.inst.inst_offset, "%ty must be a type", .{}), }; - const globals = &cg.sections.globals; - const annotations = &cg.sections.annotations; + const globals = &cg.module; + const annotations = &cg.module; const literal_words: usize = if (cc.child_bit_width <= @bitSizeOf(Word)) 1 else 2; const elem_ids = try gpa.alloc(Id, cc.values.len); @@ -465,11 +438,10 @@ fn processSpecConstVector(ass: *Assembler) !?AsmValue { elem_id_out.* = elem_id; switch (cc.child_kind) { - .bool => { - const opcode: Opcode = if (value & 1 != 0) .OpSpecConstantTrue else .OpSpecConstantFalse; - try globals.emitRaw(gpa, opcode, 2); - globals.writeOperand(Id, cc.child); - globals.writeOperand(Id, elem_id); + .bool => if (value & 1 != 0) { + try globals.emit(gpa, .OpSpecConstantTrue, .{ .id_result_type = cc.child, .id_result = elem_id }); + } else { + try globals.emit(gpa, .OpSpecConstantFalse, .{ .id_result_type = cc.child, .id_result = elem_id }); }, .int, .float => { try globals.emitRaw(gpa, .OpSpecConstant, 2 + literal_words); @@ -487,10 +459,10 @@ fn processSpecConstVector(ass: *Assembler) !?AsmValue { const spec_id_word = std.math.cast(u32, spec_id_base + i) orelse { return ass.fail(ass.inst.inst_offset, "SpecId {} does not fit in 32 bits", .{spec_id_base + i}); }; - try annotations.emitRaw(gpa, .OpDecorate, 3); - annotations.writeOperand(Id, elem_id); - annotations.writeWord(@backingInt(spec.Decoration.spec_id)); - annotations.writeWord(spec_id_word); + try annotations.emit(gpa, .OpDecorate, .{ + .target = elem_id, + .decoration = .{ .spec_id = .{ .specialization_constant_id = spec_id_word } }, + }); } const result_id = cg.allocId(); @@ -557,14 +529,13 @@ fn parseInstruction(ass: *Assembler) !void { } const opcode_text = ass.tokenText(opcode_tok); - const index = ass.inst_map.getIndex(opcode_text) orelse { + const opcode = std.meta.stringToEnum(spec.Opcode, opcode_text) orelse { return ass.fail(opcode_tok.start, "invalid opcode '{s}'", .{opcode_text}); }; - const inst = spec.InstructionSet.core.instructions()[index]; - ass.inst.opcode = @fromBackingInt(@intCast(inst.opcode)); + ass.inst.opcode = opcode; - const expected_operands = inst.operands; + const expected_operands = opcode.operands(); // This is a loop because the result-id is not always the first operand. const requires_lhs_result = for (expected_operands) |op| { if (op.kind == .id_result) break true; @@ -614,8 +585,8 @@ fn parseOperand(ass: *Assembler, kind: spec.OperandKind) Error!void { else => switch (kind) { .literal_integer => try ass.parseLiteralInteger(), .literal_string => try ass.parseString(), - .literal_context_dependent_number => try ass.parseContextDependentNumber(), - .literal_ext_inst_integer => try ass.parseLiteralExtInstInteger(), + .literal_context_dependent_number => try ass.parseNumber(), + .literal_ext_inst_integer => try ass.parseLiteralInteger(), .pair_id_ref_id_ref => try ass.parsePhiSource(), else => return ass.todo("parse operand of type {s}", .{@tagName(kind)}), }, @@ -661,12 +632,12 @@ fn parseBitEnum(ass: *Assembler, kind: spec.OperandKind) !void { if ((mask & enumerant.value) == 0) continue; - for (enumerant.parameters) |param_kind| { + for (enumerant.parameters) |param| { if (ass.isAtInstructionBoundary()) { return ass.fail(ass.currentToken().start, "missing required parameter for bit flag '{s}'", .{enumerant.name}); } - try ass.parseOperand(param_kind); + try ass.parseOperand(param.kind); } } } @@ -721,12 +692,12 @@ fn parseValueEnum(ass: *Assembler, kind: spec.OperandKind) !void { try ass.inst.operands.append(gpa, .{ .value = enumerant.value }); - for (enumerant.parameters) |param_kind| { + for (enumerant.parameters) |param| { if (ass.isAtInstructionBoundary()) { return ass.fail(ass.currentToken().start, "missing required parameter for enum variant '{s}'", .{enumerant.name}); } - try ass.parseOperand(param_kind); + try ass.parseOperand(param.kind); } } @@ -746,30 +717,26 @@ fn parseRefId(ass: *Assembler) !void { try ass.inst.operands.append(gpa, .{ .ref_id = index }); } +fn eatPlaceholderConstant(ass: *Assembler) !?u64 { + const tok = ass.currentToken(); + if (!ass.eatToken(.placeholder)) return null; + const name = ass.tokenText(tok)[1..]; + const value = ass.value_map.get(name) orelse + return ass.fail(tok.start, "invalid placeholder '${s}'", .{name}); + return switch (value) { + .constant => |literal| literal, + else => ass.fail(tok.start, "value '{s}' cannot be used as placeholder", .{name}), + }; +} + fn parseLiteralInteger(ass: *Assembler) !void { const gpa = ass.cg.gpa; const tok = ass.currentToken(); - if (ass.eatToken(.placeholder)) { - const name = ass.tokenText(tok)[1..]; - const value = ass.value_map.get(name) orelse { - return ass.fail(tok.start, "invalid placeholder '${s}'", .{name}); - }; - switch (value) { - .constant => |literal| { - const literal32 = std.math.cast(u32, literal) orelse { - return ass.fail( - tok.start, - "placeholder value {} does not fit in 32 bits", - .{literal}, - ); - }; - try ass.inst.operands.append(gpa, .{ .literal32 = literal32 }); - }, - else => { - return ass.fail(tok.start, "value '{s}' cannot be used as placeholder", .{name}); - }, - } + if (try ass.eatPlaceholderConstant()) |literal| { + const literal32 = std.math.cast(u32, literal) orelse + return ass.fail(tok.start, "placeholder value {} does not fit in 32 bits", .{literal}); + try ass.inst.operands.append(gpa, .{ .literal32 = literal32 }); return; } @@ -786,41 +753,6 @@ fn parseLiteralInteger(ass: *Assembler) !void { try ass.inst.operands.append(gpa, .{ .literal32 = value }); } -fn parseLiteralExtInstInteger(ass: *Assembler) !void { - const gpa = ass.cg.gpa; - - const tok = ass.currentToken(); - if (ass.eatToken(.placeholder)) { - const name = ass.tokenText(tok)[1..]; - const value = ass.value_map.get(name) orelse { - return ass.fail(tok.start, "invalid placeholder '${s}'", .{name}); - }; - switch (value) { - .constant => |literal| { - const literal32 = std.math.cast(u32, literal) orelse { - return ass.fail( - tok.start, - "placeholder value {} does not fit in 32 bits", - .{literal}, - ); - }; - try ass.inst.operands.append(gpa, .{ .literal32 = literal32 }); - }, - else => { - return ass.fail(tok.start, "value '{s}' cannot be used as placeholder", .{name}); - }, - } - return; - } - - try ass.expectToken(.value); - const text = ass.tokenText(tok); - const value = std.fmt.parseInt(u32, text, 0) catch { - return ass.fail(tok.start, "'{s}' is not a valid 32-bit integer literal", .{text}); - }; - try ass.inst.operands.append(gpa, .{ .literal32 = value }); -} - fn parseString(ass: *Assembler) !void { const gpa = ass.cg.gpa; @@ -844,7 +776,7 @@ fn parseString(ass: *Assembler) !void { try ass.inst.operands.append(gpa, .{ .string = string_offset }); } -fn parseContextDependentNumber(ass: *Assembler) !void { +fn parseNumber(ass: *Assembler) !void { const cg = ass.cg; assert(ass.inst.opcode == .OpConstant or ass.inst.opcode == .OpSpecConstant); @@ -852,14 +784,14 @@ fn parseContextDependentNumber(ass: *Assembler) !void { const result = try ass.resolveRef(ass.inst.operands.items[0].ref_id); const result_id = result.resultId(); - const words = cg.sections.globals.instructions.items; + const words = cg.module.instructions.items; var offset: usize = 0; while (offset < words.len) { - const word_count = words[offset] >> 16; - const opcode: Opcode = @fromBackingInt(@intCast(words[offset] & 0xFFFF)); + const header: spec.InstructionHeader = @bitCast(words[offset]); + const word_count = header.word_count; defer offset += word_count; if (word_count == 0) break; - switch (opcode) { + switch (header.opcode) { .OpTypeInt => if (word_count >= 4 and @as(Id, @fromBackingInt(@intCast(words[offset + 1]))) == result_id) { const width: u16 = @intCast(words[offset + 2]); const signedness: std.lang.Signedness = if (words[offset + 3] == 0) .unsigned else .signed; @@ -885,22 +817,11 @@ fn parseContextDependentInt(ass: *Assembler, signedness: std.lang.Signedness, wi const gpa = ass.cg.gpa; const tok = ass.currentToken(); - if (ass.eatToken(.placeholder)) { - const name = ass.tokenText(tok)[1..]; - const value = ass.value_map.get(name) orelse { - return ass.fail(tok.start, "invalid placeholder '${s}'", .{name}); - }; - switch (value) { - .constant => |literal| { - if (width <= @bitSizeOf(spec.Word)) { - try ass.inst.operands.append(gpa, .{ .literal32 = @truncate(literal) }); - } else { - try ass.inst.operands.append(gpa, .{ .literal64 = literal }); - } - }, - else => { - return ass.fail(tok.start, "value '{s}' cannot be used as placeholder", .{name}); - }, + if (try ass.eatPlaceholderConstant()) |literal| { + if (width <= @bitSizeOf(spec.Word)) { + try ass.inst.operands.append(gpa, .{ .literal32 = @truncate(literal) }); + } else { + try ass.inst.operands.append(gpa, .{ .literal64 = literal }); } return; } @@ -943,22 +864,11 @@ fn parseContextDependentFloat(ass: *Assembler, comptime width: u16) !void { const Int = @Int(.unsigned, width); const tok = ass.currentToken(); - if (ass.eatToken(.placeholder)) { - const name = ass.tokenText(tok)[1..]; - const value = ass.value_map.get(name) orelse { - return ass.fail(tok.start, "invalid placeholder '${s}'", .{name}); - }; - switch (value) { - .constant => |literal| { - if (width <= @bitSizeOf(spec.Word)) { - try ass.inst.operands.append(gpa, .{ .literal32 = @truncate(literal) }); - } else { - try ass.inst.operands.append(gpa, .{ .literal64 = literal }); - } - }, - else => { - return ass.fail(tok.start, "value '{s}' cannot be used as placeholder", .{name}); - }, + if (try ass.eatPlaceholderConstant()) |literal| { + if (width <= @bitSizeOf(spec.Word)) { + try ass.inst.operands.append(gpa, .{ .literal32 = @truncate(literal) }); + } else { + try ass.inst.operands.append(gpa, .{ .literal64 = literal }); } return; } diff --git a/src/codegen/spirv/CodeGen.zig b/src/codegen/spirv/CodeGen.zig index 57683c89553d73c83bc93a7145136f0730fd8f13..df127e949b55dcc7401d3dbb7ad53179c7f7d3ef 100644 --- a/src/codegen/spirv/CodeGen.zig +++ b/src/codegen/spirv/CodeGen.zig @@ -10,32 +10,20 @@ base_line: u32, // Module-level output (accumulated across the nav's codegen) next_result_id: Word = 1, -decls: std.ArrayList(Decl) = .empty, -decl_deps: std.ArrayList(Decl.Index) = .empty, -nav_link: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Decl.Index) = .empty, -uav_link: std.AutoHashMapUnmanaged(struct { InternPool.Index, spec.StorageClass }, Decl.Index) = .empty, -entry_points: std.array_hash_map.Auto(Id, EntryPoint) = .empty, -error_buffer: ?Decl.Index = null, +navs: std.array_hash_map.Auto(InternPool.Nav.Index, Id) = .empty, +externs: std.array_hash_map.Auto(InternPool.Nav.Index, Mir.Extern) = .empty, +entry_points: std.ArrayList(Mir.EntryPoint) = .empty, +error_buffer: ?Id = null, struct_types: std.array_hash_map.Custom(StructType, Id, StructType.HashContext, true) = .empty, /// SPIR-V ids of OpVariables whose pointee is a Block struct block_var_ids: std.AutoHashMapUnmanaged(Id, void) = .empty, -builtins: std.AutoHashMapUnmanaged(struct { spec.BuiltIn, spec.StorageClass }, Decl.Index) = .empty, -sections: struct { - // Module layout, according to SPIR-V Spec section 2.4, "Logical Layout of a Module". - extended_instruction_set: Section = .{}, - memory_model: Section = .{}, - execution_modes: Section = .{}, - debug_strings: Section = .{}, - debug_names: Section = .{}, - annotations: Section = .{}, - globals: Section = .{}, - functions: Section = .{}, -} = .{}, +builtins: std.AutoHashMapUnmanaged(struct { spec.BuiltIn, spec.StorageClass }, Id) = .empty, +module: Section = .empty, -// Per-function state (reset between top-level genNav calls) -prologue: Section = .{}, -body: Section = .{}, -args: std.ArrayList(Id) = .empty, +// Per-function state +prologue: Section = .empty, +body: Section = .empty, +args: std.MultiArrayList(struct { id: Id, ty: Id }) = .empty, next_arg_index: u32 = 0, /// Caches the limb extractions for composite integer values so repeated /// arithmetic on the same operand doesn't re-emit `OpCompositeExtract` per @@ -79,28 +67,6 @@ pub const Repr = enum { indirect, }; -/// A function or global, tracked here so the linker can order globals and build -/// per-entry-point interface lists. -pub const Decl = struct { - pub const Index = enum(u32) { _ }; - pub const Kind = enum { func, global, invocation_global }; - - kind: Kind, - /// Result-id of the associated OpFunction / OpVariable / InvocationGlobal. - result_id: Id, - /// Range into `decl_deps` for this decl's dependencies. - begin_dep: usize = 0, - end_dep: usize = 0, - /// Whether an extern-function stub has been emitted. - has_extern_stub: bool = false, -}; - -pub const EntryPoint = struct { - decl_index: Decl.Index, - name: []const u8, - cc: std.builtin.CallingConvention, -}; - const StructType = struct { fields: []const Id, ip_index: InternPool.Index, @@ -193,24 +159,16 @@ pub fn deinit(cg: *CodeGen) void { cg.prologue.deinit(gpa); cg.body.deinit(gpa); - cg.nav_link.deinit(gpa); - cg.uav_link.deinit(gpa); + cg.navs.deinit(gpa); + cg.externs.deinit(gpa); - cg.sections.extended_instruction_set.deinit(gpa); - cg.sections.memory_model.deinit(gpa); - cg.sections.execution_modes.deinit(gpa); - cg.sections.debug_strings.deinit(gpa); - cg.sections.debug_names.deinit(gpa); - cg.sections.annotations.deinit(gpa); - cg.sections.globals.deinit(gpa); - cg.sections.functions.deinit(gpa); + cg.module.deinit(gpa); cg.struct_types.deinit(gpa); cg.block_var_ids.deinit(gpa); cg.builtins.deinit(gpa); - cg.decls.deinit(gpa); - cg.decl_deps.deinit(gpa); + for (cg.entry_points.items) |ep| gpa.free(ep.name); cg.entry_points.deinit(gpa); } @@ -223,7 +181,10 @@ pub fn generate( ) codegen.Error!Mir { const zcu = pt.zcu; const gpa = zcu.gpa; - const nav = zcu.funcInfo(func_index).owner_nav; + const ip = &zcu.intern_pool; + const nav_index = zcu.funcInfo(func_index).owner_nav; + const nav = ip.getNav(nav_index); + const fn_info = zcu.typeToFunc(.fromInterned(nav.resolved.?.type)).?; var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); @@ -235,22 +196,60 @@ pub fn generate( .zcu = zcu, .air = air.*, .liveness = liveness.*.?, - .owner_nav = nav, - .base_line = zcu.navSrcLine(nav), + .owner_nav = nav_index, + .base_line = zcu.navSrcLine(nav_index), }; defer cg.deinit(); - try cg.genNav(true); + const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type)); + const is_test = zcu.test_functions.contains(nav_index); + const nav_id = try cg.resolveNav(nav_index); + const function_id = if (is_test) cg.allocId() else nav_id; - return cg.serializeToMir(gpa); + try cg.args.ensureUnusedCapacity(gpa, fn_info.param_types.len); + for (fn_info.param_types.get(ip)) |param_ty_index| { + const param_ty: Type = .fromInterned(param_ty_index); + if (!param_ty.hasRuntimeBits(zcu)) continue; + const param_ty_id = try cg.resolveType(param_ty, .direct); + cg.args.appendAssumeCapacity(.{ .id = cg.allocId(), .ty = param_ty_id }); + } + + const root_block_id = cg.allocId(); + try cg.prologue.emit(gpa, .OpLabel, .{ .id_result = root_block_id }); + cg.block_label = root_block_id; + _ = try cg.genStructuredBody(.selection, cg.air.getMainBody()); + try cg.body.emit(gpa, .OpUnreachable, {}); + try cg.body.emit(gpa, .OpFunctionEnd, {}); + + const fn_ty_id = try cg.functionType(return_ty_id, cg.args.items(.ty)); + try cg.module.emit(gpa, .OpFunction, .{ + .id_result_type = return_ty_id, + .id_result = function_id, + .function_control = .{}, + .function_type = fn_ty_id, + }); + for (cg.args.items(.id), cg.args.items(.ty)) |arg_id, arg_ty_id| { + try cg.module.emit(gpa, .OpFunctionParameter, .{ + .id_result_type = arg_ty_id, + .id_result = arg_id, + }); + } + try cg.module.append(gpa, cg.prologue); + try cg.module.append(gpa, cg.body); + if (is_test) try cg.generateTestEntryPoint(nav.fqn.toSlice(ip), nav_id, function_id); + try cg.debugName(function_id, nav.fqn.toSlice(ip)); + + return cg.toOwnedMir(); } -pub fn generateNav( - pt: Zcu.PerThread, - nav_index: InternPool.Nav.Index, -) codegen.Error!Mir { +pub fn generateNav(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) codegen.Error!Mir { const zcu = pt.zcu; const gpa = zcu.gpa; + const ip = &zcu.intern_pool; + const target = zcu.getTarget(); + const nav = ip.getNav(nav_index); + const val = zcu.navValue(nav_index); + const ty = val.typeOf(zcu); var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); @@ -267,116 +266,118 @@ pub fn generateNav( }; defer cg.deinit(); - try cg.genNav(false); + const resolved = nav.resolved.?; + if (nav.getExtern(ip)) |ext| { + if (!ty.hasRuntimeBits(zcu)) { + const child_ty = if (ty.zigTypeTag(zcu) == .pointer) ty.childType(zcu) else ty; + if (child_ty.zigTypeTag(zcu) != .spirv) return cg.toOwnedMir(); + } - return cg.serializeToMir(gpa); + const as = resolved.@"addrspace"; + const storage_class = cg.storageClass(as); + const result_id = try cg.resolveNav(nav_index); + const ty_id = try cg.pointeeType(as, ty, true); + const ptr_ty_id = try cg.ptrType(ty_id, storage_class); + + try cg.module.emit(gpa, .OpVariable, .{ + .id_result_type = ptr_ty_id, + .id_result = result_id, + .storage_class = storage_class, + }); + + switch (target.os.tag) { + .vulkan, .opengl => { + switch (storage_class) { + .uniform, + .push_constant, + .storage_buffer, + .physical_storage_buffer, + => { + if (ty.hasRuntimeBits(zcu)) { + if (!ty.isSpirvRuntimeArray(zcu)) { + try cg.decorate(ptr_ty_id, .{ .array_stride = .{ + .array_stride = @intCast(ty.abiSize(zcu)), + } }); + } + if (!cg.needsLayout(as, ty)) try cg.decorateLayout(ty, ty_id); + } + if (ext.is_const and storage_class == .storage_buffer) { + try cg.decorate(result_id, .non_writable); + } + }, + else => {}, + } + + if (ext.decoration) |decoration| switch (decoration) { + .location => |location| { + try cg.decorate(result_id, .{ + .location = .{ .location = location }, + }); + }, + .flat => |location| { + try cg.decorate(result_id, .{ .location = .{ .location = location } }); + try cg.decorate(result_id, .flat); + }, + .descriptor => |descriptor| { + try cg.decorate(result_id, .{ .binding = .{ + .binding_point = descriptor.binding, + } }); + try cg.decorate(result_id, .{ .descriptor_set = .{ + .descriptor_set = descriptor.set, + } }); + }, + }; + }, + else => {}, + } + + if (std.meta.stringToEnum(spec.BuiltIn, nav.fqn.toSlice(ip))) |built_in| { + try cg.decorate(result_id, .{ .built_in = .{ .built_in = built_in } }); + } + + try cg.debugName(result_id, nav.fqn.toSlice(ip)); + } else if (resolved.@"threadlocal" and ty.hasRuntimeBits(zcu)) { + if (target.os.tag == .opencl) return cg.todo("threadlocal variables on OpenCL", .{}); + + const ty_id = try cg.resolveType(ty, .indirect); + const ptr_ty_id = try cg.ptrType(ty_id, .private); + const init_id = try cg.constant(ty, val, .indirect); + + const result_id = try cg.resolveNav(nav_index); + try cg.module.emit(gpa, .OpVariable, .{ + .id_result_type = ptr_ty_id, + .id_result = result_id, + .storage_class = .private, + .initializer = init_id, + }); + try cg.debugName(result_id, nav.fqn.toSlice(ip)); + } + return cg.toOwnedMir(); } -fn serializeToMir(cg: *CodeGen, gpa: Allocator) codegen.Error!Mir { - const owner_entry = cg.nav_link.get(cg.owner_nav); - const owner_decl_index = owner_entry orelse return .{ - .id_bound = cg.next_result_id, - .owner_nav = cg.owner_nav, - .kind = .func, - .decl_result_id = .none, - .extended_instruction_set = &.{}, - .globals = &.{}, - .functions = &.{}, - .annotations = &.{}, - .debug_names = &.{}, - .debug_strings = &.{}, - .execution_modes = &.{}, - .nav_refs = &.{}, - .uav_refs = &.{}, - .decl_deps = &.{}, - .internal_globals = &.{}, - .entry_points = &.{}, - }; - - const owner_decl = cg.declPtr(owner_decl_index); - - var nav_refs: std.ArrayList(Mir.NavRef) = .empty; - defer nav_refs.deinit(gpa); - var nav_it = cg.nav_link.iterator(); - while (nav_it.next()) |entry| { - if (entry.key_ptr.* == cg.owner_nav) continue; - const decl = cg.declPtr(entry.value_ptr.*); - try nav_refs.append(gpa, .{ - .local_id = decl.result_id, - .nav = entry.key_ptr.*, - .kind = decl.kind, - }); - } +fn toOwnedMir(cg: *CodeGen) Allocator.Error!Mir { + const gpa = cg.gpa; - var uav_refs: std.ArrayList(Mir.UavRef) = .empty; - defer uav_refs.deinit(gpa); - var uav_it = cg.uav_link.iterator(); - while (uav_it.next()) |entry| { - const decl = cg.declPtr(entry.value_ptr.*); - try uav_refs.append(gpa, .{ - .local_id = decl.result_id, - .val = entry.key_ptr.*[0], - .storage_class = entry.key_ptr.*[1], - .kind = decl.kind, - }); + const nav_refs = try gpa.alloc(Mir.NavRef, cg.navs.count()); + errdefer gpa.free(nav_refs); + for (nav_refs, cg.navs.keys(), cg.navs.values()) |*ref, ref_nav, id| { + ref.* = .{ .id = id, .nav = ref_nav }; } - var decl_deps: std.ArrayList(Mir.DeclDep) = .empty; - defer decl_deps.deinit(gpa); - var internal_globals: std.ArrayList(Id) = .empty; - defer internal_globals.deinit(gpa); + const externs = try gpa.dupe(Mir.Extern, cg.externs.values()); + errdefer gpa.free(externs); - const deps = cg.decl_deps.items[owner_decl.begin_dep..owner_decl.end_dep]; - for (deps) |dep_index| { - const dep_decl = cg.declPtr(dep_index); - var found = false; - nav_it.index = 0; - while (nav_it.next()) |entry| { - if (entry.value_ptr.* == dep_index) { - try decl_deps.append(gpa, .{ - .kind = dep_decl.kind, - .nav = entry.key_ptr.*, - }); - found = true; - break; - } - } - if (!found and dep_decl.kind == .global) { - try internal_globals.append(gpa, dep_decl.result_id); - } - } - - var ep_list: std.ArrayList(Mir.EntryPoint) = .empty; - defer ep_list.deinit(gpa); - var ep_it = cg.entry_points.iterator(); - while (ep_it.next()) |entry| { - const ep = entry.value_ptr; - const ep_decl = cg.declPtr(ep.decl_index); - try ep_list.append(gpa, .{ - .local_id = ep_decl.result_id, - .name = try gpa.dupe(u8, ep.name), - .cc = ep.cc, - }); - } - - return .{ + const entry_points = try cg.entry_points.toOwnedSlice(gpa); + const mir: Mir = .{ .id_bound = cg.next_result_id, - .owner_nav = cg.owner_nav, - .kind = owner_decl.kind, - .decl_result_id = owner_decl.result_id, - .extended_instruction_set = try cg.sections.extended_instruction_set.instructions.toOwnedSlice(gpa), - .globals = try cg.sections.globals.instructions.toOwnedSlice(gpa), - .functions = try cg.sections.functions.instructions.toOwnedSlice(gpa), - .annotations = try cg.sections.annotations.instructions.toOwnedSlice(gpa), - .debug_names = try cg.sections.debug_names.instructions.toOwnedSlice(gpa), - .debug_strings = try cg.sections.debug_strings.instructions.toOwnedSlice(gpa), - .execution_modes = try cg.sections.execution_modes.instructions.toOwnedSlice(gpa), - .nav_refs = try nav_refs.toOwnedSlice(gpa), - .uav_refs = try uav_refs.toOwnedSlice(gpa), - .decl_deps = try decl_deps.toOwnedSlice(gpa), - .internal_globals = try internal_globals.toOwnedSlice(gpa), - .entry_points = try ep_list.toOwnedSlice(gpa), + .module = cg.module, + .nav_refs = nav_refs, + .externs = externs, + .entry_points = entry_points, }; + cg.module = .empty; + + return mir; } fn typeOf(cg: *CodeGen, inst: Air.Inst.Ref) Type { @@ -389,22 +390,50 @@ fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type { return cg.air.typeOfIndex(inst, &zcu.intern_pool); } -/// Does not generate the nav. -pub fn resolveNav(cg: *CodeGen, ip: *InternPool, nav_index: InternPool.Nav.Index) !Decl.Index { - const entry = try cg.nav_link.getOrPut(cg.gpa, nav_index); - if (!entry.found_existing) { - const nav = ip.getNav(nav_index); - // TODO: Extern fn? - const kind: Decl.Kind = if (ip.isFunctionType(nav.resolved.?.type)) - .func - else switch (nav.resolved.?.@"addrspace") { - .generic => .invocation_global, - else => .global, - }; - entry.value_ptr.* = try cg.allocDecl(kind); +pub fn resolveNav(cg: *CodeGen, nav_index: InternPool.Nav.Index) !Id { + const gop = try cg.navs.getOrPut(cg.gpa, nav_index); + if (!gop.found_existing) gop.value_ptr.* = cg.allocId(); + return gop.value_ptr.*; +} + +fn resolveExternFn(cg: *CodeGen, nav_index: InternPool.Nav.Index) !Id { + const gpa = cg.gpa; + const zcu = cg.zcu; + const ip = &zcu.intern_pool; + if (cg.externs.get(nav_index)) |ext| return ext.id; + + const nav = ip.getNav(nav_index); + const fn_ty: Type = .fromInterned(nav.resolved.?.type); + const fn_info = zcu.typeToFunc(fn_ty).?; + const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type)); + const fn_ty_id = try cg.resolveType(fn_ty, .direct); + const scratch_top = cg.id_scratch.items.len; + defer cg.id_scratch.shrinkRetainingCapacity(scratch_top); + for (fn_info.param_types.get(ip)) |param_ty_index| { + const param_ty: Type = .fromInterned(param_ty_index); + if (!param_ty.hasRuntimeBits(zcu)) continue; + try cg.id_scratch.append(gpa, try cg.resolveType(param_ty, .direct)); } - return entry.value_ptr.*; + const id = cg.allocId(); + try cg.externs.putNoClobber(gpa, nav_index, .{ + .id = id, + .name = nav.getExtern(ip).?.name.toSlice(ip), + }); + try cg.module.emit(gpa, .OpFunction, .{ + .id_result_type = return_ty_id, + .id_result = id, + .function_control = .{}, + .function_type = fn_ty_id, + }); + for (cg.id_scratch.items[scratch_top..]) |param_ty_id| { + try cg.module.emit(gpa, .OpFunctionParameter, .{ + .id_result_type = param_ty_id, + .id_result = cg.allocId(), + }); + } + try cg.module.emit(gpa, .OpFunctionEnd, {}); + return id; } pub fn allocIds(cg: *CodeGen, n: u32) spec.IdRange { @@ -420,34 +449,10 @@ pub fn idBound(cg: *const CodeGen) Word { return cg.next_result_id; } -pub fn addEntryPointDeps( - cg: *CodeGen, - decl_index: Decl.Index, - seen: *std.bit_set.Dynamic, - interface: *std.ArrayList(Id), -) !void { - const decl = cg.declPtr(decl_index); - const deps = cg.decl_deps.items[decl.begin_dep..decl.end_dep]; - - if (seen.isSet(@backingInt(decl_index))) { - return; - } - - seen.set(@backingInt(decl_index)); - - if (decl.kind == .global) { - try interface.append(cg.gpa, decl.result_id); - } - - for (deps) |dep| { - try cg.addEntryPointDeps(dep, seen, interface); - } -} - pub fn importInstructionSet(cg: *CodeGen, set: spec.InstructionSet) !Id { assert(set != .core); const result_id = cg.allocId(); - try cg.sections.extended_instruction_set.emit(cg.gpa, .OpExtInstImport, .{ + try cg.module.emit(cg.gpa, .OpExtInstImport, .{ .id_result = result_id, .name = @tagName(set), }); @@ -456,7 +461,7 @@ pub fn importInstructionSet(cg: *CodeGen, set: spec.InstructionSet) !Id { pub fn boolType(cg: *CodeGen) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypeBool, .{ + try cg.module.emit(cg.gpa, .OpTypeBool, .{ .id_result = result_id, }); return result_id; @@ -464,20 +469,17 @@ pub fn boolType(cg: *CodeGen) !Id { pub fn voidType(cg: *CodeGen) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypeVoid, .{ + try cg.module.emit(cg.gpa, .OpTypeVoid, .{ .id_result = result_id, }); try cg.debugName(result_id, "void"); return result_id; } -pub fn opaqueType(cg: *CodeGen, name: []const u8) !Id { +pub fn opaqueType(cg: *CodeGen, comptime fmt: []const u8, args: anytype) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypeOpaque, .{ - .id_result = result_id, - .literal_string = name, - }); - try cg.debugName(result_id, name); + try cg.module.emitFormatted(cg.gpa, .OpTypeOpaque, &.{result_id}, fmt, args); + try cg.debugNameFmt(result_id, fmt, args); return result_id; } @@ -513,7 +515,7 @@ pub fn intType(cg: *CodeGen, signedness: std.lang.Signedness, bits: u16) !Id { const limb_ty = try cg.intType(.unsigned, limb_bits); const len_ty = try cg.intType(.unsigned, 32); const len_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpConstant, .{ + try cg.module.emit(cg.gpa, .OpConstant, .{ .id_result_type = len_ty, .id_result = len_id, .value = .{ .uint32 = backing_bits / limb_bits }, @@ -522,7 +524,7 @@ pub fn intType(cg: *CodeGen, signedness: std.lang.Signedness, bits: u16) !Id { } const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypeInt, .{ + try cg.module.emit(cg.gpa, .OpTypeInt, .{ .id_result = result_id, .width = backing_bits, .signedness = switch (actual_signedness) { @@ -540,7 +542,7 @@ pub fn intType(cg: *CodeGen, signedness: std.lang.Signedness, bits: u16) !Id { pub fn floatType(cg: *CodeGen, bits: u16) !Id { assert(bits > 0); const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypeFloat, .{ + try cg.module.emit(cg.gpa, .OpTypeFloat, .{ .id_result = result_id, .width = bits, }); @@ -550,7 +552,7 @@ pub fn floatType(cg: *CodeGen, bits: u16) !Id { pub fn vectorType(cg: *CodeGen, len: u32, child_ty_id: Id) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypeVector, .{ + try cg.module.emit(cg.gpa, .OpTypeVector, .{ .id_result = result_id, .component_type = child_ty_id, .component_count = len, @@ -560,7 +562,7 @@ pub fn vectorType(cg: *CodeGen, len: u32, child_ty_id: Id) !Id { pub fn arrayType(cg: *CodeGen, len_id: Id, child_ty_id: Id) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypeArray, .{ + try cg.module.emit(cg.gpa, .OpTypeArray, .{ .id_result = result_id, .element_type = child_ty_id, .length = len_id, @@ -570,7 +572,7 @@ pub fn arrayType(cg: *CodeGen, len_id: Id, child_ty_id: Id) !Id { pub fn ptrType(cg: *CodeGen, child_ty_id: Id, storage_class: spec.StorageClass) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypePointer, .{ + try cg.module.emit(cg.gpa, .OpTypePointer, .{ .id_result = result_id, .storage_class = storage_class, .type = child_ty_id, @@ -589,7 +591,7 @@ pub fn structType( if (cg.struct_types.get(.{ .fields = types, .ip_index = actual_ip_index })) |id| return id; const result_id = cg.allocId(); const types_dup = try cg.arena.dupe(Id, types); - try cg.sections.globals.emit(cg.gpa, .OpTypeStruct, .{ + try cg.module.emit(cg.gpa, .OpTypeStruct, .{ .id_result = result_id, .id_ref = types_dup, }); @@ -639,7 +641,7 @@ pub fn layoutType(cg: *CodeGen, ty: Type, is_block_root: bool) Error!Id { } }); try member_types.append(gpa, try cg.layoutType(field_ty, false)); } - try cg.sections.globals.emit(gpa, .OpTypeStruct, .{ + try cg.module.emit(gpa, .OpTypeStruct, .{ .id_result = id, .id_ref = member_types.items, }); @@ -682,7 +684,7 @@ pub fn layoutType(cg: *CodeGen, ty: Type, is_block_root: bool) Error!Id { try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } }); member_types[layout.padding_index] = arr_id; } - try cg.sections.globals.emit(gpa, .OpTypeStruct, .{ + try cg.module.emit(gpa, .OpTypeStruct, .{ .id_result = id, .id_ref = member_types[0..layout.total_fields], }); @@ -705,7 +707,7 @@ pub fn layoutType(cg: *CodeGen, ty: Type, is_block_root: bool) Error!Id { const elem_ty = ty.childType(zcu); const elem_ty_id = try cg.layoutType(elem_ty, is_block_root); const id = cg.allocId(); - try cg.sections.globals.emit(gpa, .OpTypeRuntimeArray, .{ + try cg.module.emit(gpa, .OpTypeRuntimeArray, .{ .id_result = id, .element_type = elem_ty_id, }); @@ -724,7 +726,7 @@ pub fn layoutType(cg: *CodeGen, ty: Type, is_block_root: bool) Error!Id { pub fn functionType(cg: *CodeGen, return_ty_id: Id, param_type_ids: []const Id) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpTypeFunction, .{ + try cg.module.emit(cg.gpa, .OpTypeFunction, .{ .id_result = result_id, .return_type = return_ty_id, .id_ref_2 = param_type_ids, @@ -734,7 +736,7 @@ pub fn functionType(cg: *CodeGen, return_ty_id: Id, param_type_ids: []const Id) pub fn constUndef(cg: *CodeGen, ty_id: Id) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpUndef, .{ + try cg.module.emit(cg.gpa, .OpUndef, .{ .id_result_type = ty_id, .id_result = result_id, }); @@ -743,7 +745,7 @@ pub fn constUndef(cg: *CodeGen, ty_id: Id) !Id { pub fn constNull(cg: *CodeGen, ty_id: Id) !Id { const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpConstantNull, .{ + try cg.module.emit(cg.gpa, .OpConstantNull, .{ .id_result_type = ty_id, .id_result = result_id, }); @@ -755,7 +757,7 @@ pub fn decorate( target: Id, decoration: spec.Decoration.Extended, ) !void { - try cg.sections.annotations.emit(cg.gpa, .OpDecorate, .{ + try cg.module.emit(cg.gpa, .OpDecorate, .{ .target = target, .decoration = decoration, }); @@ -767,29 +769,16 @@ pub fn decorateMember( member: u32, decoration: spec.Decoration.Extended, ) !void { - try cg.sections.annotations.emit(cg.gpa, .OpMemberDecorate, .{ + try cg.module.emit(cg.gpa, .OpMemberDecorate, .{ .structure_type = structure_type, .member = member, .decoration = decoration, }); } -pub fn allocDecl(cg: *CodeGen, kind: Decl.Kind) !Decl.Index { - try cg.decls.append(cg.gpa, .{ - .kind = kind, - .result_id = cg.allocId(), - }); - - return @as(Decl.Index, @fromBackingInt(@intCast(@as(u32, @intCast(cg.decls.items.len - 1))))); -} - -pub fn declPtr(cg: *CodeGen, index: Decl.Index) *Decl { - return &cg.decls.items[@backingInt(index)]; -} - pub fn debugName(cg: *CodeGen, target: Id, name: []const u8) !void { if (cg.zcu.comp.config.root_strip) return; - try cg.sections.debug_names.emit(cg.gpa, .OpName, .{ + try cg.module.emit(cg.gpa, .OpName, .{ .target = target, .name = name, }); @@ -797,14 +786,12 @@ pub fn debugName(cg: *CodeGen, target: Id, name: []const u8) !void { pub fn debugNameFmt(cg: *CodeGen, target: Id, comptime fmt: []const u8, args: anytype) !void { if (cg.zcu.comp.config.root_strip) return; - const name = try std.fmt.allocPrint(cg.gpa, fmt, args); - defer cg.gpa.free(name); - try cg.debugName(target, name); + try cg.module.emitFormatted(cg.gpa, .OpName, &.{target}, fmt, args); } pub fn memberDebugName(cg: *CodeGen, target: Id, member: u32, name: []const u8) !void { if (cg.zcu.comp.config.root_strip) return; - try cg.sections.debug_names.emit(cg.gpa, .OpMemberName, .{ + try cg.module.emit(cg.gpa, .OpMemberName, .{ .type = target, .member = member, .name = name, @@ -827,6 +814,7 @@ pub fn storageClass(cg: *const CodeGen, as: std.lang.AddressSpace) spec.StorageC .constant => .uniform_constant, .shared => .workgroup, .local => .function, + .private => .private, .input => .input, .gs, .fs, @@ -850,228 +838,6 @@ pub fn storageClass(cg: *const CodeGen, as: std.lang.AddressSpace) spec.StorageC const Error = codegen.Error; -pub fn genNav(cg: *CodeGen, do_codegen: bool) Error!void { - const gpa = cg.gpa; - const zcu = cg.zcu; - const ip = &zcu.intern_pool; - const target = zcu.getTarget(); - - const nav = ip.getNav(cg.owner_nav); - const val = zcu.navValue(cg.owner_nav); - const ty = val.typeOf(zcu); - - if (!do_codegen and !ty.hasRuntimeBits(zcu)) { - const child_ty = if (ty.zigTypeTag(zcu) == .pointer) ty.childType(zcu) else ty; - if (child_ty.zigTypeTag(zcu) != .spirv) return; - } - - const spv_decl_index = try cg.resolveNav(ip, cg.owner_nav); - const decl = cg.declPtr(spv_decl_index); - const result_id = decl.result_id; - decl.begin_dep = cg.decl_deps.items.len; - - switch (decl.kind) { - .func => { - if (nav.resolved.?.is_extern_decl) { - _ = try cg.resolveType(ty, .direct); - try emitExternFnStub(cg, nav, decl, ty); - decl.end_dep = cg.decl_deps.items.len; - return; - } - - const fn_info = zcu.typeToFunc(ty).?; - const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type)); - const is_test = zcu.test_functions.contains(cg.owner_nav); - - const func_result_id = if (is_test) cg.allocId() else result_id; - const prototype_ty_id = try cg.resolveType(ty, .direct); - try cg.prologue.emit(gpa, .OpFunction, .{ - .id_result_type = return_ty_id, - .id_result = func_result_id, - .function_type = prototype_ty_id, - // Note: the backend will never be asked to generate an inline function - // (this is handled in sema), so we don't need to set function_control here. - .function_control = .{}, - }); - - try cg.args.ensureUnusedCapacity(gpa, fn_info.param_types.len); - for (fn_info.param_types.get(ip)) |param_ty_index| { - const param_ty: Type = .fromInterned(param_ty_index); - if (!param_ty.hasRuntimeBits(zcu)) continue; - - const param_type_id = try cg.resolveType(param_ty, .direct); - const arg_result_id = cg.allocId(); - try cg.prologue.emit(gpa, .OpFunctionParameter, .{ - .id_result_type = param_type_id, - .id_result = arg_result_id, - }); - cg.args.appendAssumeCapacity(arg_result_id); - } - - // TODO: This could probably be done in a better way... - const root_block_id = cg.allocId(); - - // The root block of a function declaration should appear before OpVariable instructions, - // so it is generated into the function's prologue. - try cg.prologue.emit(gpa, .OpLabel, .{ - .id_result = root_block_id, - }); - cg.block_label = root_block_id; - - const main_body = cg.air.getMainBody(); - _ = try cg.genStructuredBody(.selection, main_body); - // We always expect paths to here to end, but we still need the block - // to act as a dummy merge block. - try cg.body.emit(gpa, .OpUnreachable, {}); - try cg.body.emit(gpa, .OpFunctionEnd, {}); - // Append the actual code into the functions section. - try cg.sections.functions.append(gpa, cg.prologue); - try cg.sections.functions.append(gpa, cg.body); - - // Temporarily generate a test kernel declaration if this is a test function. - if (is_test) { - try cg.generateTestEntryPoint(nav.fqn.toSlice(ip), spv_decl_index, func_result_id); - } - - try cg.debugName(func_result_id, nav.fqn.toSlice(ip)); - }, - .global => { - const key = ip.indexToKey(val.toIntern()).@"extern"; - - const storage_class = cg.storageClass(nav.resolved.?.@"addrspace"); - assert(storage_class != .generic); // These should be instance globals - - const as = nav.resolved.?.@"addrspace"; - const ty_id = try cg.pointeeType(as, ty, true); - const ptr_ty_id = try cg.ptrType(ty_id, storage_class); - - try cg.sections.globals.emit(gpa, .OpVariable, .{ - .id_result_type = ptr_ty_id, - .id_result = result_id, - .storage_class = storage_class, - }); - - switch (target.os.tag) { - .vulkan, .opengl => { - switch (storage_class) { - .uniform, - .push_constant, - .storage_buffer, - .physical_storage_buffer, - => { - if (ty.hasRuntimeBits(zcu)) { - if (!ty.isSpirvRuntimeArray(zcu)) { - try cg.decorate( - ptr_ty_id, - .{ .array_stride = .{ .array_stride = @intCast(ty.abiSize(zcu)) } }, - ); - } - if (!cg.needsLayout(as, ty)) try cg.decorateLayout(ty, ty_id); - } - if (key.is_const and storage_class == .storage_buffer) { - try cg.decorate(result_id, .non_writable); - } - }, - else => {}, - } - - if (key.decoration) |decoration| switch (decoration) { - .location => |location| { - if (storage_class != .output and storage_class != .input and storage_class != .uniform_constant) { - return cg.fail("storage class must be one of (output, input, uniform_constant) but is {s}", .{@tagName(storage_class)}); - } - try cg.decorate(result_id, .{ - .location = .{ .location = location }, - }); - }, - .flat => |location| { - try cg.decorate(result_id, .{ .location = .{ .location = location } }); - try cg.decorate(result_id, .flat); - }, - .descriptor => |descriptor| { - if (storage_class != .storage_buffer and storage_class != .uniform and storage_class != .uniform_constant) { - return cg.fail("storage class must be one of (storage_buffer, uniform, uniform_constant) but is {s}", .{@tagName(storage_class)}); - } - try cg.decorate(result_id, .{ - .binding = .{ .binding_point = descriptor.binding }, - }); - - try cg.decorate(result_id, .{ - .descriptor_set = .{ .descriptor_set = descriptor.set }, - }); - }, - }; - }, - else => {}, - } - - if (std.meta.stringToEnum(spec.BuiltIn, nav.fqn.toSlice(ip))) |built_in| { - try cg.decorate(result_id, .{ .built_in = .{ .built_in = built_in } }); - } - - try cg.debugName(result_id, nav.fqn.toSlice(ip)); - }, - .invocation_global => { - // `@extern()` produces an invocation_global whose value is a - // comptime-known pointer to an underlying extern symbol's Nav. - // The pointer is inlined at use sites so we don't need a Function-scope wrapper here. - if (ip.indexToKey(val.toIntern()) == .ptr) alias: { - const ptr_key = ip.indexToKey(val.toIntern()).ptr; - if (ptr_key.base_addr != .nav or ptr_key.byte_offset != 0) break :alias; - const underlying_nav = ip.getNav(ptr_key.base_addr.nav); - if (!underlying_nav.resolved.?.is_extern_decl) break :alias; - cg.declPtr(spv_decl_index).end_dep = cg.decl_deps.items.len; - return; - } - - const ty_id = try cg.resolveType(ty, .indirect); - const ptr_ty_id = try cg.ptrType(ty_id, .function); - - // TODO: Combine with resolveAnonDecl? - const void_ty_id = try cg.resolveType(.void, .direct); - const initializer_proto_ty_id = try cg.functionType(void_ty_id, &.{}); - - const initializer_id = cg.allocId(); - try cg.prologue.emit(gpa, .OpFunction, .{ - .id_result_type = try cg.resolveType(.void, .direct), - .id_result = initializer_id, - .function_control = .{}, - .function_type = initializer_proto_ty_id, - }); - - const root_block_id = cg.allocId(); - try cg.prologue.emit(gpa, .OpLabel, .{ - .id_result = root_block_id, - }); - cg.block_label = root_block_id; - - const val_id = try cg.constant(ty, val, .indirect); - try cg.body.emit(gpa, .OpStore, .{ - .pointer = result_id, - .object = val_id, - }); - - try cg.body.emit(gpa, .OpReturn, {}); - try cg.body.emit(gpa, .OpFunctionEnd, {}); - try cg.sections.functions.append(gpa, cg.prologue); - try cg.sections.functions.append(gpa, cg.body); - - try cg.debugNameFmt(initializer_id, "initializer of {f}", .{nav.fqn.fmt(ip)}); - try cg.debugName(result_id, nav.fqn.toSlice(ip)); - - try cg.sections.globals.emit(gpa, .OpExtInst, .{ - .id_result_type = ptr_ty_id, - .id_result = result_id, - .set = try cg.importInstructionSet(.zig), - .instruction = .{ .inst = @backingInt(spec.Zig.InvocationGlobal) }, - .id_ref_4 = &.{initializer_id}, - }); - }, - } - - cg.declPtr(spv_decl_index).end_dep = cg.decl_deps.items.len; -} - fn decorateLayout(cg: *CodeGen, ty: Type, ty_id: spec.Id) Error!void { const zcu = cg.zcu; const ip = &zcu.intern_pool; @@ -1174,144 +940,57 @@ fn importExtendedSet(cg: *CodeGen) !Id { /// Fetch the result-id for a previously generated instruction or constant. fn resolve(cg: *CodeGen, inst: Air.Inst.Ref) !Id { const zcu = cg.zcu; - const ip = &zcu.intern_pool; if (inst.toInterned()) |val_ip_index| { const ty = cg.typeOf(inst); if (ty.zigTypeTag(zcu) == .@"fn") { const val_key = zcu.intern_pool.indexToKey(val_ip_index); - const fn_nav = switch (val_key) { - .@"extern" => |@"extern"| @"extern".owner_nav, - .func => |func| func.owner_nav, + return switch (val_key) { + .@"extern" => |@"extern"| try cg.resolveExternFn(@"extern".owner_nav), + .func => |func| try cg.resolveNav(func.owner_nav), else => unreachable, }; - const spv_decl_index = try cg.resolveNav(ip, fn_nav); - try cg.decl_deps.append(cg.gpa, spv_decl_index); - const decl = cg.declPtr(spv_decl_index); - if (val_key == .@"extern") { - const nav = ip.getNav(fn_nav); - const nav_ty: Type = .fromInterned(nav.resolved.?.type); - try emitExternFnStub(cg, nav, decl, nav_ty); - } - return decl.result_id; } - return try cg.constant(ty, .fromInterned(val_ip_index), .direct); + const val: Value = .fromInterned(val_ip_index); + if (ty.isPtrAtRuntime(zcu)) { + if (val.isUndef(zcu)) return try cg.constUndef(try cg.resolveType(ty, .direct)); + if (ty.zigTypeTag(zcu) == .optional) { + const payload = val.optionalValue(zcu) orelse return try cg.constNull(try cg.resolveType(ty, .direct)); + return try cg.constantPtr(payload); + } + return try cg.constantPtr(val); + } + switch (zcu.intern_pool.indexToKey(val_ip_index)) { + .ptr => unreachable, + .slice => |slice| { + const ptr_id = try cg.constantPtr(.fromInterned(slice.ptr)); + const len_id = try cg.constant(.usize, .fromInterned(slice.len), .indirect); + return try cg.constructComposite(try cg.resolveType(ty, .direct), &.{ ptr_id, len_id }); + }, + .opt => { + const payload_ty = ty.optionalChild(zcu); + if (!payload_ty.isPtrAtRuntime(zcu)) return try cg.constant(ty, val, .direct); + const payload_val = val.optionalValue(zcu); + const ptr_id = if (payload_val) |payload| + try cg.constantPtr(payload) + else + try cg.constUndef(try cg.resolveType(payload_ty, .direct)); + const valid_id = try cg.constBool(payload_val != null, .indirect); + return try cg.constructComposite(try cg.resolveType(ty, .direct), &.{ ptr_id, valid_id }); + }, + else => return try cg.constant(ty, val, .direct), + } } const index = inst.toIndex().?; return cg.inst_results.get(index).?; // Assertion means instruction does not dominate usage. } -fn resolveUav(cg: *CodeGen, val: InternPool.Index) !Id { - const gpa = cg.gpa; - - // TODO: This cannot be a function at this point, but it should probably be handled anyway. - - const zcu = cg.zcu; - const ty: Type = .fromInterned(zcu.intern_pool.typeOf(val)); - const ty_id = try cg.resolveType(ty, .indirect); - - const spv_decl_index = blk: { - const entry = try cg.uav_link.getOrPut(gpa, .{ val, .function }); - if (entry.found_existing) { - try cg.addFunctionDep(entry.value_ptr.*, .function); - return cg.declPtr(entry.value_ptr.*).result_id; - } - - const spv_decl_index = try cg.allocDecl(.invocation_global); - try cg.addFunctionDep(spv_decl_index, .function); - entry.value_ptr.* = spv_decl_index; - break :blk spv_decl_index; - }; - - // TODO: At some point we will be able to generate this all constant here, but then all of - // constant() will need to be implemented such that it doesn't generate any at-runtime code. - // NOTE: Because this is a global, we really only want to initialize it once. Therefore the - // constant lowering of this value will need to be deferred to an initializer similar to - // other globals. - - const result_id = cg.declPtr(spv_decl_index).result_id; - - { - // Save the current state so that we can temporarily generate into a different function. - // TODO: This should probably be made a little more robust. - const func_prologue = cg.prologue; - const func_body = cg.body; - const block_label = cg.block_label; - defer { - cg.prologue = func_prologue; - cg.body = func_body; - cg.block_label = block_label; - } - - cg.prologue = .{}; - cg.body = .{}; - defer { - cg.prologue.deinit(gpa); - cg.body.deinit(gpa); - } - - const void_ty_id = try cg.resolveType(.void, .direct); - const initializer_proto_ty_id = try cg.functionType(void_ty_id, &.{}); - - const initializer_id = cg.allocId(); - try cg.prologue.emit(gpa, .OpFunction, .{ - .id_result_type = try cg.resolveType(.void, .direct), - .id_result = initializer_id, - .function_control = .{}, - .function_type = initializer_proto_ty_id, - }); - const root_block_id = cg.allocId(); - try cg.prologue.emit(gpa, .OpLabel, .{ - .id_result = root_block_id, - }); - cg.block_label = root_block_id; - - const val_id = try cg.constant(ty, .fromInterned(val), .indirect); - try cg.body.emit(gpa, .OpStore, .{ - .pointer = result_id, - .object = val_id, - }); - - try cg.body.emit(gpa, .OpReturn, {}); - try cg.body.emit(gpa, .OpFunctionEnd, {}); - - try cg.sections.functions.append(gpa, cg.prologue); - try cg.sections.functions.append(gpa, cg.body); - - try cg.debugNameFmt(initializer_id, "initializer of __anon_{d}", .{@backingInt(val)}); - - const fn_decl_ptr_ty_id = try cg.ptrType(ty_id, .function); - try cg.sections.globals.emit(gpa, .OpExtInst, .{ - .id_result_type = fn_decl_ptr_ty_id, - .id_result = result_id, - .set = try cg.importInstructionSet(.zig), - .instruction = .{ .inst = @backingInt(spec.Zig.InvocationGlobal) }, - .id_ref_4 = &.{initializer_id}, - }); - } - - return result_id; -} - fn resolvePtr(cg: *CodeGen, ref: Air.Inst.Ref) !Ptr { const id = try cg.resolve(ref); if (cg.tracked_allocas.getPtr(id)) |slot| return .{ .tracked = .{ .id = id, .slot = slot } }; return .{ .id = id }; } -fn addFunctionDep(cg: *CodeGen, decl_index: Decl.Index, storage_class: StorageClass) !void { - const gpa = cg.gpa; - const target = cg.zcu.getTarget(); - if (target.cpu.has(.spirv, .v1_4)) { - try cg.decl_deps.append(gpa, decl_index); - } else { - // Before version 1.4, the interface’s storage classes are limited to the Input and Output - if (storage_class == .input or storage_class == .output) { - try cg.decl_deps.append(gpa, decl_index); - } - } -} - /// Start a new SPIR-V block, Emits the label of the new block, and stores which /// block we are currently generating. /// Note that there is no such thing as nested blocks like in ZIR or AIR, so we don't need to @@ -1439,7 +1118,7 @@ fn constBool(cg: *CodeGen, value: bool, repr: Repr) !Id { const result_ty_id = try cg.boolType(); const result_id = cg.allocId(); switch (value) { - inline else => |value_ct| try cg.sections.globals.emit( + inline else => |value_ct| try cg.module.emit( cg.gpa, if (value_ct) .OpConstantTrue else .OpConstantFalse, .{ .id_result_type = result_ty_id, .id_result = result_id }, @@ -1485,7 +1164,7 @@ fn constInt(cg: *CodeGen, ty: Type, value: anytype) !Id { if (i < 2) @as(u32, @truncate(value64 >> @intCast(i * 32))) else fill, ); } - return cg.constructComposite(result_ty_id, constituents); + return cg.constantComposite(result_ty_id, constituents); } const final_value: spec.LiteralContextDependentNumber = switch (target.os.tag) { @@ -1515,14 +1194,29 @@ fn constInt(cg: *CodeGen, ty: Type, value: anytype) !Id { }; const result_id = cg.allocId(); - try cg.sections.globals.emit(cg.gpa, .OpConstant, .{ + try cg.module.emit(cg.gpa, .OpConstant, .{ .id_result_type = result_ty_id, .id_result = result_id, .value = final_value, }); if (!ty.isVector(zcu)) return result_id; - return cg.constructCompositeSplat(ty, result_id); + + const vector_ty_id = try cg.resolveType(ty, .direct); + const constituents = try gpa.alloc(Id, @intCast(ty.arrayLen(zcu))); + defer gpa.free(constituents); + @memset(constituents, result_id); + return cg.constantComposite(vector_ty_id, constituents); +} + +fn constantComposite(cg: *CodeGen, result_ty_id: Id, constituents: []const Id) !Id { + const result_id = cg.allocId(); + try cg.module.emit(cg.gpa, .OpConstantComposite, .{ + .id_result_type = result_ty_id, + .id_result = result_id, + .constituents = constituents, + }); + return result_id; } /// Construct a composite value from its constituents. @@ -1541,7 +1235,7 @@ pub fn constructComposite(cg: *CodeGen, result_ty_id: Id, constituents: []const for (fields, constituents, 0..) |field_ty_id, constituent, i| { const field_ptr_ty_id = try cg.ptrType(field_ty_id, .function); const index_id = cg.allocId(); - try cg.sections.globals.emit(gpa, .OpConstant, .{ + try cg.module.emit(gpa, .OpConstant, .{ .id_result_type = u32_ty_id, .id_result = index_id, .value = .{ .uint32 = @intCast(i) }, @@ -1572,21 +1266,6 @@ pub fn constructComposite(cg: *CodeGen, result_ty_id: Id, constituents: []const /// Construct a composite at runtime with all lanes set to the same value. /// ty must be an aggregate type. -fn constructCompositeSplat(cg: *CodeGen, ty: Type, constituent: Id) !Id { - const gpa = cg.gpa; - const zcu = cg.zcu; - const n: usize = @intCast(ty.arrayLen(zcu)); - - const scratch_top = cg.id_scratch.items.len; - defer cg.id_scratch.shrinkRetainingCapacity(scratch_top); - - const constituents = try cg.id_scratch.addManyAsSlice(gpa, n); - @memset(constituents, constituent); - - const result_ty_id = try cg.resolveType(ty, .direct); - return cg.constructComposite(result_ty_id, constituents); -} - /// This function generates a load for a constant in direct (ie, non-memory) representation. /// When the constant is simple, it can be generated directly using OpConstant instructions. /// When the constant is more complicated however, it needs to be constructed using multiple values. This @@ -1602,288 +1281,345 @@ fn constant(cg: *CodeGen, ty: Type, val: Value, repr: Repr) Error!Id { const result_ty_id = try cg.resolveType(ty, repr); const ip = &zcu.intern_pool; - log.debug("lowering constant: ty = {f}, val = {f}, key = {s}", .{ ty.fmt(pt), val.fmtValue(pt), @tagName(ip.indexToKey(val.toIntern())) }); + log.debug("lowering constant: ty = {f}, val = {f}, key = {s}", .{ + ty.fmt(pt), + val.fmtValue(pt), + @tagName(ip.indexToKey(val.toIntern())), + }); + assert(!ty.isPtrAtRuntime(zcu)); + if (val.isUndef(zcu)) { return cg.constUndef(result_ty_id); } - const cacheable_id = cache: { - switch (ip.indexToKey(val.toIntern())) { - .int_type, - .ptr_type, - .array_type, - .vector_type, - .opt_type, - .anyframe_type, - .error_union_type, - .simple_type, - .struct_type, - .tuple_type, - .union_type, - .opaque_type, - .spirv_type, - .enum_type, - .func_type, - .error_set_type, - .inferred_error_set_type, - => unreachable, // types, not values + const cacheable_id = cache: switch (ip.indexToKey(val.toIntern())) { + .int_type, + .ptr_type, + .array_type, + .vector_type, + .opt_type, + .anyframe_type, + .error_union_type, + .simple_type, + .struct_type, + .tuple_type, + .union_type, + .opaque_type, + .spirv_type, + .enum_type, + .func_type, + .error_set_type, + .inferred_error_set_type, + => unreachable, // types, not values - .undef => unreachable, // handled above + .undef => unreachable, // handled above - .@"extern", - .func, - .enum_literal, + .@"extern", + .func, + .enum_literal, + => unreachable, // non-runtime values + + .simple_value => |simple_value| switch (simple_value) { + .void, + .null, + .@"unreachable", => unreachable, // non-runtime values - .simple_value => |simple_value| switch (simple_value) { - .void, - .null, - .@"unreachable", - => unreachable, // non-runtime values - - .false, .true => break :cache try cg.constBool(val.toBool(), repr), - }, - .int => { - const int_info = ty.intInfo(zcu); - const backing_bits, const is_big_int = cg.backingIntBits(int_info.bits); - if (is_big_int) { - const limb_bits = cg.bigIntBits(); - const n_limbs = backing_bits / limb_bits; - const big_result_ty_id = try cg.resolveType(ty, .indirect); - var bigint_space: Value.BigIntSpace = undefined; - const bigint = val.toBigInt(&bigint_space, zcu); - const limb_bytes = try gpa.alloc(u8, backing_bits / 8); - defer gpa.free(limb_bytes); - bigint.writeTwosComplement(limb_bytes, .little); - const scratch_top = cg.id_scratch.items.len; - defer cg.id_scratch.shrinkRetainingCapacity(scratch_top); - const constituents = try cg.id_scratch.addManyAsSlice(gpa, n_limbs); - switch (limb_bits) { - 32 => { - const limbs_u32: []u32 = @ptrCast(@alignCast(limb_bytes)); - for (constituents, limbs_u32) |*c, v| { - const host_v = if (builtin.cpu.arch.endian() == .big) @byteSwap(v) else v; - c.* = try cg.constInt(.u32, host_v); - } - }, - 64 => { - const limbs_u64: []u64 = @ptrCast(@alignCast(limb_bytes)); - for (constituents, limbs_u64) |*c, v| { - const host_v = if (builtin.cpu.arch.endian() == .big) @byteSwap(v) else v; - c.* = try cg.constInt(.u64, host_v); - } - }, - else => unreachable, - } - break :cache try cg.constructComposite(big_result_ty_id, constituents); - } - if (ty.isSignedInt(zcu)) { - break :cache try cg.constInt(ty, val.toSignedInt(zcu)); - } else { - break :cache try cg.constInt(ty, val.toUnsignedInt(zcu)); - } - }, - .float => { - const lit: spec.LiteralContextDependentNumber = switch (ty.floatBits(target)) { - 16 => .{ .uint32 = @as(u16, @bitCast(val.toFloat(f16, zcu))) }, - 32 => .{ .float32 = val.toFloat(f32, zcu) }, - 64 => .{ .float64 = val.toFloat(f64, zcu) }, - 80, 128 => unreachable, // TODO + .false, .true => break :cache try cg.constBool(val.toBool(), repr), + }, + .int => { + const int_info = ty.intInfo(zcu); + const backing_bits, const is_big_int = cg.backingIntBits(int_info.bits); + if (is_big_int) { + const limb_bits = cg.bigIntBits(); + const n_limbs = backing_bits / limb_bits; + const big_result_ty_id = try cg.resolveType(ty, .indirect); + var bigint_space: Value.BigIntSpace = undefined; + const bigint = val.toBigInt(&bigint_space, zcu); + const limb_bytes = try gpa.alloc(u8, backing_bits / 8); + defer gpa.free(limb_bytes); + bigint.writeTwosComplement(limb_bytes, .little); + const scratch_top = cg.id_scratch.items.len; + defer cg.id_scratch.shrinkRetainingCapacity(scratch_top); + const constituents = try cg.id_scratch.addManyAsSlice(gpa, n_limbs); + switch (limb_bits) { + 32 => { + const limbs_u32: []u32 = @ptrCast(@alignCast(limb_bytes)); + for (constituents, limbs_u32) |*c, v| { + const host_v = if (builtin.cpu.arch.endian() == .big) @byteSwap(v) else v; + c.* = try cg.constInt(.u32, host_v); + } + }, + 64 => { + const limbs_u64: []u64 = @ptrCast(@alignCast(limb_bytes)); + for (constituents, limbs_u64) |*c, v| { + const host_v = if (builtin.cpu.arch.endian() == .big) @byteSwap(v) else v; + c.* = try cg.constInt(.u64, host_v); + } + }, else => unreachable, - }; - const lit_id = cg.allocId(); - try cg.sections.globals.emit(gpa, .OpConstant, .{ - .id_result_type = result_ty_id, - .id_result = lit_id, - .value = lit, - }); - break :cache lit_id; - }, - .err => |err| { - const value = try pt.getErrorValue(err.name); - break :cache try cg.constInt(ty, value); - }, - .error_union => |error_union| { - // TODO: Error unions may be constructed with constant instructions if the payload type - // allows it. For now, just generate it here regardless. - const err_ty = ty.errorUnionSet(zcu); - const payload_ty = ty.errorUnionPayload(zcu); - const err_val_id = switch (error_union.val) { - .err_name => |err_name| try cg.constInt( - err_ty, - try pt.getErrorValue(err_name), - ), - .payload => try cg.constInt(err_ty, 0), - }; - const eu_layout = cg.errorUnionLayout(payload_ty); - if (!eu_layout.payload_has_bits) { - // We use the error type directly as the type. - break :cache err_val_id; } - - const payload_val_id = switch (error_union.val) { - .err_name => try cg.constant(payload_ty, .undef, .indirect), - .payload => |p| try cg.constant(payload_ty, .fromInterned(p), .indirect), - }; - - var constituents: [2]Id = undefined; - var types: [2]Type = undefined; - if (eu_layout.error_first) { - constituents[0] = err_val_id; - constituents[1] = payload_val_id; - types = .{ err_ty, payload_ty }; - } else { - constituents[0] = payload_val_id; - constituents[1] = err_val_id; - types = .{ payload_ty, err_ty }; - } - - const comp_ty_id = try cg.resolveType(ty, .direct); - return try cg.constructComposite(comp_ty_id, &constituents); - }, - .enum_tag => { - const int_val = val.backingInt(zcu); - const int_ty = ty.backingIntType(zcu); - break :cache try cg.constant(int_ty, int_val, repr); - }, - .ptr => return cg.constantPtr(val), - .slice => |slice| { - const ptr_id = try cg.constantPtr(.fromInterned(slice.ptr)); - const len_id = try cg.constant(.usize, .fromInterned(slice.len), .indirect); - const comp_ty_id = try cg.resolveType(ty, .direct); - return try cg.constructComposite(comp_ty_id, &.{ ptr_id, len_id }); - }, - .opt => { - const payload_ty = ty.optionalChild(zcu); - const maybe_payload_val = val.optionalValue(zcu); - - if (!payload_ty.hasRuntimeBits(zcu)) { - break :cache try cg.constBool(maybe_payload_val != null, .indirect); - } else if (ty.optionalReprIsPayload(zcu)) { - // Optional representation is a nullable pointer or slice. - if (maybe_payload_val) |payload_val| { - return try cg.constant(payload_ty, payload_val, .indirect); - } else { - break :cache try cg.constNull(result_ty_id); - } - } - - // Optional representation is a structure. - // { Payload, Bool } - - const has_pl_id = try cg.constBool(maybe_payload_val != null, .indirect); - const payload_id = if (maybe_payload_val) |payload_val| - try cg.constant(payload_ty, payload_val, .indirect) - else - try cg.constUndef(try cg.resolveType(payload_ty, .indirect)); - - const comp_ty_id = try cg.resolveType(ty, .direct); - return try cg.constructComposite(comp_ty_id, &.{ payload_id, has_pl_id }); - }, - .aggregate => |aggregate| switch (ip.indexToKey(ty.ip_index)) { - inline .array_type, .vector_type => |array_type, tag| { - const elem_ty: Type = .fromInterned(array_type.child); - - const scratch_top = cg.id_scratch.items.len; - defer cg.id_scratch.shrinkRetainingCapacity(scratch_top); - const constituents = try cg.id_scratch.addManyAsSlice(gpa, @intCast(ty.arrayLenIncludingSentinel(zcu))); - - const child_repr: Repr = switch (tag) { - .array_type => .indirect, - .vector_type => .direct, - else => unreachable, - }; - - switch (aggregate.storage) { - .bytes => |bytes| { - // TODO: This is really space inefficient, perhaps there is a better - // way to do it? - for (constituents, bytes.toSlice(constituents.len, ip)) |*constituent, byte| { - constituent.* = try cg.constInt(elem_ty, byte); - } - }, - .elems => |elems| { - for (constituents, elems) |*constituent, elem| { - constituent.* = try cg.constant(elem_ty, .fromInterned(elem), child_repr); - } - }, - .repeated_elem => |elem| { - @memset(constituents, try cg.constant(elem_ty, .fromInterned(elem), child_repr)); - }, - } - - const comp_ty_id = try cg.resolveType(ty, .direct); - return cg.constructComposite(comp_ty_id, constituents); - }, - .struct_type => { - const struct_type = zcu.typeToStruct(ty).?; - assert(struct_type.layout != .@"packed"); // packed structs use `bitpack` - - var types: std.ArrayList(Type) = .empty; - defer types.deinit(gpa); - - var constituents: std.ArrayList(Id) = .empty; - defer constituents.deinit(gpa); - - var it = struct_type.iterateRuntimeOrder(ip); - while (it.next()) |field_index| { - const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]); - if (!field_ty.hasRuntimeBits(zcu)) { - // This is a zero-bit field - we only needed it for the alignment. - continue; - } - - // TODO: Padding? - const field_val = try val.fieldValue(pt, field_index); - const field_id = try cg.constant(field_ty, field_val, .indirect); - - try types.append(gpa, field_ty); - try constituents.append(gpa, field_id); - } - - const comp_ty_id = try cg.resolveType(ty, .direct); - return try cg.constructComposite(comp_ty_id, constituents.items); - }, - .tuple_type => |tuple| { - var constituents: std.ArrayList(Id) = .empty; - defer constituents.deinit(gpa); - - for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, field_val, i| { - if (field_val != .none) continue; - const ft: Type = .fromInterned(field_ty); - if (!ft.hasRuntimeBits(zcu)) continue; - - const fv = try val.fieldValue(pt, i); - const field_id = try cg.constant(ft, fv, .indirect); - try constituents.append(gpa, field_id); - } - - const comp_ty_id = try cg.resolveType(ty, .direct); - return try cg.constructComposite(comp_ty_id, constituents.items); - }, + break :cache try cg.constantComposite(big_result_ty_id, constituents); + } + if (ty.isSignedInt(zcu)) { + break :cache try cg.constInt(ty, val.toSignedInt(zcu)); + } else { + break :cache try cg.constInt(ty, val.toUnsignedInt(zcu)); + } + }, + .float => { + const lit: spec.LiteralContextDependentNumber = switch (ty.floatBits(target)) { + 16 => .{ .uint32 = @as(u16, @bitCast(val.toFloat(f16, zcu))) }, + 32 => .{ .float32 = val.toFloat(f32, zcu) }, + 64 => .{ .float64 = val.toFloat(f64, zcu) }, + 80, 128 => unreachable, // TODO else => unreachable, + }; + const lit_id = cg.allocId(); + try cg.module.emit(gpa, .OpConstant, .{ + .id_result_type = result_ty_id, + .id_result = lit_id, + .value = lit, + }); + break :cache lit_id; + }, + .err => |err| { + const value = try pt.getErrorValue(err.name); + break :cache try cg.constInt(ty, value); + }, + .error_union => |error_union| { + // TODO: Error unions may be constructed with constant instructions if the payload type + // allows it. For now, just generate it here regardless. + const err_ty = ty.errorUnionSet(zcu); + const payload_ty = ty.errorUnionPayload(zcu); + const err_val_id = switch (error_union.val) { + .err_name => |err_name| try cg.constInt( + err_ty, + try pt.getErrorValue(err_name), + ), + .payload => try cg.constInt(err_ty, 0), + }; + const eu_layout = cg.errorUnionLayout(payload_ty); + if (!eu_layout.payload_has_bits) { + // We use the error type directly as the type. + break :cache err_val_id; + } + + const payload_val_id = switch (error_union.val) { + .err_name => try cg.constant(payload_ty, .undef, .indirect), + .payload => |p| try cg.constant(payload_ty, .fromInterned(p), .indirect), + }; + + var constituents: [2]Id = undefined; + var types: [2]Type = undefined; + if (eu_layout.error_first) { + constituents[0] = err_val_id; + constituents[1] = payload_val_id; + types = .{ err_ty, payload_ty }; + } else { + constituents[0] = payload_val_id; + constituents[1] = err_val_id; + types = .{ payload_ty, err_ty }; + } + + const comp_ty_id = try cg.resolveType(ty, .direct); + return try cg.constantComposite(comp_ty_id, &constituents); + }, + .enum_tag => { + const int_val = val.backingInt(zcu); + const int_ty = ty.backingIntType(zcu); + break :cache try cg.constant(int_ty, int_val, repr); + }, + .ptr, .slice => unreachable, + .opt => { + const payload_ty = ty.optionalChild(zcu); + const maybe_payload_val = val.optionalValue(zcu); + + if (!payload_ty.hasRuntimeBits(zcu)) { + break :cache try cg.constBool(maybe_payload_val != null, .indirect); + } else if (ty.optionalReprIsPayload(zcu)) { + // Optional representation is a nullable pointer or slice. + if (maybe_payload_val) |payload_val| { + return try cg.constant(payload_ty, payload_val, .indirect); + } else { + break :cache try cg.constNull(result_ty_id); + } + } + + // Optional representation is a structure. + // { Payload, Bool } + + const has_pl_id = try cg.constBool(maybe_payload_val != null, .indirect); + const payload_id = if (maybe_payload_val) |payload_val| + try cg.constant(payload_ty, payload_val, .indirect) + else + try cg.constUndef(try cg.resolveType(payload_ty, .indirect)); + + const comp_ty_id = try cg.resolveType(ty, .direct); + return try cg.constantComposite(comp_ty_id, &.{ payload_id, has_pl_id }); + }, + .aggregate => |aggregate| switch (ip.indexToKey(ty.ip_index)) { + inline .array_type, .vector_type => |array_type, tag| { + const elem_ty: Type = .fromInterned(array_type.child); + + const scratch_top = cg.id_scratch.items.len; + defer cg.id_scratch.shrinkRetainingCapacity(scratch_top); + const constituents = try cg.id_scratch.addManyAsSlice(gpa, @intCast(ty.arrayLenIncludingSentinel(zcu))); + + const child_repr: Repr = switch (tag) { + .array_type => .indirect, + .vector_type => .direct, + else => unreachable, + }; + + switch (aggregate.storage) { + .bytes => |bytes| { + // TODO: This is really space inefficient, perhaps there is a better + // way to do it? + for (constituents, bytes.toSlice(constituents.len, ip)) |*constituent, byte| { + constituent.* = try cg.constInt(elem_ty, byte); + } + }, + .elems => |elems| { + for (constituents, elems) |*constituent, elem| { + constituent.* = try cg.constant(elem_ty, .fromInterned(elem), child_repr); + } + }, + .repeated_elem => |elem| { + @memset(constituents, try cg.constant(elem_ty, .fromInterned(elem), child_repr)); + }, + } + + const comp_ty_id = try cg.resolveType(ty, .direct); + return cg.constantComposite(comp_ty_id, constituents); }, - .un => |un| { - assert(ty.containerLayout(zcu) != .@"packed"); // packed unions use `bitpack` - if (un.tag == .none) { - @panic("TODO"); - } - const active_field = ty.unionTagFieldIndex(.fromInterned(un.tag), zcu).?; - const union_obj = zcu.typeToUnion(ty).?; - const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[active_field]); - const payload = if (field_ty.hasRuntimeBits(zcu)) - try cg.constant(field_ty, .fromInterned(un.val), .direct) - else - null; - return try cg.unionInit(ty, active_field, payload); + .struct_type => { + const struct_type = zcu.typeToStruct(ty).?; + assert(struct_type.layout != .@"packed"); // packed structs use `bitpack` + + var types: std.ArrayList(Type) = .empty; + defer types.deinit(gpa); + + var constituents: std.ArrayList(Id) = .empty; + defer constituents.deinit(gpa); + + var it = struct_type.iterateRuntimeOrder(ip); + while (it.next()) |field_index| { + const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]); + if (!field_ty.hasRuntimeBits(zcu)) { + // This is a zero-bit field - we only needed it for the alignment. + continue; + } + + // TODO: Padding? + const field_val = try val.fieldValue(pt, field_index); + const field_id = try cg.constant(field_ty, field_val, .indirect); + + try types.append(gpa, field_ty); + try constituents.append(gpa, field_id); + } + + const comp_ty_id = try cg.resolveType(ty, .direct); + return cg.constantComposite(comp_ty_id, constituents.items); }, - .bitpack => |bitpack| { - const int_val: Value = .fromInterned(bitpack.backing_int_val); - break :cache try cg.constant(int_val.typeOf(zcu), int_val, repr); + .tuple_type => |tuple| { + var constituents: std.ArrayList(Id) = .empty; + defer constituents.deinit(gpa); + + for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, field_val, i| { + if (field_val != .none) continue; + const ft: Type = .fromInterned(field_ty); + if (!ft.hasRuntimeBits(zcu)) continue; + + const fv = try val.fieldValue(pt, i); + const field_id = try cg.constant(ft, fv, .indirect); + try constituents.append(gpa, field_id); + } + + const comp_ty_id = try cg.resolveType(ty, .direct); + return cg.constantComposite(comp_ty_id, constituents.items); }, + else => unreachable, + }, + .un => |un| { + assert(ty.containerLayout(zcu) != .@"packed"); + if (un.tag == .none) return cg.todo("constant of untagged union", .{}); + + const active_field = ty.unionTagFieldIndex(.fromInterned(un.tag), zcu).?; + const union_obj = zcu.typeToUnion(ty).?; + const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[active_field]); + if (union_obj.layout == .auto) { + if (ty.unionHasAllZeroBitFieldTypes(zcu)) { + const tag_ty = ty.unionTagTypeRuntime(zcu).?; + break :cache try cg.constant(tag_ty, .fromInterned(un.tag), .indirect); + } + + const constituents = try gpa.alloc(Id, union_obj.field_types.len + 1); + defer gpa.free(constituents); + for ( + constituents[0..union_obj.field_types.len], + union_obj.field_types.get(ip), + 0.., + ) |*constituent, member_ty_index, member| { + const member_ty: Type = .fromInterned(member_ty_index); + assert(!member_ty.isPtrAtRuntime(zcu)); + + const member_ty_id = if (member_ty.hasRuntimeBits(zcu)) + try cg.resolveType(member_ty, .indirect) + else + try cg.structType(&.{}, null, .none); + constituent.* = if (member == active_field and member_ty.hasRuntimeBits(zcu)) + try cg.constant(member_ty, .fromInterned(un.val), .indirect) + else + try cg.constUndef(member_ty_id); + } + + var len = union_obj.field_types.len; + if (ty.unionTagTypeRuntime(zcu)) |tag_ty| { + constituents[len] = try cg.constant(tag_ty, .fromInterned(un.tag), .indirect); + len += 1; + } + + return cg.constantComposite(result_ty_id, constituents[0..len]); + } + + const layout = cg.unionLayout(ty); + if (field_ty.hasRuntimeBits(zcu) and layout.payload_ty.eql(field_ty)) { + const constituents = try gpa.alloc(Id, layout.total_fields); + defer gpa.free(constituents); + constituents[layout.payload_index] = try cg.constant( + field_ty, + .fromInterned(un.val), + .indirect, + ); + + if (layout.payload_padding_size != 0) { + const u8_ty_id = try cg.resolveType(.u8, .indirect); + const len_id = try cg.constInt(.u32, layout.payload_padding_size); + constituents[layout.payload_padding_index] = try cg.constUndef(try cg.arrayType(len_id, u8_ty_id)); + } + + if (layout.padding_size != 0) { + const u8_ty_id = try cg.resolveType(.u8, .indirect); + const len_id = try cg.constInt(.u32, layout.padding_size); + constituents[layout.padding_index] = try cg.constUndef(try cg.arrayType(len_id, u8_ty_id)); + } + + return cg.constantComposite(result_ty_id, constituents); + } + + const payload: ?Id = if (field_ty.hasRuntimeBits(zcu)) + try cg.constant(field_ty, .fromInterned(un.val), .direct) + else + null; + return try cg.unionInit(ty, active_field, payload); + }, + .bitpack => |bitpack| { + const int_val: Value = .fromInterned(bitpack.backing_int_val); + break :cache try cg.constant(int_val.typeOf(zcu), int_val, repr); + }, - .memoized_call => unreachable, - } + .memoized_call => unreachable, }; return cacheable_id; } @@ -1942,46 +1678,42 @@ fn derivePtr(cg: *CodeGen, derivation: Value.PointerDeriveStep) !Id { const nav = ip.getNav(nav_index); const nav_ty: Type = .fromInterned(nav.resolved.?.type); - switch (nav.resolved.?.value) { - .none => {}, - else => |value| switch (ip.indexToKey(value)) { - // TODO: Properly lower function pointers; for now substitute undef. - .func => return try cg.constUndef(ty_id), - .@"extern" => if (ip.isFunctionType(nav_ty.toIntern())) { - const spv_decl_index = try cg.resolveNav(ip, nav_index); - const decl = cg.declPtr(spv_decl_index); - try emitExternFnStub(cg, nav, decl, nav_ty); - return decl.result_id; - }, - else => {}, - }, - } + const is_extern = nav.getExtern(ip) != null; + assert(nav.resolved.?.@"const" or is_extern); + if (ip.isFunctionType(nav_ty.toIntern())) { + if (is_extern) return try cg.resolveExternFn(nav_index); + return try cg.constUndef(ty_id); + } if (!nav_ty.hasRuntimeBits(zcu) and nav_ty.zigTypeTag(zcu) != .spirv) { return cg.constUndef(ty_id); } + if (!is_extern) { + return cg.todo("pointer to constant '{f}'", .{nav.fqn.fmt(ip)}); + } - const spv_decl_index = try cg.resolveNav(ip, nav_index); - const spv_decl = cg.declPtr(spv_decl_index); - assert(spv_decl.kind != .func); - const storage_class = cg.storageClass(nav.resolved.?.@"addrspace"); - try cg.addFunctionDep(spv_decl_index, storage_class); + const as = nav.resolved.?.@"addrspace"; + assert(as != .generic); + const storage_class = cg.storageClass(as); + const var_id = try cg.resolveNav(nav_index); const nav_ty_id = try cg.resolveType(nav_ty, .indirect); const decl_ptr_ty_id = try cg.ptrType(nav_ty_id, storage_class); - if (cg.needsLayout(nav.resolved.?.@"addrspace", nav_ty)) { - try cg.block_var_ids.put(gpa, spv_decl.result_id, {}); + if (cg.needsLayout(as, nav_ty)) { + try cg.block_var_ids.put(gpa, var_id, {}); } - if (decl_ptr_ty_id == ty_id) return spv_decl.result_id; + + if (decl_ptr_ty_id == ty_id) return var_id; switch (target.os.tag) { - .vulkan, .opengl => return spv_decl.result_id, + .vulkan, .opengl => return var_id, else => {}, } + const casted_ptr_id = cg.allocId(); try cg.body.emit(gpa, .OpBitcast, .{ .id_result_type = ty_id, .id_result = casted_ptr_id, - .operand = spv_decl.result_id, + .operand = var_id, }); return casted_ptr_id; }, @@ -2001,24 +1733,15 @@ fn derivePtr(cg: *CodeGen, derivation: Value.PointerDeriveStep) !Id { return cg.constUndef(ty_id); } - // Uav refs are always generic. - assert(result_ptr_ty.ptrAddressSpace(zcu) == .generic); + if (cg.storageClass(result_ptr_ty.ptrAddressSpace(zcu)) != .function) { + return cg.todo("pointer to an anonymous constant of type '{f}'", .{uav_ty.fmt(pt)}); + } + const uav_ty_id = try cg.resolveType(uav_ty, .indirect); - const decl_ptr_ty_id = try cg.ptrType(uav_ty_id, .function); - const ptr_id = try cg.resolveUav(uav.val); - - if (decl_ptr_ty_id == ty_id) return ptr_id; - switch (target.os.tag) { - .vulkan, .opengl => return ptr_id, - else => {}, - } - const casted_ptr_id = cg.allocId(); - try cg.body.emit(gpa, .OpBitcast, .{ - .id_result_type = ty_id, - .id_result = casted_ptr_id, - .operand = ptr_id, - }); - return casted_ptr_id; + const ptr_id = try cg.alloc(uav_ty_id, null); + const val_id = try cg.constant(uav_ty, .fromInterned(uav.val), .indirect); + try cg.body.emit(gpa, .OpStore, .{ .pointer = ptr_id, .object = val_id }); + return ptr_id; }, .eu_payload_ptr => @panic("TODO"), .opt_payload_ptr => @panic("TODO"), @@ -2102,61 +1825,6 @@ fn derivePtr(cg: *CodeGen, derivation: Value.PointerDeriveStep) !Id { } } -/// Emit a stub OpFunction/OpFunctionEnd + Import linkage decoration for an -/// extern function so the module is structurally valid. The stub will be -/// replaced by the real definition at link time. -fn emitExternFnStub(cg: *CodeGen, nav: InternPool.Nav, decl: *Decl, fn_ty: Type) !void { - if (decl.has_extern_stub) return; - decl.has_extern_stub = true; - - const gpa = cg.gpa; - const zcu = cg.zcu; - const ip = &zcu.intern_pool; - const fn_info = zcu.typeToFunc(fn_ty).?; - const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type)); - const prototype_ty_id = try cg.resolveType(fn_ty, .direct); - - var stub: Section = .{}; - defer stub.deinit(gpa); - try stub.emit(gpa, .OpFunction, .{ - .id_result_type = return_ty_id, - .id_result = decl.result_id, - .function_type = prototype_ty_id, - .function_control = .{}, - }); - for (fn_info.param_types.get(ip)) |param_ty_index| { - const param_ty: Type = .fromInterned(param_ty_index); - if (!param_ty.hasRuntimeBits(zcu)) continue; - const param_type_id = try cg.resolveType(param_ty, .direct); - try stub.emit(gpa, .OpFunctionParameter, .{ - .id_result_type = param_type_id, - .id_result = cg.allocId(), - }); - } - try stub.emit(gpa, .OpFunctionEnd, {}); - try cg.sections.functions.append(gpa, stub); - - const extern_name = nav.getExtern(ip).?.name.toSlice(ip); - try cg.sections.annotations.emit(gpa, .OpDecorate, .{ - .target = decl.result_id, - .decoration = .{ .linkage_attributes = .{ - .name = extern_name, - .linkage_type = .import, - } }, - }); - try cg.debugName(decl.result_id, extern_name); -} - -fn resolveTypeName(cg: *CodeGen, ty: Type) ![]const u8 { - const gpa = cg.gpa; - var aw: std.Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - ty.print(&aw.writer, cg.pt, null) catch |err| switch (err) { - error.WriteFailed => return error.OutOfMemory, - }; - return try aw.toOwnedSlice(); -} - /// Generate a union type. Union types are always generated with the /// most aligned field active. If the tag alignment is greater /// than that of the payload, a regular union (non-packed, with both tag and @@ -2209,7 +1877,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { .direct => return try cg.voidType(), .indirect => { if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{}); - return try cg.opaqueType("void"); + return try cg.opaqueType("void", .{}); }, }, .bool => switch (repr) { @@ -2220,7 +1888,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { if (ty.toIntern() == .u0_type) { assert(repr == .indirect); if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{}); - return try cg.opaqueType("u0"); + return try cg.opaqueType("u0", .{}); } const int_info = ty.intInfo(zcu); return try cg.intType(int_info.signedness, int_info.bits); @@ -2250,7 +1918,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { if (!elem_ty.hasRuntimeBits(zcu)) { assert(repr == .indirect); if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{}); - return try cg.opaqueType("zero-sized-array"); + return try cg.opaqueType("zero-sized-array", .{}); } else if (total_len == 0) { // The size of the array would be 0, but that is not allowed in SPIR-V. // This path can be reached for example when there is a slicing of a pointer @@ -2364,9 +2032,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { null, .none, ); - const type_name = try cg.resolveTypeName(ty); - defer gpa.free(type_name); - try cg.debugName(result_id, type_name); + try cg.debugNameFmt(result_id, "{f}", .{ty.fmt(cg.pt)}); return result_id; }, .struct_type => ip.loadStructType(ty.toIntern()), @@ -2399,9 +2065,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { ty.toIntern(), ); - const type_name = try cg.resolveTypeName(ty); - defer gpa.free(type_name); - try cg.debugName(result_id, type_name); + try cg.debugNameFmt(result_id, "{f}", .{ty.fmt(cg.pt)}); return result_id; }, @@ -2430,9 +2094,36 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { }, .@"union" => { const union_obj = zcu.typeToUnion(ty).?; - if (union_obj.layout == .@"packed") { - return try cg.intType(.unsigned, @intCast(ty.bitSize(zcu))); + switch (union_obj.layout) { + .@"extern" => {}, + .@"packed" => return try cg.intType(.unsigned, @intCast(ty.bitSize(zcu))), + .auto => { + if (ty.unionHasAllZeroBitFieldTypes(zcu)) { + return try cg.resolveType(ty.unionTagTypeRuntime(zcu).?, .indirect); + } + + const member_types = try gpa.alloc(Id, union_obj.field_types.len + 1); + defer gpa.free(member_types); + for (member_types[0..union_obj.field_types.len], union_obj.field_types.get(ip)) |*member_ty_id, field_ty_index| { + const field_ty: Type = .fromInterned(field_ty_index); + member_ty_id.* = if (field_ty.hasRuntimeBits(zcu)) + try cg.resolveType(field_ty, .indirect) + else + try cg.structType(&.{}, null, .none); + } + + var len = union_obj.field_types.len; + if (ty.unionTagTypeRuntime(zcu)) |tag_ty| { + member_types[len] = try cg.resolveType(tag_ty, .indirect); + len += 1; + } + + const result_id = try cg.structType(member_types[0..len], null, .none); + try cg.debugNameFmt(result_id, "{f}", .{ty.fmt(cg.pt)}); + return result_id; + }, } + const layout = cg.unionLayout(ty); if (!layout.has_payload) { return try cg.resolveType(.fromInterned(union_obj.enum_tag_type), .indirect); @@ -2463,9 +2154,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { member_names[0..layout.total_fields], .none, ); - const type_name = try cg.resolveTypeName(ty); - defer gpa.free(type_name); - try cg.debugName(result_id, type_name); + try cg.debugNameFmt(result_id, "{f}", .{ty.fmt(cg.pt)}); return result_id; }, .error_set => { @@ -2502,18 +2191,16 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { }, .@"opaque" => { if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{}); - const type_name = try cg.resolveTypeName(ty); - defer gpa.free(type_name); - return try cg.opaqueType(type_name); + return try cg.opaqueType("{f}", .{ty.fmt(cg.pt)}); }, .spirv => { const spirv_type = ip.loadSpirvType(ty.toIntern()); const result_id = cg.allocId(); switch (spirv_type.flags.tag) { - .sampler => try cg.sections.globals.emit(gpa, .OpTypeSampler, .{ .id_result = result_id }), + .sampler => try cg.module.emit(gpa, .OpTypeSampler, .{ .id_result = result_id }), .image => { const sampled_type_id = try cg.resolveType(.fromInterned(spirv_type.ty), .direct); - try cg.sections.globals.emit(gpa, .OpTypeImage, .{ + try cg.module.emit(gpa, .OpTypeImage, .{ .id_result = result_id, .sampled_type = sampled_type_id, .dim = switch (spirv_type.flags.dim) { @@ -2560,7 +2247,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { }, .sampled_image => { const image_ty_id = try cg.resolveType(.fromInterned(spirv_type.ty), .indirect); - try cg.sections.globals.emit(gpa, .OpTypeSampledImage, .{ + try cg.module.emit(gpa, .OpTypeSampledImage, .{ .id_result = result_id, .image_type = image_ty_id, }); @@ -2568,7 +2255,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { .runtime_array => { const elem_ty: Type = .fromInterned(spirv_type.ty); const elem_ty_id = try cg.resolveType(elem_ty, .indirect); - try cg.sections.globals.emit(gpa, .OpTypeRuntimeArray, .{ + try cg.module.emit(gpa, .OpTypeRuntimeArray, .{ .id_result = result_id, .element_type = elem_ty_id, }); @@ -4061,7 +3748,7 @@ fn buildWideMul( fn generateTestEntryPoint( cg: *CodeGen, name: []const u8, - spv_decl_index: Decl.Index, + kernel_id: Id, test_id: Id, ) !void { const gpa = cg.gpa; @@ -4075,9 +3762,8 @@ fn generateTestEntryPoint( }); const ptr_anyerror_ty_id = try cg.resolveType(ptr_anyerror_ty, .direct); - const kernel_id = cg.declPtr(spv_decl_index).result_id; - - const section = &cg.sections.functions; + var section: Section = .empty; + defer section.deinit(gpa); const p_error_id = cg.allocId(); switch (target.os.tag) { @@ -4103,11 +3789,8 @@ fn generateTestEntryPoint( }, .vulkan, .opengl => { if (cg.error_buffer == null) { - const spv_err_decl_index = try cg.allocDecl(.global); - const err_buf_result_id = cg.declPtr(spv_err_decl_index).result_id; - const buffer_struct_ty_id = cg.allocId(); - try cg.sections.globals.emit(gpa, .OpTypeStruct, .{ + try cg.module.emit(gpa, .OpTypeStruct, .{ .id_result = buffer_struct_ty_id, .id_ref = &.{anyerror_ty_id}, }); @@ -4116,21 +3799,21 @@ fn generateTestEntryPoint( try cg.decorateMember(buffer_struct_ty_id, 0, .{ .offset = .{ .byte_offset = 0 } }); const ptr_buffer_struct_ty_id = cg.allocId(); - try cg.sections.globals.emit(gpa, .OpTypePointer, .{ + try cg.module.emit(gpa, .OpTypePointer, .{ .id_result = ptr_buffer_struct_ty_id, .storage_class = cg.storageClass(.global), .type = buffer_struct_ty_id, }); - try cg.sections.globals.emit(gpa, .OpVariable, .{ + const error_buffer_id = cg.allocId(); + cg.error_buffer = error_buffer_id; + try cg.module.emit(gpa, .OpVariable, .{ .id_result_type = ptr_buffer_struct_ty_id, - .id_result = err_buf_result_id, + .id_result = error_buffer_id, .storage_class = cg.storageClass(.global), }); - try cg.decorate(err_buf_result_id, .{ .descriptor_set = .{ .descriptor_set = 0 } }); - try cg.decorate(err_buf_result_id, .{ .binding = .{ .binding_point = 0 } }); - - cg.error_buffer = spv_err_decl_index; + try cg.decorate(error_buffer_id, .{ .descriptor_set = .{ .descriptor_set = 0 } }); + try cg.decorate(error_buffer_id, .{ .binding = .{ .binding_point = 0 } }); } const void_ty_id = try cg.resolveType(.void, .direct); @@ -4145,10 +3828,7 @@ fn generateTestEntryPoint( .id_result = cg.allocId(), }); - const spv_err_decl_index = cg.error_buffer.?; - const buffer_id = cg.declPtr(spv_err_decl_index).result_id; - try cg.decl_deps.append(gpa, spv_err_decl_index); - + const buffer_id = cg.error_buffer.?; const zero_id = try cg.constInt(.u32, 0); try section.emit(gpa, .OpInBoundsAccessChain, .{ .id_result_type = ptr_anyerror_ty_id, @@ -4176,17 +3856,17 @@ fn generateTestEntryPoint( }); try section.emit(gpa, .OpReturn, {}); try section.emit(gpa, .OpFunctionEnd, {}); + try cg.module.append(gpa, section); // Just generate a quick other name because the intel runtime crashes when the entry- // point name is the same as a different OpName. - const test_name = try std.fmt.allocPrint(cg.arena, "test {s}", .{name}); - - const ep_gop = try cg.entry_points.getOrPut(cg.gpa, cg.declPtr(spv_decl_index).result_id); - ep_gop.value_ptr.* = .{ - .decl_index = spv_decl_index, + const test_name = try std.fmt.allocPrint(gpa, "test {s}", .{name}); + errdefer gpa.free(test_name); + try cg.entry_points.append(gpa, .{ + .id = kernel_id, .name = test_name, .cc = .{ .spirv_kernel = .{ .x = 1, .y = 1, .z = 1 } }, - }; + }); } fn intFromBool(cg: *CodeGen, value: Temporary, result_ty: Type) !Temporary { @@ -4456,6 +4136,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void { .ptr_elem_ptr => try cg.airPtrElemPtr(inst), .ptr_elem_val => try cg.airPtrElemVal(inst), .array_elem_val => try cg.airArrayElemVal(inst), + .runtime_nav_ptr => try cg.airRuntimeNavPtr(inst), .set_union_tag => return cg.airSetUnionTag(inst), .get_union_tag => try cg.airGetUnionTag(inst), @@ -5549,12 +5230,17 @@ fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) !?Id { } fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) !?Id { + const gpa = cg.gpa; + const zcu = cg.zcu; const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op; - const operand_id = try cg.resolve(ty_op.operand); const result_ty = cg.typeOfIndex(inst); - return try cg.constructCompositeSplat(result_ty, operand_id); + const scratch_top = cg.id_scratch.items.len; + defer cg.id_scratch.shrinkRetainingCapacity(scratch_top); + const constituents = try cg.id_scratch.addManyAsSlice(gpa, @intCast(result_ty.arrayLen(zcu))); + @memset(constituents, operand_id); + return try cg.constructComposite(try cg.resolveType(result_ty, .direct), constituents); } fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) !?Id { @@ -6767,6 +6453,11 @@ fn ptrElemPtr(cg: *CodeGen, ptr_ty: Type, ptr_id: Id, index_id: Id) !Id { } } +fn airRuntimeNavPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id { + const ty_nav = cg.air.instructions.items(.data)[@backingInt(inst)].ty_nav; + return try cg.resolveNav(ty_nav.nav); +} + fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id { const zcu = cg.zcu; const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl; @@ -6849,38 +6540,27 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !void { const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op; const un_ptr_ty = cg.typeOf(bin_op.lhs); const un_ty = un_ptr_ty.childType(zcu); - const layout = cg.unionLayout(un_ty); - - if (layout.tag_size == 0) return; - - const tag_ty = un_ty.unionTagTypeRuntime(zcu).?; + const tag_ty = un_ty.unionTagTypeRuntime(zcu) orelse return; const tag_ty_id = try cg.resolveType(tag_ty, .indirect); const tag_ptr_ty_id = try cg.ptrType(tag_ty_id, cg.storageClass(un_ptr_ty.ptrAddressSpace(zcu))); - const union_ptr_id = try cg.resolve(bin_op.lhs); const new_tag_id = try cg.resolve(bin_op.rhs); - - if (!layout.has_payload) { - try cg.store(tag_ty, union_ptr_id, new_tag_id, .{ .is_volatile = un_ptr_ty.isVolatilePtr(zcu) }); - } else { - const ptr_id = try cg.accessChain(tag_ptr_ty_id, union_ptr_id, &.{layout.tag_index}); - try cg.store(tag_ty, ptr_id, new_tag_id, .{ .is_volatile = un_ptr_ty.isVolatilePtr(zcu) }); - } + const ptr_id = if (un_ty.unionHasAllZeroBitFieldTypes(zcu)) + union_ptr_id + else + try cg.accessChain(tag_ptr_ty_id, union_ptr_id, &.{zcu.typeToUnion(un_ty).?.field_types.len}); + try cg.store(tag_ty, ptr_id, new_tag_id, .{ .is_volatile = un_ptr_ty.isVolatilePtr(zcu) }); } fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !?Id { + const zcu = cg.zcu; const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op; const un_ty = cg.typeOf(ty_op.operand); - - const zcu = cg.zcu; - const layout = cg.unionLayout(un_ty); - if (layout.tag_size == 0) return null; - + const tag_ty = un_ty.unionTagTypeRuntime(zcu) orelse return null; const union_handle = try cg.resolve(ty_op.operand); - if (!layout.has_payload) return union_handle; - - const tag_ty = un_ty.unionTagTypeRuntime(zcu).?; - return try cg.extractField(tag_ty, union_handle, layout.tag_index); + if (un_ty.unionHasAllZeroBitFieldTypes(zcu)) return union_handle; + const tag_member = zcu.typeToUnion(un_ty).?.field_types.len; + return try cg.extractField(tag_ty, union_handle, tag_member); } fn unionInit( @@ -6905,6 +6585,29 @@ fn unionInit( const payload_ty: Type = .fromInterned(union_ty.field_types.get(ip)[active_field]); assert(union_ty.layout != .@"packed"); + if (union_ty.layout == .auto) { + if (ty.unionHasAllZeroBitFieldTypes(zcu)) { + const runtime_tag_ty = ty.unionTagTypeRuntime(zcu).?; + const tag_val = try pt.enumValueFieldIndex(runtime_tag_ty, active_field); + return try cg.constant(runtime_tag_ty, tag_val, .indirect); + } + const ty_id = try cg.resolveType(ty, .indirect); + const tmp_id = try cg.alloc(ty_id, null); + if (payload) |payload_id| { + const payload_ty_id = try cg.resolveType(payload_ty, .indirect); + const ptr_ty_id = try cg.ptrType(payload_ty_id, .function); + const payload_ptr_id = try cg.accessChain(ptr_ty_id, tmp_id, &.{active_field}); + try cg.store(payload_ty, payload_ptr_id, payload_id, .{}); + } + if (ty.unionTagTypeRuntime(zcu)) |runtime_tag_ty| { + const tag_val = try pt.enumValueFieldIndex(runtime_tag_ty, active_field); + const tag_id = try cg.constant(runtime_tag_ty, tag_val, .indirect); + const tag_ptr_ty_id = try cg.ptrType(try cg.resolveType(runtime_tag_ty, .indirect), .function); + const tag_ptr_id = try cg.accessChain(tag_ptr_ty_id, tmp_id, &.{union_ty.field_types.len}); + try cg.store(runtime_tag_ty, tag_ptr_id, tag_id, .{}); + } + return try cg.load(ty, tmp_id, .{}); + } const tag_int = if (layout.tag_size != 0) blk: { const tag_val = try pt.enumValueFieldIndex(tag_ty, active_field); @@ -7033,7 +6736,8 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id { if (field_ty.isInt(zcu)) return result_id; return try cg.bitCast(field_ty, int_ty, result_id); }, - else => { + .auto => return try cg.extractField(field_ty, object_id, field_index), + .@"extern" => { // Store, ptr-elem-ptr, pointer-cast, load const layout = cg.unionLayout(object_ty); assert(layout.has_payload); @@ -7163,7 +6867,11 @@ fn structFieldPtr( }, .@"union" => switch (object_ty.containerLayout(zcu)) { .@"packed" => return cg.todo("implement field access for packed unions", .{}), - .auto, .@"extern" => { + .auto => { + if (!result_ptr_ty.childType(zcu).hasRuntimeBits(zcu)) return try cg.constUndef(result_ty_id); + return try cg.accessChain(result_ty_id, object_ptr, &.{field_index}); + }, + .@"extern" => { const layout = cg.unionLayout(object_ty); if (!layout.has_payload) { // Asked to get a pointer to a zero-sized field. Just lower this @@ -7272,7 +6980,7 @@ fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) !?Id { fn airArg(cg: *CodeGen) Id { defer cg.next_arg_index += 1; - return cg.args.items[cg.next_arg_index]; + return cg.args.items(.id)[cg.next_arg_index]; } /// Given a slice of incoming block connections, returns the block-id of the next @@ -8669,7 +8377,7 @@ fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) !void { if (zcu.comp.config.root_strip) return; const path_id = cg.allocId(); - try cg.sections.debug_strings.emit(cg.gpa, .OpString, .{ + try cg.module.emit(cg.gpa, .OpString, .{ .id_result = path_id, .string = path, }); @@ -8942,18 +8650,15 @@ fn builtin3D( const ptr_ty_id = try cg.ptrType(vec_ty_id, .input); const builtins_gop = try cg.builtins.getOrPut(gpa, .{ built_in, .input }); if (!builtins_gop.found_existing) { - builtins_gop.value_ptr.* = try cg.allocDecl(.global); - const decl = cg.declPtr(builtins_gop.value_ptr.*); - try cg.sections.globals.emit(gpa, .OpVariable, .{ + builtins_gop.value_ptr.* = cg.allocId(); + try cg.module.emit(gpa, .OpVariable, .{ .id_result_type = ptr_ty_id, - .id_result = decl.result_id, + .id_result = builtins_gop.value_ptr.*, .storage_class = .input, }); - try cg.decorate(decl.result_id, .{ .built_in = .{ .built_in = built_in } }); + try cg.decorate(builtins_gop.value_ptr.*, .{ .built_in = .{ .built_in = built_in } }); } - const spv_decl_index = builtins_gop.value_ptr.*; - try cg.decl_deps.append(gpa, spv_decl_index); - const ptr_id = cg.declPtr(spv_decl_index).result_id; + const ptr_id = builtins_gop.value_ptr.*; const vec_id = cg.allocId(); try cg.body.emit(gpa, .OpLoad, .{ .id_result_type = vec_ty_id, diff --git a/src/codegen/spirv/Mir.zig b/src/codegen/spirv/Mir.zig index c91c362426f83d8b458bce82e2b872e9d24722ff..1cc07c0f9b2e7c20b7b848bc0928d41d28b4a8cc 100644 --- a/src/codegen/spirv/Mir.zig +++ b/src/codegen/spirv/Mir.zig @@ -1,71 +1,39 @@ const std = @import("std"); const Allocator = std.mem.Allocator; - const spec = @import("spec.zig"); const Word = spec.Word; const Id = spec.Id; - +const Section = @import("Section.zig"); const InternPool = @import("../../InternPool.zig"); -const CodeGen = @import("CodeGen.zig"); - const Mir = @This(); id_bound: Word, -owner_nav: InternPool.Nav.Index, -kind: CodeGen.Decl.Kind, -decl_result_id: Id, -extended_instruction_set: []const Word, -globals: []const Word, -functions: []const Word, -annotations: []const Word, -debug_names: []const Word, -debug_strings: []const Word, -execution_modes: []const Word, +module: Section, nav_refs: []const NavRef, -uav_refs: []const UavRef, -decl_deps: []const DeclDep, -internal_globals: []const Id, +externs: []const Extern, entry_points: []const EntryPoint, pub const NavRef = struct { - local_id: Id, + id: Id, nav: InternPool.Nav.Index, - kind: CodeGen.Decl.Kind, }; -pub const UavRef = struct { - local_id: Id, - val: InternPool.Index, - storage_class: spec.StorageClass, - kind: CodeGen.Decl.Kind, -}; - -pub const DeclDep = struct { - kind: CodeGen.Decl.Kind, - nav: InternPool.Nav.Index, +pub const Extern = struct { + id: Id, + name: []const u8, }; pub const EntryPoint = struct { - local_id: Id, + id: Id, name: []const u8, cc: std.builtin.CallingConvention, }; pub fn deinit(mir: *Mir, gpa: Allocator) void { - gpa.free(mir.extended_instruction_set); - gpa.free(mir.globals); - gpa.free(mir.functions); - gpa.free(mir.annotations); - gpa.free(mir.debug_names); - gpa.free(mir.debug_strings); - gpa.free(mir.execution_modes); + mir.module.deinit(gpa); gpa.free(mir.nav_refs); - gpa.free(mir.uav_refs); - gpa.free(mir.decl_deps); - gpa.free(mir.internal_globals); - for (mir.entry_points) |ep| { - gpa.free(ep.name); - } + gpa.free(mir.externs); + for (mir.entry_points) |ep| gpa.free(ep.name); gpa.free(mir.entry_points); mir.* = undefined; } diff --git a/src/codegen/spirv/Section.zig b/src/codegen/spirv/Section.zig index fcd7739c1dd3a6916a258e79ce36dae6fec3ba2c..a594d294906e666145743db33a53c7ffc4a0d0eb 100644 --- a/src/codegen/spirv/Section.zig +++ b/src/codegen/spirv/Section.zig @@ -1,91 +1,273 @@ //! Represents a section or subsection of instructions in a SPIR-V binary. Instructions can be append //! to separate sections, which can then later be merged into the final binary. -const Section = @This(); - const std = @import("std"); const Allocator = std.mem.Allocator; -const testing = std.testing; - const spec = @import("spec.zig"); -const Word = spec.Word; -const DoubleWord = @Int(.unsigned, @bitSizeOf(Word) * 2); -const Log2Word = std.math.Log2Int(Word); - const Opcode = spec.Opcode; +const InstructionHeader = spec.InstructionHeader; +const Word = spec.Word; +const Id = spec.Id; +const Section = @This(); -instructions: std.ArrayList(Word) = .empty, +instructions: std.ArrayList(Word), +ids: std.bit_set.Dynamic, -pub fn deinit(section: *Section, allocator: Allocator) void { - section.instructions.deinit(allocator); +pub const empty: Section = .{ .instructions = .empty, .ids = .{} }; + +pub const Error = error{ + OutOfMemory, + UnknownOpcode, + InvalidInstruction, + InvalidInstructionLength, + NestedFunction, + UnterminatedFunction, + UndefinedId, +}; + +pub fn fromWords(gpa: Allocator, words: []const Word, id_bound: Word) Error!Section { + var section: Section = .empty; + errdefer section.deinit(gpa); + try section.writeWords(gpa, words); + const type_widths = try gpa.alloc(u16, id_bound); + defer gpa.free(type_widths); + @memset(type_widths, 0); + var instruction_sets: std.array_hash_map.Auto(Word, spec.InstructionSet) = .empty; + defer instruction_sets.deinit(gpa); + var results: std.bit_set.Dynamic = try .initEmpty(gpa, id_bound); + defer results.deinit(gpa); + + var in_function = false; + var offset: usize = 0; + while (offset < words.len) { + const header: InstructionHeader = @bitCast(words[offset]); + const len = header.word_count; + if (len == 0 or offset + len > words.len) return error.InvalidInstructionLength; + + const inst = words[offset..][0..len]; + const op = std.enums.fromInt(Opcode, @as(u16, @truncate(inst[0]))) orelse + return error.UnknownOpcode; + const end = offset + len; + var index = offset + 1; + const list: []const spec.Operand = switch (op) { + .OpFunction => list: { + if (in_function) return error.NestedFunction; + in_function = true; + break :list op.operands(); + }, + .OpFunctionEnd => list: { + if (!in_function) return error.InvalidInstruction; + in_function = false; + break :list op.operands(); + }, + .OpSpecConstantOp => list: { + const prefix = op.operands()[0..2]; + index = try section.parseOperands(type_widths, offset, end, prefix, index); + if (index >= end) return error.InvalidInstruction; + const inner = std.enums.fromInt(Opcode, words[index]) orelse + return error.InvalidInstruction; + index += 1; + break :list inner.operands()[2..]; + }, + .OpExtInst => list: { + if (len < 5) return error.InvalidInstruction; + const set = instruction_sets.get(inst[3]) orelse return error.InvalidInstruction; + const prefix = op.operands()[0..4]; + index = try section.parseOperands(type_widths, offset, end, prefix, index); + break :list set.operands(inst[4]) orelse return error.InvalidInstruction; + }, + else => op.operands(), + }; + const operands_end = try section.parseOperands(type_widths, offset, end, list, index); + if (operands_end != end) return error.InvalidInstruction; + if (op.resultIndex()) |result_index| { + if (results.isSet(inst[result_index])) return error.InvalidInstruction; + results.set(inst[result_index]); + } + + switch (op) { + .OpTypeInt, .OpTypeFloat => { + type_widths[inst[1]] = std.math.cast(u16, inst[2]) orelse + return error.InvalidInstruction; + }, + .OpExtInstImport => { + const set_name = std.mem.sliceTo(std.mem.sliceAsBytes(inst[2..]), 0); + if (std.meta.stringToEnum(spec.InstructionSet, set_name)) |set| { + if (set != .core) try instruction_sets.put(gpa, inst[1], set); + } + }, + else => if (op.resultIndex() == 2) { + type_widths[inst[2]] = type_widths[inst[1]]; + }, + } + offset = end; + } + if (in_function) return error.UnterminatedFunction; + var it = section.ids.iterator(.{}); + while (it.next()) |index| { + if (!results.isSet(section.instructions.items[index])) return error.UndefinedId; + } + + return section; +} + +pub fn deinit(section: *Section, gpa: Allocator) void { + section.instructions.deinit(gpa); + section.ids.deinit(gpa); section.* = undefined; } pub fn reset(section: *Section) void { section.instructions.clearRetainingCapacity(); + section.ids.unsetAll(); } -pub fn toWords(section: Section) []Word { - return section.instructions.items; +pub fn append(section: *Section, gpa: Allocator, other: Section) error{OutOfMemory}!void { + const offset = section.instructions.items.len; + try section.writeWords(gpa, other.instructions.items); + var it = other.ids.iterator(.{}); + while (it.next()) |index| section.ids.set(offset + index); } -/// Append the instructions from another section into this section. -pub fn append(section: *Section, allocator: Allocator, other_section: Section) !void { - try section.instructions.appendSlice(allocator, other_section.instructions.items); +pub fn ensureUnusedCapacity(section: *Section, gpa: Allocator, words: usize) !void { + try section.instructions.ensureUnusedCapacity(gpa, words); + try section.ids.resize(gpa, section.instructions.capacity, false); } -pub fn ensureUnusedCapacity( +fn parseOperands( section: *Section, - allocator: Allocator, - words: usize, -) !void { - try section.instructions.ensureUnusedCapacity(allocator, words); + type_widths: []const u16, + inst_start: usize, + inst_end: usize, + operands: []const spec.Operand, + index: usize, +) Error!usize { + const words = section.instructions.items; + var next = index; + var lists: [16][]const spec.Operand = undefined; + lists[0] = operands; + var depth: usize = 1; + while (depth > 0) { + const list = &lists[depth - 1]; + if (list.len == 0) { + depth -= 1; + continue; + } + const entry = list.*[0]; + if (next >= inst_end) { + if (entry.quantifier == .required) return error.InvalidInstruction; + list.* = &.{}; + continue; + } + if (entry.quantifier != .variadic) list.* = list.*[1..]; + const single = [_]spec.OperandKind{entry.kind}; + const kinds: []const spec.OperandKind = switch (entry.kind.category()) { + .composite => entry.kind.bases(), + else => &single, + }; + for (kinds) |base| { + if (next >= inst_end) return error.InvalidInstruction; + const is_pair_literal = entry.kind == .pair_literal_integer_id_ref; + const kind: spec.OperandKind = if (is_pair_literal and base == .literal_integer) + .literal_context_dependent_number + else + base; + switch (kind.category()) { + .id => { + if (words[next] == 0 or words[next] >= type_widths.len) { + return error.InvalidInstruction; + } + section.ids.set(next); + next += 1; + }, + .literal => switch (kind) { + .literal_string => { + next = for (words[next..inst_end], next + 1..) |word, after| { + const bytes: [4]u8 = @bitCast(word); + if (std.mem.findScalar(u8, &bytes, 0) != null) break after; + } else return error.InvalidInstruction; + }, + .literal_context_dependent_number => { + next += switch (type_widths[words[inst_start + 1]]) { + 1...32 => 1, + 33...64 => 2, + else => return error.InvalidInstruction, + }; + if (next > inst_end) return error.InvalidInstruction; + }, + .literal_spec_constant_op_integer => unreachable, + else => next += 1, + }, + .value_enum => { + const enumerant = for (kind.enumerants()) |enumerant| { + if (enumerant.value == words[next]) break enumerant; + } else return error.InvalidInstruction; + next += 1; + if (enumerant.parameters.len != 0) { + if (depth == lists.len) return error.InvalidInstruction; + lists[depth] = enumerant.parameters; + depth += 1; + } + }, + .bit_enum => { + const mask = words[next]; + next += 1; + const enumerants = kind.enumerants(); + var i = enumerants.len; + while (i > 0) { + i -= 1; + const enumerant = enumerants[i]; + if (mask & enumerant.value == 0) continue; + if (enumerant.parameters.len == 0) continue; + if (depth == lists.len) return error.InvalidInstruction; + lists[depth] = enumerant.parameters; + depth += 1; + } + }, + .composite => unreachable, + } + } + } + return next; } -/// Write an instruction and size, operands are to be inserted manually. pub fn emitRaw( section: *Section, - allocator: Allocator, + gpa: Allocator, opcode: Opcode, operand_words: usize, ) !void { const word_count = 1 + operand_words; if (word_count > std.math.maxInt(u16)) return error.OutOfMemory; - try section.instructions.ensureUnusedCapacity(allocator, word_count); - section.writeWord((@as(Word, @intCast(word_count)) << 16) | @backingInt(opcode)); + try section.ensureUnusedCapacity(gpa, word_count); + const header: InstructionHeader = .{ .opcode = opcode, .word_count = @intCast(word_count) }; + section.writeWord(@bitCast(header)); } -/// Write an entire instruction, including all operands -pub fn emitRawInstruction( +pub fn emitFormatted( section: *Section, - allocator: Allocator, - opcode: Opcode, - operands: []const Word, + gpa: Allocator, + comptime opcode: Opcode, + ids: []const Id, + comptime fmt: []const u8, + args: anytype, ) !void { - try section.emitRaw(allocator, opcode, operands.len); - section.writeWords(operands); -} - -pub fn emitAssumeCapacity( - section: *Section, - comptime opcode: spec.Opcode, - operands: opcode.Operands(), -) !void { - const word_count = instructionSize(opcode, operands); - if (word_count > std.math.maxInt(u16)) return error.OutOfMemory; - section.writeWord((@as(Word, @intCast(word_count)) << 16) | @backingInt(opcode)); - section.writeOperands(opcode.Operands(), operands); + const len = std.fmt.count(fmt, args); + const string_words = len / @sizeOf(Word) + 1; + try section.emitRaw(gpa, opcode, ids.len + string_words); + for (ids) |id| section.writeId(id); + const words = section.instructions.addManyAsSliceAssumeCapacity(string_words); + @memset(words, 0); + _ = std.fmt.bufPrint(std.mem.sliceAsBytes(words), fmt, args) catch unreachable; + for (words) |*word| word.* = std.mem.littleToNative(Word, word.*); } pub fn emit( section: *Section, - allocator: Allocator, + gpa: Allocator, comptime opcode: spec.Opcode, operands: opcode.Operands(), ) !void { - const word_count = instructionSize(opcode, operands); - if (word_count > std.math.maxInt(u16)) return error.OutOfMemory; - try section.instructions.ensureUnusedCapacity(allocator, word_count); - section.writeWord((@as(Word, @intCast(word_count)) << 16) | @backingInt(opcode)); + try section.emitRaw(gpa, opcode, opcode.instructionSize(operands) - 1); section.writeOperands(opcode.Operands(), operands); } @@ -93,18 +275,26 @@ pub fn writeWord(section: *Section, word: Word) void { section.instructions.appendAssumeCapacity(word); } -pub fn writeWords(section: *Section, words: []const Word) void { +pub fn writeWords(section: *Section, gpa: Allocator, words: []const Word) error{OutOfMemory}!void { + try section.ensureUnusedCapacity(gpa, words.len); section.instructions.appendSliceAssumeCapacity(words); } +pub const DoubleWord = @Int(.unsigned, @bitSizeOf(Word) * 2); + pub fn writeDoubleWord(section: *Section, dword: DoubleWord) void { - section.writeWords(&.{ + section.instructions.appendSliceAssumeCapacity(&.{ @truncate(dword), @truncate(dword >> @bitSizeOf(Word)), }); } -fn writeOperands(section: *Section, comptime Operands: type, operands: Operands) void { +pub fn writeId(section: *Section, id: Id) void { + section.ids.set(section.instructions.items.len); + section.instructions.appendAssumeCapacity(@backingInt(id)); +} + +pub fn writeOperands(section: *Section, comptime Operands: type, operands: Operands) void { const info = switch (@typeInfo(Operands)) { .@"struct" => |info| info, .void => return, @@ -118,19 +308,28 @@ fn writeOperands(section: *Section, comptime Operands: type, operands: Operands) pub fn writeOperand(section: *Section, comptime Operand: type, operand: Operand) void { switch (Operand) { spec.LiteralSpecConstantOpInteger => unreachable, - spec.Id => section.writeWord(@backingInt(operand)), + spec.Id => section.writeId(operand), spec.LiteralInteger => section.writeWord(operand), spec.LiteralString => section.writeString(operand), - spec.LiteralContextDependentNumber => section.writeContextDependentNumber(operand), + spec.LiteralContextDependentNumber => section.writeNumber(operand), spec.LiteralExtInstInteger => section.writeWord(operand.inst), - spec.PairLiteralIntegerIdRef => section.writeWords(&.{ operand.value, @fromBackingInt(@intCast(operand.label)) }), - spec.PairIdRefLiteralInteger => section.writeWords(&.{ @backingInt(operand.target), operand.member }), - spec.PairIdRefIdRef => section.writeWords(&.{ @backingInt(operand[0]), @backingInt(operand[1]) }), + spec.PairLiteralIntegerIdRef => { + section.writeWord(operand.value); + section.writeId(operand.label); + }, + spec.PairIdRefLiteralInteger => { + section.writeId(operand.target); + section.writeWord(operand.member); + }, + spec.PairIdRefIdRef => { + section.writeId(operand[0]); + section.writeId(operand[1]); + }, else => switch (@typeInfo(Operand)) { .@"enum" => section.writeWord(@backingInt(operand)), .optional => |info| if (operand) |child| section.writeOperand(info.child, child), .pointer => |info| { - std.debug.assert(info.size == .slice); // Should be no other pointer types in the spec. + std.debug.assert(info.size == .slice); for (operand) |item| { section.writeOperand(info.child, item); } @@ -148,20 +347,21 @@ pub fn writeOperand(section: *Section, comptime Operand: type, operand: Operand) } } -fn writeString(section: *Section, str: []const u8) void { +pub fn writeString(section: *Section, str: []const u8) void { const zero_terminated_len = str.len + 1; var i: usize = 0; while (i < zero_terminated_len) : (i += @sizeOf(Word)) { var word: Word = 0; var j: usize = 0; while (j < @sizeOf(Word) and i + j < str.len) : (j += 1) { - word |= @as(Word, str[i + j]) << @as(Log2Word, @intCast(j * @bitSizeOf(u8))); + const shift: std.math.Log2Int(Word) = @intCast(j * @bitSizeOf(u8)); + word |= @as(Word, str[i + j]) << shift; } section.instructions.appendAssumeCapacity(word); } } -fn writeContextDependentNumber(section: *Section, operand: spec.LiteralContextDependentNumber) void { +pub fn writeNumber(section: *Section, operand: spec.LiteralContextDependentNumber) void { switch (operand) { .int32 => |int| section.writeWord(@bitCast(int)), .uint32 => |int| section.writeWord(@bitCast(int)), @@ -172,17 +372,23 @@ fn writeContextDependentNumber(section: *Section, operand: spec.LiteralContextDe } } -fn writeExtendedMask(section: *Section, comptime Operand: type, operand: Operand) void { +pub fn writeExtendedMask(section: *Section, comptime Operand: type, operand: Operand) void { var mask: Word = 0; const info = @typeInfo(Operand).@"struct"; - inline for (info.field_names, info.field_types, 0..) |field_name, field_type, bit| { + comptime var bit: usize = 0; + inline for (info.field_names, info.field_types) |field_name, field_type| { switch (@typeInfo(field_type)) { - .optional => if (@field(operand, field_name) != null) { - mask |= 1 << @as(u5, @intCast(bit)); + .optional => { + if (@field(operand, field_name) != null) { + mask |= @as(Word, 1) << @intCast(bit); + } + bit += 1; }, - .bool => if (@field(operand, field_name)) { - mask |= 1 << @as(u5, @intCast(bit)); + .bool => { + if (@field(operand, field_name)) mask |= @as(Word, 1) << @intCast(bit); + bit += 1; }, + .int => bit += @bitSizeOf(field_type), else => unreachable, } } @@ -191,86 +397,22 @@ fn writeExtendedMask(section: *Section, comptime Operand: type, operand: Operand inline for (info.field_names, info.field_types) |field_name, field_type| { switch (@typeInfo(field_type)) { - .optional => |opt_info| if (@field(operand, field_name)) |child| { - section.writeOperands(opt_info.child, child); + .optional => |opt_info| { + if (@field(operand, field_name)) |child| { + section.writeOperands(opt_info.child, child); + } }, - .bool => {}, + .bool, .int => {}, else => unreachable, } } } -fn writeExtendedUnion(section: *Section, comptime Operand: type, operand: Operand) void { +pub fn writeExtendedUnion(section: *Section, comptime Operand: type, operand: Operand) void { return switch (operand) { inline else => |op, tag| { section.writeWord(@backingInt(tag)); - section.writeOperands( - @FieldType(Operand, @tagName(tag)), - op, - ); - }, - }; -} - -fn instructionSize(comptime opcode: spec.Opcode, operands: opcode.Operands()) usize { - return operandsSize(opcode.Operands(), operands) + 1; -} - -fn operandsSize(comptime Operands: type, operands: Operands) usize { - const info = switch (@typeInfo(Operands)) { - .@"struct" => |info| info, - .void => return 0, - else => unreachable, - }; - - var total: usize = 0; - inline for (info.field_names, info.field_types) |field_name, field_type| { - total += operandSize(field_type, @field(operands, field_name)); - } - - return total; -} - -fn operandSize(comptime Operand: type, operand: Operand) usize { - return switch (Operand) { - spec.LiteralSpecConstantOpInteger => unreachable, - spec.Id, spec.LiteralInteger, spec.LiteralExtInstInteger => 1, - spec.LiteralString => @divCeil(operand.len + 1, @sizeOf(Word)), - spec.LiteralContextDependentNumber => switch (operand) { - .int32, .uint32, .float32 => 1, - .int64, .uint64, .float64 => 2, - }, - spec.PairLiteralIntegerIdRef, spec.PairIdRefLiteralInteger, spec.PairIdRefIdRef => 2, - else => switch (@typeInfo(Operand)) { - .@"enum" => 1, - .optional => |info| if (operand) |child| operandSize(info.child, child) else 0, - .pointer => |info| blk: { - std.debug.assert(info.size == .slice); // Should be no other pointer types in the spec. - var total: usize = 0; - for (operand) |item| { - total += operandSize(info.child, item); - } - break :blk total; - }, - .@"struct" => |struct_info| { - if (struct_info.layout == .@"packed") return 1; - - var total: usize = 0; - inline for (struct_info.field_names, struct_info.field_types) |field_name, field_type| { - switch (@typeInfo(field_type)) { - .optional => |info| if (@field(operand, field_name)) |child| { - total += operandsSize(info.child, child); - }, - .bool => {}, - else => unreachable, - } - } - return total + 1; // Add one for the mask itself. - }, - .@"union" => switch (operand) { - inline else => |op, tag| operandsSize(@FieldType(Operand, @tagName(tag)), op) + 1, - }, - else => unreachable, + section.writeOperands(@FieldType(Operand, @tagName(tag)), op); }, }; } diff --git a/src/link.zig b/src/link.zig index c97c752517928551c1753933a9c512417cb9e434..0096023099d00dcdd59e42a4e3e1cae12e777e47 100644 --- a/src/link.zig +++ b/src/link.zig @@ -1319,7 +1319,7 @@ pub const File = struct { .macho => MachO, .c => C, .wasm => Wasm, - .spirv => SpirV, + .spirv => Spirv, .lld => Lld, .plan9 => comptime unreachable, .spork8 => Spork8, @@ -1423,7 +1423,7 @@ pub const File = struct { pub const Elf = @import("link/Elf.zig"); pub const Elf2 = @import("link/Elf2.zig"); pub const MachO = @import("link/MachO.zig"); - pub const SpirV = @import("link/SpirV.zig"); + pub const Spirv = @import("link/Spirv.zig"); pub const Wasm = @import("link/Wasm.zig"); pub const Dwarf = @import("link/Dwarf.zig"); pub const Dwarf2 = @import("link/Dwarf2.zig"); diff --git a/src/link/SpirV.zig b/src/link/SpirV.zig deleted file mode 100644 index 30b020985091d2b62bcb0358da3ff90d61f2b69d..0000000000000000000000000000000000000000 --- a/src/link/SpirV.zig +++ /dev/null @@ -1,1216 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Path = std.Build.Cache.Path; -const assert = std.debug.assert; -const log = std.log.scoped(.link); -const zig_version = @import("builtin").zig_version; -const Zcu = @import("../Zcu.zig"); -const InternPool = @import("../InternPool.zig"); -const Compilation = @import("../Compilation.zig"); -const link = @import("../link.zig"); -const Air = @import("../Air.zig"); -const Type = @import("../Type.zig"); -const codegen = @import("../codegen.zig"); -const CodeGen = @import("../codegen/spirv/CodeGen.zig"); -const BinaryModule = @import("SpirV/BinaryModule.zig"); -const lower_invocation_globals = @import("SpirV/lower_invocation_globals.zig"); -const dedup_types = @import("SpirV/dedup_types.zig"); -const prune_unused = @import("SpirV/prune_unused.zig"); -const spec = @import("../codegen/spirv/spec.zig"); -const Section = @import("../codegen/spirv/Section.zig"); -const Id = spec.Id; -const Word = spec.Word; -const Mir = @import("../codegen/spirv/Mir.zig"); - -const Linker = @This(); - -base: link.File, -fragments: std.array_hash_map.Auto(InternPool.Nav.Index, Mir) = .empty, -pending_navs: std.ArrayList(InternPool.Nav.Index) = .empty, -entry_points: std.ArrayList(EntryPointDecl) = .empty, -external_objects: std.ArrayList(ExternalObject) = .empty, - -const EntryPointDecl = struct { - nav: InternPool.Nav.Index, - name: []const u8, - cc: std.builtin.CallingConvention, -}; - -const ExternalObject = struct { - instructions: []const Word, - id_bound: u32, -}; - -pub fn createEmpty( - arena: Allocator, - comp: *Compilation, - emit: Path, - options: link.File.OpenOptions, -) !*Linker { - const io = comp.io; - const target = &comp.root_mod.resolved_target.result; - - assert(!comp.config.use_lld); // Caught by Compilation.Config.resolve - assert(!comp.config.use_llvm); // Caught by Compilation.Config.resolve - assert(target.ofmt == .spirv); // Caught by Compilation.Config.resolve - switch (target.cpu.arch) { - .spirv32, .spirv64 => {}, - else => unreachable, // Caught by Compilation.Config.resolve. - } - switch (target.os.tag) { - .opencl, .opengl, .vulkan => {}, - else => unreachable, // Caught by Compilation.Config.resolve. - } - - const linker = try arena.create(Linker); - linker.* = .{ - .base = .{ - .tag = .spirv, - .comp = comp, - .emit = emit, - .gc_sections = options.gc_sections orelse false, - .print_gc_sections = options.print_gc_sections, - .stack_size = options.stack_size orelse 0, - .allow_shlib_undefined = options.allow_shlib_undefined orelse false, - .file = null, - .build_id = options.build_id, - }, - }; - errdefer linker.deinit(); - - linker.base.file = try emit.root_dir.handle.createFile(io, emit.sub_path, .{ - .truncate = true, - .read = true, - }); - - return linker; -} - -pub fn open( - arena: Allocator, - comp: *Compilation, - emit: Path, - options: link.File.OpenOptions, -) !*Linker { - return createEmpty(arena, comp, emit, options); -} - -pub fn deinit(linker: *Linker) void { - const gpa = linker.base.comp.gpa; - for (linker.fragments.values()) |*mir| { - mir.deinit(gpa); - } - linker.fragments.deinit(gpa); - linker.pending_navs.deinit(gpa); - linker.entry_points.deinit(gpa); - for (linker.external_objects.items) |obj| { - gpa.free(obj.instructions); - } - linker.external_objects.deinit(gpa); -} - -pub fn loadInput(linker: *Linker, input: link.Input) !void { - switch (input) { - .object => |obj| { - const comp = linker.base.comp; - const gpa = comp.gpa; - const io = comp.io; - const diags = &comp.link_diags; - - const stat = obj.file.stat(io) catch |err| - return diags.fail("failed to stat SPIR-V object '{f}': {t}", .{ obj.path, err }); - const file_size = std.math.cast(usize, stat.size) orelse - return diags.fail("SPIR-V object '{f}' is too large", .{obj.path}); - if (file_size < 5 * @sizeOf(Word)) - return diags.fail("SPIR-V object '{f}' is too small to contain a valid header", .{obj.path}); - if (file_size % @sizeOf(Word) != 0) - return diags.fail("SPIR-V object '{f}' size is not a multiple of the word size", .{obj.path}); - - const word_count = file_size / @sizeOf(Word); - const all_words = try gpa.alloc(Word, word_count); - defer gpa.free(all_words); - - const bytes = std.mem.sliceAsBytes(all_words); - const n_read = obj.file.readPositionalAll(io, bytes, 0) catch |err| - return diags.fail("failed to read SPIR-V object '{f}': {t}", .{ obj.path, err }); - if (n_read != bytes.len) - return diags.fail("SPIR-V object '{f}': incomplete read", .{obj.path}); - - const needs_swap = all_words[0] == @byteSwap(spec.magic_number); - if (needs_swap) { - for (all_words) |*w| w.* = @byteSwap(w.*); - } - - if (all_words[0] != spec.magic_number) - return diags.fail("SPIR-V object '{f}': invalid magic number", .{obj.path}); - - const id_bound = all_words[3]; - const instructions = try gpa.dupe(Word, all_words[5..]); - errdefer gpa.free(instructions); - - // OpCapability instructions appear at the top of the module - // so we can stop scanning as soon as we hit anything else. - var it: BinaryModule.Instruction.Iterator = .init(instructions, 0); - const has_linkage = while (it.next()) |inst| switch (inst.opcode) { - .OpCapability => { - const cap: spec.Capability = @fromBackingInt(@intCast(inst.operands[0])); - if (cap == .linkage) break true; - }, - else => break false, - } else false; - if (!has_linkage) { - return diags.fail("SPIR-V object '{f}' is missing the Linkage capability and cannot be linked", .{obj.path}); - } - - try linker.external_objects.append(gpa, .{ - .instructions = instructions, - .id_bound = id_bound, - }); - }, - else => { - const diags = &linker.base.comp.link_diags; - return diags.fail("unsupported link input for SPIR-V target", .{}); - }, - } -} - -pub fn updateFunc( - linker: *Linker, - pt: Zcu.PerThread, - func_index: InternPool.Index, - mir: *codegen.AnyMir, -) !void { - const gpa = linker.base.comp.gpa; - const nav = pt.zcu.funcInfo(func_index).owner_nav; - - if (linker.fragments.getPtr(nav)) |existing| { - existing.deinit(gpa); - } - - try linker.fragments.put(gpa, nav, mir.spirv); - mir.spirv = .{ - .extended_instruction_set = &.{}, - .globals = &.{}, - .functions = &.{}, - .annotations = &.{}, - .debug_names = &.{}, - .debug_strings = &.{}, - .execution_modes = &.{}, - .id_bound = 0, - .owner_nav = mir.spirv.owner_nav, - .kind = mir.spirv.kind, - .decl_result_id = .none, - .nav_refs = &.{}, - .uav_refs = &.{}, - .decl_deps = &.{}, - .internal_globals = &.{}, - .entry_points = &.{}, - }; -} - -pub fn updateNav(linker: *Linker, pt: Zcu.PerThread, nav: InternPool.Nav.Index) link.Error!void { - const ip = &pt.zcu.intern_pool; - log.debug("deferring nav {f}({d}) to flush", .{ ip.getNav(nav).fqn.fmt(ip), nav }); - - const gpa = linker.base.comp.gpa; - linker.pending_navs.append(gpa, nav) catch return error.OutOfMemory; -} - -pub fn updateExports( - linker: *Linker, - pt: Zcu.PerThread, - export_indices: []const Zcu.Export.Index, -) link.Error!void { - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const gpa = linker.base.comp.gpa; - for (export_indices) |exp_index| { - const exp = exp_index.ptr(zcu); - const nav_index = switch (exp.exported) { - .nav => |nav| nav, - .uav => @panic("TODO: implement Linker linker code for exporting a constant value"), - }; - const nav_ty = ip.getNav(nav_index).resolved.?.type; - if (ip.isFunctionType(nav_ty)) { - const cc = Type.fromInterned(nav_ty).fnCallingConvention(zcu); - try linker.entry_points.append(gpa, .{ - .nav = nav_index, - .name = exp.opts.name.toSlice(ip), - .cc = cc, - }); - } - } -} - -pub fn flush( - linker: *Linker, - arena: Allocator, - tid: Zcu.PerThread.Id, - prog_node: std.Progress.Node, -) link.Error!void { - const sub_prog_node = prog_node.start("Flush Module", 0); - defer sub_prog_node.end(); - - const comp = linker.base.comp; - const diags = &comp.link_diags; - const gpa = comp.gpa; - const io = comp.io; - - if (comp.zcu) |zcu| { - const active = zcu.activate(tid); - defer active.deactivate(); - const pt = active.pt; - for (linker.pending_navs.items) |nav| { - if (linker.fragments.contains(nav)) continue; - - const mir = CodeGen.generateNav(pt, nav) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.AlreadyReported => continue, - error.Canceled => return error.Canceled, - }; - - linker.fragments.put(gpa, nav, mir) catch return error.OutOfMemory; - } - linker.pending_navs.clearRetainingCapacity(); - } - - const merged = mergeFragments(linker, gpa, arena) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - }; - - var binary = linkModule(arena, merged.words, merged.id_bound, sub_prog_node) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - else => |other| { - return diags.fail("error while linking: {s}", .{@errorName(other)}); - }, - }; - defer binary.deinit(arena); - - const header = [_]Word{ - spec.magic_number, - merged.version.toWord(), - merged.generator_id, - binary.id_bound, - 0, - }; - - var file_writer = linker.base.file.?.writer(io, &.{}); - file_writer.interface.writeSliceEndian(Word, &header, .little) catch |err| switch (err) { - error.WriteFailed => return diags.fail("failed to write: {t}", .{file_writer.err.?}), - }; - file_writer.interface.writeSliceEndian(Word, binary.instructions, .little) catch |err| switch (err) { - error.WriteFailed => return diags.fail("failed to write: {t}", .{file_writer.err.?}), - }; - file_writer.end() catch |err| switch (err) { - error.WriteFailed => return diags.fail("failed to write: {t}", .{file_writer.err.?}), - else => |e| return diags.fail("failed to write: {t}", .{e}), - }; -} - -fn linkModule(arena: Allocator, words: []const Word, id_bound: u32, progress: std.Progress.Node) !BinaryModule { - var parser = try BinaryModule.Parser.init(arena); - defer parser.deinit(); - var binary = try parser.initFromWords(words, id_bound); - try prune_unused.run(&parser, &binary); - try dedup_types.run(&parser, &binary); - try lower_invocation_globals.run(&parser, &binary, progress); - return binary; -} - -fn mergeFragments(linker: *Linker, gpa: Allocator, arena: Allocator) error{OutOfMemory}!MergedModule { - const comp = linker.base.comp; - const target = &comp.root_mod.resolved_target.result; - const maybe_ip: ?*InternPool = if (comp.zcu) |zcu| &zcu.intern_pool else null; - const is_obj = comp.config.output_mode == .Obj; - - var next_id: Word = 1; - - var nav_final_ids: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Id) = .empty; - defer nav_final_ids.deinit(gpa); - - var uav_final_ids: std.AutoHashMapUnmanaged(struct { InternPool.Index, spec.StorageClass }, Id) = .empty; - defer uav_final_ids.deinit(gpa); - - var frag_infos: std.ArrayList(FragmentInfo) = .empty; - defer frag_infos.deinit(gpa); - try frag_infos.ensureTotalCapacity(gpa, @intCast(linker.fragments.count())); - - for (linker.fragments.keys(), linker.fragments.values()) |nav, *mir| { - const id_offset = next_id - 1; - frag_infos.appendAssumeCapacity(.{ .id_offset = id_offset }); - if (mir.decl_result_id != .none) { - try nav_final_ids.put(gpa, nav, @fromBackingInt(@intCast(@backingInt(mir.decl_result_id) + id_offset))); - } - next_id += mir.id_bound - 1; - } - - for (linker.fragments.values(), frag_infos.items) |*mir, frag_info| { - for (mir.nav_refs) |ref| { - if (!nav_final_ids.contains(ref.nav)) { - try nav_final_ids.put(gpa, ref.nav, @fromBackingInt(@intCast(@backingInt(ref.local_id) + frag_info.id_offset))); - } - } - for (mir.uav_refs) |ref| { - const key = .{ ref.val, ref.storage_class }; - if (!uav_final_ids.contains(key)) { - try uav_final_ids.put(gpa, key, @fromBackingInt(@intCast(@backingInt(ref.local_id) + frag_info.id_offset))); - } - } - } - - // Resolve Zig extern navs against external objects. - var ext_id_offsets: std.ArrayList(Word) = .empty; - defer ext_id_offsets.deinit(gpa); - try ext_id_offsets.ensureTotalCapacity(gpa, linker.external_objects.items.len); - - var unresolved_extern_count: u32 = 0; - var resolved_ids: std.array_hash_map.Auto(Id, void) = .empty; - defer resolved_ids.deinit(gpa); - - if (maybe_ip) |ip| { - var extern_name_map: std.array_hash_map.String(InternPool.Nav.Index) = .empty; - defer extern_name_map.deinit(gpa); - - var nav_it = nav_final_ids.iterator(); - while (nav_it.next()) |entry| { - const nav = ip.getNav(entry.key_ptr.*); - if (!nav.resolved.?.is_extern_decl) continue; - const name = if (nav.getExtern(ip)) |e| e.name.toSlice(ip) else nav.fqn.toSlice(ip); - try extern_name_map.put(gpa, name, entry.key_ptr.*); - } - - for (linker.external_objects.items) |ext_obj| { - const id_offset = next_id - 1; - ext_id_offsets.appendAssumeCapacity(id_offset); - - var it: BinaryModule.Instruction.Iterator = .init(ext_obj.instructions, 0); - while (it.next()) |inst| { - const ld = LinkageDecoration.parse(inst) orelse continue; - if (ld.linkage_type != .@"export") continue; - const remapped_id: Id = @fromBackingInt(@intCast(@backingInt(ld.target_id) + id_offset)); - - if (extern_name_map.get(ld.name)) |nav_index| { - log.debug("extern resolve: '{s}' -> ext_fn_id={d}", .{ ld.name, @backingInt(remapped_id) }); - nav_final_ids.getPtr(nav_index).?.* = remapped_id; - _ = extern_name_map.swapRemove(ld.name); - try resolved_ids.put(gpa, remapped_id, {}); - } - } - next_id += ext_obj.id_bound - 1; - } - - unresolved_extern_count = @intCast(extern_name_map.count()); - } else { - for (linker.external_objects.items) |ext_obj| { - ext_id_offsets.appendAssumeCapacity(next_id - 1); - next_id += ext_obj.id_bound - 1; - } - } - - var parser = BinaryModule.Parser.init(gpa) catch return error.OutOfMemory; - defer parser.deinit(); - var sections: Sections = .{}; - defer sections.deinit(gpa); - - try mergeZigFragments(linker, gpa, &parser, §ions, frag_infos.items, &nav_final_ids, &uav_final_ids, &resolved_ids, maybe_ip); - - var has_linkage = false; - try appendExternalObjects(linker, gpa, &parser, ext_id_offsets.items, §ions, &has_linkage, linker.fragments.count() == 0, is_obj, &resolved_ids); - - if (is_obj) { - for (linker.entry_points.items) |ep| { - if (ep.cc != .spirv_device) continue; - const final_id = nav_final_ids.get(ep.nav) orelse continue; - try sections.annotations.emit(gpa, .OpDecorate, .{ - .target = final_id, - .decoration = .{ .linkage_attributes = .{ .name = ep.name, .linkage_type = .@"export" } }, - }); - has_linkage = true; - } - if (unresolved_extern_count > 0) has_linkage = true; - } - - var capabilities_section = Section{}; - defer capabilities_section.deinit(gpa); - var extensions_section = Section{}; - defer extensions_section.deinit(gpa); - var memory_model_section = Section{}; - defer memory_model_section.deinit(gpa); - - try emitPreamble( - gpa, - target, - has_linkage, - &capabilities_section, - &extensions_section, - &memory_model_section, - ); - try emitEntryPoints( - linker, - gpa, - target, - §ions.entry_points, - §ions.execution_modes, - &nav_final_ids, - &uav_final_ids, - &frag_infos, - ); - - const zig_packed_version = (zig_version.major << 12) | (zig_version.minor << 7) | zig_version.patch; - if (maybe_ip) |ip| { - try emitSourceInfo(gpa, ip, zig_packed_version, §ions.debug_strings); - } - - const version: spec.Version = .{ - .major = 1, - .minor = blk: { - if (target.cpu.has(.spirv, .v1_6)) break :blk 6; - if (target.cpu.has(.spirv, .v1_5)) break :blk 5; - if (target.cpu.has(.spirv, .v1_4)) break :blk 4; - if (target.cpu.has(.spirv, .v1_3)) break :blk 3; - if (target.cpu.has(.spirv, .v1_2)) break :blk 2; - if (target.cpu.has(.spirv, .v1_1)) break :blk 1; - break :blk 0; - }, - }; - - const buffers = &[_][]const Word{ - capabilities_section.toWords(), - extensions_section.toWords(), - sections.ext_inst.toWords(), - memory_model_section.toWords(), - sections.entry_points.toWords(), - sections.execution_modes.toWords(), - sections.debug_strings.toWords(), - sections.debug_names.toWords(), - sections.annotations.toWords(), - sections.globals.toWords(), - sections.functions.toWords(), - }; - - var total_size: usize = 0; - for (buffers) |buffer| total_size += buffer.len; - const result = try arena.alloc(Word, total_size); - - var offset: usize = 0; - for (buffers) |buffer| { - @memcpy(result[offset..][0..buffer.len], buffer); - offset += buffer.len; - } - - return .{ - .words = result, - .id_bound = next_id, - .version = version, - .generator_id = (spec.zig_generator_id << 16) | zig_packed_version, - }; -} - -fn mergeZigFragments( - linker: *Linker, - gpa: Allocator, - parser: *BinaryModule.Parser, - sections: *Sections, - frag_infos: []const FragmentInfo, - nav_final_ids: *const std.AutoHashMapUnmanaged(InternPool.Nav.Index, Id), - uav_final_ids: *const std.AutoHashMapUnmanaged(struct { InternPool.Index, spec.StorageClass }, Id), - resolved_ids: *const std.array_hash_map.Auto(Id, void), - maybe_ip: ?*InternPool, -) error{OutOfMemory}!void { - for (linker.fragments.values(), frag_infos) |*mir, frag_info| { - var id_remap: std.AutoHashMapUnmanaged(Id, Id) = .empty; - defer id_remap.deinit(gpa); - - var resolved_local_ids: std.array_hash_map.Auto(Id, void) = .empty; - defer resolved_local_ids.deinit(gpa); - - for (mir.nav_refs) |ref| { - if (nav_final_ids.get(ref.nav)) |final_id| { - try id_remap.put(gpa, ref.local_id, final_id); - if (maybe_ip) |ip| { - const nav = ip.getNav(ref.nav); - if (nav.resolved.?.is_extern_decl and resolved_ids.contains(final_id)) { - try resolved_local_ids.put(gpa, ref.local_id, {}); - } - } - } - } - for (mir.uav_refs) |ref| { - if (uav_final_ids.get(.{ ref.val, ref.storage_class })) |final_id| { - try id_remap.put(gpa, ref.local_id, final_id); - } - } - - try remapAndAppend(gpa, §ions.ext_inst, mir.extended_instruction_set, frag_info.id_offset, &id_remap, parser); - try remapAndAppend(gpa, §ions.globals, mir.globals, frag_info.id_offset, &id_remap, parser); - - try remapFilteredInsts(gpa, §ions.functions, mir.functions, frag_info.id_offset, &id_remap, parser, &resolved_local_ids, .skip_functions); - try remapFilteredInsts(gpa, §ions.annotations, mir.annotations, frag_info.id_offset, &id_remap, parser, &resolved_local_ids, .skip_linkage); - try remapFilteredInsts(gpa, §ions.debug_names, mir.debug_names, frag_info.id_offset, &id_remap, parser, &resolved_local_ids, .skip_names); - try remapAndAppend(gpa, §ions.debug_strings, mir.debug_strings, frag_info.id_offset, &id_remap, parser); - try remapAndAppend(gpa, §ions.execution_modes, mir.execution_modes, frag_info.id_offset, &id_remap, parser); - - for (mir.entry_points) |ep| { - try linker.entry_points.append(gpa, .{ .nav = mir.owner_nav, .name = ep.name, .cc = ep.cc }); - } - } -} - -const FilterMode = enum { skip_functions, skip_linkage, skip_names }; - -fn remapFilteredInsts( - gpa: Allocator, - dest: *Section, - words: []const Word, - id_offset: Word, - id_remap: *const std.AutoHashMapUnmanaged(Id, Id), - parser: *BinaryModule.Parser, - skip_ids: *const std.array_hash_map.Auto(Id, void), - mode: FilterMode, -) error{OutOfMemory}!void { - if (words.len == 0) return; - var it: BinaryModule.Instruction.Iterator = .init(words, 0); - var skip_function = false; - while (it.next()) |inst| { - switch (mode) { - .skip_functions => { - if (inst.opcode == .OpFunction) { - skip_function = skip_ids.contains(@fromBackingInt(@intCast(inst.operands[1]))); - } - if (skip_function) { - if (inst.opcode == .OpFunctionEnd) skip_function = false; - continue; - } - }, - .skip_linkage => { - if (LinkageDecoration.parse(inst)) |ld| { - if (skip_ids.contains(ld.target_id)) continue; - } - }, - .skip_names => { - if (inst.opcode == .OpName and inst.operands.len >= 1) { - if (skip_ids.contains(@fromBackingInt(@intCast(inst.operands[0])))) continue; - } - }, - } - try remapAndAppendInst(gpa, dest, words, inst, id_offset, id_remap, parser); - } -} - -fn emitPreamble( - gpa: Allocator, - target: *const std.Target, - has_linkage: bool, - capabilities: *Section, - extensions: *Section, - memory_model: *Section, -) !void { - var caps: std.EnumSet(spec.Capability) = .empty; - var exts: std.StringHashMapUnmanaged(void) = .empty; - defer exts.deinit(gpa); - - switch (target.os.tag) { - .opengl, .vulkan => caps.insert(.shader), - .opencl, .amdhsa => { - caps.insert(.kernel); - caps.insert(.addresses); - }, - else => unreachable, - } - if (target.cpu.arch == .spirv64) { - caps.insert(.int64); - if (target.os.tag == .vulkan) { - caps.insert(.physical_storage_buffer_addresses); - try exts.put(gpa, "SPV_KHR_physical_storage_buffer", {}); - } - } - if (has_linkage) caps.insert(.linkage); - - inline for (@typeInfo(spec.Capability).@"enum".field_names) |cap_name| { - if (@hasField(std.Target.spirv.Feature, cap_name)) { - if (target.cpu.has(.spirv, @field(std.Target.spirv.Feature, cap_name))) - caps.insert(@field(spec.Capability, cap_name)); - } - } - inline for (@typeInfo(spec.Extension).@"enum".field_names) |ext_name| { - switch (@field(spec.Extension, ext_name)) { - .v1_0, .v1_1, .v1_2, .v1_3, .v1_4, .v1_5, .v1_6 => {}, - else => if (@hasField(std.Target.spirv.Feature, ext_name)) { - if (target.cpu.has(.spirv, @field(std.Target.spirv.Feature, ext_name))) - try exts.put(gpa, ext_name, {}); - }, - } - } - - var cit = caps.iterator(); - while (cit.next()) |cap| try capabilities.emit(gpa, .OpCapability, .{ .capability = cap }); - var eit = exts.iterator(); - while (eit.next()) |e| try extensions.emit(gpa, .OpExtension, .{ .name = e.key_ptr.* }); - - const addressing_model: spec.AddressingModel = switch (target.os.tag) { - .opengl => .logical, - .vulkan => switch (target.cpu.arch) { - .spirv32 => .logical, - .spirv64 => .physical_storage_buffer64, - else => unreachable, - }, - .opencl => switch (target.cpu.arch) { - .spirv32 => .physical32, - .spirv64 => .physical64, - else => unreachable, - }, - .amdhsa => .physical64, - else => unreachable, - }; - try memory_model.emit(gpa, .OpMemoryModel, .{ - .addressing_model = addressing_model, - .memory_model = switch (target.os.tag) { - .opencl => .open_cl, - .vulkan, .opengl => .glsl450, - .amdhsa => unreachable, // TODO - else => unreachable, - }, - }); -} - -fn emitEntryPoints( - linker: *Linker, - gpa: Allocator, - target: *const std.Target, - entry_points_section: *Section, - execution_modes_section: *Section, - nav_final_ids: *const std.AutoHashMapUnmanaged(InternPool.Nav.Index, Id), - uav_final_ids: *const std.AutoHashMapUnmanaged(struct { InternPool.Index, spec.StorageClass }, Id), - frag_infos: *const std.ArrayList(FragmentInfo), -) error{OutOfMemory}!void { - for (linker.entry_points.items) |ep| { - const final_id = nav_final_ids.get(ep.nav) orelse continue; - - var interface: std.ArrayList(Id) = .empty; - defer interface.deinit(gpa); - var visited: std.AutoHashMapUnmanaged(InternPool.Nav.Index, void) = .empty; - defer visited.deinit(gpa); - try collectEntryPointInterface(linker, ep.nav, &interface, &visited, nav_final_ids, uav_final_ids, frag_infos, gpa); - - const exec_model: spec.ExecutionModel = switch (target.os.tag) { - .vulkan, .opengl => switch (ep.cc) { - .spirv_vertex => .vertex, - .spirv_fragment => .fragment, - .spirv_kernel => .gl_compute, - .spirv_task => .task_ext, - .spirv_mesh => .mesh_ext, - .spirv_device => continue, - else => unreachable, - }, - .opencl => switch (ep.cc) { - .spirv_kernel => .kernel, - .spirv_device => continue, - else => unreachable, - }, - else => unreachable, - }; - - try entry_points_section.emit(gpa, .OpEntryPoint, .{ - .execution_model = exec_model, - .entry_point = final_id, - .name = ep.name, - .interface = interface.items, - }); - - switch (ep.cc) { - .spirv_kernel, .spirv_task => |kernel| { - try execution_modes_section.emit(gpa, .OpExecutionMode, .{ - .entry_point = final_id, - .mode = .{ .local_size = .{ .x_size = kernel.x, .y_size = kernel.y, .z_size = kernel.z } }, - }); - }, - .spirv_fragment => |fragment| { - try execution_modes_section.emit(gpa, .OpExecutionMode, .{ - .entry_point = final_id, - .mode = if (target.os.tag == .vulkan) .origin_upper_left else .origin_lower_left, - }); - if (fragment.pixel_centered_integer) { - try execution_modes_section.emit(gpa, .OpExecutionMode, .{ - .entry_point = final_id, - .mode = .pixel_center_integer, - }); - } - const exec_mode: ?spec.ExecutionMode.Extended = switch (fragment.depth_assumption) { - .none => null, - .greater => .depth_greater, - .less => .depth_less, - .unchanged => .depth_unchanged, - }; - if (exec_mode) |mode| { - try execution_modes_section.emit(gpa, .OpExecutionMode, .{ - .entry_point = final_id, - .mode = mode, - }); - } - }, - .spirv_mesh => |mesh| { - try execution_modes_section.emit(gpa, .OpExecutionMode, .{ - .entry_point = final_id, - .mode = .{ .output_vertices = .{ .vertex_count = mesh.max_vertices } }, - }); - try execution_modes_section.emit(gpa, .OpExecutionMode, .{ - .entry_point = final_id, - .mode = .{ .output_primitives_ext = .{ .primitive_count = mesh.max_primitives } }, - }); - try execution_modes_section.emit(gpa, .OpExecutionMode, .{ - .entry_point = final_id, - .mode = .{ .local_size = .{ .x_size = mesh.x, .y_size = mesh.y, .z_size = mesh.z } }, - }); - try execution_modes_section.emit(gpa, .OpExecutionMode, .{ - .entry_point = final_id, - .mode = switch (mesh.stage_output) { - .output_points => .output_points, - .output_lines => .output_lines_ext, - .output_triangles => .output_triangles_ext, - }, - }); - }, - else => {}, - } - } -} - -fn emitSourceInfo(gpa: Allocator, ip: *InternPool, version: u32, debug_strings: *Section) error{OutOfMemory}!void { - var error_info: std.Io.Writer.Allocating = .init(gpa); - defer error_info.deinit(); - error_info.writer.writeAll("zig_errors:") catch return error.OutOfMemory; - for (ip.global_error_set.getNamesFromMainThread()) |name| { - error_info.writer.writeByte(':') catch return error.OutOfMemory; - std.Uri.Component.percentEncode( - &error_info.writer, - name.toSlice(ip), - struct { - fn isValidChar(c: u8) bool { - return switch (c) { - 0, '%', ':' => false, - else => true, - }; - } - }.isValidChar, - ) catch return error.OutOfMemory; - } - try debug_strings.emit(gpa, .OpSourceExtension, .{ .extension = error_info.written() }); - try debug_strings.emit(gpa, .OpSource, .{ .source_language = .zig, .version = version, .file = null, .source = null }); -} - -const MergedModule = struct { - words: []const Word, - id_bound: Word, - version: spec.Version, - generator_id: u32, -}; - -const FragmentInfo = struct { - id_offset: Word, -}; - -const LinkageDecoration = struct { - target_id: Id, - name: []const u8, - linkage_type: spec.LinkageType, - - fn parse(inst: BinaryModule.Instruction) ?LinkageDecoration { - if (inst.opcode != .OpDecorate) return null; - if (inst.operands.len < 3) return null; - if (inst.operands[1] != @backingInt(spec.Decoration.linkage_attributes)) return null; - return .{ - .target_id = @fromBackingInt(@intCast(inst.operands[0])), - .name = std.mem.sliceTo(std.mem.sliceAsBytes(inst.operands[2 .. inst.operands.len - 1]), 0), - .linkage_type = @fromBackingInt(@intCast(inst.operands[inst.operands.len - 1])), - }; - } -}; - -const Sections = struct { - ext_inst: Section = .{}, - globals: Section = .{}, - functions: Section = .{}, - annotations: Section = .{}, - debug_names: Section = .{}, - debug_strings: Section = .{}, - entry_points: Section = .{}, - execution_modes: Section = .{}, - - fn deinit(self: *Sections, gpa: Allocator) void { - self.ext_inst.deinit(gpa); - self.globals.deinit(gpa); - self.functions.deinit(gpa); - self.annotations.deinit(gpa); - self.debug_names.deinit(gpa); - self.debug_strings.deinit(gpa); - self.entry_points.deinit(gpa); - self.execution_modes.deinit(gpa); - } - - const SectionClass = enum { ext_inst, debug_name, debug_string, annotation, global }; - - fn classifyPreambleInst(opcode: spec.Opcode) SectionClass { - return switch (opcode) { - .OpExtInstImport => .ext_inst, - .OpName, .OpMemberName => .debug_name, - .OpString => .debug_string, - .OpDecorate, - .OpMemberDecorate, - .OpGroupDecorate, - .OpGroupMemberDecorate, - .OpDecorationGroup, - .OpDecorateId, - .OpDecorateString, - .OpMemberDecorateString, - => .annotation, - else => .global, - }; - } - - fn getSection(self: *Sections, class: SectionClass) *Section { - return switch (class) { - .ext_inst => &self.ext_inst, - .debug_name => &self.debug_names, - .debug_string => &self.debug_strings, - .annotation => &self.annotations, - .global => &self.globals, - }; - } -}; - -fn appendExternalObjects( - linker: *Linker, - gpa: Allocator, - parser: *BinaryModule.Parser, - ext_id_offsets: []const Word, - sections: *Sections, - has_linkage: *bool, - keep_entry_points: bool, - is_obj: bool, - resolved_ids: *const std.array_hash_map.Auto(Id, void), -) error{OutOfMemory}!void { - var export_map: std.array_hash_map.String(Id) = .empty; - defer export_map.deinit(gpa); - - for (linker.external_objects.items, ext_id_offsets) |ext_obj, id_offset| { - var it: BinaryModule.Instruction.Iterator = .init(ext_obj.instructions, 0); - while (it.next()) |inst| { - const ld = LinkageDecoration.parse(inst) orelse continue; - if (ld.linkage_type != .@"export") continue; - try export_map.put(gpa, ld.name, @fromBackingInt(@intCast(@backingInt(ld.target_id) + id_offset))); - } - } - - var per_obj_remaps = try gpa.alloc(std.AutoHashMapUnmanaged(Id, Id), linker.external_objects.items.len); - defer { - for (per_obj_remaps) |*m| m.deinit(gpa); - gpa.free(per_obj_remaps); - } - for (per_obj_remaps) |*m| m.* = .empty; - - var resolved_linkage_ids: std.array_hash_map.Auto(Id, void) = .empty; - defer resolved_linkage_ids.deinit(gpa); - - for (resolved_ids.keys()) |id| { - try resolved_linkage_ids.put(gpa, id, {}); - } - - for (linker.external_objects.items, ext_id_offsets, 0..) |ext_obj, id_offset, obj_idx| { - var it: BinaryModule.Instruction.Iterator = .init(ext_obj.instructions, 0); - while (it.next()) |inst| { - const ld = LinkageDecoration.parse(inst) orelse continue; - if (ld.linkage_type != .import) continue; - const remapped_import: Id = @fromBackingInt(@intCast(@backingInt(ld.target_id) + id_offset)); - - if (export_map.get(ld.name)) |export_id| { - try per_obj_remaps[obj_idx].put(gpa, ld.target_id, export_id); - try resolved_linkage_ids.put(gpa, remapped_import, {}); - try resolved_linkage_ids.put(gpa, export_id, {}); - log.debug("cross-object resolve: '{s}' import={d} -> export={d}", .{ - ld.name, @backingInt(remapped_import), @backingInt(export_id), - }); - } else { - has_linkage.* = true; - } - } - } - - for (linker.external_objects.items, ext_id_offsets, 0..) |ext_obj, id_offset, obj_idx| { - var binary = parser.initFromWords(ext_obj.instructions, ext_obj.id_bound) catch - return error.OutOfMemory; - defer binary.deinit(gpa); - - const id_remap = &per_obj_remaps[obj_idx]; - - var preamble_it: BinaryModule.Instruction.Iterator = .init(ext_obj.instructions, 0); - while (preamble_it.next()) |inst| { - if (inst.offset >= binary.functions_start) break; - - switch (inst.opcode) { - .OpCapability, - .OpExtension, - .OpMemoryModel, - .OpSource, - .OpSourceExtension, - .OpSourceContinued, - => continue, - .OpEntryPoint => { - if (keep_entry_points) - try remapAndAppendInst(gpa, §ions.entry_points, ext_obj.instructions, inst, id_offset, id_remap, parser); - continue; - }, - .OpExecutionMode, .OpExecutionModeId => { - if (keep_entry_points) - try remapAndAppendInst(gpa, §ions.execution_modes, ext_obj.instructions, inst, id_offset, id_remap, parser); - continue; - }, - else => {}, - } - - if (LinkageDecoration.parse(inst)) |ld| { - const remapped: Id = @fromBackingInt(@intCast(@backingInt(ld.target_id) + id_offset)); - if (resolved_linkage_ids.contains(remapped)) { - if (ld.linkage_type == .@"export" and is_obj) { - has_linkage.* = true; - } else { - continue; - } - } - } - - if (inst.opcode == .OpName and inst.operands.len >= 1) { - if (id_remap.contains(@fromBackingInt(@intCast(inst.operands[0])))) continue; - } - - const dest = sections.getSection(Sections.classifyPreambleInst(inst.opcode)); - try remapAndAppendInst(gpa, dest, ext_obj.instructions, inst, id_offset, id_remap, parser); - } - - var fn_it: BinaryModule.Instruction.Iterator = .init(ext_obj.instructions, binary.functions_start); - var skip_function = false; - while (fn_it.next()) |inst| { - if (inst.opcode == .OpFunction) { - skip_function = id_remap.contains(@fromBackingInt(@intCast(inst.operands[1]))); - } - if (!skip_function) { - try remapAndAppendInst(gpa, §ions.functions, ext_obj.instructions, inst, id_offset, id_remap, parser); - } - if (inst.opcode == .OpFunctionEnd) { - skip_function = false; - } - } - } -} - -fn collectEntryPointInterface( - linker: *Linker, - nav: InternPool.Nav.Index, - interface: *std.ArrayList(Id), - visited: *std.AutoHashMapUnmanaged(InternPool.Nav.Index, void), - nav_final_ids: *const std.AutoHashMapUnmanaged(InternPool.Nav.Index, Id), - uav_final_ids: *const std.AutoHashMapUnmanaged(struct { InternPool.Index, spec.StorageClass }, Id), - frag_infos: *const std.ArrayList(FragmentInfo), - gpa: Allocator, -) error{OutOfMemory}!void { - const visited_gop = try visited.getOrPut(gpa, nav); - if (visited_gop.found_existing) return; - - const frag_index = linker.fragments.getIndex(nav) orelse return; - const mir = &linker.fragments.values()[frag_index]; - const id_offset = frag_infos.items[frag_index].id_offset; - - if (mir.kind == .global) { - if (nav_final_ids.get(nav)) |final_id| { - try interface.append(gpa, final_id); - } - } - - for (mir.uav_refs) |ref| { - if (ref.kind == .global) { - if (uav_final_ids.get(.{ ref.val, ref.storage_class })) |final_id| { - try interface.append(gpa, final_id); - } - } - } - - for (mir.internal_globals) |local_id| { - const global_id: Id = @fromBackingInt(@intCast(@backingInt(local_id) + id_offset)); - try interface.append(gpa, global_id); - } - - for (mir.decl_deps) |dep| { - try collectEntryPointInterface(linker, dep.nav, interface, visited, nav_final_ids, uav_final_ids, frag_infos, gpa); - } -} - -fn remapAndAppend( - gpa: Allocator, - dest: *Section, - words: []const Word, - id_offset: Word, - id_remap: *const std.AutoHashMapUnmanaged(Id, Id), - parser: *BinaryModule.Parser, -) error{OutOfMemory}!void { - if (words.len == 0) return; - - try dest.instructions.ensureUnusedCapacity(gpa, words.len); - - var it: BinaryModule.Instruction.Iterator = .init(words, 0); - while (it.next()) |inst| { - try remapAndAppendInst(gpa, dest, words, inst, id_offset, id_remap, parser); - } -} - -fn remapAndAppendInst( - gpa: Allocator, - dest: *Section, - words: []const Word, - inst: BinaryModule.Instruction, - id_offset: Word, - id_remap: *const std.AutoHashMapUnmanaged(Id, Id), - parser: *BinaryModule.Parser, -) error{OutOfMemory}!void { - const inst_words = words[inst.offset..][0..((words[inst.offset] >> 16))]; - try dest.instructions.ensureUnusedCapacity(gpa, inst_words.len); - const dest_start = dest.instructions.items.len; - dest.instructions.appendSliceAssumeCapacity(inst_words); - const inst_slice = dest.instructions.items[dest_start..][0..inst_words.len]; - - const inst_spec = parser.getInstSpec(inst.opcode) orelse return; - var offset: usize = 0; - for (inst_spec.operands) |operand| { - const cat = operand.kind.category(); - switch (operand.quantifier) { - .required, .optional => { - if (offset >= inst.operands.len) break; - offset += remapOperand(operand.kind, cat, inst, inst_slice, offset, id_offset, id_remap); - }, - .variadic => { - while (offset < inst.operands.len) { - offset += remapOperand(operand.kind, cat, inst, inst_slice, offset, id_offset, id_remap); - } - }, - } - } -} - -fn remapOperand( - kind: spec.OperandKind, - cat: spec.OperandCategory, - inst: BinaryModule.Instruction, - inst_slice: []Word, - offset: usize, - id_offset: Word, - id_remap: *const std.AutoHashMapUnmanaged(Id, Id), -) usize { - switch (cat) { - .id => { - remapSingleId(&inst_slice[1 + offset], id_offset, id_remap); - return 1; - }, - .literal => return operandLiteralWordCount(kind, inst, offset), - .composite => { - remapCompositeOperand(kind, inst_slice, offset, id_offset, id_remap); - return 2; - }, - .bit_enum => { - const mask = inst_slice[1 + offset]; - var consumed: usize = 1; - for (kind.enumerants()) |e| { - if ((mask & e.value) == 0) continue; - for (e.parameters) |param_kind| { - if (offset + consumed >= inst.operands.len) return consumed; - consumed += remapOperand( - param_kind, - param_kind.category(), - inst, - inst_slice, - offset + consumed, - id_offset, - id_remap, - ); - } - } - return consumed; - }, - .value_enum => { - const value = inst_slice[1 + offset]; - var consumed: usize = 1; - for (kind.enumerants()) |e| { - if (e.value != value) continue; - for (e.parameters) |param_kind| { - if (offset + consumed >= inst.operands.len) return consumed; - consumed += remapOperand( - param_kind, - param_kind.category(), - inst, - inst_slice, - offset + consumed, - id_offset, - id_remap, - ); - } - break; - } - return consumed; - }, - } -} - -fn remapCompositeOperand( - kind: spec.OperandKind, - inst_slice: []Word, - offset: usize, - id_offset: Word, - id_remap: *const std.AutoHashMapUnmanaged(Id, Id), -) void { - switch (kind) { - .pair_literal_integer_id_ref => { - remapSingleId(&inst_slice[1 + offset + 1], id_offset, id_remap); - }, - .pair_id_ref_literal_integer => { - remapSingleId(&inst_slice[1 + offset], id_offset, id_remap); - }, - .pair_id_ref_id_ref => { - remapSingleId(&inst_slice[1 + offset], id_offset, id_remap); - remapSingleId(&inst_slice[1 + offset + 1], id_offset, id_remap); - }, - else => {}, - } -} - -fn operandLiteralWordCount(kind: spec.OperandKind, inst: BinaryModule.Instruction, offset: usize) usize { - return switch (kind) { - .literal_integer, .literal_float => 1, - .literal_string => blk: { - var count: usize = 0; - var off = offset; - while (off < inst.operands.len) { - const word = inst.operands[off]; - count += 1; - off += 1; - if (word & 0xFF000000 == 0 or - word & 0x00FF0000 == 0 or - word & 0x0000FF00 == 0 or - word & 0x000000FF == 0) - { - break; - } - } - break :blk count; - }, - .literal_context_dependent_number => inst.operands.len - offset, - .literal_ext_inst_integer => 1, - else => 1, - }; -} - -fn remapSingleId(word: *Word, id_offset: Word, id_remap: *const std.AutoHashMapUnmanaged(Id, Id)) void { - const id: Id = @fromBackingInt(@intCast(word.*)); - if (id == .none) return; - if (id_remap.get(id)) |final_id| { - word.* = @backingInt(final_id); - } else { - word.* = @backingInt(id) + id_offset; - } -} diff --git a/src/link/SpirV/BinaryModule.zig b/src/link/SpirV/BinaryModule.zig deleted file mode 100644 index b19cf7c4a2b4249940ffdbe025c167aefe57b883..0000000000000000000000000000000000000000 --- a/src/link/SpirV/BinaryModule.zig +++ /dev/null @@ -1,347 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; -const log = std.log.scoped(.spirv_parse); - -const spec = @import("../../codegen/spirv/spec.zig"); -const Opcode = spec.Opcode; -const Word = spec.Word; -const InstructionSet = spec.InstructionSet; -const ResultId = spec.Id; - -const BinaryModule = @This(); - -/// The result-id bound of this SPIR-V module. -id_bound: u32, - -/// The instructions of this module (no header). -instructions: []const Word, - -/// Maps OpExtInstImport result-ids to their InstructionSet. -ext_inst_map: std.AutoHashMapUnmanaged(ResultId, InstructionSet), - -/// Width of arithmetic types (OpTypeInt/OpTypeFloat). Needed to correctly -/// parse operands of Op(Spec)Constant and OpSwitch. -arith_type_width: std.AutoHashMapUnmanaged(ResultId, u16), - -functions_start: usize, - -pub fn deinit(bm: *BinaryModule, gpa: Allocator) void { - bm.ext_inst_map.deinit(gpa); - bm.arith_type_width.deinit(gpa); - bm.* = undefined; -} - -pub fn iterateInstructions(bm: BinaryModule) Instruction.Iterator { - return Instruction.Iterator.init(bm.instructions, 0); -} - -pub fn iterateInstructionsFrom(bm: BinaryModule, offset: usize) Instruction.Iterator { - return Instruction.Iterator.init(bm.instructions, offset); -} - -pub const Instruction = struct { - pub const Iterator = struct { - words: []const Word, - offset: usize = 0, - - pub fn init(words: []const Word, start_offset: usize) Iterator { - return .{ .words = words, .offset = start_offset }; - } - - pub fn next(it: *Iterator) ?Instruction { - if (it.offset >= it.words.len) return null; - - const instruction_len = it.words[it.offset] >> 16; - defer it.offset += instruction_len; - assert(instruction_len != 0); - assert(it.offset < it.words.len); - - return Instruction{ - .opcode = @fromBackingInt(@intCast(it.words[it.offset] & 0xFFFF)), - .offset = it.offset, - .operands = it.words[it.offset..][1..instruction_len], - }; - } - }; - - opcode: Opcode, - offset: usize, - operands: []const Word, -}; - -pub const Parser = struct { - gpa: Allocator, - opcode_table: std.AutoHashMapUnmanaged(u32, u16) = .empty, - - pub fn init(gpa: Allocator) !Parser { - var parser = Parser{ .gpa = gpa }; - errdefer parser.deinit(); - - inline for (std.meta.tags(InstructionSet)) |set| { - const instructions = set.instructions(); - try parser.opcode_table.ensureUnusedCapacity(gpa, @intCast(instructions.len)); - for (instructions, 0..) |inst, i| { - const entry = parser.opcode_table.getOrPutAssumeCapacity(mapSetAndOpcode(set, @intCast(inst.opcode))); - if (!entry.found_existing) { - entry.value_ptr.* = @intCast(i); - } - } - } - - return parser; - } - - pub fn deinit(parser: *Parser) void { - parser.opcode_table.deinit(parser.gpa); - } - - fn mapSetAndOpcode(set: InstructionSet, opcode: u16) u32 { - return (@as(u32, @backingInt(set)) << 16) | opcode; - } - - pub fn getInstSpec(parser: Parser, opcode: Opcode) ?spec.Instruction { - const index = parser.opcode_table.get(mapSetAndOpcode(.core, @backingInt(opcode))) orelse return null; - return InstructionSet.core.instructions()[index]; - } - - /// Build a BinaryModule from raw instruction words (no header). - /// Scans for ext_inst_map, arith_type_width, and the functions section offset. - pub fn initFromWords(parser: *Parser, words: []const Word, id_bound: u32) !BinaryModule { - var binary = BinaryModule{ - .id_bound = id_bound, - .instructions = words, - .ext_inst_map = .{}, - .arith_type_width = .{}, - .functions_start = undefined, - }; - - var maybe_function_section: ?usize = null; - var it = binary.iterateInstructions(); - while (it.next()) |inst| { - const inst_spec = parser.getInstSpec(inst.opcode) orelse continue; - const operands = inst.operands; - - switch (inst.opcode) { - .OpExtInstImport => { - const set_name = std.mem.sliceTo(std.mem.sliceAsBytes(operands[1..]), 0); - const set = std.meta.stringToEnum(InstructionSet, set_name) orelse continue; - if (set == .core) continue; - try binary.ext_inst_map.put(parser.gpa, @fromBackingInt(@intCast(operands[0])), set); - }, - .OpTypeInt, .OpTypeFloat => { - try binary.arith_type_width.put(parser.gpa, @fromBackingInt(@intCast(operands[0])), @intCast(operands[1])); - }, - .OpFunction => if (maybe_function_section == null) { - maybe_function_section = inst.offset; - }, - else => {}, - } - - // propagate arith type widths through instructions that return int/float - const spec_operands = inst_spec.operands; - if (spec_operands.len >= 2 and - spec_operands[0].kind == .id_result_type and - spec_operands[1].kind == .id_result) - { - if (operands.len >= 2) { - if (binary.arith_type_width.get(@fromBackingInt(@intCast(operands[0])))) |width| { - try binary.arith_type_width.put(parser.gpa, @fromBackingInt(@intCast(operands[1])), width); - } - } - } - } - - binary.functions_start = maybe_function_section orelse binary.instructions.len; - - return binary; - } - - /// Parse offsets in the instruction that contain result-ids. - /// Returned offsets are relative to inst.operands. - pub fn parseInstructionResultIds( - parser: *Parser, - binary: BinaryModule, - inst: Instruction, - offsets: *std.ArrayList(u16), - ) !void { - const index = parser.opcode_table.get(mapSetAndOpcode(.core, @backingInt(inst.opcode))).?; - const operands = InstructionSet.core.instructions()[index].operands; - - var offset: usize = 0; - switch (inst.opcode) { - .OpSpecConstantOp => { - assert(operands[0].kind == .id_result_type); - assert(operands[1].kind == .id_result); - offset = try parser.parseOperandsResultIds(binary, inst, operands[0..2], offset, offsets); - - if (offset >= inst.operands.len) return error.InvalidPhysicalFormat; - const spec_opcode = std.math.cast(u16, inst.operands[offset]) orelse return error.InvalidPhysicalFormat; - const spec_index = parser.opcode_table.get(mapSetAndOpcode(.core, spec_opcode)) orelse - return error.InvalidPhysicalFormat; - const spec_operands = InstructionSet.core.instructions()[spec_index].operands; - assert(spec_operands[0].kind == .id_result_type); - assert(spec_operands[1].kind == .id_result); - offset = try parser.parseOperandsResultIds(binary, inst, spec_operands[2..], offset + 1, offsets); - }, - .OpExtInst => { - assert(operands[0].kind == .id_result_type); - assert(operands[1].kind == .id_result); - offset = try parser.parseOperandsResultIds(binary, inst, operands[0..2], offset, offsets); - - if (offset + 1 >= inst.operands.len) return error.InvalidPhysicalFormat; - const set_id: ResultId = @fromBackingInt(@intCast(inst.operands[offset])); - try offsets.append(parser.gpa, @intCast(offset)); - const set = binary.ext_inst_map.get(set_id) orelse { - log.err("invalid instruction set {}", .{@backingInt(set_id)}); - return error.InvalidId; - }; - const ext_opcode = std.math.cast(u16, inst.operands[offset + 1]) orelse return error.InvalidPhysicalFormat; - const ext_index = parser.opcode_table.get(mapSetAndOpcode(set, ext_opcode)) orelse - return error.InvalidPhysicalFormat; - const ext_operands = set.instructions()[ext_index].operands; - offset = try parser.parseOperandsResultIds(binary, inst, ext_operands, offset + 2, offsets); - }, - else => { - offset = try parser.parseOperandsResultIds(binary, inst, operands, offset, offsets); - }, - } - - if (offset != inst.operands.len) return error.InvalidPhysicalFormat; - } - - fn parseOperandsResultIds( - parser: *Parser, - binary: BinaryModule, - inst: Instruction, - operands: []const spec.Operand, - start_offset: usize, - offsets: *std.ArrayList(u16), - ) !usize { - var offset = start_offset; - for (operands) |operand| { - offset = try parser.parseOperandResultIds(binary, inst, operand, offset, offsets); - } - return offset; - } - - fn parseOperandResultIds( - parser: *Parser, - binary: BinaryModule, - inst: Instruction, - operand: spec.Operand, - start_offset: usize, - offsets: *std.ArrayList(u16), - ) !usize { - var offset = start_offset; - switch (operand.quantifier) { - .variadic => while (offset < inst.operands.len) { - offset = try parser.parseOperandKindResultIds(binary, inst, operand.kind, offset, offsets); - }, - .optional => if (offset < inst.operands.len) { - offset = try parser.parseOperandKindResultIds(binary, inst, operand.kind, offset, offsets); - }, - .required => { - offset = try parser.parseOperandKindResultIds(binary, inst, operand.kind, offset, offsets); - }, - } - return offset; - } - - fn parseOperandKindResultIds( - parser: *Parser, - binary: BinaryModule, - inst: Instruction, - kind: spec.OperandKind, - start_offset: usize, - offsets: *std.ArrayList(u16), - ) !usize { - var offset = start_offset; - if (offset >= inst.operands.len) return error.InvalidPhysicalFormat; - - switch (kind.category()) { - .bit_enum => { - const mask = inst.operands[offset]; - offset += 1; - for (kind.enumerants()) |enumerant| { - if ((mask & enumerant.value) != 0) { - for (enumerant.parameters) |param_kind| { - offset = try parser.parseOperandKindResultIds(binary, inst, param_kind, offset, offsets); - } - } - } - }, - .value_enum => { - const value = inst.operands[offset]; - offset += 1; - for (kind.enumerants()) |enumerant| { - if (value == enumerant.value) { - for (enumerant.parameters) |param_kind| { - offset = try parser.parseOperandKindResultIds(binary, inst, param_kind, offset, offsets); - } - break; - } - } - }, - .id => { - try offsets.append(parser.gpa, @intCast(offset)); - offset += 1; - }, - else => switch (kind) { - .literal_integer, .literal_float => offset += 1, - .literal_string => while (true) { - if (offset >= inst.operands.len) return error.InvalidPhysicalFormat; - const word = inst.operands[offset]; - offset += 1; - - if (word & 0xFF000000 == 0 or - word & 0x00FF0000 == 0 or - word & 0x0000FF00 == 0 or - word & 0x000000FF == 0) - { - break; - } - }, - .literal_context_dependent_number => { - assert(inst.opcode == .OpConstant or inst.opcode == .OpSpecConstant); - const bit_width = binary.arith_type_width.get(@fromBackingInt(@intCast(inst.operands[0]))) orelse { - log.err("invalid LiteralContextDependentNumber type {}", .{inst.operands[0]}); - return error.InvalidId; - }; - offset += switch (bit_width) { - 1...32 => 1, - 33...64 => 2, - else => unreachable, - }; - }, - .literal_ext_inst_integer => unreachable, - .literal_spec_constant_op_integer => unreachable, - .pair_literal_integer_id_ref => { - assert(inst.opcode == .OpSwitch); - const bit_width = binary.arith_type_width.get(@fromBackingInt(@intCast(inst.operands[0]))) orelse { - log.err("invalid OpSwitch type {}", .{inst.operands[0]}); - return error.InvalidId; - }; - offset += switch (bit_width) { - 1...32 => 1, - 33...64 => 2, - else => unreachable, - }; - try offsets.append(parser.gpa, @intCast(offset)); - offset += 1; - }, - .pair_id_ref_literal_integer => { - try offsets.append(parser.gpa, @intCast(offset)); - offset += 2; - }, - .pair_id_ref_id_ref => { - try offsets.append(parser.gpa, @intCast(offset)); - try offsets.append(parser.gpa, @intCast(offset + 1)); - offset += 2; - }, - else => unreachable, - }, - } - return offset; - } -}; diff --git a/src/link/SpirV/dedup_types.zig b/src/link/SpirV/dedup_types.zig deleted file mode 100644 index 8841c4185033c8ace7131236f0e549ef10745118..0000000000000000000000000000000000000000 --- a/src/link/SpirV/dedup_types.zig +++ /dev/null @@ -1,244 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; -const BinaryModule = @import("BinaryModule.zig"); - -const spec = @import("../../codegen/spirv/spec.zig"); -const Word = spec.Word; -const Id = spec.Id; -const Opcode = spec.Opcode; -const Instruction = BinaryModule.Instruction; - -/// Deduplicate types and constants in a SPIR-V binary module. -/// -/// The SPIR-V spec requires that non-aggregate types be unique. -/// When merging fragments from parallel codegen, duplicate type definitions -/// may exist. This pass identifies structurally identical types/constants, -/// keeps one canonical instance, and remaps all references to duplicates. -pub fn run(parser: *BinaryModule.Parser, binary: *BinaryModule) !void { - const gpa = parser.gpa; - - const Decoration = struct { offset: usize, len: usize }; - var decorations_by_id: std.array_hash_map.Auto(Id, std.ArrayList(Decoration)) = .empty; - defer { - for (decorations_by_id.values()) |*list| list.deinit(gpa); - decorations_by_id.deinit(gpa); - } - - var it = binary.iterateInstructions(); - while (it.next()) |inst| { - if (inst.offset >= binary.functions_start) break; - switch (inst.opcode) { - .OpName, .OpMemberName => continue, - else => switch (inst.opcode.class()) { - .annotation => {}, - else => continue, - }, - } - if (inst.operands.len == 0) continue; - const target_id: Id = @fromBackingInt(@intCast(inst.operands[0])); - - const gop = try decorations_by_id.getOrPut(gpa, target_id); - if (!gop.found_existing) gop.value_ptr.* = .empty; - try gop.value_ptr.append(gpa, .{ - .offset = inst.offset, - .len = 1 + inst.operands.len, - }); - } - - var canonical_map: std.array_hash_map.Custom(TypeKey, Id, TypeKey.HashContext, true) = .empty; - defer { - for (canonical_map.keys()) |key| gpa.free(key.words); - canonical_map.deinit(gpa); - } - - var id_remap: std.AutoHashMapUnmanaged(Id, Id) = .empty; - defer id_remap.deinit(gpa); - - var id_offsets: std.ArrayList(u16) = .empty; - defer id_offsets.deinit(gpa); - - var key_words: std.ArrayList(Word) = .empty; - defer key_words.deinit(gpa); - - var dec_hashes: std.ArrayList(u64) = .empty; - defer dec_hashes.deinit(gpa); - - // first pass: build canonical map, identify duplicates - it = binary.iterateInstructions(); - while (it.next()) |inst| { - if (inst.offset >= binary.functions_start) break; - if (!canDeduplicate(inst.opcode)) continue; - - const result_id_index: usize = switch (inst.opcode.class()) { - .type_declaration, .extension => 0, - .constant_creation => 1, - else => continue, - }; - if (result_id_index >= inst.operands.len) continue; - const result_id: Id = @fromBackingInt(@intCast(inst.operands[result_id_index])); - - key_words.items.len = 0; - try key_words.append(gpa, @backingInt(inst.opcode)); - - id_offsets.items.len = 0; - parser.parseInstructionResultIds(binary.*, inst, &id_offsets) catch continue; - - for (inst.operands, 0..) |word, i| { - if (i == result_id_index) continue; - if (std.mem.findScalar(u16, id_offsets.items, @intCast(i)) != null) { - const canonical = id_remap.get(@fromBackingInt(@intCast(word))) orelse @as(Id, @fromBackingInt(@intCast(word))); - try key_words.append(gpa, @backingInt(canonical)); - } else { - try key_words.append(gpa, word); - } - } - - if (decorations_by_id.getPtr(result_id)) |dec_list| { - dec_hashes.items.len = 0; - for (dec_list.items) |dec| { - const dec_words = binary.instructions[dec.offset..][0..dec.len]; - var hasher = std.hash.Wyhash.init(0); - hasher.update(std.mem.asBytes(&dec_words[0])); - for (dec_words[2..]) |w| { - const w_val = if (id_remap.get(@fromBackingInt(@intCast(w)))) |c| @backingInt(c) else w; - hasher.update(std.mem.asBytes(&w_val)); - } - try dec_hashes.append(gpa, hasher.final()); - } - std.mem.sort(u64, dec_hashes.items, {}, std.sort.asc(u64)); - var prev: u64 = 0; - for (dec_hashes.items) |h| { - if (h == prev) continue; - prev = h; - try key_words.append(gpa, @truncate(h)); - try key_words.append(gpa, @truncate(h >> 32)); - } - } - - const key = TypeKey{ .words = try gpa.dupe(Word, key_words.items) }; - const gop = try canonical_map.getOrPut(gpa, key); - if (gop.found_existing) { - try id_remap.put(gpa, result_id, gop.value_ptr.*); - gpa.free(key.words); - } else { - gop.value_ptr.* = result_id; - } - } - - if (id_remap.count() == 0) return; - - // second pass: rewrite id references, remove duplicates and redundant annotations - var new_words: std.ArrayList(Word) = .empty; - defer new_words.deinit(gpa); - try new_words.ensureTotalCapacity(gpa, binary.instructions.len); - - var emitted_annotations: std.AutoHashMapUnmanaged(u64, void) = .empty; - defer emitted_annotations.deinit(gpa); - - var new_functions_offset: ?usize = null; - var max_id: Word = 0; - - it = binary.iterateInstructions(); - while (it.next()) |inst| { - if (new_functions_offset == null and inst.offset >= binary.functions_start) { - new_functions_offset = new_words.items.len; - } - - if (canDeduplicate(inst.opcode)) { - const result_id_index: usize = switch (inst.opcode.class()) { - .type_declaration, .extension => 0, - .constant_creation => 1, - else => unreachable, - }; - if (result_id_index < inst.operands.len) { - const result_id: Id = @fromBackingInt(@intCast(inst.operands[result_id_index])); - if (id_remap.contains(result_id)) continue; - } - } - - switch (inst.opcode.class()) { - .annotation, .debug => { - if (inst.operands.len > 0) { - const target: Id = @fromBackingInt(@intCast(inst.operands[0])); - if (id_remap.contains(target)) continue; - } - }, - else => {}, - } - - const inst_start = new_words.items.len; - new_words.appendAssumeCapacity(binary.instructions[inst.offset]); - new_words.appendSliceAssumeCapacity(inst.operands); - const inst_slice = new_words.items[inst_start + 1 ..]; - - id_offsets.items.len = 0; - parser.parseInstructionResultIds(binary.*, inst, &id_offsets) catch continue; - - const inst_spec = parser.getInstSpec(inst.opcode); - const maybe_result_id_index: ?usize = if (inst_spec) |ispec| blk: { - break :blk for (0..@min(2, ispec.operands.len)) |i| { - if (ispec.operands[i].kind == .id_result) break @intCast(i); - } else null; - } else null; - - for (inst_slice, 0..) |*word, i| { - if (std.mem.findScalar(u16, id_offsets.items, @intCast(i)) == null) continue; - max_id = @max(max_id, word.*); - if (maybe_result_id_index != null and i == maybe_result_id_index.?) continue; - - if (id_remap.get(@fromBackingInt(@intCast(word.*)))) |canonical| { - word.* = @backingInt(canonical); - max_id = @max(max_id, word.*); - } - } - - switch (inst.opcode.class()) { - .annotation, .debug => { - const ann_words = new_words.items[inst_start..]; - const ann_hash = std.hash.Wyhash.hash(0, std.mem.sliceAsBytes(ann_words)); - const gop = try emitted_annotations.getOrPut(gpa, ann_hash); - if (gop.found_existing) { - new_words.items.len = inst_start; - continue; - } - }, - else => {}, - } - } - - var remap_it = id_remap.iterator(); - while (remap_it.next()) |entry| { - _ = binary.ext_inst_map.remove(entry.key_ptr.*); - _ = binary.arith_type_width.remove(entry.key_ptr.*); - } - - binary.instructions = try gpa.dupe(Word, new_words.items); - binary.functions_start = new_functions_offset orelse new_words.items.len; - binary.id_bound = max_id + 1; -} - -fn canDeduplicate(opcode: Opcode) bool { - return switch (opcode) { - .OpTypeForwardPointer => false, - .OpGroupDecorate, .OpGroupMemberDecorate => false, - else => switch (opcode.class()) { - .type_declaration, .constant_creation => true, - .extension => opcode == .OpExtInstImport, - else => false, - }, - }; -} - -const TypeKey = struct { - words: []const Word, - - const HashContext = struct { - pub fn hash(_: @This(), key: TypeKey) u32 { - return @truncate(std.hash.Wyhash.hash(0, std.mem.sliceAsBytes(key.words))); - } - - pub fn eql(_: @This(), a: TypeKey, b: TypeKey, _: usize) bool { - return std.mem.eql(Word, a.words, b.words); - } - }; -}; diff --git a/src/link/SpirV/lower_invocation_globals.zig b/src/link/SpirV/lower_invocation_globals.zig deleted file mode 100644 index 7ea41763d3cc3c0dad0fe244c60fafbc01c58ed0..0000000000000000000000000000000000000000 --- a/src/link/SpirV/lower_invocation_globals.zig +++ /dev/null @@ -1,803 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const log = std.log.scoped(.spirv_link); - -const BinaryModule = @import("BinaryModule.zig"); -const Section = @import("../../codegen/spirv/Section.zig"); -const spec = @import("../../codegen/spirv/spec.zig"); -const ResultId = spec.Id; -const Word = spec.Word; - -/// This structure contains all the stuff that we need to parse from the module in -/// order to run this pass, as well as some functions to ease its use. -const ModuleInfo = struct { - /// Information about a particular function. - const Fn = struct { - /// The index of the first callee in `callee_store`. - first_callee: usize, - /// The return type id of this function - return_type: ResultId, - /// The parameter types of this function - param_types: []const ResultId, - /// The set of (result-id's of) invocation globals that are accessed - /// in this function, or after resolution, that are accessed in this - /// function or any of it's callees. - invocation_globals: std.array_hash_map.Auto(ResultId, void), - }; - - /// Information about a particular invocation global - const InvocationGlobal = struct { - /// The list of invocation globals that this invocation global - /// depends on. - dependencies: std.array_hash_map.Auto(ResultId, void), - /// The invocation global's type - ty: ResultId, - /// Initializer function. May be `none`. - /// Note that if the initializer is `none`, then `dependencies` is empty. - initializer: ResultId, - }; - - /// Maps function result-id -> Fn information structure. - functions: std.array_hash_map.Auto(ResultId, Fn), - /// Set of OpFunction result-ids in this module. - entry_points: std.array_hash_map.Auto(ResultId, void), - /// For each function, a list of function result-ids that it calls. - callee_store: []const ResultId, - /// Maps each invocation global result-id to a type-id. - invocation_globals: std.array_hash_map.Auto(ResultId, InvocationGlobal), - /// Subset of `invocation_globals` reachable from any entry point. - live_invocation_globals: std.array_hash_map.Auto(ResultId, void), - /// Initializer functions of unreachable invocation globals. Their - /// OpFunction...OpFunctionEnd ranges are skipped during rewriteFunctions. - dead_initializers: std.array_hash_map.Auto(ResultId, void), - - /// Fetch the list of callees per function. Guaranteed to contain only unique IDs. - fn callees(self: ModuleInfo, fn_id: ResultId) []const ResultId { - const fn_index = self.functions.getIndex(fn_id).?; - const values = self.functions.values(); - const first_callee = values[fn_index].first_callee; - if (fn_index == values.len - 1) { - return self.callee_store[first_callee..]; - } else { - const next_first_callee = values[fn_index + 1].first_callee; - return self.callee_store[first_callee..next_first_callee]; - } - } - - /// Extract most of the required information from the binary. The remaining info is - /// constructed by `resolve()`. - fn parse( - arena: Allocator, - parser: *BinaryModule.Parser, - binary: BinaryModule, - ) !ModuleInfo { - var entry_points: std.array_hash_map.Auto(ResultId, void) = .empty; - var functions: std.array_hash_map.Auto(ResultId, Fn) = .empty; - var fn_types = std.AutoHashMap(ResultId, struct { - return_type: ResultId, - param_types: []const ResultId, - }).init(arena); - var calls: std.array_hash_map.Auto(ResultId, void) = .empty; - var callee_store: std.ArrayList(ResultId) = .empty; - var function_invocation_globals: std.array_hash_map.Auto(ResultId, void) = .empty; - var result_id_offsets: std.ArrayList(u16) = .empty; - var invocation_globals: std.array_hash_map.Auto(ResultId, InvocationGlobal) = .empty; - - var maybe_current_function: ?ResultId = null; - var fn_ty_id: ResultId = undefined; - - var it = binary.iterateInstructions(); - while (it.next()) |inst| { - result_id_offsets.items.len = 0; - try parser.parseInstructionResultIds(binary, inst, &result_id_offsets); - - switch (inst.opcode) { - .OpEntryPoint => { - const entry_point: ResultId = @fromBackingInt(@intCast(inst.operands[1])); - const entry = try entry_points.getOrPut(arena, entry_point); - if (entry.found_existing) { - log.err("Entry point type {f} has duplicate definition", .{entry_point}); - return error.DuplicateId; - } - }, - .OpTypeFunction => { - const fn_type: ResultId = @fromBackingInt(@intCast(inst.operands[0])); - const return_type: ResultId = @fromBackingInt(@intCast(inst.operands[1])); - const param_types: []const ResultId = @ptrCast(inst.operands[2..]); - - const entry = try fn_types.getOrPut(fn_type); - if (entry.found_existing) { - log.err("Function type {f} has duplicate definition", .{fn_type}); - return error.DuplicateId; - } - - entry.value_ptr.* = .{ - .return_type = return_type, - .param_types = param_types, - }; - }, - .OpExtInst => { - // Note: format and set are already verified by parseInstructionResultIds(). - const global_type: ResultId = @fromBackingInt(@intCast(inst.operands[0])); - const result_id: ResultId = @fromBackingInt(@intCast(inst.operands[1])); - const set_id: ResultId = @fromBackingInt(@intCast(inst.operands[2])); - const set_inst = inst.operands[3]; - - const set = binary.ext_inst_map.get(set_id).?; - if (set == .zig and set_inst == 0) { - const initializer: ResultId = if (inst.operands.len >= 5) - @fromBackingInt(@intCast(inst.operands[4])) - else - .none; - - try invocation_globals.put(arena, result_id, .{ - .dependencies = .{}, - .ty = global_type, - .initializer = initializer, - }); - } - }, - .OpFunction => { - if (maybe_current_function) |current_function| { - log.err("OpFunction {f} does not have an OpFunctionEnd", .{current_function}); - return error.InvalidPhysicalFormat; - } - - maybe_current_function = @fromBackingInt(@intCast(inst.operands[1])); - fn_ty_id = @fromBackingInt(@intCast(inst.operands[3])); - function_invocation_globals.clearRetainingCapacity(); - }, - .OpFunctionCall => { - const callee: ResultId = @fromBackingInt(@intCast(inst.operands[2])); - try calls.put(arena, callee, {}); - }, - .OpFunctionEnd => { - const current_function = maybe_current_function orelse { - log.err("encountered OpFunctionEnd without corresponding OpFunction", .{}); - return error.InvalidPhysicalFormat; - }; - const entry = try functions.getOrPut(arena, current_function); - if (entry.found_existing) { - log.err("Function {f} has duplicate definition", .{current_function}); - return error.DuplicateId; - } - - const first_callee = callee_store.items.len; - try callee_store.appendSlice(arena, calls.keys()); - - const fn_type = fn_types.get(fn_ty_id) orelse { - log.err("Function {f} has invalid OpFunction type", .{current_function}); - return error.InvalidId; - }; - - entry.value_ptr.* = .{ - .first_callee = first_callee, - .return_type = fn_type.return_type, - .param_types = fn_type.param_types, - .invocation_globals = try function_invocation_globals.clone(arena), - }; - maybe_current_function = null; - calls.clearRetainingCapacity(); - }, - else => {}, - } - - for (result_id_offsets.items) |off| { - const result_id: ResultId = @fromBackingInt(@intCast(inst.operands[off])); - if (invocation_globals.contains(result_id)) { - try function_invocation_globals.put(arena, result_id, {}); - } - } - } - - if (maybe_current_function) |current_function| { - log.err("OpFunction {f} does not have an OpFunctionEnd", .{current_function}); - return error.InvalidPhysicalFormat; - } - - return .{ - .functions = functions, - .entry_points = entry_points, - .callee_store = callee_store.items, - .invocation_globals = invocation_globals, - .live_invocation_globals = .empty, - .dead_initializers = .empty, - }; - } - - /// Derive the remaining info from the structures filled in by parsing. - fn resolve(self: *ModuleInfo, arena: Allocator) !void { - try self.resolveInvocationGlobalUsage(arena); - try self.resolveInvocationGlobalDependencies(arena); - try self.resolveLiveSet(arena); - } - - fn resolveLiveSet(self: *ModuleInfo, arena: Allocator) !void { - for (self.entry_points.keys()) |ep_id| { - const ep_info = self.functions.get(ep_id) orelse continue; - for (ep_info.invocation_globals.keys()) |g| { - try self.live_invocation_globals.put(arena, g, {}); - const g_info = self.invocation_globals.get(g).?; - for (g_info.dependencies.keys()) |dep| { - try self.live_invocation_globals.put(arena, dep, {}); - } - } - } - for (self.invocation_globals.keys(), self.invocation_globals.values()) |g, info| { - if (info.initializer == .none) continue; - if (self.live_invocation_globals.contains(g)) continue; - try self.dead_initializers.put(arena, info.initializer, {}); - } - } - - /// For each function, extend the list of `invocation_globals` with the - /// invocation globals that ALL of its dependencies use. - fn resolveInvocationGlobalUsage(self: *ModuleInfo, arena: Allocator) !void { - var seen: std.bit_set.Dynamic = try .initEmpty(arena, self.functions.count()); - - for (self.functions.keys()) |id| { - try self.resolveInvocationGlobalUsageStep(arena, id, &seen); - } - } - - fn resolveInvocationGlobalUsageStep( - self: *ModuleInfo, - arena: Allocator, - id: ResultId, - seen: *std.bit_set.Dynamic, - ) !void { - const index = self.functions.getIndex(id) orelse { - log.err("function calls invalid function {f}", .{id}); - return error.InvalidId; - }; - - if (seen.isSet(index)) { - return; - } - seen.set(index); - - const info = &self.functions.values()[index]; - for (self.callees(id)) |callee| { - try self.resolveInvocationGlobalUsageStep(arena, callee, seen); - const callee_info = self.functions.get(callee).?; - for (callee_info.invocation_globals.keys()) |global| { - try info.invocation_globals.put(arena, global, {}); - } - } - } - - /// For each invocation global, populate and fully resolve the `dependencies` set. - /// This requires `resolveInvocationGlobalUsage()` to be already done. - fn resolveInvocationGlobalDependencies( - self: *ModuleInfo, - arena: Allocator, - ) !void { - var seen: std.bit_set.Dynamic = try .initEmpty(arena, self.invocation_globals.count()); - - for (self.invocation_globals.keys()) |id| { - try self.resolveInvocationGlobalDependenciesStep(arena, id, &seen); - } - } - - fn resolveInvocationGlobalDependenciesStep( - self: *ModuleInfo, - arena: Allocator, - id: ResultId, - seen: *std.bit_set.Dynamic, - ) !void { - const index = self.invocation_globals.getIndex(id) orelse { - log.err("invalid invocation global {f}", .{id}); - return error.InvalidId; - }; - - if (seen.isSet(index)) { - return; - } - seen.set(index); - - const info = &self.invocation_globals.values()[index]; - if (info.initializer == .none) { - return; - } - - const initializer = self.functions.get(info.initializer) orelse { - log.err("invocation global {f} has invalid initializer {f}", .{ id, info.initializer }); - return error.InvalidId; - }; - - for (initializer.invocation_globals.keys()) |dependency| { - if (dependency == id) { - // The set of invocation global dependencies includes the dependency itself, - // so we need to skip that case. - continue; - } - - try info.dependencies.put(arena, dependency, {}); - try self.resolveInvocationGlobalDependenciesStep(arena, dependency, seen); - - const dep_info = self.invocation_globals.getPtr(dependency).?; - - for (dep_info.dependencies.keys()) |global| { - try info.dependencies.put(arena, global, {}); - } - } - } -}; - -const ModuleBuilder = struct { - const FunctionType = struct { - return_type: ResultId, - param_types: []const ResultId, - - const Context = struct { - pub fn hash(_: @This(), ty: FunctionType) u32 { - var hasher = std.hash.Wyhash.init(0); - hasher.update(std.mem.asBytes(&ty.return_type)); - hasher.update(std.mem.sliceAsBytes(ty.param_types)); - return @truncate(hasher.final()); - } - - pub fn eql(_: @This(), a: FunctionType, b: FunctionType, _: usize) bool { - if (a.return_type != b.return_type) return false; - return std.mem.eql(ResultId, a.param_types, b.param_types); - } - }; - }; - - const FunctionNewInfo = struct { - /// This is here just so that we don't need to allocate the new - /// param_types multiple times. - new_function_type: ResultId, - /// The first ID of the parameters for the invocation globals. - /// Each global is allocate here according to the index in - /// `ModuleInfo.Fn.invocation_globals`. - global_id_base: u32, - - fn invocationGlobalId(self: FunctionNewInfo, index: usize) ResultId { - return @fromBackingInt(@intCast(self.global_id_base + @as(u32, @intCast(index)))); - } - }; - - arena: Allocator, - section: Section, - /// The ID bound of the new module. - id_bound: u32, - /// The first ID of the new entry points. Entry points are allocated from - /// here according to their index in `info.entry_points`. - entry_point_new_id_base: u32, - /// OpName operands saved for invocation globals to re-emit. - global_names: std.array_hash_map.Auto(ResultId, []const Word) = .empty, - /// A set of all function types in the new program. SPIR-V mandates that these are unique, - /// and until a general type deduplication pass is programmed, we just handle it here via this. - function_types: std.array_hash_map.Custom(FunctionType, ResultId, FunctionType.Context, true) = .empty, - /// Maps functions to new information required for creating the module - function_new_info: std.array_hash_map.Auto(ResultId, FunctionNewInfo) = .empty, - /// Offset of the functions section in the new binary. - new_functions_section: ?usize, - - fn init(arena: Allocator, binary: BinaryModule, info: ModuleInfo) !ModuleBuilder { - var self = ModuleBuilder{ - .arena = arena, - .section = .{}, - .id_bound = binary.id_bound, - .entry_point_new_id_base = undefined, - .new_functions_section = null, - }; - self.entry_point_new_id_base = @backingInt(self.allocIds(@intCast(info.entry_points.count()))); - return self; - } - - fn allocId(self: *ModuleBuilder) ResultId { - return self.allocIds(1); - } - - fn allocIds(self: *ModuleBuilder, n: u32) ResultId { - defer self.id_bound += n; - return @fromBackingInt(@intCast(self.id_bound)); - } - - fn finalize(self: *ModuleBuilder, arena: Allocator, binary: *BinaryModule) !void { - binary.id_bound = self.id_bound; - binary.instructions = try arena.dupe(Word, self.section.instructions.items); - // Nothing is removed in this pass so we don't need to change any of the maps, - // just make sure the section is updated. - binary.functions_start = self.new_functions_section orelse binary.instructions.len; - } - - fn emitGlobalNames(self: *ModuleBuilder, info: ModuleInfo) !void { - for (info.functions.keys(), info.functions.values()) |func, fn_info| { - if (info.dead_initializers.contains(func)) continue; - const new_info = self.function_new_info.get(func) orelse continue; - for (fn_info.invocation_globals.keys(), 0..) |global, i| { - if (!info.live_invocation_globals.contains(global)) continue; - const name_words = self.global_names.get(global) orelse continue; - const id = new_info.invocationGlobalId(i); - try self.section.emitRaw(self.arena, .OpName, 1 + name_words.len); - self.section.writeOperand(ResultId, id); - self.section.writeWords(name_words); - } - } - } - - /// Process everything from `binary` up to the first function and emit it into the builder. - fn processPreamble(self: *ModuleBuilder, binary: BinaryModule, info: ModuleInfo) !void { - var emitted_global_names = false; - var it = binary.iterateInstructions(); - while (it.next()) |inst| { - if (!emitted_global_names) switch (inst.opcode.class()) { - .annotation, .type_declaration, .constant_creation => { - try self.emitGlobalNames(info); - emitted_global_names = true; - }, - else => {}, - }; - - switch (inst.opcode) { - .OpName => { - const id: ResultId = @fromBackingInt(@intCast(inst.operands[0])); - if (info.invocation_globals.contains(id)) { - try self.global_names.put(self.arena, id, inst.operands[1..]); - continue; - } - if (info.dead_initializers.contains(id)) continue; - }, - .OpExtInstImport => { - const set_id: ResultId = @fromBackingInt(@intCast(inst.operands[0])); - const set = binary.ext_inst_map.get(set_id).?; - if (set == .zig) continue; - }, - .OpExtInst => { - const set_id: ResultId = @fromBackingInt(@intCast(inst.operands[2])); - const set_inst = inst.operands[3]; - const set = binary.ext_inst_map.get(set_id).?; - if (set == .zig and set_inst == 0) { - continue; - } - }, - .OpEntryPoint => { - const original_id: ResultId = @fromBackingInt(@intCast(inst.operands[1])); - const fn_info = info.functions.get(original_id).?; - if (fn_info.invocation_globals.count() > 0) { - const new_id_index = info.entry_points.getIndex(original_id).?; - const new_id: ResultId = @fromBackingInt(@intCast(self.entry_point_new_id_base + new_id_index)); - try self.section.emitRaw(self.arena, .OpEntryPoint, inst.operands.len); - self.section.writeWord(inst.operands[0]); - self.section.writeOperand(ResultId, new_id); - self.section.writeWords(inst.operands[2..]); - } else { - try self.section.emitRawInstruction(self.arena, inst.opcode, inst.operands); - } - continue; - }, - .OpExecutionMode, .OpExecutionModeId => { - const original_id: ResultId = @fromBackingInt(@intCast(inst.operands[0])); - const fn_info = info.functions.get(original_id).?; - if (fn_info.invocation_globals.count() > 0) { - const new_id_index = info.entry_points.getIndex(original_id).?; - const new_id: ResultId = @fromBackingInt(@intCast(self.entry_point_new_id_base + new_id_index)); - try self.section.emitRaw(self.arena, inst.opcode, inst.operands.len); - self.section.writeOperand(ResultId, new_id); - self.section.writeWords(inst.operands[1..]); - } else { - try self.section.emitRawInstruction(self.arena, inst.opcode, inst.operands); - } - continue; - }, - .OpTypeFunction => { - continue; - }, - .OpFunction => break, - else => {}, - } - - try self.section.emitRawInstruction(self.arena, inst.opcode, inst.operands); - } - - if (!emitted_global_names) { - try self.emitGlobalNames(info); - } - } - - /// Derive new information required for further emitting this module, - fn deriveNewFnInfo(self: *ModuleBuilder, info: ModuleInfo) !void { - for (info.functions.keys(), info.functions.values()) |func, fn_info| { - const invocation_global_count = fn_info.invocation_globals.count(); - const new_param_types = try self.arena.alloc(ResultId, fn_info.param_types.len + invocation_global_count); - for (fn_info.invocation_globals.keys(), 0..) |global, i| { - new_param_types[i] = info.invocation_globals.get(global).?.ty; - } - @memcpy(new_param_types[invocation_global_count..], fn_info.param_types); - - const new_type = try self.internFunctionType(fn_info.return_type, new_param_types); - try self.function_new_info.put(self.arena, func, .{ - .new_function_type = new_type, - .global_id_base = @backingInt(self.allocIds(@intCast(invocation_global_count))), - }); - } - } - - /// Emit the new function types, which include the parameters for the invocation globals. - /// Currently, this function re-emits ALL function types to ensure that there are - /// no duplicates in the final program. - /// TODO: The above should be resolved by a generalized deduplication pass, and then - /// we only need to emit the new function pointers type here. - fn emitFunctionTypes(self: *ModuleBuilder, info: ModuleInfo) !void { - // TODO: Handle decorators. Function types usually don't have those - // though, but stuff like OpName could be a possibility. - - // Entry points retain their old function type, so make sure to emit - // those in the `function_types` set. - for (info.entry_points.keys()) |func| { - const fn_info = info.functions.get(func).?; - _ = try self.internFunctionType(fn_info.return_type, fn_info.param_types); - } - - for (self.function_types.keys(), self.function_types.values()) |fn_type, result_id| { - try self.section.emit(self.arena, .OpTypeFunction, .{ - .id_result = result_id, - .return_type = fn_type.return_type, - .id_ref_2 = fn_type.param_types, - }); - } - } - - fn internFunctionType(self: *ModuleBuilder, return_type: ResultId, param_types: []const ResultId) !ResultId { - const entry = try self.function_types.getOrPut(self.arena, .{ - .return_type = return_type, - .param_types = param_types, - }); - - if (!entry.found_existing) { - const new_id = self.allocId(); - entry.value_ptr.* = new_id; - } - - return entry.value_ptr.*; - } - - /// Rewrite the modules functions and emit them with the new parameter types. - fn rewriteFunctions( - self: *ModuleBuilder, - parser: *BinaryModule.Parser, - binary: BinaryModule, - info: ModuleInfo, - ) !void { - var result_id_offsets: std.ArrayList(u16) = .empty; - var operands: std.ArrayList(u32) = .empty; - - var maybe_current_function: ?ResultId = null; - var skip_until_end: bool = false; - var it = binary.iterateInstructionsFrom(binary.functions_start); - self.new_functions_section = self.section.instructions.items.len; - while (it.next()) |inst| { - if (skip_until_end) { - if (inst.opcode == .OpFunctionEnd) skip_until_end = false; - continue; - } - result_id_offsets.items.len = 0; - try parser.parseInstructionResultIds(binary, inst, &result_id_offsets); - - operands.items.len = 0; - try operands.appendSlice(self.arena, inst.operands); - - // Replace the result-ids with the global's new result-id if required. - for (result_id_offsets.items) |off| { - const result_id: ResultId = @fromBackingInt(@intCast(operands.items[off])); - if (info.invocation_globals.contains(result_id)) { - const func = maybe_current_function.?; - const new_info = self.function_new_info.get(func).?; - const fn_info = info.functions.get(func).?; - const index = fn_info.invocation_globals.getIndex(result_id).?; - operands.items[off] = @backingInt(new_info.invocationGlobalId(index)); - } - } - - switch (inst.opcode) { - .OpFunction => { - // Re-declare the function with the new parameters. - const func: ResultId = @fromBackingInt(@intCast(operands.items[1])); - if (info.dead_initializers.contains(func)) { - skip_until_end = true; - continue; - } - const fn_info = info.functions.get(func).?; - const new_info = self.function_new_info.get(func).?; - - try self.section.emitRaw(self.arena, .OpFunction, 4); - self.section.writeOperand(ResultId, fn_info.return_type); - self.section.writeOperand(ResultId, func); - self.section.writeWord(operands.items[2]); - self.section.writeOperand(ResultId, new_info.new_function_type); - - // Emit the OpFunctionParameters for the invocation globals. The functions - // actual parameters are emitted unchanged from their original form, so - // we don't need to handle those here. - - for (fn_info.invocation_globals.keys(), 0..) |global, index| { - const ty = info.invocation_globals.get(global).?.ty; - const id = new_info.invocationGlobalId(index); - try self.section.emit(self.arena, .OpFunctionParameter, .{ - .id_result_type = ty, - .id_result = id, - }); - } - - maybe_current_function = func; - }, - .OpFunctionCall => { - // Add the required invocation globals to the function's new parameter list. - const caller = maybe_current_function.?; - const callee: ResultId = @fromBackingInt(@intCast(operands.items[2])); - const caller_info = info.functions.get(caller).?; - const callee_info = info.functions.get(callee).?; - const caller_new_info = self.function_new_info.get(caller).?; - const total_params = callee_info.invocation_globals.count() + callee_info.param_types.len; - - try self.section.emitRaw(self.arena, .OpFunctionCall, 3 + total_params); - self.section.writeWord(operands.items[0]); // Copy result type-id - self.section.writeWord(operands.items[1]); // Copy result-id - self.section.writeOperand(ResultId, callee); - - // Add the new arguments - for (callee_info.invocation_globals.keys()) |global| { - const caller_global_index = caller_info.invocation_globals.getIndex(global).?; - const id = caller_new_info.invocationGlobalId(caller_global_index); - self.section.writeOperand(ResultId, id); - } - - // Add the original arguments - self.section.writeWords(operands.items[3..]); - }, - else => { - try self.section.emitRawInstruction(self.arena, inst.opcode, operands.items); - }, - } - } - } - - fn emitNewEntryPoints(self: *ModuleBuilder, info: ModuleInfo) !void { - const arena = self.arena; - var all_function_invocation_globals: std.array_hash_map.Auto(ResultId, void) = .empty; - - for (info.entry_points.keys(), 0..) |func, entry_point_index| { - const fn_info = info.functions.get(func).?; - if (fn_info.invocation_globals.count() == 0) continue; - const ep_id: ResultId = @fromBackingInt(@intCast(self.entry_point_new_id_base + @as(u32, @intCast(entry_point_index)))); - const fn_type = self.function_types.get(.{ - .return_type = fn_info.return_type, - .param_types = fn_info.param_types, - }).?; - - try self.section.emit(arena, .OpFunction, .{ - .id_result_type = fn_info.return_type, - .id_result = ep_id, - .function_control = .{}, // TODO: Copy the attributes from the original function maybe? - .function_type = fn_type, - }); - - // Emit OpFunctionParameter instructions for the original kernel's parameters. - const params_id_base: u32 = @backingInt(self.allocIds(@intCast(fn_info.param_types.len))); - for (fn_info.param_types, 0..) |param_type, i| { - const id: ResultId = @fromBackingInt(@intCast(params_id_base + @as(u32, @intCast(i)))); - try self.section.emit(arena, .OpFunctionParameter, .{ - .id_result_type = param_type, - .id_result = id, - }); - } - - try self.section.emit(arena, .OpLabel, .{ - .id_result = self.allocId(), - }); - - // Besides the IDs of the main kernel, we also need the - // dependencies of the globals. - // Just quickly construct that set here. - all_function_invocation_globals.clearRetainingCapacity(); - for (fn_info.invocation_globals.keys()) |global| { - try all_function_invocation_globals.put(arena, global, {}); - const global_info = info.invocation_globals.get(global).?; - for (global_info.dependencies.keys()) |dependency| { - try all_function_invocation_globals.put(arena, dependency, {}); - } - } - - // Declare the IDs of the invocation globals. - const global_id_base: u32 = @backingInt(self.allocIds(@intCast(all_function_invocation_globals.count()))); - for (all_function_invocation_globals.keys(), 0..) |global, i| { - const global_info = info.invocation_globals.get(global).?; - - const id: ResultId = @fromBackingInt(@intCast(global_id_base + @as(u32, @intCast(i)))); - try self.section.emit(arena, .OpVariable, .{ - .id_result_type = global_info.ty, - .id_result = id, - .storage_class = .function, - .initializer = null, - }); - } - - // Call initializers for invocation globals that need it - for (all_function_invocation_globals.keys()) |global| { - const global_info = info.invocation_globals.get(global).?; - if (global_info.initializer == .none) continue; - - const initializer_info = info.functions.get(global_info.initializer).?; - assert(initializer_info.param_types.len == 0); - - try self.callWithGlobalsAndLinearParams( - &all_function_invocation_globals, - global_info.initializer, - initializer_info, - global_id_base, - undefined, - ); - } - - // Call the main kernel entry - try self.callWithGlobalsAndLinearParams( - &all_function_invocation_globals, - func, - fn_info, - global_id_base, - params_id_base, - ); - - try self.section.emit(arena, .OpReturn, {}); - try self.section.emit(arena, .OpFunctionEnd, {}); - } - } - - fn callWithGlobalsAndLinearParams( - self: *ModuleBuilder, - all_globals: *const std.array_hash_map.Auto(ResultId, void), - func: ResultId, - callee_info: ModuleInfo.Fn, - global_id_base: u32, - params_id_base: u32, - ) !void { - const total_arguments = callee_info.invocation_globals.count() + callee_info.param_types.len; - try self.section.emitRaw(self.arena, .OpFunctionCall, 3 + total_arguments); - self.section.writeOperand(ResultId, callee_info.return_type); - self.section.writeOperand(ResultId, self.allocId()); - self.section.writeOperand(ResultId, func); - - // Add the invocation globals - for (callee_info.invocation_globals.keys()) |global| { - const index = all_globals.getIndex(global).?; - const id: ResultId = @fromBackingInt(@intCast(global_id_base + @as(u32, @intCast(index)))); - self.section.writeOperand(ResultId, id); - } - - // Add the arguments - for (0..callee_info.param_types.len) |index| { - const id: ResultId = @fromBackingInt(@intCast(params_id_base + @as(u32, @intCast(index)))); - self.section.writeOperand(ResultId, id); - } - } -}; - -pub fn run(parser: *BinaryModule.Parser, binary: *BinaryModule, progress: std.Progress.Node) !void { - const sub_node = progress.start("Lower invocation globals", 6); - defer sub_node.end(); - - var arena_state = std.heap.ArenaAllocator.init(parser.gpa); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - var info = try ModuleInfo.parse(arena, parser, binary.*); - try info.resolve(arena); - - var builder = try ModuleBuilder.init(arena, binary.*, info); - sub_node.completeOne(); - try builder.deriveNewFnInfo(info); - sub_node.completeOne(); - try builder.processPreamble(binary.*, info); - sub_node.completeOne(); - try builder.emitFunctionTypes(info); - sub_node.completeOne(); - try builder.rewriteFunctions(parser, binary.*, info); - sub_node.completeOne(); - try builder.emitNewEntryPoints(info); - sub_node.completeOne(); - try builder.finalize(parser.gpa, binary); -} diff --git a/src/link/SpirV/prune_unused.zig b/src/link/SpirV/prune_unused.zig deleted file mode 100644 index a41b0d878d07b7c3e3179429dae74e27aaf044b8..0000000000000000000000000000000000000000 --- a/src/link/SpirV/prune_unused.zig +++ /dev/null @@ -1,257 +0,0 @@ -const std = @import("std"); -const BinaryModule = @import("BinaryModule.zig"); -const spec = @import("../../codegen/spirv/spec.zig"); -const Opcode = spec.Opcode; -const ResultId = spec.Id; -const Word = spec.Word; - -pub fn run(parser: *BinaryModule.Parser, binary: *BinaryModule) !void { - const gpa = parser.gpa; - - // map result-id → index in id_offsets for preamble instructions and function headers - var id_to_index: std.AutoHashMapUnmanaged(ResultId, u32) = .empty; - defer id_to_index.deinit(gpa); - - // for each indexed instruction, its offset in the binary - var code_offsets: std.ArrayList(usize) = .empty; - defer code_offsets.deinit(gpa); - - var it = binary.iterateInstructions(); - while (it.next()) |inst| { - const inst_spec = parser.getInstSpec(inst.opcode) orelse continue; - const result_id = getResultId(inst, inst_spec) orelse continue; - - // only index preamble instructions and function headers - if (inst.offset < binary.functions_start or inst.opcode == .OpFunction) { - const index: u32 = @intCast(code_offsets.items.len); - try id_to_index.put(gpa, result_id, index); - try code_offsets.append(gpa, inst.offset); - } - } - - var alive: std.bit_set.Dynamic = try .initEmpty(gpa, code_offsets.items.len); - defer alive.deinit(gpa); - - var id_offset_buf: std.ArrayList(u16) = .empty; - defer id_offset_buf.deinit(gpa); - - // Mark non-prunable preamble instructions alive - // OpExtInst in the preamble is metadata (e.g. Zig error info) that references - // functions. skip it here so it doesn't root dead functions alive. - // These instructions are handled as prunable during the rewrite phase. - it = binary.iterateInstructions(); - while (it.next()) |inst| { - if (inst.offset >= binary.functions_start) break; - if (canPrune(inst.opcode) or inst.opcode == .OpExtInst) continue; - try markAlive( - parser, - binary.*, - inst, - &alive, - &id_to_index, - &code_offsets, - &id_offset_buf, - ); - } - - // mark functions with LinkageAttributes Export alive - it = binary.iterateInstructions(); - while (it.next()) |inst| { - if (inst.offset >= binary.functions_start) break; - if (inst.opcode == .OpDecorate and inst.operands.len >= 2 and - inst.operands[1] == @backingInt(spec.Decoration.linkage_attributes)) - { - // Last word after the string is the linkage type; Export = 0. - if (inst.operands[inst.operands.len - 1] == @backingInt(spec.LinkageType.@"export")) { - const target: ResultId = @fromBackingInt(@intCast(inst.operands[0])); - if (id_to_index.get(target)) |index| { - alive.set(index); - } - } - } - } - - // mark alive functions' contents alive - it = binary.iterateInstructionsFrom(binary.functions_start); - while (it.next()) |inst| { - if (inst.opcode == .OpFunction) { - const inst_spec = parser.getInstSpec(inst.opcode) orelse continue; - const result_id = getResultId(inst, inst_spec) orelse continue; - const index = id_to_index.get(result_id) orelse continue; - if (!alive.isSet(index)) { - // skip dead function - while (it.next()) |inner| { - if (inner.opcode == .OpFunctionEnd) break; - } - continue; - } - - // mark the function's type operands alive - try markAlive(parser, binary.*, inst, &alive, &id_to_index, &code_offsets, &id_offset_buf); - continue; - } - - // mark operands of alive function contents - try markAlive(parser, binary.*, inst, &alive, &id_to_index, &code_offsets, &id_offset_buf); - } - - // rewrite - var new_words: std.ArrayList(Word) = .empty; - defer new_words.deinit(gpa); - try new_words.ensureTotalCapacity(gpa, binary.instructions.len); - - var new_functions_start: ?usize = null; - - it = binary.iterateInstructions(); - while (it.next()) |inst| { - if (inst.offset >= binary.functions_start and inst.opcode == .OpFunction) { - const inst_spec = parser.getInstSpec(inst.opcode) orelse continue; - const result_id = getResultId(inst, inst_spec) orelse continue; - const index = id_to_index.get(result_id) orelse continue; - if (!alive.isSet(index)) { - while (it.next()) |inner| { - if (inner.opcode == .OpFunctionEnd) break; - } - continue; - } - } - - const is_prunable = canPrune(inst.opcode) or - (inst.opcode == .OpExtInst and inst.offset < binary.functions_start); - if (is_prunable) { - const inst_spec = parser.getInstSpec(inst.opcode) orelse { - appendInst(&new_words, binary, inst, &new_functions_start); - continue; - }; - - if (getResultId(inst, inst_spec)) |result_id| { - const index = id_to_index.get(result_id) orelse { - appendInst(&new_words, binary, inst, &new_functions_start); - continue; - }; - if (!alive.isSet(index)) continue; - } else { - // annotation-style: emit only if the target id is alive - if (inst.operands.len > 0) { - const target: ResultId = @fromBackingInt(@intCast(inst.operands[0])); - if (id_to_index.get(target)) |idx| { - if (!alive.isSet(idx)) continue; - } else continue; - } - } - } - - appendInst(&new_words, binary, inst, &new_functions_start); - } - - { - var to_remove: std.ArrayList(ResultId) = .empty; - defer to_remove.deinit(gpa); - - var ext_it = binary.ext_inst_map.iterator(); - while (ext_it.next()) |entry| { - if (id_to_index.get(entry.key_ptr.*)) |index| { - if (!alive.isSet(index)) try to_remove.append(gpa, entry.key_ptr.*); - } - } - for (to_remove.items) |id| _ = binary.ext_inst_map.remove(id); - - to_remove.items.len = 0; - var arith_it = binary.arith_type_width.iterator(); - while (arith_it.next()) |entry| { - if (id_to_index.get(entry.key_ptr.*)) |index| { - if (!alive.isSet(index)) try to_remove.append(gpa, entry.key_ptr.*); - } - } - for (to_remove.items) |id| _ = binary.arith_type_width.remove(id); - } - - binary.instructions = try gpa.dupe(Word, new_words.items); - binary.functions_start = new_functions_start orelse new_words.items.len; -} - -fn appendInst( - new_words: *std.ArrayList(Word), - binary: *const BinaryModule, - inst: BinaryModule.Instruction, - new_functions_start: *?usize, -) void { - if (new_functions_start.* == null and inst.offset >= binary.functions_start) { - new_functions_start.* = new_words.items.len; - } - const len = @as(usize, binary.instructions[inst.offset] >> 16); - new_words.appendSliceAssumeCapacity(binary.instructions[inst.offset..][0..len]); -} - -fn markAlive( - parser: *BinaryModule.Parser, - binary: BinaryModule, - inst: BinaryModule.Instruction, - alive: *std.bit_set.Dynamic, - id_to_index: *const std.AutoHashMapUnmanaged(ResultId, u32), - code_offsets: *const std.ArrayList(usize), - id_offset_buf: *std.ArrayList(u16), -) !void { - const start = id_offset_buf.items.len; - try parser.parseInstructionResultIds(binary, inst, id_offset_buf); - const end = id_offset_buf.items.len; - - var i = start; - while (i < end) : (i += 1) { - const off = id_offset_buf.items[i]; - const id: ResultId = @fromBackingInt(@intCast(inst.operands[off])); - const index = id_to_index.get(id) orelse continue; - if (alive.isSet(index)) continue; - alive.set(index); - - const offset = code_offsets.items[index]; - const ref_inst = BinaryModule.Instruction{ - .opcode = @fromBackingInt(@intCast(binary.instructions[offset] & 0xFFFF)), - .offset = offset, - .operands = blk: { - const l = binary.instructions[offset] >> 16; - break :blk binary.instructions[offset..][1..l]; - }, - }; - - if (ref_inst.opcode == .OpFunction) { - var fn_it = binary.iterateInstructionsFrom(ref_inst.offset); - _ = fn_it.next(); - while (fn_it.next()) |fn_inst| { - if (fn_inst.opcode == .OpFunctionEnd) break; - try markAlive(parser, binary, fn_inst, alive, id_to_index, code_offsets, id_offset_buf); - } - try markAlive(parser, binary, ref_inst, alive, id_to_index, code_offsets, id_offset_buf); - } else { - try markAlive(parser, binary, ref_inst, alive, id_to_index, code_offsets, id_offset_buf); - } - } -} - -fn getResultId(inst: BinaryModule.Instruction, inst_spec: spec.Instruction) ?ResultId { - for (0..@min(2, inst_spec.operands.len)) |i| { - if (inst_spec.operands[i].kind == .id_result) { - if (i < inst.operands.len) return @fromBackingInt(@intCast(inst.operands[i])); - } - } - return null; -} - -fn canPrune(op: Opcode) bool { - return switch (op.class()) { - .type_declaration, - .constant_creation, - .annotation, - => true, - else => switch (op) { - .OpFunction, - .OpUndef, - .OpString, - .OpName, - .OpMemberName, - .OpVariable, - => true, - else => false, - }, - }; -} diff --git a/src/link/Spirv.zig b/src/link/Spirv.zig new file mode 100644 index 0000000000000000000000000000000000000000..b40a3ae6e9cf99dabb5f1e7456a5b8b119393e72 --- /dev/null +++ b/src/link/Spirv.zig @@ -0,0 +1,252 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Path = std.Build.Cache.Path; +const assert = std.debug.assert; +const Zcu = @import("../Zcu.zig"); +const InternPool = @import("../InternPool.zig"); +const Compilation = @import("../Compilation.zig"); +const link = @import("../link.zig"); +const codegen = @import("../codegen.zig"); +const CodeGen = @import("../codegen/spirv/CodeGen.zig"); +const Mir = @import("../codegen/spirv/Mir.zig"); +const spec = @import("../codegen/spirv/spec.zig"); +const Section = @import("../codegen/spirv/Section.zig"); +const Flush = @import("Spirv/Flush.zig"); +const Word = spec.Word; +const Linker = @This(); + +base: link.File, +mirs: std.array_hash_map.Auto(InternPool.Nav.Index, Mir), +objects: std.ArrayList(Object), +exports: std.array_hash_map.Auto(InternPool.Nav.Index, InternPool.NullTerminatedString), + +pub const Object = struct { path: Path, mir: Mir }; + +pub fn createEmpty( + arena: Allocator, + comp: *Compilation, + emit: Path, + options: link.File.OpenOptions, +) !*Linker { + const file = try emit.root_dir.handle.createFile(comp.io, emit.sub_path, .{}); + errdefer file.close(comp.io); + const linker = try arena.create(Linker); + linker.* = .{ + .base = .{ + .tag = .spirv, + .comp = comp, + .emit = emit, + .gc_sections = options.gc_sections orelse false, + .print_gc_sections = options.print_gc_sections, + .stack_size = options.stack_size orelse 0, + .allow_shlib_undefined = options.allow_shlib_undefined orelse false, + .file = file, + .build_id = options.build_id, + }, + .mirs = .empty, + .objects = .empty, + .exports = .empty, + }; + return linker; +} + +pub fn open( + arena: Allocator, + comp: *Compilation, + emit: Path, + options: link.File.OpenOptions, +) !*Linker { + return createEmpty(arena, comp, emit, options); +} + +pub fn deinit(linker: *Linker) void { + const gpa = linker.base.comp.gpa; + for (linker.mirs.values()) |*mir| mir.deinit(gpa); + linker.mirs.deinit(gpa); + for (linker.objects.items) |*object| object.mir.deinit(gpa); + linker.objects.deinit(gpa); + linker.exports.deinit(gpa); +} + +pub fn loadInput(linker: *Linker, input: link.Input) !void { + const diags = &linker.base.comp.link_diags; + switch (input) { + .object => |obj| try linker.loadObject(obj), + else => return diags.fail("unsupported link input for SPIR-V target", .{}), + } +} + +fn loadObject(linker: *Linker, obj: link.Input.Object) !void { + const comp = linker.base.comp; + const gpa = comp.gpa; + const io = comp.io; + const diags = &comp.link_diags; + + const file_size = obj.file.length(io) catch |err| { + return diags.failParse(obj.path, "failed to get file size: {t}", .{err}); + }; + if (file_size % @sizeOf(Word) != 0) { + return diags.failParse(obj.path, "file size is not a multiple of {d}", .{ + @sizeOf(Word), + }); + } + + const words = try gpa.alloc(Word, @intCast(file_size / @sizeOf(Word))); + defer gpa.free(words); + const bytes = std.mem.sliceAsBytes(words); + const n_read = obj.file.readPositionalAll(io, bytes, 0) catch |err| { + return diags.failParse(obj.path, "failed to read: {t}", .{err}); + }; + + if (n_read != bytes.len) { + return diags.failParse(obj.path, "incomplete read", .{}); + } + const header_len = @sizeOf(spec.Header) / @sizeOf(Word); + if (words.len < header_len) { + return diags.failParse(obj.path, "header too small", .{}); + } + const header = std.mem.bytesToValue(spec.Header, std.mem.sliceAsBytes(words[0..header_len])); + if (header.magic != spec.magic_number) { + return diags.failParse(obj.path, "invalid magic number", .{}); + } + if (header.id_bound == 0) { + return diags.failParse(obj.path, "invalid id bound", .{}); + } + + const instructions = words[header_len..]; + var module = Section.fromWords(gpa, instructions, header.id_bound) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + else => return diags.failParse(obj.path, "failed to parse instructions: {t}", .{err}), + }; + errdefer module.deinit(gpa); + try linker.objects.append(gpa, .{ .path = obj.path, .mir = .{ + .id_bound = header.id_bound, + .module = module, + .nav_refs = &.{}, + .externs = &.{}, + .entry_points = &.{}, + } }); +} + +pub fn updateFunc( + linker: *Linker, + pt: Zcu.PerThread, + func_index: InternPool.Index, + mir: *codegen.AnyMir, +) !void { + const gpa = linker.base.comp.gpa; + const nav_index = pt.zcu.funcInfo(func_index).owner_nav; + const gop = try linker.mirs.getOrPut(gpa, nav_index); + if (gop.found_existing) gop.value_ptr.deinit(gpa); + gop.value_ptr.* = mir.spirv; + mir.spirv = .{ + .id_bound = 1, + .module = .empty, + .nav_refs = &.{}, + .externs = &.{}, + .entry_points = &.{}, + }; +} + +pub fn updateNav( + linker: *Linker, + pt: Zcu.PerThread, + nav_index: InternPool.Nav.Index, +) link.Error!void { + const gpa = linker.base.comp.gpa; + const ip = &pt.zcu.intern_pool; + const resolved = ip.getNav(nav_index).resolved.?; + switch (ip.indexToKey(resolved.value)) { + .func => return, + .@"extern" => |@"extern"| if (ip.isFunctionType(@"extern".ty)) return, + else => if (!resolved.@"const") assert(resolved.@"threadlocal") else return, + } + var mir = CodeGen.generateNav(pt, nav_index) catch |err| switch (err) { + error.AlreadyReported => return, + else => |e| return e, + }; + errdefer mir.deinit(gpa); + const gop = try linker.mirs.getOrPut(gpa, nav_index); + if (gop.found_existing) gop.value_ptr.deinit(gpa); + gop.value_ptr.* = mir; +} + +pub fn updateExports( + linker: *Linker, + pt: Zcu.PerThread, + export_indices: []const Zcu.Export.Index, +) link.Error!void { + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const gpa = linker.base.comp.gpa; + const diags = &linker.base.comp.link_diags; + + linker.exports.clearRetainingCapacity(); + try linker.exports.ensureUnusedCapacity(gpa, export_indices.len); + for (export_indices) |exp_index| { + const exp = exp_index.ptr(zcu); + const nav_index = switch (exp.exported) { + .nav => |nav| nav, + .uav => return diags.fail("cannot export a uav constant for SPIR-V target", .{}), + }; + const nav = ip.getNav(nav_index); + const resolved = nav.resolved.?; + if (!ip.isFunctionType(resolved.type) and !resolved.@"threadlocal") { + return diags.fail("cannot export constant '{f}' for SPIR-V target", .{ + nav.fqn.fmt(ip), + }); + } + linker.exports.putAssumeCapacity(nav_index, exp.opts.name); + } +} + +pub fn flush( + linker: *Linker, + arena: Allocator, + tid: Zcu.PerThread.Id, + prog_node: std.Progress.Node, +) link.Error!void { + _ = tid; + const comp = linker.base.comp; + const gpa = comp.gpa; + const diags = &comp.link_diags; + + const sub_node = prog_node.start("Flush Module", 1); + defer sub_node.end(); + + var f: Flush = .{ + .linker = linker, + .arena = arena, + .id_bound = 1, + .navs = .empty, + .symbols = .empty, + .interned = .empty, + .variables = .empty, + .function_references = .empty, + .entry_points = .empty, + .deps = .empty, + .scratch = .empty, + .extended_instruction_sets = .empty, + .object_entry_points = .empty, + .object_execution_modes = .empty, + .debug_strings = .empty, + .names = .empty, + .decorations = .empty, + .globals = .empty, + .declarations = .empty, + .functions = .empty, + }; + var out: Section = .empty; + defer out.deinit(gpa); + + try f.emitModule(&out); + sub_node.completeOne(); + + var writer = linker.base.file.?.writer(comp.io, &.{}); + writer.interface.writeSliceEndian(Word, out.instructions.items, .little) catch { + return diags.fail("failed to write {f}: {t}", .{ linker.base.emit, writer.err.? }); + }; + writer.end() catch |err| { + return diags.fail("failed to write {f}: {t}", .{ linker.base.emit, err }); + }; +} diff --git a/src/link/Spirv/Flush.zig b/src/link/Spirv/Flush.zig new file mode 100644 index 0000000000000000000000000000000000000000..a72fcf23f523d235f9acf1b5b0776df42edd2699 --- /dev/null +++ b/src/link/Spirv/Flush.zig @@ -0,0 +1,664 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Path = std.Build.Cache.Path; +const assert = std.debug.assert; +const InternPool = @import("../../InternPool.zig"); +const link = @import("../../link.zig"); +const Type = @import("../../Type.zig"); +const Linker = @import("../Spirv.zig"); +const Mir = @import("../../codegen/spirv/Mir.zig"); +const spec = @import("../../codegen/spirv/spec.zig"); +const Section = @import("../../codegen/spirv/Section.zig"); +const zig_version = @import("builtin").zig_version; +const Id = spec.Id; +const Word = spec.Word; +const Flush = @This(); + +linker: *Linker, +arena: Allocator, +id_bound: Word, +navs: std.array_hash_map.Auto(InternPool.Nav.Index, Id), +symbols: std.array_hash_map.String(Symbol), +interned: std.array_hash_map.String(Id), +variables: std.array_hash_map.Auto(Id, spec.StorageClass), +function_references: std.array_hash_map.Auto(Id, []const Id), +entry_points: std.ArrayList(Mir.EntryPoint), +deps: std.ArrayList(Id), +scratch: Section, +extended_instruction_sets: Section, +object_entry_points: Section, +object_execution_modes: Section, +debug_strings: Section, +names: Section, +decorations: Section, +globals: Section, +declarations: Section, +functions: Section, + +const Symbol = struct { + id: Id, + defined: bool, + declaration: []const Word, +}; + +const Annotation = struct { + target: Word, + class: spec.Class, + start: usize, + end: usize, +}; + +const Input = struct { + mir: *const Mir, + ids: []Id, + marks: []u32, + mark: u32, + + fn copy( + input: *Input, + flush: *Flush, + dest: *Section, + start: usize, + end: usize, + result_index: ?usize, + ) !void { + const words = input.mir.module.instructions.items; + try dest.ensureUnusedCapacity(flush.arena, end - start); + for (words[start..end], start..) |word, i| { + if (i == result_index) { + dest.writeWord(0); + continue; + } + if (!input.mir.module.ids.isSet(i)) { + dest.writeWord(word); + continue; + } + if (input.marks[word] != input.mark) { + input.marks[word] = input.mark; + try flush.deps.append(flush.arena, input.ids[word]); + } + dest.writeWord(@backingInt(input.ids[word])); + } + } +}; + +fn allocId(flush: *Flush) Id { + const id: Id = @fromBackingInt(flush.id_bound); + flush.id_bound += 1; + return id; +} + +fn navId(flush: *Flush, nav: InternPool.Nav.Index) !Id { + const gop = try flush.navs.getOrPut(flush.arena, nav); + if (!gop.found_existing) gop.value_ptr.* = flush.allocId(); + return gop.value_ptr.*; +} + +pub fn emitModule(flush: *Flush, out: *Section) link.Error!void { + const linker = flush.linker; + const comp = linker.base.comp; + const zcu = comp.zcu.?; + const ip = &zcu.intern_pool; + const gpa = comp.gpa; + const arena = flush.arena; + const diags = &comp.link_diags; + const target = &comp.root_mod.resolved_target.result; + const is_object = comp.config.output_mode == .Obj; + + for (linker.exports.keys(), linker.exports.values()) |nav_index, name| { + if (!linker.mirs.contains(nav_index)) { + assert(zcu.failed_codegen.contains(nav_index)); + return error.AlreadyReported; + } + + const id = try flush.navId(nav_index); + const nav_ty: Type = .fromInterned(ip.getNav(nav_index).resolved.?.type); + if (nav_ty.zigTypeTag(zcu) == .@"fn") { + const cc = nav_ty.fnCallingConvention(zcu); + if (cc != .spirv_device) { + try flush.entry_points.append(arena, .{ + .id = id, + .name = name.toSlice(ip), + .cc = cc, + }); + continue; + } + } + if (!is_object) continue; + + try flush.symbols.putNoClobber(arena, name.toSlice(ip), .{ + .id = id, + .defined = true, + .declaration = &.{}, + }); + } + + for (linker.objects.items) |*object| { + try flush.load(&object.mir, object.path); + } + + const references = try zcu.resolveReferences(); + for (linker.mirs.keys(), linker.mirs.values()) |nav_index, *mir| { + const nav = ip.getNav(nav_index); + if (nav.getExtern(ip) != null) continue; + + const unit: InternPool.AnalUnit = switch (ip.indexToKey(nav.resolved.?.type)) { + .func_type => .wrap(.{ .func = zcu.navValue(nav_index).toIntern() }), + else => .wrap(.{ .nav_val = nav_index }), + }; + if (references.contains(unit)) try flush.load(mir, null); + } + + for (linker.mirs.keys(), linker.mirs.values()) |nav_index, *mir| { + if (ip.getNav(nav_index).getExtern(ip) == null) continue; + if (flush.navs.contains(nav_index)) try flush.load(mir, null); + } + + var failed = false; + for (flush.navs.keys()) |nav_index| { + if (linker.mirs.contains(nav_index) or zcu.failed_codegen.contains(nav_index)) continue; + diags.addError("undefined reference to '{f}'", .{ip.getNav(nav_index).fqn.fmt(ip)}); + failed = true; + } + for (flush.symbols.keys(), flush.symbols.values()) |name, symbol| { + if (is_object) { + try flush.decorations.emit(arena, .OpDecorate, .{ + .target = symbol.id, + .decoration = .{ .linkage_attributes = .{ + .name = name, + .linkage_type = if (symbol.defined) .@"export" else .import, + } }, + }); + } + if (symbol.defined) continue; + if (!is_object or symbol.declaration.len == 0) { + diags.addError("undefined symbol '{s}'", .{name}); + failed = true; + continue; + } + const header: spec.InstructionHeader = @bitCast(symbol.declaration[0]); + const section = if (header.opcode == .OpVariable) &flush.globals else &flush.declarations; + try section.writeWords(arena, symbol.declaration); + } + if (failed) return error.AlreadyReported; + + var version: spec.Version = .{ .major = 1, .minor = 0 }; + for ([_]std.Target.spirv.Feature{ + .v1_1, + .v1_2, + .v1_3, + .v1_4, + .v1_5, + .v1_6, + }) |feature| { + if (target.cpu.has(.spirv, feature)) version.minor += 1; + } + + const zig_packed_version = (zig_version.major << 12) | (zig_version.minor << 7) | zig_version.patch; + const header: spec.Header = .{ + .magic = spec.magic_number, + .version = version, + .generator = .{ .tool = spec.zig_generator_id, .version = zig_packed_version }, + .id_bound = flush.id_bound, + .schema = 0, + }; + try out.writeWords(gpa, @ptrCast(@alignCast(&header))); + try flush.emitCapabilities(out); + try out.writeWords(gpa, flush.extended_instruction_sets.instructions.items); + try flush.emitMemoryModel(out); + try flush.emitEntryPoints(out); + try out.writeWords(gpa, flush.object_entry_points.instructions.items); + try flush.emitExecutionModes(out); + try out.writeWords(gpa, flush.object_execution_modes.instructions.items); + try out.writeWords(gpa, flush.debug_strings.instructions.items); + try flush.emitSourceInfo(out, zig_packed_version); + try out.writeWords(gpa, flush.names.instructions.items); + try out.writeWords(gpa, flush.decorations.instructions.items); + try out.writeWords(gpa, flush.globals.instructions.items); + try out.writeWords(gpa, flush.declarations.instructions.items); + try out.writeWords(gpa, flush.functions.instructions.items); +} + +fn load(flush: *Flush, mir: *const Mir, path: ?Path) link.Error!void { + const comp = flush.linker.base.comp; + const gpa = comp.gpa; + const arena = flush.arena; + const diags = &comp.link_diags; + + const ids = try gpa.alloc(Id, mir.id_bound); + defer gpa.free(ids); + ids[0] = .none; + for (ids[1..], flush.id_bound..) |*id, i| id.* = @fromBackingInt(@intCast(i)); + flush.id_bound += mir.id_bound - 1; + + const marks = try gpa.alloc(u32, mir.id_bound); + defer gpa.free(marks); + @memset(marks, 0); + var input: Input = .{ .mir = mir, .ids = ids, .marks = marks, .mark = 0 }; + + var forward_pointers: std.bit_set.Dynamic = try .initEmpty(gpa, mir.id_bound); + defer forward_pointers.deinit(gpa); + var imports: std.array_hash_map.Auto(Word, u32) = .empty; + defer imports.deinit(gpa); + var annotations: std.ArrayList(Annotation) = .empty; + defer annotations.deinit(gpa); + + for (mir.nav_refs) |ref| ids[@backingInt(ref.id)] = try flush.navId(ref.nav); + for (mir.externs) |ext| { + const gop = try flush.symbols.getOrPut(arena, ext.name); + if (!gop.found_existing) gop.value_ptr.* = .{ + .id = ids[@backingInt(ext.id)], + .defined = false, + .declaration = &.{}, + }; + ids[@backingInt(ext.id)] = gop.value_ptr.id; + try imports.put(gpa, @backingInt(ext.id), @intCast(gop.index)); + } + + const words = mir.module.instructions.items; + var offset: usize = 0; + while (offset < words.len) { + const start = offset; + const header: spec.InstructionHeader = @bitCast(words[start]); + const inst = words[start..][0..header.word_count]; + offset += inst.len; + + if (header.opcode == .OpDecorate) { + const target = inst[1]; + const decoration: spec.Decoration = @fromBackingInt(inst[2]); + if (decoration == .linkage_attributes) { + const parameters = inst[3..]; + const name_words = parameters[0 .. parameters.len - 1]; + const name = std.mem.sliceTo(std.mem.sliceAsBytes(name_words), 0); + const linkage: spec.LinkageType = @fromBackingInt(parameters[parameters.len - 1]); + const gop = try flush.symbols.getOrPut(arena, name); + if (!gop.found_existing) gop.value_ptr.* = .{ + .id = ids[target], + .defined = false, + .declaration = &.{}, + }; + + switch (linkage) { + .@"export" => { + if (gop.value_ptr.defined) { + return diags.failParse(path.?, "duplicate export '{s}'", .{name}); + } + gop.value_ptr.defined = true; + }, + .import => try imports.put(gpa, target, @intCast(gop.index)), + .link_once_odr => { + return diags.failParse(path.?, "unsupported linkage '{t}'", .{linkage}); + }, + } + + ids[target] = gop.value_ptr.id; + continue; + } + } + + const class = header.opcode.class(); + const target: Word = switch (header.opcode) { + .OpName, .OpMemberName => inst[1], + .OpDecorationGroup, .OpGroupDecorate, .OpGroupMemberDecorate => { + return diags.failParse(path.?, "decoration groups are not supported", .{}); + }, + else => if (class == .annotation) inst[1] else continue, + }; + try annotations.append(gpa, .{ + .target = target, + .class = class, + .start = start, + .end = offset, + }); + } + + const starts = try gpa.alloc(u32, mir.id_bound + 1); + defer gpa.free(starts); + @memset(starts, 0); + + for (annotations.items) |annotation| starts[annotation.target + 1] += 1; + for (1..starts.len) |i| starts[i] += starts[i - 1]; + + const cursors = try gpa.dupe(u32, starts); + defer gpa.free(cursors); + const by_target = try gpa.alloc(u32, annotations.items.len); + defer gpa.free(by_target); + for (annotations.items, 0..) |annotation, i| { + by_target[cursors[annotation.target]] = @intCast(i); + cursors[annotation.target] += 1; + } + + var results: std.ArrayList(Word) = .empty; + defer results.deinit(gpa); + + offset = 0; + while (offset < words.len) { + const start = offset; + const header: spec.InstructionHeader = @bitCast(words[start]); + const inst = words[start..][0..header.word_count]; + offset += inst.len; + input.mark += 1; + flush.deps.clearRetainingCapacity(); + + const result_index = header.opcode.resultIndex() orelse { + switch (header.opcode) { + .OpEntryPoint => { + try input.copy(flush, &flush.object_entry_points, start, offset, null); + }, + .OpExecutionMode, .OpExecutionModeId => { + try input.copy(flush, &flush.object_execution_modes, start, offset, null); + }, + .OpTypeForwardPointer => { + forward_pointers.set(inst[1]); + try input.copy(flush, &flush.globals, start, offset, null); + }, + else => {}, + } + continue; + }; + const result = inst[result_index]; + results.clearRetainingCapacity(); + try results.append(gpa, result); + + var has_body = false; + if (header.opcode == .OpFunction) { + while (true) { + const body: spec.InstructionHeader = @bitCast(words[offset]); + if (body.opcode == .OpLabel) has_body = true; + if (body.opcode.resultIndex()) |index| { + try results.append(gpa, words[offset + index]); + } + offset += body.word_count; + if (body.opcode == .OpFunctionEnd) break; + } + } + + for (results.items) |own| input.marks[own] = input.mark; + + if (imports.get(result)) |symbol_index| { + flush.scratch.reset(); + try input.copy(flush, &flush.scratch, start, offset, null); + const declaration = try arena.dupe(Word, flush.scratch.instructions.items); + flush.symbols.values()[symbol_index].declaration = declaration; + continue; + } + + switch (header.opcode) { + .OpFunction => if (!has_body) return diags.failParse(path.?, "function declaration without linkage", .{}), + .OpVariable => try flush.variables.put(arena, ids[result], @fromBackingInt(inst[3])), + else => if (!forward_pointers.isSet(result)) { + flush.scratch.reset(); + try input.copy(flush, &flush.scratch, start, offset, start + result_index); + for (by_target[starts[result]..starts[result + 1]]) |index| { + const annotation = annotations.items[index]; + if (annotation.class != .annotation) continue; + const target = annotation.start + 1; + try input.copy(flush, &flush.scratch, annotation.start, annotation.end, target); + } + + const key = std.mem.sliceAsBytes(flush.scratch.instructions.items); + const gop = try flush.interned.getOrPut(arena, key); + if (gop.found_existing) { + ids[result] = gop.value_ptr.*; + continue; + } + gop.key_ptr.* = try arena.dupe(u8, key); + gop.value_ptr.* = ids[result]; + }, + } + + for (results.items) |target| { + for (by_target[starts[target]..starts[target + 1]]) |index| { + const annotation = annotations.items[index]; + const section = switch (annotation.class) { + .debug => &flush.names, + .annotation => &flush.decorations, + else => unreachable, + }; + try input.copy(flush, section, annotation.start, annotation.end, null); + } + } + + const section = switch (header.opcode) { + .OpFunction => &flush.functions, + .OpExtInstImport => &flush.extended_instruction_sets, + .OpString => &flush.debug_strings, + else => &flush.globals, + }; + try input.copy(flush, section, start, offset, null); + if (header.opcode == .OpFunction) { + const references = try arena.dupe(Id, flush.deps.items); + try flush.function_references.put(arena, ids[result], references); + } + } + + for (mir.entry_points) |entry_point| { + try flush.entry_points.append(arena, .{ + .id = ids[@backingInt(entry_point.id)], + .name = entry_point.name, + .cc = entry_point.cc, + }); + } +} + +fn emitCapabilities(flush: *Flush, out: *Section) link.Error!void { + const comp = flush.linker.base.comp; + const target = &comp.root_mod.resolved_target.result; + const gpa = comp.gpa; + + switch (target.os.tag) { + .opengl, .vulkan => try out.emit(gpa, .OpCapability, .{ .capability = .shader }), + .opencl => { + try out.emit(gpa, .OpCapability, .{ .capability = .kernel }); + try out.emit(gpa, .OpCapability, .{ .capability = .addresses }); + }, + else => unreachable, + } + + if (target.cpu.arch == .spirv64) { + if (!target.cpu.has(.spirv, .int64)) { + try out.emit(gpa, .OpCapability, .{ .capability = .int64 }); + } + if (target.os.tag == .vulkan) { + try out.emit(gpa, .OpCapability, .{ .capability = .physical_storage_buffer_addresses }); + } + } + + if (comp.config.output_mode == .Obj and flush.symbols.count() > 0) { + try out.emit(gpa, .OpCapability, .{ .capability = .linkage }); + } + + inline for (@typeInfo(spec.Capability).@"enum".field_names) |name| { + if (@hasField(std.Target.spirv.Feature, name)) { + const feature = @field(std.Target.spirv.Feature, name); + if (target.cpu.has(.spirv, feature)) { + try out.emit(gpa, .OpCapability, .{ .capability = @field(spec.Capability, name) }); + } + } + } + + if (target.cpu.arch == .spirv64 and target.os.tag == .vulkan) { + try out.emit(gpa, .OpExtension, .{ .name = "SPV_KHR_physical_storage_buffer" }); + } + + inline for (@typeInfo(spec.Extension).@"enum".field_names) |name| { + if (@hasField(std.Target.spirv.Feature, name)) { + const feature = @field(std.Target.spirv.Feature, name); + if (target.cpu.has(.spirv, feature)) try out.emit(gpa, .OpExtension, .{ .name = name }); + } + } +} + +fn emitMemoryModel(flush: *Flush, out: *Section) link.Error!void { + const comp = flush.linker.base.comp; + const target = &comp.root_mod.resolved_target.result; + const addressing_model: spec.AddressingModel = switch (target.os.tag) { + .opengl => .logical, + .vulkan => switch (target.cpu.arch) { + .spirv32 => .logical, + .spirv64 => .physical_storage_buffer64, + else => unreachable, + }, + .opencl => switch (target.cpu.arch) { + .spirv32 => .physical32, + .spirv64 => .physical64, + else => unreachable, + }, + else => unreachable, + }; + try out.emit(comp.gpa, .OpMemoryModel, .{ + .addressing_model = addressing_model, + .memory_model = switch (target.os.tag) { + .opencl => .open_cl, + .vulkan, .opengl => .glsl450, + else => unreachable, + }, + }); +} + +fn emitEntryPoints(flush: *Flush, out: *Section) link.Error!void { + const comp = flush.linker.base.comp; + const gpa = comp.gpa; + const arena = flush.arena; + const target = &comp.root_mod.resolved_target.result; + const v1_4 = target.cpu.has(.spirv, .v1_4); + + var visited: std.bit_set.Dynamic = try .initEmpty(arena, flush.id_bound); + var pending: std.ArrayList(Id) = .empty; + var interface: std.ArrayList(Id) = .empty; + for (flush.entry_points.items) |entry_point| { + const execution_model: spec.ExecutionModel = switch (entry_point.cc) { + .spirv_kernel => if (target.os.tag == .opencl) .kernel else .gl_compute, + .spirv_vertex => .vertex, + .spirv_fragment => .fragment, + .spirv_task => .task_ext, + .spirv_mesh => .mesh_ext, + else => unreachable, + }; + + visited.unsetAll(); + interface.clearRetainingCapacity(); + try pending.append(arena, entry_point.id); + while (pending.pop()) |id| { + if (visited.isSet(@backingInt(id))) continue; + visited.set(@backingInt(id)); + if (flush.variables.get(id)) |storage_class| { + if (storage_class == .function) continue; + if (!v1_4 and storage_class != .input and storage_class != .output) continue; + try interface.append(arena, id); + } else if (flush.function_references.get(id)) |references| { + try pending.appendSlice(arena, references); + } + } + + try out.emit(gpa, .OpEntryPoint, .{ + .execution_model = execution_model, + .entry_point = entry_point.id, + .name = entry_point.name, + .interface = interface.items, + }); + } +} + +fn emitExecutionModes(flush: *Flush, out: *Section) link.Error!void { + const comp = flush.linker.base.comp; + const gpa = comp.gpa; + const target = &comp.root_mod.resolved_target.result; + + for (flush.entry_points.items) |entry_point| switch (entry_point.cc) { + .spirv_kernel, .spirv_task => |kernel| { + try out.emit(gpa, .OpExecutionMode, .{ + .entry_point = entry_point.id, + .mode = .{ .local_size = .{ + .x_size = kernel.x, + .y_size = kernel.y, + .z_size = kernel.z, + } }, + }); + }, + .spirv_fragment => |fragment| { + try out.emit(gpa, .OpExecutionMode, .{ + .entry_point = entry_point.id, + .mode = if (target.os.tag == .vulkan) .origin_upper_left else .origin_lower_left, + }); + if (fragment.pixel_centered_integer) { + try out.emit(gpa, .OpExecutionMode, .{ + .entry_point = entry_point.id, + .mode = .pixel_center_integer, + }); + } + if (fragment.depth_assumption != .none) { + try out.emit(gpa, .OpExecutionMode, .{ + .entry_point = entry_point.id, + .mode = switch (fragment.depth_assumption) { + .none => unreachable, + .greater => .depth_greater, + .less => .depth_less, + .unchanged => .depth_unchanged, + }, + }); + } + }, + .spirv_mesh => |mesh| { + try out.emit(gpa, .OpExecutionMode, .{ + .entry_point = entry_point.id, + .mode = .{ .output_vertices = .{ .vertex_count = mesh.max_vertices } }, + }); + try out.emit(gpa, .OpExecutionMode, .{ + .entry_point = entry_point.id, + .mode = .{ .output_primitives_ext = .{ .primitive_count = mesh.max_primitives } }, + }); + try out.emit(gpa, .OpExecutionMode, .{ + .entry_point = entry_point.id, + .mode = .{ .local_size = .{ + .x_size = mesh.x, + .y_size = mesh.y, + .z_size = mesh.z, + } }, + }); + try out.emit(gpa, .OpExecutionMode, .{ + .entry_point = entry_point.id, + .mode = switch (mesh.stage_output) { + .output_points => .output_points, + .output_lines => .output_lines_ext, + .output_triangles => .output_triangles_ext, + }, + }); + }, + else => {}, + }; +} + +fn emitSourceInfo(flush: *Flush, out: *Section, version: u32) link.Error!void { + const comp = flush.linker.base.comp; + const gpa = comp.gpa; + const zcu = comp.zcu orelse return; + const ip = &zcu.intern_pool; + + var error_info: std.Io.Writer.Allocating = .init(gpa); + defer error_info.deinit(); + error_info.writer.writeAll("zig_errors:") catch return error.OutOfMemory; + for (ip.global_error_set.getNamesFromMainThread()) |name| { + error_info.writer.writeByte(':') catch return error.OutOfMemory; + std.Uri.Component.percentEncode( + &error_info.writer, + name.toSlice(ip), + struct { + fn isValidChar(c: u8) bool { + return switch (c) { + 0, '%', ':' => false, + else => true, + }; + } + }.isValidChar, + ) catch return error.OutOfMemory; + } + + try out.emit(gpa, .OpSourceExtension, .{ .extension = error_info.written() }); + try out.emit(gpa, .OpSource, .{ + .source_language = .zig, + .version = version, + .file = null, + .source = null, + }); +} diff --git a/src/target.zig b/src/target.zig index 5439374628c8f5d29c68a1e6aa55ad272e02e920..01fe63f4ef1bc77aaa56c4246298deb17715d8a7 100644 --- a/src/target.zig +++ b/src/target.zig @@ -604,6 +604,7 @@ pub fn defaultAddressSpace( // The default address space for functions on AVR is .flash to produce // correct fixups into progmem. if (context == .function and target.cpu.arch == .avr) return .flash; + if (context == .global_mutable and target.os.tag == .vulkan) return .private; return .generic; } @@ -649,6 +650,7 @@ pub fn shouldBlockPointerOps(target: *const std.Target, as: AddressSpace) bool { // Logical pointers that never support operations .constant, .local, + .private, .input, .output, .uniform, diff --git a/test/cases/compile_errors/SpirvType_vulkan_target.zig b/test/cases/compile_errors/SpirvType_vulkan_target.zig index fdb53678e3139870df23563a9b2ac5494e524cea..7871365f4ed83ecbda72abff178b24704b6496b8 100644 --- a/test/cases/compile_errors/SpirvType_vulkan_target.zig +++ b/test/cases/compile_errors/SpirvType_vulkan_target.zig @@ -50,7 +50,7 @@ comptime { // backend=selfhosted // target=spirv32-vulkan // -// :2:21: error: access qualifier '.read_only' is only valid under the 'opencl' os +// :2:21: error: access qualifier '.read_only' is only valid under the 'opencl' OS // :14:21: error: invalid 'sampled' field value 'bool' -// :26:21: error: 'void' type for 'sampled' field is only valid under the 'opencl' os -// :38:21: error: 'sampled' field value must be a 32-bit int, 64-bit int or 32-bit float under the 'vulkan' os +// :26:21: error: 'void' type for 'sampled' field is only valid under the 'opencl' OS +// :38:21: error: 'sampled' field value must be a 32-bit int, 64-bit int or 32-bit float under the 'vulkan' OS diff --git a/test/cases/compile_errors/extern_spirv_decoration_validation.zig b/test/cases/compile_errors/extern_spirv_decoration_validation.zig index a7cdd98b58e6195fe62281c9ed025659bc9c6a57..9d48f73bf50e2f653de0b690296c3bdccf4a3cd1 100644 --- a/test/cases/compile_errors/extern_spirv_decoration_validation.zig +++ b/test/cases/compile_errors/extern_spirv_decoration_validation.zig @@ -1,13 +1,30 @@ -const x = @extern(*addrspace(.push_constant) u32, .{ - .name = "x", +const flat = @extern(*addrspace(.push_constant) u32, .{ + .name = "flat", .decoration = .{ .flat = 0 }, }); +const location = @extern(*addrspace(.uniform) u32, .{ + .name = "location", + .decoration = .{ .location = 0 }, +}); +const descriptor = @extern(*addrspace(.input) u32, .{ + .name = "descriptor", + .decoration = .{ .descriptor = .{ .set = 0, .binding = 0 } }, +}); + comptime { - _ = x; + _ = flat; +} +comptime { + _ = location; +} +comptime { + _ = descriptor; } // error // backend=selfhosted // target=spirv32-vulkan // -// :1:52: error: "flat" decoration requires "input" or "output" address space +// :1:55: error: 'flat' decoration requires 'input' or 'output' address space +// :5:53: error: 'location' decoration requires 'input' or 'output' address space +// :9:53: error: 'descriptor' decoration requires 'storage_buffer', 'uniform', 'constant' or 'global' address space diff --git a/test/cases/compile_errors/spirv_global_var_must_be_threadlocal.zig b/test/cases/compile_errors/spirv_global_var_must_be_threadlocal.zig new file mode 100644 index 0000000000000000000000000000000000000000..107895a577d30a54735826ef23d1d4c245b1e845 --- /dev/null +++ b/test/cases/compile_errors/spirv_global_var_must_be_threadlocal.zig @@ -0,0 +1,12 @@ +var x: u32 = 0; + +comptime { + _ = x; +} + +// error +// backend=selfhosted +// target=spirv32-vulkan +// +// :1:1: error: SPIR-V target does not support global variables +// :1:1: note: consider using 'threadlocal' diff --git a/test/cases/compile_errors/spirv_vulkan_threadlocal_addrspace.zig b/test/cases/compile_errors/spirv_vulkan_threadlocal_addrspace.zig new file mode 100644 index 0000000000000000000000000000000000000000..517d2c7cb3a25afe41f8b7fe248d8a5c8c4f2df1 --- /dev/null +++ b/test/cases/compile_errors/spirv_vulkan_threadlocal_addrspace.zig @@ -0,0 +1,11 @@ +threadlocal var counter: u32 addrspace(.generic) = 0; + +export fn main() callconv(.kernel) void { + counter += 1; +} + +// error +// backend=selfhosted +// target=spirv32-vulkan +// +// :1:41: error: threadlocal variables with address space 'generic' are not supported on vulkan