authorgravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2024-03-09 12:00:34+01:00
committergravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2024-03-18 19:13:46+01:00
log3d5721da239502a23cb7b3629111f40b54455ddf
tree93ffa28225bb5edbb2badf1295fbf071ec05d9bb
parent3bffa58012bbe298c1f99f27e55d6f088d5d2078
signaturebadge-check Signed by SSH key SHA256:ZS52FNyUv2WUXvO4njmVaFVO46RHojFuOrxRc4LuKzg

spirv: update spec generator

For module parsing and assembling, we will also need to know all of the SPIR-V extensions and their instructions. This commit updates the generator to generate those. Because there are multiple instruction sets that each have a separate list of Opcodes, no separate enum is generated for these opcodes. Additionally, the previous mechanism for runtime instruction information, `Opcode`'s `fn operands()`, has been removed in favor for `InstructionSet.core.instructions()`. Any mapping from operand to instruction is to be done at runtime. Using a runtime populated hashmap should also be more efficient than the previous mechanism using `stringToEnum`.

2 files changed, 251 insertions(+), 81 deletions(-)

tools/gen_spirv_spec.zig+247-79
......@@ -1,45 +1,110 @@
11const std = @import("std");
2const g = @import("spirv/grammar.zig");
32const Allocator = std.mem.Allocator;
3const g = @import("spirv/grammar.zig");
4const CoreRegistry = g.CoreRegistry;
5const ExtensionRegistry = g.ExtensionRegistry;
6const Instruction = g.Instruction;
7const OperandKind = g.OperandKind;
8const Enumerant = g.Enumerant;
9const Operand = g.Operand;
410
511const ExtendedStructSet = std.StringHashMap(void);
612
13const Extension = struct {
14 name: []const u8,
15 spec: ExtensionRegistry,
16};
17
18const CmpInst = struct {
19 fn lt(_: CmpInst, a: Instruction, b: Instruction) bool {
20 return a.opcode < b.opcode;
21 }
22};
23
24const StringPair = struct { []const u8, []const u8 };
25
26const StringPairContext = struct {
27 pub fn hash(_: @This(), a: StringPair) u32 {
28 var hasher = std.hash.Wyhash.init(0);
29 const x, const y = a;
30 hasher.update(x);
31 hasher.update(y);
32 return @truncate(hasher.final());
33 }
34
35 pub fn eql(_: @This(), a: StringPair, b: StringPair, b_index: usize) bool {
36 _ = b_index;
37 const a_x, const a_y = a;
38 const b_x, const b_y = b;
39 return std.mem.eql(u8, a_x, b_x) and std.mem.eql(u8, a_y, b_y);
40 }
41};
42
43const OperandKindMap = std.ArrayHashMap(StringPair, OperandKind, StringPairContext, true);
44
745pub fn main() !void {
846 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
947 defer arena.deinit();
10 const allocator = arena.allocator();
48 const a = arena.allocator();
1149
12 const args = try std.process.argsAlloc(allocator);
50 const args = try std.process.argsAlloc(a);
1351 if (args.len != 2) {
14 usageAndExit(std.io.getStdErr(), args[0], 1);
52 usageAndExit(args[0], 1);
53 }
54
55 const json_path = try std.fs.path.join(a, &.{ args[1], "include/spirv/unified1/" });
56 const dir = try std.fs.cwd().openDir(json_path, .{ .iterate = true });
57
58 // const spec_path = try std.fs.path.join(a, &.{spirv_headers_dir_path, "spirv.core.grammar.json"});
59 // const core_spec = try std.fs.cwd().readFileAlloc(a, spec_path, std.math.maxInt(usize));
60
61 const core_spec = try readRegistry(CoreRegistry, a, dir, "spirv.core.grammar.json");
62 std.sort.block(Instruction, core_spec.instructions, CmpInst{}, CmpInst.lt);
63
64 var exts = std.ArrayList(Extension).init(a);
65
66 var it = dir.iterate();
67 while (try it.next()) |entry| {
68 if (entry.kind != .file or !std.mem.startsWith(u8, entry.name, "extinst.")) {
69 continue;
70 }
71
72 std.debug.assert(std.mem.endsWith(u8, entry.name, ".grammar.json"));
73 const name = entry.name["extinst.".len .. entry.name.len - ".grammar.json".len];
74 const spec = try readRegistry(ExtensionRegistry, a, dir, entry.name);
75
76 std.sort.block(Instruction, spec.instructions, CmpInst{}, CmpInst.lt);
77
78 try exts.append(.{ .name = try a.dupe(u8, name), .spec = spec });
1579 }
1680
17 const spec_path = args[1];
18 const spec = try std.fs.cwd().readFileAlloc(allocator, spec_path, std.math.maxInt(usize));
81 var bw = std.io.bufferedWriter(std.io.getStdOut().writer());
82 try render(bw.writer(), a, core_spec, exts.items);
83 try bw.flush();
84}
1985
86fn readRegistry(comptime RegistryType: type, a: Allocator, dir: std.fs.Dir, path: []const u8) !RegistryType {
87 const spec = try dir.readFileAlloc(a, path, std.math.maxInt(usize));
2088 // Required for json parsing.
2189 @setEvalBranchQuota(10000);
2290
23 var scanner = std.json.Scanner.initCompleteInput(allocator, spec);
91 var scanner = std.json.Scanner.initCompleteInput(a, spec);
2492 var diagnostics = std.json.Diagnostics{};
2593 scanner.enableDiagnostics(&diagnostics);
26 const parsed = std.json.parseFromTokenSource(g.CoreRegistry, allocator, &scanner, .{}) catch |err| {
27 std.debug.print("line,col: {},{}\n", .{ diagnostics.getLine(), diagnostics.getColumn() });
94 const parsed = std.json.parseFromTokenSource(RegistryType, a, &scanner, .{}) catch |err| {
95 std.debug.print("{s}:{}:{}:\n", .{ path, diagnostics.getLine(), diagnostics.getColumn() });
2896 return err;
2997 };
30
31 var bw = std.io.bufferedWriter(std.io.getStdOut().writer());
32 try render(bw.writer(), allocator, parsed.value);
33 try bw.flush();
98 return parsed.value;
3499}
35100
36101/// Returns a set with types that require an extra struct for the `Instruction` interface
37102/// to the spir-v spec, or whether the original type can be used.
38103fn extendedStructs(
39 arena: Allocator,
40 kinds: []const g.OperandKind,
104 a: Allocator,
105 kinds: []const OperandKind,
41106) !ExtendedStructSet {
42 var map = ExtendedStructSet.init(arena);
107 var map = ExtendedStructSet.init(a);
43108 try map.ensureTotalCapacity(@as(u32, @intCast(kinds.len)));
44109
45110 for (kinds) |kind| {
......@@ -73,7 +138,7 @@ fn tagPriorityScore(tag: []const u8) usize {
73138 }
74139}
75140
76fn render(writer: anytype, allocator: Allocator, registry: g.CoreRegistry) !void {
141fn render(writer: anytype, a: Allocator, registry: CoreRegistry, extensions: []const Extension) !void {
77142 try writer.writeAll(
78143 \\//! This file is auto-generated by tools/gen_spirv_spec.zig.
79144 \\
......@@ -99,6 +164,7 @@ fn render(writer: anytype, allocator: Allocator, registry: g.CoreRegistry) !void
99164 \\pub const IdScope = IdRef;
100165 \\
101166 \\pub const LiteralInteger = Word;
167 \\pub const LiteralFloat = Word;
102168 \\pub const LiteralString = []const u8;
103169 \\pub const LiteralContextDependentNumber = union(enum) {
104170 \\ int32: i32,
......@@ -139,6 +205,12 @@ fn render(writer: anytype, allocator: Allocator, registry: g.CoreRegistry) !void
139205 \\ parameters: []const OperandKind,
140206 \\};
141207 \\
208 \\pub const Instruction = struct {
209 \\ name: []const u8,
210 \\ opcode: Word,
211 \\ operands: []const Operand,
212 \\};
213 \\
142214 \\
143215 );
144216
......@@ -151,15 +223,123 @@ fn render(writer: anytype, allocator: Allocator, registry: g.CoreRegistry) !void
151223 .{ registry.major_version, registry.minor_version, registry.revision, registry.magic_number },
152224 );
153225
154 const extended_structs = try extendedStructs(allocator, registry.operand_kinds);
155 try renderClass(writer, allocator, registry.instructions);
156 try renderOperandKind(writer, registry.operand_kinds);
157 try renderOpcodes(writer, allocator, registry.instructions, extended_structs);
158 try renderOperandKinds(writer, allocator, registry.operand_kinds, extended_structs);
226 // Merge the operand kinds from all extensions together.
227 // var all_operand_kinds = std.ArrayList(OperandKind).init(a);
228 // try all_operand_kinds.appendSlice(registry.operand_kinds);
229 var all_operand_kinds = OperandKindMap.init(a);
230 for (registry.operand_kinds) |kind| {
231 try all_operand_kinds.putNoClobber(.{ "core", kind.kind }, kind);
232 }
233 for (extensions) |ext| {
234 // Note: extensions may define the same operand kind, with different
235 // parameters. Instead of trying to merge them, just discriminate them
236 // using the name of the extension. This is similar to what
237 // the official headers do.
238
239 try all_operand_kinds.ensureUnusedCapacity(ext.spec.operand_kinds.len);
240 for (ext.spec.operand_kinds) |kind| {
241 var new_kind = kind;
242 new_kind.kind = try std.mem.join(a, ".", &.{ ext.name, kind.kind });
243 try all_operand_kinds.putNoClobber(.{ ext.name, kind.kind }, new_kind);
244 }
245 }
246
247 const extended_structs = try extendedStructs(a, all_operand_kinds.values());
248 // Note: extensions don't seem to have class.
249 try renderClass(writer, a, registry.instructions);
250 try renderOperandKind(writer, all_operand_kinds.values());
251 try renderOpcodes(writer, a, registry.instructions, extended_structs);
252 try renderOperandKinds(writer, a, all_operand_kinds.values(), extended_structs);
253 try renderInstructionSet(writer, a, registry, extensions, all_operand_kinds);
254}
255
256fn renderInstructionSet(
257 writer: anytype,
258 a: Allocator,
259 core: CoreRegistry,
260 extensions: []const Extension,
261 all_operand_kinds: OperandKindMap,
262) !void {
263 _ = a;
264 try writer.writeAll(
265 \\pub const InstructionSet = enum {
266 \\ core,
267 );
268
269 for (extensions) |ext| {
270 try writer.print("{},\n", .{std.zig.fmtId(ext.name)});
271 }
272
273 try writer.writeAll(
274 \\
275 \\ pub fn instructions(self: InstructionSet) []const Instruction {
276 \\ return switch (self) {
277 \\
278 );
279
280 try renderInstructionsCase(writer, "core", core.instructions, all_operand_kinds);
281 for (extensions) |ext| {
282 try renderInstructionsCase(writer, ext.name, ext.spec.instructions, all_operand_kinds);
283 }
284
285 try writer.writeAll(
286 \\ };
287 \\ }
288 \\};
289 \\
290 );
291}
292
293fn renderInstructionsCase(
294 writer: anytype,
295 set_name: []const u8,
296 instructions: []const Instruction,
297 all_operand_kinds: OperandKindMap,
298) !void {
299 // Note: theoretically we could dedup from tags and give every instruction a list of aliases,
300 // but there aren't so many total aliases and that would add more overhead in total. We will
301 // just filter those out when needed.
302
303 try writer.print(".{} => &[_]Instruction{{\n", .{std.zig.fmtId(set_name)});
304
305 for (instructions) |inst| {
306 try writer.print(
307 \\.{{
308 \\ .name = "{s}",
309 \\ .opcode = {},
310 \\ .operands = &[_]Operand{{
311 \\
312 , .{ inst.opname, inst.opcode });
313
314 for (inst.operands) |operand| {
315 const quantifier = if (operand.quantifier) |q|
316 switch (q) {
317 .@"?" => "optional",
318 .@"*" => "variadic",
319 }
320 else
321 "required";
322
323 const kind = all_operand_kinds.get(.{ set_name, operand.kind }) orelse
324 all_operand_kinds.get(.{ "core", operand.kind }).?;
325 try writer.print(".{{.kind = .{}, .quantifier = .{s}}},\n", .{ std.zig.fmtId(kind.kind), quantifier });
326 }
327
328 try writer.writeAll(
329 \\ },
330 \\},
331 \\
332 );
333 }
334
335 try writer.writeAll(
336 \\},
337 \\
338 );
159339}
160340
161fn renderClass(writer: anytype, allocator: Allocator, instructions: []const g.Instruction) !void {
162 var class_map = std.StringArrayHashMap(void).init(allocator);
341fn renderClass(writer: anytype, a: Allocator, instructions: []const Instruction) !void {
342 var class_map = std.StringArrayHashMap(void).init(a);
163343
164344 for (instructions) |inst| {
165345 if (std.mem.eql(u8, inst.class.?, "@exclude")) {
......@@ -173,7 +353,7 @@ fn renderClass(writer: anytype, allocator: Allocator, instructions: []const g.In
173353 try renderInstructionClass(writer, class);
174354 try writer.writeAll(",\n");
175355 }
176 try writer.writeAll("};\n");
356 try writer.writeAll("};\n\n");
177357}
178358
179359fn renderInstructionClass(writer: anytype, class: []const u8) !void {
......@@ -192,7 +372,7 @@ fn renderInstructionClass(writer: anytype, class: []const u8) !void {
192372 }
193373}
194374
195fn renderOperandKind(writer: anytype, operands: []const g.OperandKind) !void {
375fn renderOperandKind(writer: anytype, operands: []const OperandKind) !void {
196376 try writer.writeAll("pub const OperandKind = enum {\n");
197377 for (operands) |operand| {
198378 try writer.print("{},\n", .{std.zig.fmtId(operand.kind)});
......@@ -242,7 +422,7 @@ fn renderOperandKind(writer: anytype, operands: []const g.OperandKind) !void {
242422 try writer.writeAll("};\n}\n};\n");
243423}
244424
245fn renderEnumerant(writer: anytype, enumerant: g.Enumerant) !void {
425fn renderEnumerant(writer: anytype, enumerant: Enumerant) !void {
246426 try writer.print(".{{.name = \"{s}\", .value = ", .{enumerant.enumerant});
247427 switch (enumerant.value) {
248428 .bitflag => |flag| try writer.writeAll(flag),
......@@ -260,14 +440,14 @@ fn renderEnumerant(writer: anytype, enumerant: g.Enumerant) !void {
260440
261441fn renderOpcodes(
262442 writer: anytype,
263 allocator: Allocator,
264 instructions: []const g.Instruction,
443 a: Allocator,
444 instructions: []const Instruction,
265445 extended_structs: ExtendedStructSet,
266446) !void {
267 var inst_map = std.AutoArrayHashMap(u32, usize).init(allocator);
447 var inst_map = std.AutoArrayHashMap(u32, usize).init(a);
268448 try inst_map.ensureTotalCapacity(instructions.len);
269449
270 var aliases = std.ArrayList(struct { inst: usize, alias: usize }).init(allocator);
450 var aliases = std.ArrayList(struct { inst: usize, alias: usize }).init(a);
271451 try aliases.ensureTotalCapacity(instructions.len);
272452
273453 for (instructions, 0..) |inst, i| {
......@@ -323,31 +503,6 @@ fn renderOpcodes(
323503 try renderOperand(writer, .instruction, inst.opname, inst.operands, extended_structs);
324504 }
325505
326 try writer.writeAll(
327 \\};
328 \\}
329 \\pub fn operands(self: Opcode) []const Operand {
330 \\return switch (self) {
331 \\
332 );
333
334 for (instructions_indices) |i| {
335 const inst = instructions[i];
336 try writer.print(".{} => &[_]Operand{{", .{std.zig.fmtId(inst.opname)});
337 for (inst.operands) |operand| {
338 const quantifier = if (operand.quantifier) |q|
339 switch (q) {
340 .@"?" => "optional",
341 .@"*" => "variadic",
342 }
343 else
344 "required";
345
346 try writer.print(".{{.kind = .{s}, .quantifier = .{s}}},", .{ operand.kind, quantifier });
347 }
348 try writer.writeAll("},\n");
349 }
350
351506 try writer.writeAll(
352507 \\};
353508 \\}
......@@ -368,14 +523,14 @@ fn renderOpcodes(
368523
369524fn renderOperandKinds(
370525 writer: anytype,
371 allocator: Allocator,
372 kinds: []const g.OperandKind,
526 a: Allocator,
527 kinds: []const OperandKind,
373528 extended_structs: ExtendedStructSet,
374529) !void {
375530 for (kinds) |kind| {
376531 switch (kind.category) {
377 .ValueEnum => try renderValueEnum(writer, allocator, kind, extended_structs),
378 .BitEnum => try renderBitEnum(writer, allocator, kind, extended_structs),
532 .ValueEnum => try renderValueEnum(writer, a, kind, extended_structs),
533 .BitEnum => try renderBitEnum(writer, a, kind, extended_structs),
379534 else => {},
380535 }
381536 }
......@@ -383,20 +538,26 @@ fn renderOperandKinds(
383538
384539fn renderValueEnum(
385540 writer: anytype,
386 allocator: Allocator,
387 enumeration: g.OperandKind,
541 a: Allocator,
542 enumeration: OperandKind,
388543 extended_structs: ExtendedStructSet,
389544) !void {
390545 const enumerants = enumeration.enumerants orelse return error.InvalidRegistry;
391546
392 var enum_map = std.AutoArrayHashMap(u32, usize).init(allocator);
547 var enum_map = std.AutoArrayHashMap(u32, usize).init(a);
393548 try enum_map.ensureTotalCapacity(enumerants.len);
394549
395 var aliases = std.ArrayList(struct { enumerant: usize, alias: usize }).init(allocator);
550 var aliases = std.ArrayList(struct { enumerant: usize, alias: usize }).init(a);
396551 try aliases.ensureTotalCapacity(enumerants.len);
397552
398553 for (enumerants, 0..) |enumerant, i| {
399 const result = enum_map.getOrPutAssumeCapacity(enumerant.value.int);
554 try writer.context.flush();
555 const value: u31 = switch (enumerant.value) {
556 .int => |value| value,
557 // Some extensions declare ints as string
558 .bitflag => |value| try std.fmt.parseInt(u31, value, 10),
559 };
560 const result = enum_map.getOrPutAssumeCapacity(value);
400561 if (!result.found_existing) {
401562 result.value_ptr.* = i;
402563 continue;
......@@ -422,9 +583,12 @@ fn renderValueEnum(
422583
423584 for (enum_indices) |i| {
424585 const enumerant = enumerants[i];
425 if (enumerant.value != .int) return error.InvalidRegistry;
586 // if (enumerant.value != .int) return error.InvalidRegistry;
426587
427 try writer.print("{} = {},\n", .{ std.zig.fmtId(enumerant.enumerant), enumerant.value.int });
588 switch (enumerant.value) {
589 .int => |value| try writer.print("{} = {},\n", .{ std.zig.fmtId(enumerant.enumerant), value }),
590 .bitflag => |value| try writer.print("{} = {s},\n", .{ std.zig.fmtId(enumerant.enumerant), value }),
591 }
428592 }
429593
430594 try writer.writeByte('\n');
......@@ -454,8 +618,8 @@ fn renderValueEnum(
454618
455619fn renderBitEnum(
456620 writer: anytype,
457 allocator: Allocator,
458 enumeration: g.OperandKind,
621 a: Allocator,
622 enumeration: OperandKind,
459623 extended_structs: ExtendedStructSet,
460624) !void {
461625 try writer.print("pub const {s} = packed struct {{\n", .{std.zig.fmtId(enumeration.kind)});
......@@ -463,7 +627,7 @@ fn renderBitEnum(
463627 var flags_by_bitpos = [_]?usize{null} ** 32;
464628 const enumerants = enumeration.enumerants orelse return error.InvalidRegistry;
465629
466 var aliases = std.ArrayList(struct { flag: usize, alias: u5 }).init(allocator);
630 var aliases = std.ArrayList(struct { flag: usize, alias: u5 }).init(a);
467631 try aliases.ensureTotalCapacity(enumerants.len);
468632
469633 for (enumerants, 0..) |enumerant, i| {
......@@ -471,6 +635,10 @@ fn renderBitEnum(
471635 const value = try parseHexInt(enumerant.value.bitflag);
472636 if (value == 0) {
473637 continue; // Skip 'none' items
638 } else if (std.mem.eql(u8, enumerant.enumerant, "FlagIsPublic")) {
639 // This flag is special and poorly defined in the json files.
640 // Just skip it for now
641 continue;
474642 }
475643
476644 std.debug.assert(@popCount(value) == 1);
......@@ -540,7 +708,7 @@ fn renderOperand(
540708 mask,
541709 },
542710 field_name: []const u8,
543 parameters: []const g.Operand,
711 parameters: []const Operand,
544712 extended_structs: ExtendedStructSet,
545713) !void {
546714 if (kind == .instruction) {
......@@ -606,7 +774,7 @@ fn renderOperand(
606774 try writer.writeAll(",\n");
607775}
608776
609fn renderFieldName(writer: anytype, operands: []const g.Operand, field_index: usize) !void {
777fn renderFieldName(writer: anytype, operands: []const Operand, field_index: usize) !void {
610778 const operand = operands[field_index];
611779
612780 // Should be enough for all names - adjust as needed.
......@@ -673,16 +841,16 @@ fn parseHexInt(text: []const u8) !u31 {
673841 return try std.fmt.parseInt(u31, text[prefix.len..], 16);
674842}
675843
676fn usageAndExit(file: std.fs.File, arg0: []const u8, code: u8) noreturn {
677 file.writer().print(
678 \\Usage: {s} <spirv json spec>
844fn usageAndExit(arg0: []const u8, code: u8) noreturn {
845 std.io.getStdErr().writer().print(
846 \\Usage: {s} <SPIRV-Headers repository path>
679847 \\
680 \\Generates Zig bindings for a SPIR-V specification .json (either core or
681 \\extinst versions). The result, printed to stdout, should be used to update
848 \\Generates Zig bindings for SPIR-V specifications found in the SPIRV-Headers
849 \\repository. The result, printed to stdout, should be used to update
682850 \\files in src/codegen/spirv. Don't forget to format the output.
683851 \\
684 \\The relevant specifications can be obtained from the SPIR-V registry:
685 \\https://github.com/KhronosGroup/SPIRV-Headers/blob/master/include/spirv/unified1/
852 \\<SPIRV-Headers repository path> should point to a clone of
853 \\https://github.com/KhronosGroup/SPIRV-Headers/
686854 \\
687855 , .{arg0}) catch std.process.exit(1);
688856 std.process.exit(code);
tools/spirv/grammar.zig+4-2
......@@ -22,8 +22,8 @@ pub const CoreRegistry = struct {
2222};
2323
2424pub const ExtensionRegistry = struct {
25 copyright: [][]const u8,
26 version: u32,
25 copyright: ?[][]const u8 = null,
26 version: ?u32 = null,
2727 revision: u32,
2828 instructions: []Instruction,
2929 operand_kinds: []OperandKind = &[_]OperandKind{},
......@@ -40,6 +40,8 @@ pub const Instruction = struct {
4040 opcode: u32,
4141 operands: []Operand = &[_]Operand{},
4242 capabilities: [][]const u8 = &[_][]const u8{},
43 // DebugModuleINTEL has this...
44 capability: ?[]const u8 = null,
4345 extensions: [][]const u8 = &[_][]const u8{},
4446 version: ?[]const u8 = null,
4547