authorgravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2021-05-13 01:19:32+02:00
committergravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2021-05-14 19:49:32+02:00
log4fe0a0b82b531d4ac44ed9a249137e9f6c186d0b
treeab16715f5702b25125b613aa882586e1edcb8f50
parent25329ca8521fc34a5cedc837a1142e1111d4e844

SPIR-V: SPIR-V feature generation tool

There is a lot left to be desired for this tool, as currently dependencies of extensions and dependencies of capabilities on extensions are not included: - There is no machine-readable definition of dependencies of extensions. - A capability may depend on either of a multitude of extensions, which as of yet cannot be properly modelled in the target system.

1 files changed, 321 insertions(+), 0 deletions(-)

tools/update_spirv_features.zig created+321
......@@ -0,0 +1,321 @@
1const std = @import("std");
2const fs = std.fs;
3const Allocator = std.mem.Allocator;
4const g = @import("spirv/grammar.zig");
5
6//! This tool generates SPIR-V features from the grammar files in the SPIRV-Headers
7//! (https://github.com/KhronosGroup/SPIRV-Headers/) and SPIRV-Registry (https://github.com/KhronosGroup/SPIRV-Registry/)
8//! repositories. Currently it only generates a basic feature set definition consisting of versions, extensions and capabilities.
9//! There is a lot left to be desired, as currently dependencies of extensions and dependencies on extensions aren't generated.
10//! This is because there are some peculiarities in the SPIR-V registries:
11//! - Capabilities may depend on multiple extensions, which cannot be modelled yet by std.Target.
12//! - Extension dependencies are not documented in a machine-readable manner.
13//! - Note that the grammar spec also contains definitions from extensions which aren't actually official. Most of these seem to be
14//! from an intel project (https://github.com/intel/llvm/, https://github.com/intel/llvm/tree/sycl/sycl/doc/extensions/SPIRV),
15//! and so ONLY extensions in the SPIRV-Registry should be included.
16
17const Version = struct {
18 major: u32,
19 minor: u32,
20
21 fn parse(str: []const u8) !Version {
22 var it = std.mem.split(str, ".");
23
24 const major = it.next() orelse return error.InvalidVersion;
25 const minor = it.next() orelse return error.InvalidVersion;
26
27 if (it.next() != null) return error.InvalidVersion;
28
29 return Version{
30 .major = std.fmt.parseInt(u32, major, 10) catch return error.InvalidVersion,
31 .minor = std.fmt.parseInt(u32, minor, 10) catch return error.InvalidVersion,
32 };
33 }
34
35 fn eql(a: Version, b: Version) bool {
36 return a.major == b.major and a.minor == b.minor;
37 }
38
39 fn lessThan(ctx: void, a: Version, b: Version) bool {
40 return if (a.major == b.major)
41 a.minor < b.minor
42 else
43 a.major < b.major;
44 }
45};
46
47pub fn main() !void {
48 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
49 defer arena.deinit();
50 const allocator = &arena.allocator;
51
52 const args = try std.process.argsAlloc(allocator);
53
54 if (args.len <= 1) {
55 usageAndExit(std.io.getStdErr(), args[0], 1);
56 }
57 if (std.mem.eql(u8, args[1], "--help")) {
58 usageAndExit(std.io.getStdErr(), args[0], 0);
59 }
60 if (args.len != 3) {
61 usageAndExit(std.io.getStdErr(), args[0], 1);
62 }
63
64 const spirv_headers_root = args[1];
65 const spirv_registry_root = args[2];
66
67 if (std.mem.startsWith(u8, spirv_headers_root, "-") or std.mem.startsWith(u8, spirv_registry_root, "-")) {
68 usageAndExit(std.io.getStdErr(), args[0], 1);
69 }
70
71 const registry_path = try fs.path.join(allocator, &.{ spirv_headers_root, "include", "spirv", "unified1", "spirv.core.grammar.json" });
72 const registry_json = try std.fs.cwd().readFileAlloc(allocator, registry_path, std.math.maxInt(usize));
73 var tokens = std.json.TokenStream.init(registry_json);
74 const registry = try std.json.parse(g.CoreRegistry, &tokens, .{ .allocator = allocator });
75
76 const capabilities = for (registry.operand_kinds) |opkind| {
77 if (std.mem.eql(u8, opkind.kind, "Capability"))
78 break opkind.enumerants orelse return error.InvalidRegistry;
79 } else return error.InvalidRegistry;
80
81 const extensions = try gather_extensions(allocator, spirv_registry_root);
82 const versions = try gatherVersions(allocator, registry);
83
84 var bw = std.io.bufferedWriter(std.io.getStdOut().writer());
85 const w = bw.writer();
86
87 try w.writeAll(
88 \\//! This file is auto-generated by tools/update_spirv_features.zig.
89 \\//! TODO: Dependencies of capabilities on extensions.
90 \\//! TODO: Dependencies of extensions on extensions.
91 \\//! TODO: Dependencies of extensions on versions.
92 \\
93 \\const std = @import("../std.zig");
94 \\const CpuFeature = std.Target.Cpu.Feature;
95 \\const CpuModel = std.Target.Cpu.Model;
96 \\
97 \\pub const Feature = enum {
98 \\
99 );
100
101 for (versions) |ver| {
102 try w.print(" v{}_{},\n", .{ ver.major, ver.minor });
103 }
104
105 for (extensions) |ext| {
106 try w.print(" {},\n", .{ std.zig.fmtId(ext) });
107 }
108
109 for (capabilities) |cap| {
110 try w.print(" {},\n", .{ std.zig.fmtId(cap.enumerant) });
111 }
112
113 try w.writeAll(
114 \\};
115 \\
116 \\pub usingnamespace CpuFeature.feature_set_fns(Feature);
117 \\
118 \\pub const all_features = blk: {
119 \\ @setEvalBranchQuota(2000);
120 \\ const len = @typeInfo(Feature).Enum.fields.len;
121 \\ std.debug.assert(len <= CpuFeature.Set.needed_bit_count);
122 \\ var result: [len]CpuFeature = undefined;
123 \\
124 );
125
126 for (versions) |ver, i| {
127 try w.print(
128 \\ result[@enumToInt(Feature.v{0}_{1})] = .{{
129 \\ .llvm_name = null,
130 \\ .description = "SPIR-V version {0}.{1}",
131 \\
132 , .{ ver.major, ver.minor }
133 );
134
135 if (i == 0) {
136 try w.writeAll(
137 \\ .dependencies = featureSet(&[_]Feature{}),
138 \\ };
139 \\
140 );
141 } else {
142 try w.print(
143 \\ .dependencies = featureSet(&[_]Feature{{
144 \\ .v{}_{},
145 \\ }}),
146 \\ }};
147 \\
148 , .{ versions[i - 1].major, versions[i - 1].minor }
149 );
150 }
151 }
152
153 // TODO: Extension dependencies.
154 for (extensions) |ext| {
155 try w.print(
156 \\ result[@enumToInt(Feature.{s})] = .{{
157 \\ .llvm_name = null,
158 \\ .description = "SPIR-V extension {s}",
159 \\ .dependencies = featureSet(&[_]Feature{{}}),
160 \\ }};
161 \\
162 , .{
163 std.zig.fmtId(ext),
164 ext,
165 }
166 );
167 }
168
169 // TODO: Capability extension dependencies.
170 for (capabilities) |cap| {
171 try w.print(
172 \\ result[@enumToInt(Feature.{s})] = .{{
173 \\ .llvm_name = null,
174 \\ .description = "Enable SPIR-V capability {s}",
175 \\ .dependencies = featureSet(&[_]Feature{{
176 \\
177 , .{
178 std.zig.fmtId(cap.enumerant),
179 cap.enumerant,
180 }
181 );
182
183 if (cap.version) |ver_str| {
184 if (!std.mem.eql(u8, ver_str, "None")) {
185 const ver = try Version.parse(ver_str);
186 try w.print(" .v{}_{},\n", .{ ver.major, ver.minor });
187 }
188 }
189
190 for (cap.capabilities) |cap_dep| {
191 try w.print(" .{},\n", .{ std.zig.fmtId(cap_dep) });
192 }
193
194 try w.writeAll(
195 \\ }),
196 \\ };
197 \\
198 );
199 }
200
201 try w.writeAll(
202 \\ const ti = @typeInfo(Feature);
203 \\ for (result) |*elem, i| {
204 \\ elem.index = i;
205 \\ elem.name = ti.Enum.fields[i].name;
206 \\ }
207 \\ break :blk result;
208 \\};
209 \\
210 );
211
212 try bw.flush();
213}
214
215/// SPIRV-Registry should hold all extensions currently registered for SPIR-V.
216/// The *.grammar.json in SPIRV-Headers should have most of these as well, but with this we're sure to get only the actually
217/// registered ones.
218/// TODO: Unfortunately, neither repository contains a machine-readable list of extension dependencies.
219fn gather_extensions(allocator: *Allocator, spirv_registry_root: []const u8) ![]const []const u8 {
220 const extensions_path = try fs.path.join(allocator, &.{spirv_registry_root, "extensions"});
221 var extensions_dir = try fs.cwd().openDir(extensions_path, .{ .iterate = true });
222 defer extensions_dir.close();
223
224 var extensions = std.ArrayList([]const u8).init(allocator);
225
226 var vendor_it = extensions_dir.iterate();
227 while (try vendor_it.next()) |vendor_entry| {
228 std.debug.assert(vendor_entry.kind == .Directory); // If this fails, the structure of SPIRV-Registry has changed.
229
230 const vendor_dir = try extensions_dir.openDir(vendor_entry.name, .{ .iterate = true });
231 var ext_it = vendor_dir.iterate();
232 while (try ext_it.next()) |ext_entry| {
233 // There is both a HTML and asciidoc version of every spec (as well as some other directories),
234 // we need just the name, but to avoid duplicates here we will just skip anything thats not asciidoc.
235 if (!std.mem.endsWith(u8, ext_entry.name, ".asciidoc"))
236 continue;
237
238 // Unfortunately, some extension filenames are incorrect, so we need to look for the string in tne 'Name Strings' section.
239 // This has the following format:
240 // ```
241 // Name Strings
242 // ------------
243 //
244 // SPV_EXT_name
245 // ```
246 // OR
247 // ```
248 // == Name Strings
249 //
250 // SPV_EXT_name
251 // ```
252
253 const ext_spec = try vendor_dir.readFileAlloc(allocator, ext_entry.name, std.math.maxInt(usize));
254 const name_strings = "Name Strings";
255
256 const name_strings_offset = std.mem.indexOf(u8, ext_spec, name_strings) orelse return error.InvalidRegistry;
257
258 // As the specs are inconsistent on this next part, just skip any newlines/minuses
259 var ext_start = name_strings_offset + name_strings.len + 1;
260 while (ext_spec[ext_start] == '\n' or ext_spec[ext_start] == '-') {
261 ext_start += 1;
262 }
263
264 const ext_end = std.mem.indexOfScalarPos(u8, ext_spec, ext_start, '\n') orelse return error.InvalidRegistry;
265 const ext = ext_spec[ext_start .. ext_end];
266
267 std.debug.assert(std.mem.startsWith(u8, ext, "SPV_")); // Sanity check, all extensions should have a name like SPV_VENDOR_extension.
268
269 try extensions.append(try allocator.dupe(u8, ext));
270 }
271 }
272
273 return extensions.items;
274}
275
276fn insertVersion(versions: *std.ArrayList(Version), version: ?[]const u8) !void {
277 const ver_str = version orelse return;
278 if (std.mem.eql(u8, ver_str, "None"))
279 return;
280
281 const ver = try Version.parse(ver_str);
282 for (versions.items) |existing_ver| {
283 if (ver.eql(existing_ver)) return;
284 }
285
286 try versions.append(ver);
287}
288
289fn gatherVersions(allocator: *Allocator, registry: g.CoreRegistry) ![]const Version {
290 // Expected number of versions is small
291 var versions = std.ArrayList(Version).init(allocator);
292
293 for (registry.instructions) |inst| {
294 try insertVersion(&versions, inst.version);
295 }
296
297 for (registry.operand_kinds) |opkind| {
298 const enumerants = opkind.enumerants orelse continue;
299 for (enumerants) |enumerant| {
300 try insertVersion(&versions, enumerant.version);
301 }
302 }
303
304 std.sort.sort(Version, versions.items, {}, Version.lessThan);
305
306 return versions.items;
307}
308
309fn usageAndExit(file: fs.File, arg0: []const u8, code: u8) noreturn {
310 file.writer().print(
311 \\Usage: {s} /path/git/SPIRV-Headers /path/git/SPIRV-Registry
312 \\
313 \\Prints to stdout Zig code which can be used to replace the file lib/std/target/spirv.zig.
314 \\
315 \\SPIRV-Headers can be cloned from https://github.com/KhronosGroup/SPIRV-Headers,
316 \\SPIRV-Registry can be cloned from https://github.com/KhronosGroup/SPIRV-Registry.
317 \\
318 , .{arg0}
319 ) catch std.process.exit(1);
320 std.process.exit(code);
321}