authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-02-26 21:16:04-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-02-26 21:16:04-07:00
log2733ca91b29cf9f0f26305635602fe4087f1a677
tree56b7955c9a79a216462ca4ae9a3f12a69c1919dc
parent3519b74c0632ce86aaab93d44d68659975ab6e21

introduce tools/update_cpu_features.zig

This replaces the previous target cpu features tool, taking advantage of llvm-tblgen --dump-json instead of trying to use python to parse the .td files. This is an initial version that has the basics working, including a simple feature override system, as well as multi-threaded processing. Follow-up commits will do clean ups to make the diff of the newly generated source files against previous versions be as desired.

1 files changed, 525 insertions(+), 0 deletions(-)

tools/update_cpu_features.zig created+525
......@@ -0,0 +1,525 @@
1const std = @import("std");
2const fs = std.fs;
3const mem = std.mem;
4const json = std.json;
5const assert = std.debug.assert;
6
7const FeatureOverride = struct {
8 llvm_name: []const u8,
9 omit: bool = false,
10 zig_name: ?[]const u8 = null,
11 desc: ?[]const u8 = null,
12};
13
14const LlvmTarget = struct {
15 zig_name: []const u8,
16 llvm_name: []const u8,
17 td_name: []const u8,
18 feature_overrides: []const FeatureOverride = &.{},
19 branch_quota: ?usize = null,
20};
21
22const llvm_targets = [_]LlvmTarget{
23 .{
24 .zig_name = "aarch64",
25 .llvm_name = "AArch64",
26 .td_name = "AArch64.td",
27 .feature_overrides = &.{
28 .{
29 .llvm_name = "CONTEXTIDREL2",
30 .zig_name = "contextidr_el2",
31 .desc = "Enable RW operand Context ID Register (EL2)",
32 },
33 },
34 },
35 .{
36 .zig_name = "amdgpu",
37 .llvm_name = "AMDGPU",
38 .td_name = "AMDGPU.td",
39 },
40 .{
41 .zig_name = "arc",
42 .llvm_name = "ARC",
43 .td_name = "ARC.td",
44 },
45 .{
46 .zig_name = "arm",
47 .llvm_name = "ARM",
48 .td_name = "ARM.td",
49 },
50 .{
51 .zig_name = "avr",
52 .llvm_name = "AVR",
53 .td_name = "AVR.td",
54 },
55 .{
56 .zig_name = "bpf",
57 .llvm_name = "BPF",
58 .td_name = "BPF.td",
59 },
60 .{
61 .zig_name = "csky",
62 .llvm_name = "CSKY",
63 .td_name = "CSKY.td",
64 },
65 .{
66 .zig_name = "hexagon",
67 .llvm_name = "Hexagon",
68 .td_name = "Hexagon.td",
69 },
70 .{
71 .zig_name = "lanai",
72 .llvm_name = "Lanai",
73 .td_name = "Lanai.td",
74 },
75 .{
76 .zig_name = "msp430",
77 .llvm_name = "MSP430",
78 .td_name = "MSP430.td",
79 },
80 .{
81 .zig_name = "mips",
82 .llvm_name = "Mips",
83 .td_name = "Mips.td",
84 },
85 .{
86 .zig_name = "nvptx",
87 .llvm_name = "NVPTX",
88 .td_name = "NVPTX.td",
89 },
90 .{
91 .zig_name = "powerpc",
92 .llvm_name = "PowerPC",
93 .td_name = "PPC.td",
94 },
95 .{
96 .zig_name = "riscv",
97 .llvm_name = "RISCV",
98 .td_name = "RISCV.td",
99 },
100 .{
101 .zig_name = "sparc",
102 .llvm_name = "Sparc",
103 .td_name = "Sparc.td",
104 },
105 .{
106 .zig_name = "systemz",
107 .llvm_name = "SystemZ",
108 .td_name = "SystemZ.td",
109 },
110 .{
111 .zig_name = "ve",
112 .llvm_name = "VE",
113 .td_name = "VE.td",
114 },
115 .{
116 .zig_name = "wasm",
117 .llvm_name = "WebAssembly",
118 .td_name = "WebAssembly.td",
119 },
120 .{
121 .zig_name = "x86",
122 .llvm_name = "X86",
123 .td_name = "X86.td",
124 .feature_overrides = &.{
125 .{
126 .llvm_name = "64bit-mode",
127 .omit = true,
128 },
129 .{
130 .llvm_name = "i386",
131 .zig_name = "_i386",
132 },
133 .{
134 .llvm_name = "i486",
135 .zig_name = "_i486",
136 },
137 .{
138 .llvm_name = "i586",
139 .zig_name = "_i586",
140 },
141 .{
142 .llvm_name = "i686",
143 .zig_name = "_i686",
144 },
145 },
146 },
147 .{
148 .zig_name = "xcore",
149 .llvm_name = "XCore",
150 .td_name = "XCore.td",
151 },
152};
153
154pub fn main() anyerror!void {
155 var arena_state = std.heap.ArenaAllocator.init(std.heap.page_allocator);
156 defer arena_state.deinit();
157 const arena = &arena_state.allocator;
158
159 const args = try std.process.argsAlloc(arena);
160 if (args.len <= 1) {
161 usageAndExit(std.io.getStdErr(), args[0], 1);
162 }
163 if (std.mem.eql(u8, args[1], "--help")) {
164 usageAndExit(std.io.getStdOut(), args[0], 0);
165 }
166 if (args.len < 4) {
167 usageAndExit(std.io.getStdErr(), args[0], 1);
168 }
169
170 const llvm_tblgen_exe = args[1];
171 if (std.mem.startsWith(u8, llvm_tblgen_exe, "-")) {
172 usageAndExit(std.io.getStdErr(), args[0], 1);
173 }
174
175 const llvm_src_root = args[2];
176 if (std.mem.startsWith(u8, llvm_src_root, "-")) {
177 usageAndExit(std.io.getStdErr(), args[0], 1);
178 }
179
180 const zig_src_root = args[3];
181 if (std.mem.startsWith(u8, zig_src_root, "-")) {
182 usageAndExit(std.io.getStdErr(), args[0], 1);
183 }
184
185 var zig_src_dir = try fs.cwd().openDir(zig_src_root, .{});
186 defer zig_src_dir.close();
187
188 var progress = std.Progress{};
189 const root_progress = try progress.start("", llvm_targets.len);
190 defer root_progress.end();
191
192 if (std.builtin.single_threaded) {
193 for (llvm_targets) |llvm_target| {
194 try processOneTarget(Job{
195 .llvm_tblgen_exe = llvm_tblgen_exe,
196 .llvm_src_root = llvm_src_root,
197 .zig_src_dir = zig_src_dir,
198 .root_progress = &root_progress,
199 .llvm_target = llvm_target,
200 });
201 }
202 } else {
203 var threads = try arena.alloc(*std.Thread, llvm_targets.len);
204 for (llvm_targets) |llvm_target, i| {
205 threads[i] = try std.Thread.spawn(Job{
206 .llvm_tblgen_exe = llvm_tblgen_exe,
207 .llvm_src_root = llvm_src_root,
208 .zig_src_dir = zig_src_dir,
209 .root_progress = root_progress,
210 .llvm_target = llvm_target,
211 }, processOneTarget);
212 }
213 for (threads) |thread| {
214 thread.wait();
215 }
216 }
217}
218
219const Job = struct {
220 llvm_tblgen_exe: []const u8,
221 llvm_src_root: []const u8,
222 zig_src_dir: std.fs.Dir,
223 root_progress: *std.Progress.Node,
224 llvm_target: LlvmTarget,
225};
226
227fn processOneTarget(job: Job) anyerror!void {
228 const llvm_target = job.llvm_target;
229
230 var arena_state = std.heap.ArenaAllocator.init(std.heap.page_allocator);
231 defer arena_state.deinit();
232 const arena = &arena_state.allocator;
233
234 var progress_node = job.root_progress.start(llvm_target.zig_name, 3);
235 progress_node.activate();
236 defer progress_node.end();
237
238 var tblgen_progress = progress_node.start("invoke llvm-tblgen", 0);
239 tblgen_progress.activate();
240
241 const child_args = [_][]const u8{
242 job.llvm_tblgen_exe,
243 "--dump-json",
244 try std.fmt.allocPrint(arena, "{s}/llvm/lib/Target/{s}/{s}", .{
245 job.llvm_src_root,
246 llvm_target.llvm_name,
247 llvm_target.td_name,
248 }),
249 try std.fmt.allocPrint(arena, "-I={s}/llvm/include", .{job.llvm_src_root}),
250 try std.fmt.allocPrint(arena, "-I={s}/llvm/lib/Target/{s}", .{
251 job.llvm_src_root, llvm_target.llvm_name,
252 }),
253 };
254
255 const child_result = try std.ChildProcess.exec(.{
256 .allocator = arena,
257 .argv = &child_args,
258 .max_output_bytes = 200 * 1024 * 1024,
259 });
260 tblgen_progress.end();
261 if (child_result.stderr.len != 0) {
262 std.debug.warn("{s}\n", .{child_result.stderr});
263 }
264
265 const json_text = switch (child_result.term) {
266 .Exited => |code| if (code == 0) child_result.stdout else {
267 std.debug.warn("llvm-tblgen exited with code {d}\n", .{code});
268 std.process.exit(1);
269 },
270 else => {
271 std.debug.warn("llvm-tblgen crashed\n", .{});
272 std.process.exit(1);
273 },
274 };
275
276 var json_parse_progress = progress_node.start("parse JSON", 0);
277 json_parse_progress.activate();
278
279 var parser = json.Parser.init(arena, false);
280 const tree = try parser.parse(json_text);
281 json_parse_progress.end();
282
283 var render_progress = progress_node.start("render zig code", 0);
284 render_progress.activate();
285
286 const root_map = &tree.root.Object;
287 var all_features = std.ArrayList(*json.ObjectMap).init(arena);
288 var all_cpus = std.ArrayList(*json.ObjectMap).init(arena);
289 {
290 var it = root_map.iterator();
291 root_it: while (it.next()) |kv| {
292 if (kv.key.len == 0) continue;
293 if (kv.key[0] == '!') continue;
294 if (kv.value != .Object) continue;
295 if (hasSuperclass(&kv.value.Object, "SubtargetFeature")) {
296 const llvm_name = kv.value.Object.get("Name").?.String;
297 if (llvm_name.len == 0) continue;
298 for (llvm_target.feature_overrides) |feature_override| {
299 if (mem.eql(u8, llvm_name, feature_override.llvm_name)) {
300 if (feature_override.omit) {
301 continue :root_it;
302 }
303 }
304 }
305
306 try all_features.append(&kv.value.Object);
307 }
308 if (hasSuperclass(&kv.value.Object, "Processor")) {
309 try all_cpus.append(&kv.value.Object);
310 }
311 }
312 }
313 std.sort.sort(*json.ObjectMap, all_features.items, {}, objectLessThan);
314 std.sort.sort(*json.ObjectMap, all_cpus.items, {}, objectLessThan);
315
316 const target_sub_path = try fs.path.join(arena, &.{ "lib", "std", "target" });
317 var target_dir = try job.zig_src_dir.makeOpenPath(target_sub_path, .{});
318 defer target_dir.close();
319
320 const zig_code_basename = try std.fmt.allocPrint(arena, "{s}.zig", .{llvm_target.zig_name});
321 var zig_code_file = try target_dir.createFile(zig_code_basename, .{});
322 defer zig_code_file.close();
323
324 var bw = std.io.bufferedWriter(zig_code_file.writer());
325 const w = bw.writer();
326
327 try w.writeAll(
328 \\//! This file is auto-generated by tools/update_cpu_features.zig.
329 \\
330 \\const std = @import("../std.zig");
331 \\const CpuFeature = std.Target.Cpu.Feature;
332 \\const CpuModel = std.Target.Cpu.Model;
333 \\
334 \\pub const Feature = enum {
335 \\
336 );
337
338 for (all_features.items) |obj| {
339 const llvm_name = obj.get("Name").?.String;
340 const zig_name = try llvmNameToZigName(arena, llvm_target, llvm_name);
341 try w.print(" {s},\n", .{std.zig.fmtId(zig_name)});
342 }
343
344 try w.writeAll(
345 \\};
346 \\
347 \\pub usingnamespace CpuFeature.feature_set_fns(Feature);
348 \\
349 \\pub const all_features = blk: {
350 );
351 if (llvm_target.branch_quota) |branch_quota| {
352 try w.print(" @setEvalBranchQuota({d});\n", .{branch_quota});
353 }
354 try w.writeAll(
355 \\ const len = @typeInfo(Feature).Enum.fields.len;
356 \\ std.debug.assert(len <= CpuFeature.Set.needed_bit_count);
357 \\ var result: [len]CpuFeature = undefined;
358 \\
359 );
360
361 for (all_features.items) |obj| {
362 const llvm_name = obj.get("Name").?.String;
363 const description = obj.get("Desc").?.String;
364 const zig_name = try llvmNameToZigName(arena, llvm_target, llvm_name);
365 try w.print(
366 \\ result[@enumToInt(Feature.{s})] = .{{
367 \\ .llvm_name = "{s}",
368 \\ .description = "{s}",
369 \\ .dependencies = featureSet(&[_]Feature{{
370 ,
371 .{
372 std.zig.fmtId(zig_name),
373 std.zig.fmtEscapes(llvm_name),
374 std.zig.fmtEscapes(description),
375 },
376 );
377 const implies = obj.get("Implies").?.Array;
378 var dependencies = std.ArrayList(*json.ObjectMap).init(arena);
379 for (implies.items) |imply| {
380 const other_key = imply.Object.get("def").?.String;
381 const other_obj = &root_map.getEntry(other_key).?.value.Object;
382 try dependencies.append(other_obj);
383 }
384 std.sort.sort(*json.ObjectMap, dependencies.items, {}, objectLessThan);
385
386 if (dependencies.items.len == 0) {
387 try w.writeAll(
388 \\}),
389 \\ };
390 \\
391 );
392 } else {
393 try w.writeAll("\n");
394 for (dependencies.items) |dep| {
395 const other_llvm_name = dep.get("Name").?.String;
396 const other_zig_name = try llvmNameToZigName(arena, llvm_target, other_llvm_name);
397 try w.print(" .{s},\n", .{std.zig.fmtId(other_zig_name)});
398 }
399 try w.writeAll(
400 \\ }),
401 \\ };
402 \\
403 );
404 }
405 }
406 try w.writeAll(
407 \\ const ti = @typeInfo(Feature);
408 \\ for (result) |*elem, i| {
409 \\ elem.index = i;
410 \\ elem.name = ti.Enum.fields[i].name;
411 \\ }
412 \\ break :blk result;
413 \\};
414 \\
415 \\pub const cpu = struct {
416 \\
417 );
418
419 for (all_cpus.items) |obj| {
420 var cpu_features = std.ArrayList(*json.ObjectMap).init(arena);
421
422 const llvm_name = obj.get("Name").?.String;
423 const features = obj.get("Features").?.Array;
424 for (features.items) |feature| {
425 const feature_key = feature.Object.get("def").?.String;
426 const feature_obj = &root_map.getEntry(feature_key).?.value.Object;
427 const feature_llvm_name = feature_obj.get("Name").?.String;
428 if (feature_llvm_name.len == 0) continue;
429 try cpu_features.append(feature_obj);
430 }
431 const tune_features = obj.get("TuneFeatures").?.Array;
432 for (tune_features.items) |feature| {
433 const feature_key = feature.Object.get("def").?.String;
434 const feature_obj = &root_map.getEntry(feature_key).?.value.Object;
435 const feature_llvm_name = feature_obj.get("Name").?.String;
436 if (feature_llvm_name.len == 0) continue;
437 try cpu_features.append(feature_obj);
438 }
439 std.sort.sort(*json.ObjectMap, cpu_features.items, {}, objectLessThan);
440 const zig_cpu_name = try llvmNameToZigName(arena, llvm_target, llvm_name);
441 try w.print(
442 \\ pub const {s} = CpuModel{{
443 \\ .name = "{s}",
444 \\ .llvm_name = "{s}",
445 \\ .features = featureSet(&[_]Feature{{
446 , .{
447 std.zig.fmtId(zig_cpu_name),
448 std.zig.fmtEscapes(zig_cpu_name),
449 std.zig.fmtEscapes(llvm_name),
450 });
451 if (cpu_features.items.len == 0) {
452 try w.writeAll(
453 \\}),
454 \\ };
455 \\
456 );
457 } else {
458 try w.writeAll("\n");
459 for (cpu_features.items) |feature_obj| {
460 const feature_llvm_name = feature_obj.get("Name").?.String;
461 const feature_zig_name = try llvmNameToZigName(arena, llvm_target, feature_llvm_name);
462 try w.print(" .{s},\n", .{std.zig.fmtId(feature_zig_name)});
463 }
464 try w.writeAll(
465 \\ }),
466 \\ };
467 \\
468 );
469 }
470 }
471
472 try w.writeAll(
473 \\};
474 \\
475 );
476 try bw.flush();
477
478 render_progress.end();
479}
480
481fn usageAndExit(file: fs.File, arg0: []const u8, code: u8) noreturn {
482 file.writer().print(
483 \\Usage: {s} /path/to/llvm-tblgen /path/git/llvm-project /path/git/zig
484 \\
485 \\Prints to stdout Zig code which you can use to replace the file src/clang_options_data.zig.
486 \\
487 , .{arg0}) catch std.process.exit(1);
488 std.process.exit(code);
489}
490
491fn objectLessThan(context: void, a: *json.ObjectMap, b: *json.ObjectMap) bool {
492 const a_key = a.get("Name").?.String;
493 const b_key = b.get("Name").?.String;
494 return std.mem.lessThan(u8, a_key, b_key);
495}
496
497fn llvmNameToZigName(
498 arena: *mem.Allocator,
499 llvm_target: LlvmTarget,
500 llvm_name: []const u8,
501) ![]const u8 {
502 for (llvm_target.feature_overrides) |feature_override| {
503 if (mem.eql(u8, feature_override.llvm_name, llvm_name)) {
504 assert(!feature_override.omit);
505 return feature_override.zig_name orelse break;
506 }
507 }
508 const duped = try arena.dupe(u8, llvm_name);
509 for (duped) |*byte| switch (byte.*) {
510 '-', '.' => byte.* = '_',
511 else => continue,
512 };
513 return duped;
514}
515
516fn hasSuperclass(obj: *json.ObjectMap, class_name: []const u8) bool {
517 const superclasses_json = obj.get("!superclasses") orelse return false;
518 for (superclasses_json.Array.items) |superclass_json| {
519 const superclass = superclass_json.String;
520 if (std.mem.eql(u8, superclass, class_name)) {
521 return true;
522 }
523 }
524 return false;
525}