authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-05-29 06:11:28-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-05-31 18:54:28-04:00
logc1e9ef9eaabb2219a3762c5957b1c63ad20bf1ed
treee7f1c0f1aa7dc8268283a0ef71ef15b888ce9173
parentc907866d55dd92352e26f7501e5a441210bedaba

Legalize: implement scalarization of unary operations

8 files changed, 635 insertions(+), 147 deletions(-)

src/Air/Legalize.zig+372-98
...@@ -1,21 +1,48 @@...@@ -1,21 +1,48 @@
1zcu: *const Zcu,1pt: Zcu.PerThread,
2air: Air,2air_instructions: std.MultiArrayList(Air.Inst),
3features: std.enums.EnumSet(Feature),3air_extra: std.ArrayListUnmanaged(u32),
4features: *const Features,
45
5pub const Feature = enum {6pub const Feature = enum {
7 scalarize_not,
8 scalarize_clz,
9 scalarize_ctz,
10 scalarize_popcount,
11 scalarize_byte_swap,
12 scalarize_bit_reverse,
13 scalarize_sqrt,
14 scalarize_sin,
15 scalarize_cos,
16 scalarize_tan,
17 scalarize_exp,
18 scalarize_exp2,
19 scalarize_log,
20 scalarize_log2,
21 scalarize_log10,
22 scalarize_abs,
23 scalarize_floor,
24 scalarize_ceil,
25 scalarize_round,
26 scalarize_trunc_float,
27 scalarize_neg,
28 scalarize_neg_optimized,
29
6 /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)30 /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
7 remove_shift_vector_rhs_splat,31 remove_shift_vector_rhs_splat,
8 /// Legalize reduce of a one element vector to a bitcast32 /// Legalize reduce of a one element vector to a bitcast
9 reduce_one_elem_to_bitcast,33 reduce_one_elem_to_bitcast,
10};34};
1135
12pub const Features = std.enums.EnumFieldStruct(Feature, bool, false);36pub const Features = std.enums.EnumSet(Feature);
37
38pub const Error = std.mem.Allocator.Error;
1339
14pub fn legalize(air: *Air, backend: std.builtin.CompilerBackend, zcu: *const Zcu) std.mem.Allocator.Error!void {40pub fn legalize(air: *Air, backend: std.builtin.CompilerBackend, pt: Zcu.PerThread) Error!void {
15 var l: Legalize = .{41 var l: Legalize = .{
16 .zcu = zcu,42 .pt = pt,
17 .air = air.*,43 .air_instructions = air.instructions.toMultiArrayList(),
18 .features = features: switch (backend) {44 .air_extra = air.extra,
45 .features = &features: switch (backend) {
19 .other, .stage1 => unreachable,46 .other, .stage1 => unreachable,
20 inline .stage2_llvm,47 inline .stage2_llvm,
21 .stage2_c,48 .stage2_c,
...@@ -30,118 +57,365 @@ pub fn legalize(air: *Air, backend: std.builtin.CompilerBackend, zcu: *const Zcu...@@ -30,118 +57,365 @@ pub fn legalize(air: *Air, backend: std.builtin.CompilerBackend, zcu: *const Zcu
30 .stage2_powerpc,57 .stage2_powerpc,
31 => |ct_backend| {58 => |ct_backend| {
32 const Backend = codegen.importBackend(ct_backend) orelse break :features .initEmpty();59 const Backend = codegen.importBackend(ct_backend) orelse break :features .initEmpty();
33 break :features if (@hasDecl(Backend, "legalize_features"))60 break :features if (@hasDecl(Backend, "legalize_features")) Backend.legalize_features else .initEmpty();
34 .init(Backend.legalize_features)
35 else
36 .initEmpty();
37 },61 },
38 _ => unreachable,62 _ => unreachable,
39 },63 },
40 };64 };
41 defer air.* = l.air;65 if (l.features.bits.eql(.initEmpty())) return;
42 if (!l.features.bits.eql(.initEmpty())) try l.legalizeBody(l.air.getMainBody());66 defer air.* = l.getTmpAir();
67 const main_extra = l.extraData(Air.Block, l.air_extra.items[@intFromEnum(Air.ExtraIndex.main_block)]);
68 try l.legalizeBody(main_extra.end, main_extra.data.body_len);
69}
70
71fn getTmpAir(l: *const Legalize) Air {
72 return .{
73 .instructions = l.air_instructions.slice(),
74 .extra = l.air_extra,
75 };
76}
77
78fn typeOf(l: *const Legalize, ref: Air.Inst.Ref) Type {
79 return l.getTmpAir().typeOf(ref, &l.pt.zcu.intern_pool);
80}
81
82fn typeOfIndex(l: *const Legalize, inst: Air.Inst.Index) Type {
83 return l.getTmpAir().typeOfIndex(inst, &l.pt.zcu.intern_pool);
43}84}
4485
45fn legalizeBody(l: *Legalize, body: []const Air.Inst.Index) std.mem.Allocator.Error!void {86fn extraData(l: *const Legalize, comptime T: type, index: usize) @TypeOf(Air.extraData(undefined, T, undefined)) {
46 const zcu = l.zcu;87 return l.getTmpAir().extraData(T, index);
88}
89
90fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
91 const zcu = l.pt.zcu;
47 const ip = &zcu.intern_pool;92 const ip = &zcu.intern_pool;
48 const tags = l.air.instructions.items(.tag);93 for (body_start..body_start + body_len) |inst_extra_index| {
49 const data = l.air.instructions.items(.data);94 const inst: Air.Inst.Index = @enumFromInt(l.air_extra.items[inst_extra_index]);
50 for (body) |inst| inst: switch (tags[@intFromEnum(inst)]) {95 inst: switch (l.air_instructions.items(.tag)[@intFromEnum(inst)]) {
51 else => {},96 else => {},
5297
53 .shl,98 inline .not,
54 .shl_exact,99 .clz,
55 .shl_sat,100 .ctz,
56 .shr,101 .popcount,
57 .shr_exact,102 .byte_swap,
58 => |air_tag| if (l.features.contains(.remove_shift_vector_rhs_splat)) done: {103 .bit_reverse,
59 const bin_op = data[@intFromEnum(inst)].bin_op;104 .abs,
60 const ty = l.air.typeOf(bin_op.rhs, ip);105 => |air_tag| if (l.features.contains(@field(Feature, "scalarize_" ++ @tagName(air_tag)))) done: {
61 if (!ty.isVector(zcu)) break :done;106 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
62 if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) {107 if (!ty_op.ty.toType().isVector(zcu)) break :done;
63 else => {},108 continue :inst try l.scalarizeUnary(inst, .ty_op, ty_op.operand);
64 .aggregate => |aggregate| switch (aggregate.storage) {109 },
65 else => {},110 inline .sqrt,
66 .repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{111 .sin,
67 .lhs = bin_op.lhs,112 .cos,
68 .rhs = Air.internedToRef(splat),113 .tan,
69 } }),114 .exp,
70 },115 .exp2,
71 } else {116 .log,
72 const rhs_inst = bin_op.rhs.toIndex().?;117 .log2,
73 switch (tags[@intFromEnum(rhs_inst)]) {118 .log10,
119 .floor,
120 .ceil,
121 .round,
122 .trunc_float,
123 .neg,
124 .neg_optimized,
125 => |air_tag| if (l.features.contains(@field(Feature, "scalarize_" ++ @tagName(air_tag)))) done: {
126 const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;
127 if (!l.typeOf(un_op).isVector(zcu)) break :done;
128 continue :inst try l.scalarizeUnary(inst, .un_op, un_op);
129 },
130
131 .shl,
132 .shl_exact,
133 .shl_sat,
134 .shr,
135 .shr_exact,
136 => |air_tag| if (l.features.contains(.remove_shift_vector_rhs_splat)) done: {
137 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
138 const ty = l.typeOf(bin_op.rhs);
139 if (!ty.isVector(zcu)) break :done;
140 if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) {
74 else => {},141 else => {},
75 .splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{142 .aggregate => |aggregate| switch (aggregate.storage) {
76 .lhs = bin_op.lhs,143 else => {},
77 .rhs = data[@intFromEnum(rhs_inst)].ty_op.operand,144 .repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{
145 .lhs = bin_op.lhs,
146 .rhs = Air.internedToRef(splat),
147 } }),
148 },
149 } else {
150 const rhs_inst = bin_op.rhs.toIndex().?;
151 switch (l.air_instructions.items(.tag)[@intFromEnum(rhs_inst)]) {
152 else => {},
153 .splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{
154 .lhs = bin_op.lhs,
155 .rhs = l.air_instructions.items(.data)[@intFromEnum(rhs_inst)].ty_op.operand,
156 } }),
157 }
158 }
159 },
160
161 .reduce,
162 .reduce_optimized,
163 => if (l.features.contains(.reduce_one_elem_to_bitcast)) done: {
164 const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;
165 const vector_ty = l.typeOf(reduce.operand);
166 switch (vector_ty.vectorLen(zcu)) {
167 0 => unreachable,
168 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
169 .ty = Air.internedToRef(vector_ty.scalarType(zcu).toIntern()),
170 .operand = reduce.operand,
78 } }),171 } }),
172 else => break :done,
79 }173 }
80 }174 },
81 },
82175
83 .reduce,176 .@"try", .try_cold => {
84 .reduce_optimized,177 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
85 => if (l.features.contains(.reduce_one_elem_to_bitcast)) done: {178 const extra = l.extraData(Air.Try, pl_op.payload);
86 const reduce = data[@intFromEnum(inst)].reduce;179 try l.legalizeBody(extra.end, extra.data.body_len);
87 const vector_ty = l.air.typeOf(reduce.operand, ip);180 },
88 switch (vector_ty.vectorLen(zcu)) {181 .try_ptr, .try_ptr_cold => {
89 0 => unreachable,182 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
90 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{183 const extra = l.extraData(Air.TryPtr, ty_pl.payload);
91 .ty = Air.internedToRef(vector_ty.scalarType(zcu).toIntern()),184 try l.legalizeBody(extra.end, extra.data.body_len);
92 .operand = reduce.operand,185 },
93 } }),186 .block, .loop => {
94 else => break :done,187 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
95 }188 const extra = l.extraData(Air.Block, ty_pl.payload);
96 },189 try l.legalizeBody(extra.end, extra.data.body_len);
190 },
191 .dbg_inline_block => {
192 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
193 const extra = l.extraData(Air.DbgInlineBlock, ty_pl.payload);
194 try l.legalizeBody(extra.end, extra.data.body_len);
195 },
196 .cond_br => {
197 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
198 const extra = l.extraData(Air.CondBr, pl_op.payload);
199 try l.legalizeBody(extra.end, extra.data.then_body_len);
200 try l.legalizeBody(extra.end + extra.data.then_body_len, extra.data.else_body_len);
201 },
202 .switch_br, .loop_switch_br => {
203 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
204 const extra = l.extraData(Air.SwitchBr, pl_op.payload);
205 const hint_bag_count = std.math.divCeil(usize, extra.data.cases_len + 1, 10) catch unreachable;
206 var extra_index = extra.end + hint_bag_count;
207 for (0..extra.data.cases_len) |_| {
208 const case_extra = l.extraData(Air.SwitchBr.Case, extra_index);
209 const case_body_start = case_extra.end + case_extra.data.items_len + case_extra.data.ranges_len * 2;
210 try l.legalizeBody(case_body_start, case_extra.data.body_len);
211 extra_index = case_body_start + case_extra.data.body_len;
212 }
213 try l.legalizeBody(extra_index, extra.data.else_body_len);
214 },
215 }
216 }
217}
97218
98 .@"try", .try_cold => {219const UnaryDataTag = enum { un_op, ty_op };
99 const pl_op = data[@intFromEnum(inst)].pl_op;220inline fn scalarizeUnary(l: *Legalize, inst: Air.Inst.Index, data_tag: UnaryDataTag, un_op: Air.Inst.Ref) Error!Air.Inst.Tag {
100 const extra = l.air.extraData(Air.Try, pl_op.payload);221 return l.replaceInst(inst, .block, try l.scalarizeUnaryBlockPayload(inst, data_tag, un_op));
101 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));222}
102 },223fn scalarizeUnaryBlockPayload(
103 .try_ptr, .try_ptr_cold => {224 l: *Legalize,
104 const ty_pl = data[@intFromEnum(inst)].ty_pl;225 inst: Air.Inst.Index,
105 const extra = l.air.extraData(Air.TryPtr, ty_pl.payload);226 data_tag: UnaryDataTag,
106 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));227 un_op: Air.Inst.Ref,
107 },228) Error!Air.Inst.Data {
108 .block, .loop => {229 const pt = l.pt;
109 const ty_pl = data[@intFromEnum(inst)].ty_pl;230 const zcu = pt.zcu;
110 const extra = l.air.extraData(Air.Block, ty_pl.payload);231 const gpa = zcu.gpa;
111 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));232
112 },233 const res_ty = l.typeOfIndex(inst);
113 .dbg_inline_block => {234 try l.air_instructions.ensureUnusedCapacity(gpa, 15);
114 const ty_pl = data[@intFromEnum(inst)].ty_pl;235 const res_alloc_inst = l.addInstAssumeCapacity(.{
115 const extra = l.air.extraData(Air.DbgInlineBlock, ty_pl.payload);236 .tag = .alloc,
116 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));237 .data = .{ .ty = try pt.singleMutPtrType(res_ty) },
117 },238 });
118 .cond_br => {239 const index_alloc_inst = l.addInstAssumeCapacity(.{
119 const pl_op = data[@intFromEnum(inst)].pl_op;240 .tag = .alloc,
120 const extra = l.air.extraData(Air.CondBr, pl_op.payload);241 .data = .{ .ty = try pt.singleMutPtrType(.usize) },
121 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.then_body_len]));242 });
122 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));243 const index_init_inst = l.addInstAssumeCapacity(.{
123 },244 .tag = .store,
124 .switch_br, .loop_switch_br => {245 .data = .{ .bin_op = .{
125 const switch_br = l.air.unwrapSwitch(inst);246 .lhs = index_alloc_inst.toRef(),
126 var it = switch_br.iterateCases();247 .rhs = try pt.intRef(.usize, 0),
127 while (it.next()) |case| try l.legalizeBody(case.body);248 } },
128 try l.legalizeBody(it.elseBody());249 });
250 const cur_index_inst = l.addInstAssumeCapacity(.{
251 .tag = .load,
252 .data = .{ .ty_op = .{
253 .ty = .usize_type,
254 .operand = index_alloc_inst.toRef(),
255 } },
256 });
257 const get_elem_inst = l.addInstAssumeCapacity(.{
258 .tag = .array_elem_val,
259 .data = .{ .bin_op = .{
260 .lhs = un_op,
261 .rhs = cur_index_inst.toRef(),
262 } },
263 });
264 const op_elem_inst = l.addInstAssumeCapacity(.{
265 .tag = l.air_instructions.items(.tag)[@intFromEnum(inst)],
266 .data = switch (data_tag) {
267 .un_op => .{ .un_op = get_elem_inst.toRef() },
268 .ty_op => .{ .ty_op = .{
269 .ty = Air.internedToRef(res_ty.scalarType(zcu).toIntern()),
270 .operand = get_elem_inst.toRef(),
271 } },
129 },272 },
130 };273 });
274 const set_elem_inst = l.addInstAssumeCapacity(.{
275 .tag = .vector_store_elem,
276 .data = .{ .vector_store_elem = .{
277 .vector_ptr = res_alloc_inst.toRef(),
278 .payload = try l.addExtra(Air.Bin, .{
279 .lhs = cur_index_inst.toRef(),
280 .rhs = op_elem_inst.toRef(),
281 }),
282 } },
283 });
284 const not_done_inst = l.addInstAssumeCapacity(.{
285 .tag = .cmp_lt,
286 .data = .{ .bin_op = .{
287 .lhs = cur_index_inst.toRef(),
288 .rhs = try pt.intRef(.usize, res_ty.vectorLen(zcu)),
289 } },
290 });
291 const next_index_inst = l.addInstAssumeCapacity(.{
292 .tag = .add,
293 .data = .{ .bin_op = .{
294 .lhs = cur_index_inst.toRef(),
295 .rhs = try pt.intRef(.usize, 1),
296 } },
297 });
298 const set_index_inst = l.addInstAssumeCapacity(.{
299 .tag = .store,
300 .data = .{ .bin_op = .{
301 .lhs = index_alloc_inst.toRef(),
302 .rhs = next_index_inst.toRef(),
303 } },
304 });
305 const loop_inst: Air.Inst.Index = @enumFromInt(l.air_instructions.len + 4);
306 const repeat_inst = l.addInstAssumeCapacity(.{
307 .tag = .repeat,
308 .data = .{ .repeat = .{ .loop_inst = loop_inst } },
309 });
310 const final_res_inst = l.addInstAssumeCapacity(.{
311 .tag = .load,
312 .data = .{ .ty_op = .{
313 .ty = Air.internedToRef(res_ty.toIntern()),
314 .operand = res_alloc_inst.toRef(),
315 } },
316 });
317 const br_res_inst = l.addInstAssumeCapacity(.{
318 .tag = .br,
319 .data = .{ .br = .{
320 .block_inst = inst,
321 .operand = final_res_inst.toRef(),
322 } },
323 });
324 const done_br_inst = l.addInstAssumeCapacity(.{
325 .tag = .cond_br,
326 .data = .{ .pl_op = .{
327 .operand = not_done_inst.toRef(),
328 .payload = try l.addCondBrBodies(&.{
329 next_index_inst,
330 set_index_inst,
331 repeat_inst,
332 }, &.{
333 final_res_inst,
334 br_res_inst,
335 }),
336 } },
337 });
338 assert(loop_inst == l.addInstAssumeCapacity(.{
339 .tag = .loop,
340 .data = .{ .ty_pl = .{
341 .ty = .noreturn_type,
342 .payload = try l.addBlockBody(&.{
343 cur_index_inst,
344 get_elem_inst,
345 op_elem_inst,
346 set_elem_inst,
347 not_done_inst,
348 done_br_inst,
349 }),
350 } },
351 }));
352 return .{ .ty_pl = .{
353 .ty = Air.internedToRef(res_ty.toIntern()),
354 .payload = try l.addBlockBody(&.{
355 res_alloc_inst,
356 index_alloc_inst,
357 index_init_inst,
358 loop_inst,
359 }),
360 } };
361}
362
363fn addInstAssumeCapacity(l: *Legalize, inst: Air.Inst) Air.Inst.Index {
364 defer l.air_instructions.appendAssumeCapacity(inst);
365 return @enumFromInt(l.air_instructions.len);
366}
367
368fn addExtra(l: *Legalize, comptime Extra: type, extra: Extra) Error!u32 {
369 const extra_fields = @typeInfo(Extra).@"struct".fields;
370 try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, extra_fields.len);
371 defer inline for (extra_fields) |field| l.air_extra.appendAssumeCapacity(switch (field.type) {
372 u32 => @field(extra, field.name),
373 Air.Inst.Ref => @intFromEnum(@field(extra, field.name)),
374 else => @compileError(@typeName(field.type)),
375 });
376 return @intCast(l.air_extra.items.len);
377}
378
379fn addBlockBody(l: *Legalize, body: []const Air.Inst.Index) Error!u32 {
380 try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, 1 + body.len);
381 defer {
382 l.air_extra.appendAssumeCapacity(@intCast(body.len));
383 l.air_extra.appendSliceAssumeCapacity(@ptrCast(body));
384 }
385 return @intCast(l.air_extra.items.len);
386}
387
388fn addCondBrBodies(l: *Legalize, then_body: []const Air.Inst.Index, else_body: []const Air.Inst.Index) Error!u32 {
389 try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, 3 + then_body.len + else_body.len);
390 defer {
391 l.air_extra.appendSliceAssumeCapacity(&.{
392 @intCast(then_body.len),
393 @intCast(else_body.len),
394 @bitCast(Air.CondBr.BranchHints{
395 .true = .none,
396 .false = .none,
397 .then_cov = .none,
398 .else_cov = .none,
399 }),
400 });
401 l.air_extra.appendSliceAssumeCapacity(@ptrCast(then_body));
402 l.air_extra.appendSliceAssumeCapacity(@ptrCast(else_body));
403 }
404 return @intCast(l.air_extra.items.len);
131}405}
132406
133// inline to propagate comptime `tag`s407// inline to propagate comptime `tag`s
134inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag {408inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag {
135 const ip = &l.zcu.intern_pool;409 const orig_ty = if (std.debug.runtime_safety) l.typeOfIndex(inst) else {};
136 const orig_ty = if (std.debug.runtime_safety) l.air.typeOfIndex(inst, ip) else {};410 l.air_instructions.set(@intFromEnum(inst), .{ .tag = tag, .data = data });
137 l.air.instructions.items(.tag)[@intFromEnum(inst)] = tag;411 if (std.debug.runtime_safety) assert(l.typeOfIndex(inst).toIntern() == orig_ty.toIntern());
138 l.air.instructions.items(.data)[@intFromEnum(inst)] = data;
139 if (std.debug.runtime_safety) std.debug.assert(l.air.typeOfIndex(inst, ip).toIntern() == orig_ty.toIntern());
140 return tag;412 return tag;
141}413}
142414
143const Air = @import("../Air.zig");415const Air = @import("../Air.zig");
416const assert = std.debug.assert;
144const codegen = @import("../codegen.zig");417const codegen = @import("../codegen.zig");
145const Legalize = @This();418const Legalize = @This();
146const std = @import("std");419const std = @import("std");
420const Type = @import("../Type.zig");
147const Zcu = @import("../Zcu.zig");421const Zcu = @import("../Zcu.zig");
src/Zcu/PerThread.zig+1-1
...@@ -1742,7 +1742,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *A...@@ -1742,7 +1742,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *A
1742 }1742 }
17431743
1744 const backend = target_util.zigBackend(zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm);1744 const backend = target_util.zigBackend(zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm);
1745 try air.legalize(backend, zcu);1745 try air.legalize(backend, pt);
17461746
1747 var liveness = try Air.Liveness.analyze(gpa, air.*, ip);1747 var liveness = try Air.Liveness.analyze(gpa, air.*, ip);
1748 defer liveness.deinit(gpa);1748 defer liveness.deinit(gpa);
src/arch/x86_64/CodeGen.zig+250-40
...@@ -32,10 +32,15 @@ const FrameIndex = bits.FrameIndex;...@@ -32,10 +32,15 @@ const FrameIndex = bits.FrameIndex;
3232
33const InnerError = codegen.CodeGenError || error{OutOfRegisters};33const InnerError = codegen.CodeGenError || error{OutOfRegisters};
3434
35pub const legalize_features: Air.Legalize.Features = .{35pub const legalize_features: Air.Legalize.Features = .init(.{
36 .scalarize_ctz = true,
37 .scalarize_popcount = true,
38 .scalarize_byte_swap = true,
39 .scalarize_bit_reverse = true,
40
36 .remove_shift_vector_rhs_splat = false,41 .remove_shift_vector_rhs_splat = false,
37 .reduce_one_elem_to_bitcast = true,42 .reduce_one_elem_to_bitcast = true,
38};43});
3944
40/// Set this to `false` to uncover Sema OPV bugs.45/// Set this to `false` to uncover Sema OPV bugs.
41/// https://github.com/ziglang/zig/issues/2241946/// https://github.com/ziglang/zig/issues/22419
...@@ -63352,14 +63357,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -63352,14 +63357,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
63352 defer assert(cg.loops.remove(inst));63357 defer assert(cg.loops.remove(inst));
63353 try cg.genBodyBlock(@ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len]));63358 try cg.genBodyBlock(@ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len]));
63354 },63359 },
63355 .repeat => if (use_old) try cg.airRepeat(inst) else {63360 .repeat => {
63356 const repeat = air_datas[@intFromEnum(inst)].repeat;63361 const repeat = air_datas[@intFromEnum(inst)].repeat;
63357 const loop = cg.loops.get(repeat.loop_inst).?;63362 const loop = cg.loops.get(repeat.loop_inst).?;
63358 try cg.restoreState(loop.state, &.{}, .{63363 try cg.restoreState(loop.state, &.{}, .{
63359 .emit_instructions = true,63364 .emit_instructions = true,
63360 .update_tracking = false,63365 .update_tracking = false,
63361 .resurrect = false,63366 .resurrect = false,
63362 .close_scope = true,63367 .close_scope = false,
63363 });63368 });
63364 _ = try cg.asmJmpReloc(loop.target);63369 _ = try cg.asmJmpReloc(loop.target);
63365 },63370 },
...@@ -162356,6 +162361,136 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -162356,6 +162361,136 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
162356 .each = .{ .once = &.{162361 .each = .{ .once = &.{
162357 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .src2w, ._, ._ },162362 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .src2w, ._, ._ },
162358 } },162363 } },
162364 }, .{
162365 .required_features = .{ .avx, null, null, null },
162366 .src_constraints = .{ .any, .any, .{ .float = .word } },
162367 .patterns = &.{
162368 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162369 },
162370 .each = .{ .once = &.{
162371 .{ ._, .vp_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ },
162372 } },
162373 }, .{
162374 .required_features = .{ .sse4_1, null, null, null },
162375 .src_constraints = .{ .any, .any, .{ .float = .word } },
162376 .patterns = &.{
162377 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162378 },
162379 .each = .{ .once = &.{
162380 .{ ._, .p_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ },
162381 } },
162382 }, .{
162383 .required_features = .{ .sse2, null, null, null },
162384 .src_constraints = .{ .any, .any, .{ .float = .word } },
162385 .patterns = &.{
162386 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162387 },
162388 .extra_temps = .{
162389 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
162390 .unused,
162391 .unused,
162392 .unused,
162393 .unused,
162394 .unused,
162395 .unused,
162396 .unused,
162397 .unused,
162398 .unused,
162399 .unused,
162400 },
162401 .each = .{ .once = &.{
162402 .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ },
162403 .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp0w, ._, ._ },
162404 } },
162405 }, .{
162406 .required_features = .{ .sse, null, null, null },
162407 .src_constraints = .{ .any, .any, .{ .float = .word } },
162408 .patterns = &.{
162409 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162410 },
162411 .extra_temps = .{
162412 .{ .type = .f32, .kind = .mem },
162413 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
162414 .unused,
162415 .unused,
162416 .unused,
162417 .unused,
162418 .unused,
162419 .unused,
162420 .unused,
162421 .unused,
162422 .unused,
162423 },
162424 .each = .{ .once = &.{
162425 .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ },
162426 .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ },
162427 .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp1w, ._, ._ },
162428 } },
162429 }, .{
162430 .required_features = .{ .avx, null, null, null },
162431 .src_constraints = .{ .any, .any, .{ .float = .word } },
162432 .patterns = &.{
162433 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
162434 },
162435 .each = .{ .once = &.{
162436 .{ ._, .vp_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ },
162437 } },
162438 }, .{
162439 .required_features = .{ .sse4_1, null, null, null },
162440 .src_constraints = .{ .any, .any, .{ .float = .word } },
162441 .patterns = &.{
162442 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
162443 },
162444 .each = .{ .once = &.{
162445 .{ ._, .p_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ },
162446 } },
162447 }, .{
162448 .required_features = .{ .sse2, null, null, null },
162449 .src_constraints = .{ .any, .any, .{ .float = .word } },
162450 .patterns = &.{
162451 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162452 },
162453 .extra_temps = .{
162454 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
162455 .unused,
162456 .unused,
162457 .unused,
162458 .unused,
162459 .unused,
162460 .unused,
162461 .unused,
162462 .unused,
162463 .unused,
162464 .unused,
162465 },
162466 .each = .{ .once = &.{
162467 .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ },
162468 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp0w, ._, ._ },
162469 } },
162470 }, .{
162471 .required_features = .{ .sse, null, null, null },
162472 .src_constraints = .{ .any, .any, .{ .float = .word } },
162473 .patterns = &.{
162474 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162475 },
162476 .extra_temps = .{
162477 .{ .type = .f32, .kind = .mem },
162478 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
162479 .unused,
162480 .unused,
162481 .unused,
162482 .unused,
162483 .unused,
162484 .unused,
162485 .unused,
162486 .unused,
162487 .unused,
162488 },
162489 .each = .{ .once = &.{
162490 .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ },
162491 .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ },
162492 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp1w, ._, ._ },
162493 } },
162359 }, .{162494 }, .{
162360 .src_constraints = .{ .any, .any, .{ .int = .dword } },162495 .src_constraints = .{ .any, .any, .{ .int = .dword } },
162361 .patterns = &.{162496 .patterns = &.{
...@@ -162374,30 +162509,120 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -162374,30 +162509,120 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
162374 .each = .{ .once = &.{162509 .each = .{ .once = &.{
162375 .{ ._, ._, .mov, .leasi(.src0d, .@"4", .src1), .src2d, ._, ._ },162510 .{ ._, ._, .mov, .leasi(.src0d, .@"4", .src1), .src2d, ._, ._ },
162376 } },162511 } },
162512 }, .{
162513 .required_features = .{ .avx, null, null, null },
162514 .src_constraints = .{ .any, .any, .{ .float = .dword } },
162515 .patterns = &.{
162516 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162517 },
162518 .each = .{ .once = &.{
162519 .{ ._, .v_ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
162520 } },
162521 }, .{
162522 .required_features = .{ .sse, null, null, null },
162523 .src_constraints = .{ .any, .any, .{ .float = .dword } },
162524 .patterns = &.{
162525 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162526 },
162527 .each = .{ .once = &.{
162528 .{ ._, ._ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
162529 } },
162530 }, .{
162531 .required_features = .{ .avx, null, null, null },
162532 .src_constraints = .{ .any, .any, .{ .float = .dword } },
162533 .patterns = &.{
162534 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
162535 },
162536 .each = .{ .once = &.{
162537 .{ ._, .v_ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ },
162538 } },
162539 }, .{
162540 .required_features = .{ .sse, null, null, null },
162541 .src_constraints = .{ .any, .any, .{ .float = .dword } },
162542 .patterns = &.{
162543 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
162544 },
162545 .each = .{ .once = &.{
162546 .{ ._, ._ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ },
162547 } },
162377 }, .{162548 }, .{
162378 .required_features = .{ .@"64bit", null, null, null },162549 .required_features = .{ .@"64bit", null, null, null },
162379 .dst_constraints = .{ .{ .int = .qword }, .any },162550 .src_constraints = .{ .any, .any, .{ .int = .qword } },
162380 .patterns = &.{162551 .patterns = &.{
162381 .{ .src = .{ .to_mem, .simm32, .simm32 } },162552 .{ .src = .{ .to_gpr, .simm32, .simm32 } },
162382 .{ .src = .{ .to_mem, .simm32, .to_gpr } },162553 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
162383 },162554 },
162384 .each = .{ .once = &.{162555 .each = .{ .once = &.{
162385 .{ ._, ._, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2q, ._, ._ },162556 .{ ._, ._, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2q, ._, ._ },
162386 } },162557 } },
162387 }, .{162558 }, .{
162388 .required_features = .{ .@"64bit", null, null, null },162559 .required_features = .{ .@"64bit", null, null, null },
162389 .dst_constraints = .{ .{ .int = .qword }, .any },162560 .src_constraints = .{ .any, .any, .{ .int = .qword } },
162390 .patterns = &.{162561 .patterns = &.{
162391 .{ .src = .{ .to_mem, .to_gpr, .simm32 } },162562 .{ .src = .{ .to_gpr, .to_gpr, .simm32 } },
162392 .{ .src = .{ .to_mem, .to_gpr, .to_gpr } },162563 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
162393 },162564 },
162394 .each = .{ .once = &.{162565 .each = .{ .once = &.{
162395 .{ ._, ._, .mov, .leasi(.src0q, .@"8", .src1), .src2q, ._, ._ },162566 .{ ._, ._, .mov, .leasi(.src0q, .@"8", .src1), .src2q, ._, ._ },
162396 } },162567 } },
162568 }, .{
162569 .required_features = .{ .avx, null, null, null },
162570 .src_constraints = .{ .any, .any, .{ .float = .qword } },
162571 .patterns = &.{
162572 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162573 },
162574 .each = .{ .once = &.{
162575 .{ ._, .v_sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
162576 } },
162577 }, .{
162578 .required_features = .{ .sse2, null, null, null },
162579 .src_constraints = .{ .any, .any, .{ .float = .qword } },
162580 .patterns = &.{
162581 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162582 },
162583 .each = .{ .once = &.{
162584 .{ ._, ._sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
162585 } },
162586 }, .{
162587 .required_features = .{ .sse, null, null, null },
162588 .src_constraints = .{ .any, .any, .{ .float = .qword } },
162589 .patterns = &.{
162590 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
162591 },
162592 .each = .{ .once = &.{
162593 .{ ._, ._ps, .movl, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
162594 } },
162595 }, .{
162596 .required_features = .{ .avx, null, null, null },
162597 .src_constraints = .{ .any, .any, .{ .float = .qword } },
162598 .patterns = &.{
162599 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
162600 },
162601 .each = .{ .once = &.{
162602 .{ ._, .v_sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
162603 } },
162604 }, .{
162605 .required_features = .{ .sse2, null, null, null },
162606 .src_constraints = .{ .any, .any, .{ .float = .qword } },
162607 .patterns = &.{
162608 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
162609 },
162610 .each = .{ .once = &.{
162611 .{ ._, ._sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
162612 } },
162613 }, .{
162614 .required_features = .{ .sse, null, null, null },
162615 .src_constraints = .{ .any, .any, .{ .float = .qword } },
162616 .patterns = &.{
162617 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
162618 },
162619 .each = .{ .once = &.{
162620 .{ ._, ._ps, .movl, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
162621 } },
162397 } }) catch |err| switch (err) {162622 } }) catch |err| switch (err) {
162398 error.SelectFailed => {162623 error.SelectFailed => {
162399 const elem_size = cg.typeOf(bin_op.rhs).abiSize(zcu);162624 const elem_size = cg.typeOf(bin_op.rhs).abiSize(zcu);
162400 while (try ops[0].toBase(false, cg) or162625 while (try ops[0].toRegClass(true, .general_purpose, cg) or
162401 try ops[1].toRegClass(true, .general_purpose, cg))162626 try ops[1].toRegClass(true, .general_purpose, cg))
162402 {}162627 {}
162403 const base_reg = ops[0].tracking(cg).short.register.to64();162628 const base_reg = ops[0].tracking(cg).short.register.to64();
...@@ -162410,11 +162635,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -162410,11 +162635,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
162410 rhs_reg,162635 rhs_reg,
162411 .u(elem_size),162636 .u(elem_size),
162412 );162637 );
162413 try cg.asmRegisterMemory(162638 try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
162414 .{ ._, .lea },162639 .base = .{ .reg = base_reg },
162415 base_reg,162640 .mod = .{ .rm = .{ .index = rhs_reg } },
162416 try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }),162641 });
162417 );
162418 } else if (elem_size > 8) {162642 } else if (elem_size > 8) {
162419 try cg.spillEflagsIfOccupied();162643 try cg.spillEflagsIfOccupied();
162420 try cg.asmRegisterImmediate(162644 try cg.asmRegisterImmediate(
...@@ -162422,20 +162646,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -162422,20 +162646,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
162422 rhs_reg,162646 rhs_reg,
162423 .u(std.math.log2_int(u64, elem_size)),162647 .u(std.math.log2_int(u64, elem_size)),
162424 );162648 );
162425 try cg.asmRegisterMemory(162649 try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
162426 .{ ._, .lea },162650 .base = .{ .reg = base_reg },
162427 base_reg,162651 .mod = .{ .rm = .{ .index = rhs_reg } },
162428 try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }),162652 });
162429 );162653 } else try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
162430 } else try cg.asmRegisterMemory(162654 .base = .{ .reg = base_reg },
162431 .{ ._, .lea },162655 .mod = .{ .rm = .{
162432 base_reg,
162433 try ops[0].tracking(cg).short.mem(cg, .{
162434 .index = rhs_reg,162656 .index = rhs_reg,
162435 .scale = .fromFactor(@intCast(elem_size)),162657 .scale = .fromFactor(@intCast(elem_size)),
162436 }),162658 } },
162437 );162659 });
162438 try ops[0].store(&ops[1], .{}, cg);162660 try ops[0].store(&ops[2], .{}, cg);
162439 },162661 },
162440 else => |e| return e,162662 else => |e| return e,
162441 };162663 };
...@@ -174453,18 +174675,6 @@ fn airBr(self: *CodeGen, inst: Air.Inst.Index) !void {...@@ -174453,18 +174675,6 @@ fn airBr(self: *CodeGen, inst: Air.Inst.Index) !void {
174453 try self.freeValue(block_tracking.short);174675 try self.freeValue(block_tracking.short);
174454}174676}
174455174677
174456fn airRepeat(self: *CodeGen, inst: Air.Inst.Index) !void {
174457 const loop_inst = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst;
174458 const repeat_info = self.loops.get(loop_inst).?;
174459 try self.restoreState(repeat_info.state, &.{}, .{
174460 .emit_instructions = true,
174461 .update_tracking = false,
174462 .resurrect = false,
174463 .close_scope = true,
174464 });
174465 _ = try self.asmJmpReloc(repeat_info.target);
174466}
174467
174468fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {174678fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
174469 @setEvalBranchQuota(1_100);174679 @setEvalBranchQuota(1_100);
174470 const pt = self.pt;174680 const pt = self.pt;
test/behavior/bitreverse.zig+3-3
...@@ -123,12 +123,12 @@ fn vector8() !void {...@@ -123,12 +123,12 @@ fn vector8() !void {
123123
124test "bitReverse vectors u8" {124test "bitReverse vectors u8" {
125 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;125 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
126 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
127 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;126 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
128 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;127 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
129 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO128 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
130 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;129 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
131 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;130 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
131 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
132132
133 try comptime vector8();133 try comptime vector8();
134 try vector8();134 try vector8();
...@@ -144,12 +144,12 @@ fn vector16() !void {...@@ -144,12 +144,12 @@ fn vector16() !void {
144144
145test "bitReverse vectors u16" {145test "bitReverse vectors u16" {
146 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;146 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
147 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
148 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;147 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
149 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;148 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
150 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO149 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
151 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;150 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
152 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;151 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
152 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
153153
154 try comptime vector16();154 try comptime vector16();
155 try vector16();155 try vector16();
...@@ -165,12 +165,12 @@ fn vector24() !void {...@@ -165,12 +165,12 @@ fn vector24() !void {
165165
166test "bitReverse vectors u24" {166test "bitReverse vectors u24" {
167 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;167 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
168 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
169 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;168 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
170 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;169 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
171 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO170 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
172 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;171 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
173 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;172 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
173 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
174174
175 try comptime vector24();175 try comptime vector24();
176 try vector24();176 try vector24();
test/behavior/byteswap.zig+3-3
...@@ -95,12 +95,12 @@ fn vector8() !void {...@@ -95,12 +95,12 @@ fn vector8() !void {
9595
96test "@byteSwap vectors u8" {96test "@byteSwap vectors u8" {
97 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;97 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
98 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
99 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;98 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
100 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;99 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
101 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO100 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
102 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;101 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
103 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;102 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
103 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
104104
105 try comptime vector8();105 try comptime vector8();
106 try vector8();106 try vector8();
...@@ -116,12 +116,12 @@ fn vector16() !void {...@@ -116,12 +116,12 @@ fn vector16() !void {
116116
117test "@byteSwap vectors u16" {117test "@byteSwap vectors u16" {
118 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;118 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
119 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
120 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;119 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
121 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;120 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
122 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO121 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
123 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;122 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
124 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;123 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
124 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
125125
126 try comptime vector16();126 try comptime vector16();
127 try vector16();127 try vector16();
...@@ -137,12 +137,12 @@ fn vector24() !void {...@@ -137,12 +137,12 @@ fn vector24() !void {
137137
138test "@byteSwap vectors u24" {138test "@byteSwap vectors u24" {
139 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;139 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
140 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
141 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;140 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
142 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;141 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
143 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO142 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
144 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;143 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
145 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;144 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
145 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
146146
147 try comptime vector24();147 try comptime vector24();
148 try vector24();148 try vector24();
test/behavior/math.zig+1-1
...@@ -193,12 +193,12 @@ fn testCtz128() !void {...@@ -193,12 +193,12 @@ fn testCtz128() !void {
193193
194test "@ctz vectors" {194test "@ctz vectors" {
195 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO195 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
196 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
197 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO196 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
198 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO197 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
199 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO198 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
200 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;199 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
201 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;200 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
201 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
202202
203 try testCtzVectors();203 try testCtzVectors();
204 try comptime testCtzVectors();204 try comptime testCtzVectors();
test/behavior/popcount.zig+1-1
...@@ -77,12 +77,12 @@ fn testPopCountIntegers() !void {...@@ -77,12 +77,12 @@ fn testPopCountIntegers() !void {
7777
78test "@popCount vectors" {78test "@popCount vectors" {
79 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO79 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
80 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
81 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO80 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
82 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO81 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
83 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO82 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
84 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;83 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
85 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;84 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
85 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
8686
87 try comptime testPopCountVectors();87 try comptime testPopCountVectors();
88 try testPopCountVectors();88 try testPopCountVectors();
test/behavior/x86_64/unary.zig+4
...@@ -4828,6 +4828,7 @@ inline fn ctz(comptime Type: type, rhs: Type) @TypeOf(@ctz(rhs)) {...@@ -4828,6 +4828,7 @@ inline fn ctz(comptime Type: type, rhs: Type) @TypeOf(@ctz(rhs)) {
4828test ctz {4828test ctz {
4829 const test_ctz = unary(ctz, .{});4829 const test_ctz = unary(ctz, .{});
4830 try test_ctz.testInts();4830 try test_ctz.testInts();
4831 try test_ctz.testIntVectors();
4831}4832}
48324833
4833inline fn popCount(comptime Type: type, rhs: Type) @TypeOf(@popCount(rhs)) {4834inline fn popCount(comptime Type: type, rhs: Type) @TypeOf(@popCount(rhs)) {
...@@ -4836,6 +4837,7 @@ inline fn popCount(comptime Type: type, rhs: Type) @TypeOf(@popCount(rhs)) {...@@ -4836,6 +4837,7 @@ inline fn popCount(comptime Type: type, rhs: Type) @TypeOf(@popCount(rhs)) {
4836test popCount {4837test popCount {
4837 const test_pop_count = unary(popCount, .{});4838 const test_pop_count = unary(popCount, .{});
4838 try test_pop_count.testInts();4839 try test_pop_count.testInts();
4840 try test_pop_count.testIntVectors();
4839}4841}
48404842
4841inline fn byteSwap(comptime Type: type, rhs: Type) RoundBitsUp(Type, 8) {4843inline fn byteSwap(comptime Type: type, rhs: Type) RoundBitsUp(Type, 8) {
...@@ -4844,6 +4846,7 @@ inline fn byteSwap(comptime Type: type, rhs: Type) RoundBitsUp(Type, 8) {...@@ -4844,6 +4846,7 @@ inline fn byteSwap(comptime Type: type, rhs: Type) RoundBitsUp(Type, 8) {
4844test byteSwap {4846test byteSwap {
4845 const test_byte_swap = unary(byteSwap, .{});4847 const test_byte_swap = unary(byteSwap, .{});
4846 try test_byte_swap.testInts();4848 try test_byte_swap.testInts();
4849 try test_byte_swap.testIntVectors();
4847}4850}
48484851
4849inline fn bitReverse(comptime Type: type, rhs: Type) @TypeOf(@bitReverse(rhs)) {4852inline fn bitReverse(comptime Type: type, rhs: Type) @TypeOf(@bitReverse(rhs)) {
...@@ -4852,6 +4855,7 @@ inline fn bitReverse(comptime Type: type, rhs: Type) @TypeOf(@bitReverse(rhs)) {...@@ -4852,6 +4855,7 @@ inline fn bitReverse(comptime Type: type, rhs: Type) @TypeOf(@bitReverse(rhs)) {
4852test bitReverse {4855test bitReverse {
4853 const test_bit_reverse = unary(bitReverse, .{});4856 const test_bit_reverse = unary(bitReverse, .{});
4854 try test_bit_reverse.testInts();4857 try test_bit_reverse.testInts();
4858 try test_bit_reverse.testIntVectors();
4855}4859}
48564860
4857inline fn sqrt(comptime Type: type, rhs: Type) @TypeOf(@sqrt(rhs)) {4861inline fn sqrt(comptime Type: type, rhs: Type) @TypeOf(@sqrt(rhs)) {