| ... | @@ -1,7 +1,36 @@ | ... | @@ -1,7 +1,36 @@ |
| 1 | pt: Zcu.PerThread, | 1 | pt: Zcu.PerThread, |
| 2 | air_instructions: std.MultiArrayList(Air.Inst), | 2 | air_instructions: std.MultiArrayList(Air.Inst), |
| 3 | air_extra: std.ArrayListUnmanaged(u32), | 3 | air_extra: std.ArrayListUnmanaged(u32), |
| 4 | features: *const Features, | 4 | features: if (switch (dev.env) { |
| | 5 | .bootstrap => @import("../codegen/c.zig").legalizeFeatures(undefined), |
| | 6 | else => null, |
| | 7 | }) |bootstrap_features| struct { |
| | 8 | fn init(features: *const Features) @This() { |
| | 9 | assert(features.eql(bootstrap_features.*)); |
| | 10 | return .{}; |
| | 11 | } |
| | 12 | /// `inline` to propagate comptime-known result. |
| | 13 | inline fn has(_: @This(), comptime feature: Feature) bool { |
| | 14 | return comptime bootstrap_features.contains(feature); |
| | 15 | } |
| | 16 | /// `inline` to propagate comptime-known result. |
| | 17 | fn hasAny(_: @This(), comptime features: []const Feature) bool { |
| | 18 | return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.initEmpty()); |
| | 19 | } |
| | 20 | } else struct { |
| | 21 | features: *const Features, |
| | 22 | /// `inline` to propagate whether `dev.check` returns. |
| | 23 | inline fn init(features: *const Features) @This() { |
| | 24 | dev.check(.legalize); |
| | 25 | return .{ .features = features }; |
| | 26 | } |
| | 27 | fn has(rt: @This(), comptime feature: Feature) bool { |
| | 28 | return rt.features.contains(feature); |
| | 29 | } |
| | 30 | fn hasAny(rt: @This(), comptime features: []const Feature) bool { |
| | 31 | return !rt.features.intersectWith(comptime .initMany(features)).eql(comptime .initEmpty()); |
| | 32 | } |
| | 33 | }, |
| 5 | | 34 | |
| 6 | pub const Feature = enum { | 35 | pub const Feature = enum { |
| 7 | scalarize_add, | 36 | scalarize_add, |
| ... | @@ -199,7 +228,7 @@ pub const Feature = enum { | ... | @@ -199,7 +228,7 @@ pub const Feature = enum { |
| 199 | .float_from_int => .scalarize_float_from_int, | 228 | .float_from_int => .scalarize_float_from_int, |
| 200 | .shuffle_one => .scalarize_shuffle_one, | 229 | .shuffle_one => .scalarize_shuffle_one, |
| 201 | .shuffle_two => .scalarize_shuffle_two, | 230 | .shuffle_two => .scalarize_shuffle_two, |
| 202 | .select => .scalarize_selects, | 231 | .select => .scalarize_select, |
| 203 | .mul_add => .scalarize_mul_add, | 232 | .mul_add => .scalarize_mul_add, |
| 204 | }; | 233 | }; |
| 205 | } | 234 | } |
| ... | @@ -210,13 +239,12 @@ pub const Features = std.enums.EnumSet(Feature); | ... | @@ -210,13 +239,12 @@ pub const Features = std.enums.EnumSet(Feature); |
| 210 | pub const Error = std.mem.Allocator.Error; | 239 | pub const Error = std.mem.Allocator.Error; |
| 211 | | 240 | |
| 212 | pub fn legalize(air: *Air, pt: Zcu.PerThread, features: *const Features) Error!void { | 241 | pub fn legalize(air: *Air, pt: Zcu.PerThread, features: *const Features) Error!void { |
| 213 | dev.check(.legalize); | | |
| 214 | assert(!features.eql(comptime .initEmpty())); // backend asked to run legalize, but no features were enabled | 242 | assert(!features.eql(comptime .initEmpty())); // backend asked to run legalize, but no features were enabled |
| 215 | var l: Legalize = .{ | 243 | var l: Legalize = .{ |
| 216 | .pt = pt, | 244 | .pt = pt, |
| 217 | .air_instructions = air.instructions.toMultiArrayList(), | 245 | .air_instructions = air.instructions.toMultiArrayList(), |
| 218 | .air_extra = air.extra, | 246 | .air_extra = air.extra, |
| 219 | .features = features, | 247 | .features = .init(features), |
| 220 | }; | 248 | }; |
| 221 | defer air.* = l.getTmpAir(); | 249 | defer air.* = l.getTmpAir(); |
| 222 | const main_extra = l.extraData(Air.Block, l.air_extra.items[@intFromEnum(Air.ExtraIndex.main_block)]); | 250 | const main_extra = l.extraData(Air.Block, l.air_extra.items[@intFromEnum(Air.ExtraIndex.main_block)]); |
| ... | @@ -278,28 +306,28 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -278,28 +306,28 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 278 | .bit_and, | 306 | .bit_and, |
| 279 | .bit_or, | 307 | .bit_or, |
| 280 | .xor, | 308 | .xor, |
| 281 | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { | 309 | => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) { |
| 282 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; | 310 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 283 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); | 311 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); |
| 284 | }, | 312 | }, |
| 285 | .add_safe => if (l.features.contains(.expand_add_safe)) { | 313 | .add_safe => if (l.features.has(.expand_add_safe)) { |
| 286 | assert(!l.features.contains(.scalarize_add_safe)); // it doesn't make sense to do both | 314 | assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both |
| 287 | continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow)); | 315 | continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow)); |
| 288 | } else if (l.features.contains(.scalarize_add_safe)) { | 316 | } else if (l.features.has(.scalarize_add_safe)) { |
| 289 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; | 317 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 290 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); | 318 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); |
| 291 | }, | 319 | }, |
| 292 | .sub_safe => if (l.features.contains(.expand_sub_safe)) { | 320 | .sub_safe => if (l.features.has(.expand_sub_safe)) { |
| 293 | assert(!l.features.contains(.scalarize_sub_safe)); // it doesn't make sense to do both | 321 | assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both |
| 294 | continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow)); | 322 | continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow)); |
| 295 | } else if (l.features.contains(.scalarize_sub_safe)) { | 323 | } else if (l.features.has(.scalarize_sub_safe)) { |
| 296 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; | 324 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 297 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); | 325 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); |
| 298 | }, | 326 | }, |
| 299 | .mul_safe => if (l.features.contains(.expand_mul_safe)) { | 327 | .mul_safe => if (l.features.has(.expand_mul_safe)) { |
| 300 | assert(!l.features.contains(.scalarize_mul_safe)); // it doesn't make sense to do both | 328 | assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both |
| 301 | continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow)); | 329 | continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow)); |
| 302 | } else if (l.features.contains(.scalarize_mul_safe)) { | 330 | } else if (l.features.has(.scalarize_mul_safe)) { |
| 303 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; | 331 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 304 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); | 332 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); |
| 305 | }, | 333 | }, |
| ... | @@ -308,7 +336,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -308,7 +336,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 308 | .sub_with_overflow, | 336 | .sub_with_overflow, |
| 309 | .mul_with_overflow, | 337 | .mul_with_overflow, |
| 310 | .shl_with_overflow, | 338 | .shl_with_overflow, |
| 311 | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { | 339 | => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) { |
| 312 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 340 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 313 | if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst)); | 341 | if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst)); |
| 314 | }, | 342 | }, |
| ... | @@ -320,13 +348,13 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -320,13 +348,13 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 320 | .shl, | 348 | .shl, |
| 321 | .shl_exact, | 349 | .shl_exact, |
| 322 | .shl_sat, | 350 | .shl_sat, |
| 323 | => |air_tag| if (!l.features.intersectWith(comptime .initMany(&.{ | 351 | => |air_tag| if (l.features.hasAny(&.{ |
| 324 | .unsplat_shift_rhs, | 352 | .unsplat_shift_rhs, |
| 325 | .scalarize(air_tag), | 353 | .scalarize(air_tag), |
| 326 | })).eql(comptime .initEmpty())) { | 354 | })) { |
| 327 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; | 355 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 328 | if (l.typeOf(bin_op.rhs).isVector(zcu)) { | 356 | if (l.typeOf(bin_op.rhs).isVector(zcu)) { |
| 329 | if (l.features.contains(.unsplat_shift_rhs)) { | 357 | if (l.features.has(.unsplat_shift_rhs)) { |
| 330 | if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) { | 358 | if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) { |
| 331 | else => {}, | 359 | else => {}, |
| 332 | .aggregate => |aggregate| switch (aggregate.storage) { | 360 | .aggregate => |aggregate| switch (aggregate.storage) { |
| ... | @@ -347,7 +375,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -347,7 +375,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 347 | } | 375 | } |
| 348 | } | 376 | } |
| 349 | } | 377 | } |
| 350 | if (l.features.contains(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op); | 378 | if (l.features.has(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op); |
| 351 | } | 379 | } |
| 352 | }, | 380 | }, |
| 353 | inline .not, | 381 | inline .not, |
| ... | @@ -364,11 +392,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -364,11 +392,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 364 | .int_from_float, | 392 | .int_from_float, |
| 365 | .int_from_float_optimized, | 393 | .int_from_float_optimized, |
| 366 | .float_from_int, | 394 | .float_from_int, |
| 367 | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { | 395 | => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) { |
| 368 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; | 396 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 369 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); | 397 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); |
| 370 | }, | 398 | }, |
| 371 | .bitcast => if (l.features.contains(.scalarize_bitcast)) { | 399 | .bitcast => if (l.features.has(.scalarize_bitcast)) { |
| 372 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; | 400 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 373 | | 401 | |
| 374 | const to_ty = ty_op.ty.toType(); | 402 | const to_ty = ty_op.ty.toType(); |
| ... | @@ -404,10 +432,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -404,10 +432,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 404 | }; | 432 | }; |
| 405 | if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst)); | 433 | if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst)); |
| 406 | }, | 434 | }, |
| 407 | .intcast_safe => if (l.features.contains(.expand_intcast_safe)) { | 435 | .intcast_safe => if (l.features.has(.expand_intcast_safe)) { |
| 408 | assert(!l.features.contains(.scalarize_intcast_safe)); // it doesn't make sense to do both | 436 | assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both |
| 409 | continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst)); | 437 | continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst)); |
| 410 | } else if (l.features.contains(.scalarize_intcast_safe)) { | 438 | } else if (l.features.has(.scalarize_intcast_safe)) { |
| 411 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; | 439 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 412 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); | 440 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); |
| 413 | }, | 441 | }, |
| ... | @@ -442,7 +470,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -442,7 +470,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 442 | .trunc_float, | 470 | .trunc_float, |
| 443 | .neg, | 471 | .neg, |
| 444 | .neg_optimized, | 472 | .neg_optimized, |
| 445 | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { | 473 | => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) { |
| 446 | const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op; | 474 | const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op; |
| 447 | if (l.typeOf(un_op).isVector(zcu)) continue :inst try l.scalarize(inst, .un_op); | 475 | if (l.typeOf(un_op).isVector(zcu)) continue :inst try l.scalarize(inst, .un_op); |
| 448 | }, | 476 | }, |
| ... | @@ -459,7 +487,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -459,7 +487,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 459 | .cmp_neq, | 487 | .cmp_neq, |
| 460 | .cmp_neq_optimized, | 488 | .cmp_neq_optimized, |
| 461 | => {}, | 489 | => {}, |
| 462 | inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { | 490 | inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) { |
| 463 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 491 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 464 | if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector); | 492 | if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector); |
| 465 | }, | 493 | }, |
| ... | @@ -513,13 +541,13 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -513,13 +541,13 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 513 | .bool_and, | 541 | .bool_and, |
| 514 | .bool_or, | 542 | .bool_or, |
| 515 | => {}, | 543 | => {}, |
| 516 | .load => if (l.features.contains(.expand_packed_load)) { | 544 | .load => if (l.features.has(.expand_packed_load)) { |
| 517 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; | 545 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 518 | const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu); | 546 | const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu); |
| 519 | if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst)); | 547 | if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst)); |
| 520 | }, | 548 | }, |
| 521 | .ret, .ret_safe, .ret_load => {}, | 549 | .ret, .ret_safe, .ret_load => {}, |
| 522 | .store, .store_safe => if (l.features.contains(.expand_packed_store)) { | 550 | .store, .store_safe => if (l.features.has(.expand_packed_store)) { |
| 523 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; | 551 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 524 | const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu); | 552 | const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu); |
| 525 | if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst)); | 553 | if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst)); |
| ... | @@ -542,7 +570,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -542,7 +570,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 542 | .struct_field_ptr_index_2, | 570 | .struct_field_ptr_index_2, |
| 543 | .struct_field_ptr_index_3, | 571 | .struct_field_ptr_index_3, |
| 544 | => {}, | 572 | => {}, |
| 545 | .struct_field_val => if (l.features.contains(.expand_packed_struct_field_val)) { | 573 | .struct_field_val => if (l.features.has(.expand_packed_struct_field_val)) { |
| 546 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 574 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 547 | const extra = l.extraData(Air.StructField, ty_pl.payload).data; | 575 | const extra = l.extraData(Air.StructField, ty_pl.payload).data; |
| 548 | switch (l.typeOf(extra.struct_operand).containerLayout(zcu)) { | 576 | switch (l.typeOf(extra.struct_operand).containerLayout(zcu)) { |
| ... | @@ -564,7 +592,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -564,7 +592,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 564 | .ptr_elem_ptr, | 592 | .ptr_elem_ptr, |
| 565 | .array_to_slice, | 593 | .array_to_slice, |
| 566 | => {}, | 594 | => {}, |
| 567 | .reduce, .reduce_optimized => if (l.features.contains(.reduce_one_elem_to_bitcast)) { | 595 | .reduce, .reduce_optimized => if (l.features.has(.reduce_one_elem_to_bitcast)) { |
| 568 | const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce; | 596 | const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce; |
| 569 | const vector_ty = l.typeOf(reduce.operand); | 597 | const vector_ty = l.typeOf(reduce.operand); |
| 570 | switch (vector_ty.vectorLen(zcu)) { | 598 | switch (vector_ty.vectorLen(zcu)) { |
| ... | @@ -577,9 +605,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -577,9 +605,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 577 | } | 605 | } |
| 578 | }, | 606 | }, |
| 579 | .splat => {}, | 607 | .splat => {}, |
| 580 | .shuffle_one => if (l.features.contains(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one), | 608 | .shuffle_one => if (l.features.has(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one), |
| 581 | .shuffle_two => if (l.features.contains(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two), | 609 | .shuffle_two => if (l.features.has(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two), |
| 582 | .select => if (l.features.contains(.scalarize_select)) continue :inst try l.scalarize(inst, .select), | 610 | .select => if (l.features.has(.scalarize_select)) continue :inst try l.scalarize(inst, .select), |
| 583 | .memset, | 611 | .memset, |
| 584 | .memset_safe, | 612 | .memset_safe, |
| 585 | .memcpy, | 613 | .memcpy, |
| ... | @@ -597,7 +625,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -597,7 +625,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 597 | .error_name, | 625 | .error_name, |
| 598 | .error_set_has_value, | 626 | .error_set_has_value, |
| 599 | => {}, | 627 | => {}, |
| 600 | .aggregate_init => if (l.features.contains(.expand_packed_aggregate_init)) { | 628 | .aggregate_init => if (l.features.has(.expand_packed_aggregate_init)) { |
| 601 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 629 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 602 | const agg_ty = ty_pl.ty.toType(); | 630 | const agg_ty = ty_pl.ty.toType(); |
| 603 | switch (agg_ty.zigTypeTag(zcu)) { | 631 | switch (agg_ty.zigTypeTag(zcu)) { |
| ... | @@ -609,7 +637,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -609,7 +637,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 609 | } | 637 | } |
| 610 | }, | 638 | }, |
| 611 | .union_init, .prefetch => {}, | 639 | .union_init, .prefetch => {}, |
| 612 | .mul_add => if (l.features.contains(.scalarize_mul_add)) { | 640 | .mul_add => if (l.features.has(.scalarize_mul_add)) { |
| 613 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; | 641 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 614 | if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin); | 642 | if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin); |
| 615 | }, | 643 | }, |
| ... | @@ -636,6 +664,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -636,6 +664,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 636 | } | 664 | } |
| 637 | | 665 | |
| 638 | const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select }; | 666 | const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select }; |
| | 667 | /// inline to propagate comptime-known `replaceInst` result. |
| 639 | inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag { | 668 | inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag { |
| 640 | return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form)); | 669 | return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form)); |
| 641 | } | 670 | } |
| ... | @@ -2691,7 +2720,7 @@ fn addBlockBody(l: *Legalize, body: []const Air.Inst.Index) Error!u32 { | ... | @@ -2691,7 +2720,7 @@ fn addBlockBody(l: *Legalize, body: []const Air.Inst.Index) Error!u32 { |
| 2691 | } | 2720 | } |
| 2692 | | 2721 | |
| 2693 | /// Returns `tag` to remind the caller to `continue :inst` the result. | 2722 | /// Returns `tag` to remind the caller to `continue :inst` the result. |
| 2694 | /// This is inline to propagate the comptime-known `tag`. | 2723 | /// `inline` to propagate the comptime-known `tag` result. |
| 2695 | inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, comptime tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag { | 2724 | inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, comptime tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag { |
| 2696 | const orig_ty = if (std.debug.runtime_safety) l.typeOfIndex(inst) else {}; | 2725 | const orig_ty = if (std.debug.runtime_safety) l.typeOfIndex(inst) else {}; |
| 2697 | l.air_instructions.set(@intFromEnum(inst), .{ .tag = tag, .data = data }); | 2726 | l.air_instructions.set(@intFromEnum(inst), .{ .tag = tag, .data = data }); |
Jacob Young <jacobly@ziglang.org> 2025-06-15 11:36:15-04:00
src