| ... | @@ -46,6 +46,12 @@ pub const Feature = enum { | ... | @@ -46,6 +46,12 @@ pub const Feature = enum { |
| 46 | scalarize_shl_sat, | 46 | scalarize_shl_sat, |
| 47 | scalarize_xor, | 47 | scalarize_xor, |
| 48 | scalarize_not, | 48 | scalarize_not, |
| | 49 | /// Scalarize `bitcast` from or to an array or vector type to `bitcast`s of the elements. |
| | 50 | /// This does not apply if `@bitSizeOf(Elem) == 8 * @sizeOf(Elem)`. |
| | 51 | /// When this feature is enabled, all remaining `bitcast`s can be lowered using the old bitcast |
| | 52 | /// semantics (reinterpret memory) instead of the new bitcast semantics (copy logical bits) and |
| | 53 | /// the behavior will be equivalent. However, the behavior of `@bitSize` on arrays must be |
| | 54 | /// changed in `Type.zig` before enabling this feature to conform to the new bitcast semantics. |
| 49 | scalarize_bitcast, | 55 | scalarize_bitcast, |
| 50 | scalarize_clz, | 56 | scalarize_clz, |
| 51 | scalarize_ctz, | 57 | scalarize_ctz, |
| ... | @@ -101,6 +107,19 @@ pub const Feature = enum { | ... | @@ -101,6 +107,19 @@ pub const Feature = enum { |
| 101 | /// Not compatible with `scalarize_mul_safe`. | 107 | /// Not compatible with `scalarize_mul_safe`. |
| 102 | expand_mul_safe, | 108 | expand_mul_safe, |
| 103 | | 109 | |
| | 110 | /// Replace `load` from a packed pointer with a non-packed `load`, `shr`, `truncate`. |
| | 111 | /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the |
| | 112 | /// first byte of memory until bit pointers know their backing type. |
| | 113 | expand_packed_load, |
| | 114 | /// Replace `store` and `store_safe` to a packed pointer with a non-packed `load`/`store`, `bit_and`, `bit_or`, and `shl`. |
| | 115 | /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the |
| | 116 | /// first byte of memory until bit pointers know their backing type. |
| | 117 | expand_packed_store, |
| | 118 | /// Replace `struct_field_val` of a packed field with a `store` and packed `load`. |
| | 119 | expand_packed_struct_field_val, |
| | 120 | /// Replace `aggregate_init` of a packed aggregate with a series a packed `store`s followed by a `load`. |
| | 121 | expand_packed_aggregate_init, |
| | 122 | |
| 104 | fn scalarize(tag: Air.Inst.Tag) Feature { | 123 | fn scalarize(tag: Air.Inst.Tag) Feature { |
| 105 | return switch (tag) { | 124 | return switch (tag) { |
| 106 | else => unreachable, | 125 | else => unreachable, |
| ... | @@ -192,7 +211,7 @@ pub const Error = std.mem.Allocator.Error; | ... | @@ -192,7 +211,7 @@ pub const Error = std.mem.Allocator.Error; |
| 192 | | 211 | |
| 193 | pub fn legalize(air: *Air, pt: Zcu.PerThread, features: *const Features) Error!void { | 212 | pub fn legalize(air: *Air, pt: Zcu.PerThread, features: *const Features) Error!void { |
| 194 | dev.check(.legalize); | 213 | dev.check(.legalize); |
| 195 | assert(!features.bits.eql(.initEmpty())); // backend asked to run legalize, but no features were enabled | 214 | assert(!features.eql(comptime .initEmpty())); // backend asked to run legalize, but no features were enabled |
| 196 | var l: Legalize = .{ | 215 | var l: Legalize = .{ |
| 197 | .pt = pt, | 216 | .pt = pt, |
| 198 | .air_instructions = air.instructions.toMultiArrayList(), | 217 | .air_instructions = air.instructions.toMultiArrayList(), |
| ... | @@ -229,8 +248,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -229,8 +248,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 229 | for (0..body_len) |body_index| { | 248 | for (0..body_len) |body_index| { |
| 230 | const inst: Air.Inst.Index = @enumFromInt(l.air_extra.items[body_start + body_index]); | 249 | const inst: Air.Inst.Index = @enumFromInt(l.air_extra.items[body_start + body_index]); |
| 231 | inst: switch (l.air_instructions.items(.tag)[@intFromEnum(inst)]) { | 250 | inst: switch (l.air_instructions.items(.tag)[@intFromEnum(inst)]) { |
| 232 | .arg, | 251 | .arg => {}, |
| 233 | => {}, | | |
| 234 | inline .add, | 252 | inline .add, |
| 235 | .add_optimized, | 253 | .add_optimized, |
| 236 | .add_wrap, | 254 | .add_wrap, |
| ... | @@ -285,9 +303,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -285,9 +303,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 285 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; | 303 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 286 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); | 304 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); |
| 287 | }, | 305 | }, |
| 288 | .ptr_add, | 306 | .ptr_add, .ptr_sub => {}, |
| 289 | .ptr_sub, | | |
| 290 | => {}, | | |
| 291 | inline .add_with_overflow, | 307 | inline .add_with_overflow, |
| 292 | .sub_with_overflow, | 308 | .sub_with_overflow, |
| 293 | .mul_with_overflow, | 309 | .mul_with_overflow, |
| ... | @@ -296,44 +312,43 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -296,44 +312,43 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 296 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 312 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 297 | if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst)); | 313 | if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst)); |
| 298 | }, | 314 | }, |
| 299 | .alloc, | 315 | .alloc => {}, |
| 300 | => {}, | 316 | .inferred_alloc, .inferred_alloc_comptime => unreachable, |
| 301 | .inferred_alloc, | 317 | .ret_ptr, .assembly => {}, |
| 302 | .inferred_alloc_comptime, | | |
| 303 | => unreachable, | | |
| 304 | .ret_ptr, | | |
| 305 | .assembly, | | |
| 306 | => {}, | | |
| 307 | inline .shr, | 318 | inline .shr, |
| 308 | .shr_exact, | 319 | .shr_exact, |
| 309 | .shl, | 320 | .shl, |
| 310 | .shl_exact, | 321 | .shl_exact, |
| 311 | .shl_sat, | 322 | .shl_sat, |
| 312 | => |air_tag| done: { | 323 | => |air_tag| if (!l.features.intersectWith(comptime .initMany(&.{ |
| | 324 | .unsplat_shift_rhs, |
| | 325 | .scalarize(air_tag), |
| | 326 | })).eql(comptime .initEmpty())) { |
| 313 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; | 327 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 314 | if (!l.typeOf(bin_op.rhs).isVector(zcu)) break :done; | 328 | if (l.typeOf(bin_op.rhs).isVector(zcu)) { |
| 315 | if (l.features.contains(.unsplat_shift_rhs)) { | 329 | if (l.features.contains(.unsplat_shift_rhs)) { |
| 316 | if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) { | 330 | if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) { |
| 317 | else => {}, | | |
| 318 | .aggregate => |aggregate| switch (aggregate.storage) { | | |
| 319 | else => {}, | | |
| 320 | .repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ | | |
| 321 | .lhs = bin_op.lhs, | | |
| 322 | .rhs = Air.internedToRef(splat), | | |
| 323 | } }), | | |
| 324 | }, | | |
| 325 | } else { | | |
| 326 | const rhs_inst = bin_op.rhs.toIndex().?; | | |
| 327 | switch (l.air_instructions.items(.tag)[@intFromEnum(rhs_inst)]) { | | |
| 328 | else => {}, | 331 | else => {}, |
| 329 | .splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ | 332 | .aggregate => |aggregate| switch (aggregate.storage) { |
| 330 | .lhs = bin_op.lhs, | 333 | else => {}, |
| 331 | .rhs = l.air_instructions.items(.data)[@intFromEnum(rhs_inst)].ty_op.operand, | 334 | .repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ |
| 332 | } }), | 335 | .lhs = bin_op.lhs, |
| | 336 | .rhs = Air.internedToRef(splat), |
| | 337 | } }), |
| | 338 | }, |
| | 339 | } else { |
| | 340 | const rhs_inst = bin_op.rhs.toIndex().?; |
| | 341 | switch (l.air_instructions.items(.tag)[@intFromEnum(rhs_inst)]) { |
| | 342 | else => {}, |
| | 343 | .splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ |
| | 344 | .lhs = bin_op.lhs, |
| | 345 | .rhs = l.air_instructions.items(.data)[@intFromEnum(rhs_inst)].ty_op.operand, |
| | 346 | } }), |
| | 347 | } |
| 333 | } | 348 | } |
| 334 | } | 349 | } |
| | 350 | if (l.features.contains(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op); |
| 335 | } | 351 | } |
| 336 | if (l.features.contains(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op); | | |
| 337 | }, | 352 | }, |
| 338 | inline .not, | 353 | inline .not, |
| 339 | .clz, | 354 | .clz, |
| ... | @@ -353,13 +368,41 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -353,13 +368,41 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 353 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; | 368 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 354 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); | 369 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); |
| 355 | }, | 370 | }, |
| 356 | inline .bitcast, | 371 | .bitcast => if (l.features.contains(.scalarize_bitcast)) { |
| 357 | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { | | |
| 358 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; | 372 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| | 373 | |
| 359 | const to_ty = ty_op.ty.toType(); | 374 | const to_ty = ty_op.ty.toType(); |
| | 375 | const to_ty_tag = to_ty.zigTypeTag(zcu); |
| | 376 | const to_ty_legal = legal: switch (to_ty_tag) { |
| | 377 | else => true, |
| | 378 | .array, .vector => { |
| | 379 | if (to_ty.arrayLen(zcu) == 1) break :legal true; |
| | 380 | const to_elem_ty = to_ty.childType(zcu); |
| | 381 | break :legal to_elem_ty.bitSize(zcu) == 8 * to_elem_ty.abiSize(zcu); |
| | 382 | }, |
| | 383 | }; |
| | 384 | |
| 360 | const from_ty = l.typeOf(ty_op.operand); | 385 | const from_ty = l.typeOf(ty_op.operand); |
| 361 | if (to_ty.isVector(zcu) and from_ty.isVector(zcu) and to_ty.vectorLen(zcu) == from_ty.vectorLen(zcu)) | 386 | const from_ty_legal = legal: switch (from_ty.zigTypeTag(zcu)) { |
| 362 | continue :inst try l.scalarize(inst, .ty_op); | 387 | else => true, |
| | 388 | .array, .vector => { |
| | 389 | if (from_ty.arrayLen(zcu) == 1) break :legal true; |
| | 390 | const from_elem_ty = from_ty.childType(zcu); |
| | 391 | break :legal from_elem_ty.bitSize(zcu) == 8 * from_elem_ty.abiSize(zcu); |
| | 392 | }, |
| | 393 | }; |
| | 394 | |
| | 395 | if (!to_ty_legal and !from_ty_legal and to_ty.arrayLen(zcu) == from_ty.arrayLen(zcu)) switch (to_ty_tag) { |
| | 396 | else => unreachable, |
| | 397 | .array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastToArrayBlockPayload(inst)), |
| | 398 | .vector => continue :inst try l.scalarize(inst, .bitcast), |
| | 399 | }; |
| | 400 | if (!to_ty_legal) switch (to_ty_tag) { |
| | 401 | else => unreachable, |
| | 402 | .array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultArrayBlockPayload(inst)), |
| | 403 | .vector => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultVectorBlockPayload(inst)), |
| | 404 | }; |
| | 405 | if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst)); |
| 363 | }, | 406 | }, |
| 364 | .intcast_safe => if (l.features.contains(.expand_intcast_safe)) { | 407 | .intcast_safe => if (l.features.contains(.expand_intcast_safe)) { |
| 365 | assert(!l.features.contains(.scalarize_intcast_safe)); // it doesn't make sense to do both | 408 | assert(!l.features.contains(.scalarize_intcast_safe)); // it doesn't make sense to do both |
| ... | @@ -368,9 +411,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -368,9 +411,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 368 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; | 411 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 369 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); | 412 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); |
| 370 | }, | 413 | }, |
| 371 | .block, | 414 | .block, .loop => { |
| 372 | .loop, | | |
| 373 | => { | | |
| 374 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 415 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 375 | const extra = l.extraData(Air.Block, ty_pl.payload); | 416 | const extra = l.extraData(Air.Block, ty_pl.payload); |
| 376 | try l.legalizeBody(extra.end, extra.data.body_len); | 417 | try l.legalizeBody(extra.end, extra.data.body_len); |
| ... | @@ -418,22 +459,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -418,22 +459,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 418 | .cmp_neq, | 459 | .cmp_neq, |
| 419 | .cmp_neq_optimized, | 460 | .cmp_neq_optimized, |
| 420 | => {}, | 461 | => {}, |
| 421 | inline .cmp_vector, | 462 | inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { |
| 422 | .cmp_vector_optimized, | | |
| 423 | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { | | |
| 424 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 463 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 425 | if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_pl_vector_cmp); | 464 | if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector); |
| 426 | }, | 465 | }, |
| 427 | .cond_br, | 466 | .cond_br => { |
| 428 | => { | | |
| 429 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; | 467 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 430 | const extra = l.extraData(Air.CondBr, pl_op.payload); | 468 | const extra = l.extraData(Air.CondBr, pl_op.payload); |
| 431 | try l.legalizeBody(extra.end, extra.data.then_body_len); | 469 | try l.legalizeBody(extra.end, extra.data.then_body_len); |
| 432 | try l.legalizeBody(extra.end + extra.data.then_body_len, extra.data.else_body_len); | 470 | try l.legalizeBody(extra.end + extra.data.then_body_len, extra.data.else_body_len); |
| 433 | }, | 471 | }, |
| 434 | .switch_br, | 472 | .switch_br, .loop_switch_br => { |
| 435 | .loop_switch_br, | | |
| 436 | => { | | |
| 437 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; | 473 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 438 | const extra = l.extraData(Air.SwitchBr, pl_op.payload); | 474 | const extra = l.extraData(Air.SwitchBr, pl_op.payload); |
| 439 | const hint_bag_count = std.math.divCeil(usize, extra.data.cases_len + 1, 10) catch unreachable; | 475 | const hint_bag_count = std.math.divCeil(usize, extra.data.cases_len + 1, 10) catch unreachable; |
| ... | @@ -446,27 +482,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -446,27 +482,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 446 | } | 482 | } |
| 447 | try l.legalizeBody(extra_index, extra.data.else_body_len); | 483 | try l.legalizeBody(extra_index, extra.data.else_body_len); |
| 448 | }, | 484 | }, |
| 449 | .switch_dispatch, | 485 | .switch_dispatch => {}, |
| 450 | => {}, | 486 | .@"try", .try_cold => { |
| 451 | .@"try", | | |
| 452 | .try_cold, | | |
| 453 | => { | | |
| 454 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; | 487 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 455 | const extra = l.extraData(Air.Try, pl_op.payload); | 488 | const extra = l.extraData(Air.Try, pl_op.payload); |
| 456 | try l.legalizeBody(extra.end, extra.data.body_len); | 489 | try l.legalizeBody(extra.end, extra.data.body_len); |
| 457 | }, | 490 | }, |
| 458 | .try_ptr, | 491 | .try_ptr, .try_ptr_cold => { |
| 459 | .try_ptr_cold, | | |
| 460 | => { | | |
| 461 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 492 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 462 | const extra = l.extraData(Air.TryPtr, ty_pl.payload); | 493 | const extra = l.extraData(Air.TryPtr, ty_pl.payload); |
| 463 | try l.legalizeBody(extra.end, extra.data.body_len); | 494 | try l.legalizeBody(extra.end, extra.data.body_len); |
| 464 | }, | 495 | }, |
| 465 | .dbg_stmt, | 496 | .dbg_stmt, .dbg_empty_stmt => {}, |
| 466 | .dbg_empty_stmt, | 497 | .dbg_inline_block => { |
| 467 | => {}, | | |
| 468 | .dbg_inline_block, | | |
| 469 | => { | | |
| 470 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 498 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 471 | const extra = l.extraData(Air.DbgInlineBlock, ty_pl.payload); | 499 | const extra = l.extraData(Air.DbgInlineBlock, ty_pl.payload); |
| 472 | try l.legalizeBody(extra.end, extra.data.body_len); | 500 | try l.legalizeBody(extra.end, extra.data.body_len); |
| ... | @@ -484,14 +512,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -484,14 +512,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 484 | .is_non_err_ptr, | 512 | .is_non_err_ptr, |
| 485 | .bool_and, | 513 | .bool_and, |
| 486 | .bool_or, | 514 | .bool_or, |
| 487 | .load, | | |
| 488 | .ret, | | |
| 489 | .ret_safe, | | |
| 490 | .ret_load, | | |
| 491 | .store, | | |
| 492 | .store_safe, | | |
| 493 | .unreach, | | |
| 494 | => {}, | 515 | => {}, |
| | 516 | .load => if (l.features.contains(.expand_packed_load)) { |
| | 517 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| | 518 | const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu); |
| | 519 | if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst)); |
| | 520 | }, |
| | 521 | .ret, .ret_safe, .ret_load => {}, |
| | 522 | .store, .store_safe => if (l.features.contains(.expand_packed_store)) { |
| | 523 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| | 524 | const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu); |
| | 525 | if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst)); |
| | 526 | }, |
| | 527 | .unreach, |
| 495 | .optional_payload, | 528 | .optional_payload, |
| 496 | .optional_payload_ptr, | 529 | .optional_payload_ptr, |
| 497 | .optional_payload_ptr_set, | 530 | .optional_payload_ptr_set, |
| ... | @@ -508,7 +541,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -508,7 +541,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 508 | .struct_field_ptr_index_1, | 541 | .struct_field_ptr_index_1, |
| 509 | .struct_field_ptr_index_2, | 542 | .struct_field_ptr_index_2, |
| 510 | .struct_field_ptr_index_3, | 543 | .struct_field_ptr_index_3, |
| 511 | .struct_field_val, | 544 | => {}, |
| | 545 | .struct_field_val => if (l.features.contains(.expand_packed_struct_field_val)) { |
| | 546 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| | 547 | const extra = l.extraData(Air.StructField, ty_pl.payload).data; |
| | 548 | switch (l.typeOf(extra.struct_operand).containerLayout(zcu)) { |
| | 549 | .auto, .@"extern" => {}, |
| | 550 | .@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedStructFieldValBlockPayload(inst)), |
| | 551 | } |
| | 552 | }, |
| 512 | .set_union_tag, | 553 | .set_union_tag, |
| 513 | .get_union_tag, | 554 | .get_union_tag, |
| 514 | .slice, | 555 | .slice, |
| ... | @@ -523,9 +564,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -523,9 +564,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 523 | .ptr_elem_ptr, | 564 | .ptr_elem_ptr, |
| 524 | .array_to_slice, | 565 | .array_to_slice, |
| 525 | => {}, | 566 | => {}, |
| 526 | .reduce, | 567 | .reduce, .reduce_optimized => if (l.features.contains(.reduce_one_elem_to_bitcast)) { |
| 527 | .reduce_optimized, | | |
| 528 | => if (l.features.contains(.reduce_one_elem_to_bitcast)) done: { | | |
| 529 | const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce; | 568 | const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce; |
| 530 | const vector_ty = l.typeOf(reduce.operand); | 569 | const vector_ty = l.typeOf(reduce.operand); |
| 531 | switch (vector_ty.vectorLen(zcu)) { | 570 | switch (vector_ty.vectorLen(zcu)) { |
| ... | @@ -534,11 +573,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -534,11 +573,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 534 | .ty = Air.internedToRef(vector_ty.childType(zcu).toIntern()), | 573 | .ty = Air.internedToRef(vector_ty.childType(zcu).toIntern()), |
| 535 | .operand = reduce.operand, | 574 | .operand = reduce.operand, |
| 536 | } }), | 575 | } }), |
| 537 | else => break :done, | 576 | else => {}, |
| 538 | } | 577 | } |
| 539 | }, | 578 | }, |
| 540 | .splat, | 579 | .splat => {}, |
| 541 | => {}, | | |
| 542 | .shuffle_one => if (l.features.contains(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one), | 580 | .shuffle_one => if (l.features.contains(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one), |
| 543 | .shuffle_two => if (l.features.contains(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two), | 581 | .shuffle_two => if (l.features.contains(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two), |
| 544 | .select => if (l.features.contains(.scalarize_select)) continue :inst try l.scalarize(inst, .select), | 582 | .select => if (l.features.contains(.scalarize_select)) continue :inst try l.scalarize(inst, .select), |
| ... | @@ -558,12 +596,20 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -558,12 +596,20 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 558 | .tag_name, | 596 | .tag_name, |
| 559 | .error_name, | 597 | .error_name, |
| 560 | .error_set_has_value, | 598 | .error_set_has_value, |
| 561 | .aggregate_init, | | |
| 562 | .union_init, | | |
| 563 | .prefetch, | | |
| 564 | => {}, | 599 | => {}, |
| 565 | inline .mul_add, | 600 | .aggregate_init => if (l.features.contains(.expand_packed_aggregate_init)) { |
| 566 | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { | 601 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| | 602 | const agg_ty = ty_pl.ty.toType(); |
| | 603 | switch (agg_ty.zigTypeTag(zcu)) { |
| | 604 | else => {}, |
| | 605 | .@"struct", .@"union" => switch (agg_ty.containerLayout(zcu)) { |
| | 606 | .auto, .@"extern" => {}, |
| | 607 | .@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)), |
| | 608 | }, |
| | 609 | } |
| | 610 | }, |
| | 611 | .union_init, .prefetch => {}, |
| | 612 | .mul_add => if (l.features.contains(.scalarize_mul_add)) { |
| 567 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; | 613 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 568 | if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin); | 614 | if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin); |
| 569 | }, | 615 | }, |
| ... | @@ -589,7 +635,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -589,7 +635,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 589 | } | 635 | } |
| 590 | } | 636 | } |
| 591 | | 637 | |
| 592 | const ScalarizeForm = enum { un_op, ty_op, bin_op, ty_pl_vector_cmp, pl_op_bin, shuffle_one, shuffle_two, select }; | 638 | const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select }; |
| 593 | inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag { | 639 | inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag { |
| 594 | return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form)); | 640 | return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form)); |
| 595 | } | 641 | } |
| ... | @@ -602,8 +648,8 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: | ... | @@ -602,8 +648,8 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: |
| 602 | const res_len = res_ty.vectorLen(zcu); | 648 | const res_len = res_ty.vectorLen(zcu); |
| 603 | | 649 | |
| 604 | const extra_insts = switch (form) { | 650 | const extra_insts = switch (form) { |
| 605 | .un_op, .ty_op => 1, | 651 | .un_op, .ty_op, .bitcast => 1, |
| 606 | .bin_op, .ty_pl_vector_cmp => 2, | 652 | .bin_op, .cmp_vector => 2, |
| 607 | .pl_op_bin => 3, | 653 | .pl_op_bin => 3, |
| 608 | .shuffle_one, .shuffle_two => 13, | 654 | .shuffle_one, .shuffle_two => 13, |
| 609 | .select => 6, | 655 | .select => 6, |
| ... | @@ -688,32 +734,6 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: | ... | @@ -688,32 +734,6 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: |
| 688 | }).toRef(), | 734 | }).toRef(), |
| 689 | } }, | 735 | } }, |
| 690 | }).toRef(), | 736 | }).toRef(), |
| 691 | .ty_pl_vector_cmp => { | | |
| 692 | const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data; | | |
| 693 | break :res_elem (try loop.block.addCmp( | | |
| 694 | l, | | |
| 695 | extra.compareOperator(), | | |
| 696 | loop.block.add(l, .{ | | |
| 697 | .tag = .array_elem_val, | | |
| 698 | .data = .{ .bin_op = .{ | | |
| 699 | .lhs = extra.lhs, | | |
| 700 | .rhs = cur_index_inst.toRef(), | | |
| 701 | } }, | | |
| 702 | }).toRef(), | | |
| 703 | loop.block.add(l, .{ | | |
| 704 | .tag = .array_elem_val, | | |
| 705 | .data = .{ .bin_op = .{ | | |
| 706 | .lhs = extra.rhs, | | |
| 707 | .rhs = cur_index_inst.toRef(), | | |
| 708 | } }, | | |
| 709 | }).toRef(), | | |
| 710 | .{ .optimized = switch (orig.tag) { | | |
| 711 | else => unreachable, | | |
| 712 | .cmp_vector => false, | | |
| 713 | .cmp_vector_optimized => true, | | |
| 714 | } }, | | |
| 715 | )).toRef(); | | |
| 716 | }, | | |
| 717 | .pl_op_bin => { | 737 | .pl_op_bin => { |
| 718 | const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data; | 738 | const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data; |
| 719 | break :res_elem loop.block.add(l, .{ | 739 | break :res_elem loop.block.add(l, .{ |
| ... | @@ -745,6 +765,39 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: | ... | @@ -745,6 +765,39 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: |
| 745 | } }, | 765 | } }, |
| 746 | }).toRef(); | 766 | }).toRef(); |
| 747 | }, | 767 | }, |
| | 768 | .bitcast => loop.block.addBitCast(l, res_ty.childType(zcu), loop.block.add(l, .{ |
| | 769 | .tag = .array_elem_val, |
| | 770 | .data = .{ .bin_op = .{ |
| | 771 | .lhs = orig.data.ty_op.operand, |
| | 772 | .rhs = cur_index_inst.toRef(), |
| | 773 | } }, |
| | 774 | }).toRef()), |
| | 775 | .cmp_vector => { |
| | 776 | const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data; |
| | 777 | break :res_elem (try loop.block.addCmp( |
| | 778 | l, |
| | 779 | extra.compareOperator(), |
| | 780 | loop.block.add(l, .{ |
| | 781 | .tag = .array_elem_val, |
| | 782 | .data = .{ .bin_op = .{ |
| | 783 | .lhs = extra.lhs, |
| | 784 | .rhs = cur_index_inst.toRef(), |
| | 785 | } }, |
| | 786 | }).toRef(), |
| | 787 | loop.block.add(l, .{ |
| | 788 | .tag = .array_elem_val, |
| | 789 | .data = .{ .bin_op = .{ |
| | 790 | .lhs = extra.rhs, |
| | 791 | .rhs = cur_index_inst.toRef(), |
| | 792 | } }, |
| | 793 | }).toRef(), |
| | 794 | .{ .optimized = switch (orig.tag) { |
| | 795 | else => unreachable, |
| | 796 | .cmp_vector => false, |
| | 797 | .cmp_vector_optimized => true, |
| | 798 | } }, |
| | 799 | )).toRef(); |
| | 800 | }, |
| 748 | .shuffle_one, .shuffle_two => { | 801 | .shuffle_one, .shuffle_two => { |
| 749 | const ip = &zcu.intern_pool; | 802 | const ip = &zcu.intern_pool; |
| 750 | const unwrapped = switch (form) { | 803 | const unwrapped = switch (form) { |
| ... | @@ -1056,17 +1109,16 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: | ... | @@ -1056,17 +1109,16 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: |
| 1056 | .payload = try l.addBlockBody(res_block.body()), | 1109 | .payload = try l.addBlockBody(res_block.body()), |
| 1057 | } }; | 1110 | } }; |
| 1058 | } | 1111 | } |
| 1059 | fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { | 1112 | fn scalarizeBitcastToArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1060 | const pt = l.pt; | 1113 | const pt = l.pt; |
| 1061 | const zcu = pt.zcu; | 1114 | const zcu = pt.zcu; |
| 1062 | | 1115 | |
| 1063 | const orig = l.air_instructions.get(@intFromEnum(orig_inst)); | 1116 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1064 | const res_ty = l.typeOfIndex(orig_inst); | 1117 | const res_ty = orig_ty_op.ty.toType(); |
| 1065 | const wrapped_res_ty = res_ty.fieldType(0, zcu); | 1118 | const res_elem_ty = res_ty.childType(zcu); |
| 1066 | const wrapped_res_scalar_ty = wrapped_res_ty.childType(zcu); | 1119 | const res_len = res_ty.arrayLen(zcu); |
| 1067 | const res_len = wrapped_res_ty.vectorLen(zcu); | | |
| 1068 | | 1120 | |
| 1069 | var inst_buf: [21]Air.Inst.Index = undefined; | 1121 | var inst_buf: [16]Air.Inst.Index = undefined; |
| 1070 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); | 1122 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 1071 | | 1123 | |
| 1072 | var res_block: Block = .init(&inst_buf); | 1124 | var res_block: Block = .init(&inst_buf); |
| ... | @@ -1075,20 +1127,6 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! | ... | @@ -1075,20 +1127,6 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! |
| 1075 | .tag = .alloc, | 1127 | .tag = .alloc, |
| 1076 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, | 1128 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, |
| 1077 | }); | 1129 | }); |
| 1078 | const ptr_wrapped_res_inst = res_block.add(l, .{ | | |
| 1079 | .tag = .struct_field_ptr_index_0, | | |
| 1080 | .data = .{ .ty_op = .{ | | |
| 1081 | .ty = Air.internedToRef((try pt.singleMutPtrType(wrapped_res_ty)).toIntern()), | | |
| 1082 | .operand = res_alloc_inst.toRef(), | | |
| 1083 | } }, | | |
| 1084 | }); | | |
| 1085 | const ptr_overflow_res_inst = res_block.add(l, .{ | | |
| 1086 | .tag = .struct_field_ptr_index_1, | | |
| 1087 | .data = .{ .ty_op = .{ | | |
| 1088 | .ty = Air.internedToRef((try pt.singleMutPtrType(res_ty.fieldType(1, zcu))).toIntern()), | | |
| 1089 | .operand = res_alloc_inst.toRef(), | | |
| 1090 | } }, | | |
| 1091 | }); | | |
| 1092 | const index_alloc_inst = res_block.add(l, .{ | 1130 | const index_alloc_inst = res_block.add(l, .{ |
| 1093 | .tag = .alloc, | 1131 | .tag = .alloc, |
| 1094 | .data = .{ .ty = .ptr_usize }, | 1132 | .data = .{ .ty = .ptr_usize }, |
| ... | @@ -1111,68 +1149,26 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! | ... | @@ -1111,68 +1149,26 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! |
| 1111 | .operand = index_alloc_inst.toRef(), | 1149 | .operand = index_alloc_inst.toRef(), |
| 1112 | } }, | 1150 | } }, |
| 1113 | }); | 1151 | }); |
| 1114 | const extra = l.extraData(Air.Bin, orig.data.ty_pl.payload).data; | | |
| 1115 | const res_elem = loop.block.add(l, .{ | | |
| 1116 | .tag = orig.tag, | | |
| 1117 | .data = .{ .ty_pl = .{ | | |
| 1118 | .ty = Air.internedToRef(try zcu.intern_pool.getTupleType(zcu.gpa, pt.tid, .{ | | |
| 1119 | .types = &.{ wrapped_res_scalar_ty.toIntern(), .u1_type }, | | |
| 1120 | .values = &(.{.none} ** 2), | | |
| 1121 | })), | | |
| 1122 | .payload = try l.addExtra(Air.Bin, .{ | | |
| 1123 | .lhs = loop.block.add(l, .{ | | |
| 1124 | .tag = .array_elem_val, | | |
| 1125 | .data = .{ .bin_op = .{ | | |
| 1126 | .lhs = extra.lhs, | | |
| 1127 | .rhs = cur_index_inst.toRef(), | | |
| 1128 | } }, | | |
| 1129 | }).toRef(), | | |
| 1130 | .rhs = loop.block.add(l, .{ | | |
| 1131 | .tag = .array_elem_val, | | |
| 1132 | .data = .{ .bin_op = .{ | | |
| 1133 | .lhs = extra.rhs, | | |
| 1134 | .rhs = cur_index_inst.toRef(), | | |
| 1135 | } }, | | |
| 1136 | }).toRef(), | | |
| 1137 | }), | | |
| 1138 | } }, | | |
| 1139 | }); | | |
| 1140 | _ = loop.block.add(l, .{ | 1152 | _ = loop.block.add(l, .{ |
| 1141 | .tag = .vector_store_elem, | 1153 | .tag = .store, |
| 1142 | .data = .{ .vector_store_elem = .{ | 1154 | .data = .{ .bin_op = .{ |
| 1143 | .vector_ptr = ptr_overflow_res_inst.toRef(), | 1155 | .lhs = loop.block.add(l, .{ |
| 1144 | .payload = try l.addExtra(Air.Bin, .{ | 1156 | .tag = .ptr_elem_ptr, |
| 1145 | .lhs = cur_index_inst.toRef(), | 1157 | .data = .{ .ty_pl = .{ |
| 1146 | .rhs = loop.block.add(l, .{ | 1158 | .ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()), |
| 1147 | .tag = .struct_field_val, | 1159 | .payload = try l.addExtra(Air.Bin, .{ |
| 1148 | .data = .{ .ty_pl = .{ | 1160 | .lhs = res_alloc_inst.toRef(), |
| 1149 | .ty = .u1_type, | 1161 | .rhs = cur_index_inst.toRef(), |
| 1150 | .payload = try l.addExtra(Air.StructField, .{ | 1162 | }), |
| 1151 | .struct_operand = res_elem.toRef(), | 1163 | } }, |
| 1152 | .field_index = 1, | 1164 | }).toRef(), |
| 1153 | }), | 1165 | .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{ |
| 1154 | } }, | 1166 | .tag = .array_elem_val, |
| 1155 | }).toRef(), | 1167 | .data = .{ .bin_op = .{ |
| 1156 | }), | 1168 | .lhs = orig_ty_op.operand, |
| 1157 | } }, | 1169 | .rhs = cur_index_inst.toRef(), |
| 1158 | }); | 1170 | } }, |
| 1159 | _ = loop.block.add(l, .{ | 1171 | }).toRef()), |
| 1160 | .tag = .vector_store_elem, | | |
| 1161 | .data = .{ .vector_store_elem = .{ | | |
| 1162 | .vector_ptr = ptr_wrapped_res_inst.toRef(), | | |
| 1163 | .payload = try l.addExtra(Air.Bin, .{ | | |
| 1164 | .lhs = cur_index_inst.toRef(), | | |
| 1165 | .rhs = loop.block.add(l, .{ | | |
| 1166 | .tag = .struct_field_val, | | |
| 1167 | .data = .{ .ty_pl = .{ | | |
| 1168 | .ty = Air.internedToRef(wrapped_res_scalar_ty.toIntern()), | | |
| 1169 | .payload = try l.addExtra(Air.StructField, .{ | | |
| 1170 | .struct_operand = res_elem.toRef(), | | |
| 1171 | .field_index = 0, | | |
| 1172 | }), | | |
| 1173 | } }, | | |
| 1174 | }).toRef(), | | |
| 1175 | }), | | |
| 1176 | } }, | 1172 | } }, |
| 1177 | }); | 1173 | }); |
| 1178 | | 1174 | |
| ... | @@ -1226,48 +1222,630 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! | ... | @@ -1226,48 +1222,630 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! |
| 1226 | .payload = try l.addBlockBody(res_block.body()), | 1222 | .payload = try l.addBlockBody(res_block.body()), |
| 1227 | } }; | 1223 | } }; |
| 1228 | } | 1224 | } |
| 1229 | | 1225 | fn scalarizeBitcastOperandBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1230 | fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { | | |
| 1231 | const pt = l.pt; | 1226 | const pt = l.pt; |
| 1232 | const zcu = pt.zcu; | 1227 | const zcu = pt.zcu; |
| 1233 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; | | |
| 1234 | | | |
| 1235 | const operand_ref = ty_op.operand; | | |
| 1236 | const operand_ty = l.typeOf(operand_ref); | | |
| 1237 | const dest_ty = ty_op.ty.toType(); | | |
| 1238 | | 1228 | |
| 1239 | const is_vector = operand_ty.zigTypeTag(zcu) == .vector; | 1229 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1240 | const operand_scalar_ty = operand_ty.scalarType(zcu); | 1230 | const res_ty = orig_ty_op.ty.toType(); |
| 1241 | const dest_scalar_ty = dest_ty.scalarType(zcu); | 1231 | const operand_ty = l.typeOf(orig_ty_op.operand); |
| | 1232 | const int_bits: u16 = @intCast(operand_ty.bitSize(zcu)); |
| | 1233 | const int_ty = try pt.intType(.unsigned, int_bits); |
| | 1234 | const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits)); |
| | 1235 | const elem_bits: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu)); |
| | 1236 | const elem_int_ty = try pt.intType(.unsigned, elem_bits); |
| 1242 | | 1237 | |
| 1243 | assert(operand_scalar_ty.zigTypeTag(zcu) == .int); | 1238 | var inst_buf: [22]Air.Inst.Index = undefined; |
| 1244 | const dest_is_enum = switch (dest_scalar_ty.zigTypeTag(zcu)) { | 1239 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 1245 | .int => false, | | |
| 1246 | .@"enum" => true, | | |
| 1247 | else => unreachable, | | |
| 1248 | }; | | |
| 1249 | | 1240 | |
| 1250 | const operand_info = operand_scalar_ty.intInfo(zcu); | 1241 | var res_block: Block = .init(&inst_buf); |
| 1251 | const dest_info = dest_scalar_ty.intInfo(zcu); | 1242 | { |
| | 1243 | const int_alloc_inst = res_block.add(l, .{ |
| | 1244 | .tag = .alloc, |
| | 1245 | .data = .{ .ty = try pt.singleMutPtrType(int_ty) }, |
| | 1246 | }); |
| | 1247 | _ = res_block.add(l, .{ |
| | 1248 | .tag = .store, |
| | 1249 | .data = .{ .bin_op = .{ |
| | 1250 | .lhs = int_alloc_inst.toRef(), |
| | 1251 | .rhs = try pt.intRef(int_ty, 0), |
| | 1252 | } }, |
| | 1253 | }); |
| | 1254 | const index_alloc_inst = res_block.add(l, .{ |
| | 1255 | .tag = .alloc, |
| | 1256 | .data = .{ .ty = .ptr_usize }, |
| | 1257 | }); |
| | 1258 | _ = res_block.add(l, .{ |
| | 1259 | .tag = .store, |
| | 1260 | .data = .{ .bin_op = .{ |
| | 1261 | .lhs = index_alloc_inst.toRef(), |
| | 1262 | .rhs = .zero_usize, |
| | 1263 | } }, |
| | 1264 | }); |
| 1252 | | 1265 | |
| 1253 | const have_min_check, const have_max_check = c: { | 1266 | var loop: Loop = .init(l, &res_block); |
| 1254 | const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed); | 1267 | loop.block = .init(res_block.stealRemainingCapacity()); |
| 1255 | const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed); | 1268 | { |
| 1256 | const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0; | 1269 | const cur_index_inst = loop.block.add(l, .{ |
| 1257 | const operand_allows_neg = operand_info.signedness == .signed and operand_info.bits > 0; | 1270 | .tag = .load, |
| 1258 | break :c .{ | 1271 | .data = .{ .ty_op = .{ |
| 1259 | operand_allows_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits), | 1272 | .ty = .usize_type, |
| 1260 | dest_pos_bits < operand_pos_bits, | 1273 | .operand = index_alloc_inst.toRef(), |
| 1261 | }; | 1274 | } }, |
| 1262 | }; | 1275 | }); |
| | 1276 | const cur_int_inst = loop.block.add(l, .{ |
| | 1277 | .tag = .bit_or, |
| | 1278 | .data = .{ .bin_op = .{ |
| | 1279 | .lhs = loop.block.add(l, .{ |
| | 1280 | .tag = .shl_exact, |
| | 1281 | .data = .{ .bin_op = .{ |
| | 1282 | .lhs = loop.block.add(l, .{ |
| | 1283 | .tag = .intcast, |
| | 1284 | .data = .{ .ty_op = .{ |
| | 1285 | .ty = Air.internedToRef(int_ty.toIntern()), |
| | 1286 | .operand = loop.block.addBitCast(l, elem_int_ty, loop.block.add(l, .{ |
| | 1287 | .tag = .array_elem_val, |
| | 1288 | .data = .{ .bin_op = .{ |
| | 1289 | .lhs = orig_ty_op.operand, |
| | 1290 | .rhs = cur_index_inst.toRef(), |
| | 1291 | } }, |
| | 1292 | }).toRef()), |
| | 1293 | } }, |
| | 1294 | }).toRef(), |
| | 1295 | .rhs = loop.block.add(l, .{ |
| | 1296 | .tag = .mul, |
| | 1297 | .data = .{ .bin_op = .{ |
| | 1298 | .lhs = loop.block.add(l, .{ |
| | 1299 | .tag = .intcast, |
| | 1300 | .data = .{ .ty_op = .{ |
| | 1301 | .ty = Air.internedToRef(shift_ty.toIntern()), |
| | 1302 | .operand = cur_index_inst.toRef(), |
| | 1303 | } }, |
| | 1304 | }).toRef(), |
| | 1305 | .rhs = try pt.intRef(shift_ty, elem_bits), |
| | 1306 | } }, |
| | 1307 | }).toRef(), |
| | 1308 | } }, |
| | 1309 | }).toRef(), |
| | 1310 | .rhs = loop.block.add(l, .{ |
| | 1311 | .tag = .load, |
| | 1312 | .data = .{ .ty_op = .{ |
| | 1313 | .ty = Air.internedToRef(int_ty.toIntern()), |
| | 1314 | .operand = int_alloc_inst.toRef(), |
| | 1315 | } }, |
| | 1316 | }).toRef(), |
| | 1317 | } }, |
| | 1318 | }); |
| 1263 | | 1319 | |
| 1264 | // The worst-case scenario in terms of total instructions and total condbrs is the case where | 1320 | var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp( |
| 1265 | // the result type is an exhaustive enum whose tag type is smaller than the operand type: | 1321 | l, |
| 1266 | // | 1322 | .lt, |
| 1267 | // %x = block({ | 1323 | cur_index_inst.toRef(), |
| 1268 | // %1 = cmp_lt(%y, @min_allowed_int) | 1324 | try pt.intRef(.usize, operand_ty.arrayLen(zcu) - 1), |
| 1269 | // %2 = cmp_gt(%y, @max_allowed_int) | 1325 | .{}, |
| 1270 | // %3 = bool_or(%1, %2) | 1326 | )).toRef(), &loop.block, .{}); |
| | 1327 | loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity()); |
| | 1328 | { |
| | 1329 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1330 | .tag = .store, |
| | 1331 | .data = .{ .bin_op = .{ |
| | 1332 | .lhs = int_alloc_inst.toRef(), |
| | 1333 | .rhs = cur_int_inst.toRef(), |
| | 1334 | } }, |
| | 1335 | }); |
| | 1336 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1337 | .tag = .store, |
| | 1338 | .data = .{ .bin_op = .{ |
| | 1339 | .lhs = index_alloc_inst.toRef(), |
| | 1340 | .rhs = loop_cond_br.then_block.add(l, .{ |
| | 1341 | .tag = .add, |
| | 1342 | .data = .{ .bin_op = .{ |
| | 1343 | .lhs = cur_index_inst.toRef(), |
| | 1344 | .rhs = .one_usize, |
| | 1345 | } }, |
| | 1346 | }).toRef(), |
| | 1347 | } }, |
| | 1348 | }); |
| | 1349 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1350 | .tag = .repeat, |
| | 1351 | .data = .{ .repeat = .{ .loop_inst = loop.inst } }, |
| | 1352 | }); |
| | 1353 | } |
| | 1354 | loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity()); |
| | 1355 | _ = loop_cond_br.else_block.add(l, .{ |
| | 1356 | .tag = .br, |
| | 1357 | .data = .{ .br = .{ |
| | 1358 | .block_inst = orig_inst, |
| | 1359 | .operand = loop_cond_br.else_block.addBitCast(l, res_ty, cur_int_inst.toRef()), |
| | 1360 | } }, |
| | 1361 | }); |
| | 1362 | try loop_cond_br.finish(l); |
| | 1363 | } |
| | 1364 | try loop.finish(l); |
| | 1365 | } |
| | 1366 | return .{ .ty_pl = .{ |
| | 1367 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 1368 | .payload = try l.addBlockBody(res_block.body()), |
| | 1369 | } }; |
| | 1370 | } |
| | 1371 | fn scalarizeBitcastResultArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 1372 | const pt = l.pt; |
| | 1373 | const zcu = pt.zcu; |
| | 1374 | |
| | 1375 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| | 1376 | const res_ty = orig_ty_op.ty.toType(); |
| | 1377 | const int_bits: u16 = @intCast(res_ty.bitSize(zcu)); |
| | 1378 | const int_ty = try pt.intType(.unsigned, int_bits); |
| | 1379 | const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits)); |
| | 1380 | const res_elem_ty = res_ty.childType(zcu); |
| | 1381 | const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu)); |
| | 1382 | const elem_int_ty = try pt.intType(.unsigned, elem_bits); |
| | 1383 | |
| | 1384 | var inst_buf: [20]Air.Inst.Index = undefined; |
| | 1385 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| | 1386 | |
| | 1387 | var res_block: Block = .init(&inst_buf); |
| | 1388 | { |
| | 1389 | const res_alloc_inst = res_block.add(l, .{ |
| | 1390 | .tag = .alloc, |
| | 1391 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, |
| | 1392 | }); |
| | 1393 | const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand); |
| | 1394 | const index_alloc_inst = res_block.add(l, .{ |
| | 1395 | .tag = .alloc, |
| | 1396 | .data = .{ .ty = .ptr_usize }, |
| | 1397 | }); |
| | 1398 | _ = res_block.add(l, .{ |
| | 1399 | .tag = .store, |
| | 1400 | .data = .{ .bin_op = .{ |
| | 1401 | .lhs = index_alloc_inst.toRef(), |
| | 1402 | .rhs = .zero_usize, |
| | 1403 | } }, |
| | 1404 | }); |
| | 1405 | |
| | 1406 | var loop: Loop = .init(l, &res_block); |
| | 1407 | loop.block = .init(res_block.stealRemainingCapacity()); |
| | 1408 | { |
| | 1409 | const cur_index_inst = loop.block.add(l, .{ |
| | 1410 | .tag = .load, |
| | 1411 | .data = .{ .ty_op = .{ |
| | 1412 | .ty = .usize_type, |
| | 1413 | .operand = index_alloc_inst.toRef(), |
| | 1414 | } }, |
| | 1415 | }); |
| | 1416 | _ = loop.block.add(l, .{ |
| | 1417 | .tag = .store, |
| | 1418 | .data = .{ .bin_op = .{ |
| | 1419 | .lhs = loop.block.add(l, .{ |
| | 1420 | .tag = .ptr_elem_ptr, |
| | 1421 | .data = .{ .ty_pl = .{ |
| | 1422 | .ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()), |
| | 1423 | .payload = try l.addExtra(Air.Bin, .{ |
| | 1424 | .lhs = res_alloc_inst.toRef(), |
| | 1425 | .rhs = cur_index_inst.toRef(), |
| | 1426 | }), |
| | 1427 | } }, |
| | 1428 | }).toRef(), |
| | 1429 | .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{ |
| | 1430 | .tag = .trunc, |
| | 1431 | .data = .{ .ty_op = .{ |
| | 1432 | .ty = Air.internedToRef(elem_int_ty.toIntern()), |
| | 1433 | .operand = loop.block.add(l, .{ |
| | 1434 | .tag = .shr, |
| | 1435 | .data = .{ .bin_op = .{ |
| | 1436 | .lhs = int_ref, |
| | 1437 | .rhs = loop.block.add(l, .{ |
| | 1438 | .tag = .mul, |
| | 1439 | .data = .{ .bin_op = .{ |
| | 1440 | .lhs = loop.block.add(l, .{ |
| | 1441 | .tag = .intcast, |
| | 1442 | .data = .{ .ty_op = .{ |
| | 1443 | .ty = Air.internedToRef(shift_ty.toIntern()), |
| | 1444 | .operand = cur_index_inst.toRef(), |
| | 1445 | } }, |
| | 1446 | }).toRef(), |
| | 1447 | .rhs = try pt.intRef(shift_ty, elem_bits), |
| | 1448 | } }, |
| | 1449 | }).toRef(), |
| | 1450 | } }, |
| | 1451 | }).toRef(), |
| | 1452 | } }, |
| | 1453 | }).toRef()), |
| | 1454 | } }, |
| | 1455 | }); |
| | 1456 | |
| | 1457 | var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp( |
| | 1458 | l, |
| | 1459 | .lt, |
| | 1460 | cur_index_inst.toRef(), |
| | 1461 | try pt.intRef(.usize, res_ty.arrayLen(zcu) - 1), |
| | 1462 | .{}, |
| | 1463 | )).toRef(), &loop.block, .{}); |
| | 1464 | loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity()); |
| | 1465 | { |
| | 1466 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1467 | .tag = .store, |
| | 1468 | .data = .{ .bin_op = .{ |
| | 1469 | .lhs = index_alloc_inst.toRef(), |
| | 1470 | .rhs = loop_cond_br.then_block.add(l, .{ |
| | 1471 | .tag = .add, |
| | 1472 | .data = .{ .bin_op = .{ |
| | 1473 | .lhs = cur_index_inst.toRef(), |
| | 1474 | .rhs = .one_usize, |
| | 1475 | } }, |
| | 1476 | }).toRef(), |
| | 1477 | } }, |
| | 1478 | }); |
| | 1479 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1480 | .tag = .repeat, |
| | 1481 | .data = .{ .repeat = .{ .loop_inst = loop.inst } }, |
| | 1482 | }); |
| | 1483 | } |
| | 1484 | loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity()); |
| | 1485 | _ = loop_cond_br.else_block.add(l, .{ |
| | 1486 | .tag = .br, |
| | 1487 | .data = .{ .br = .{ |
| | 1488 | .block_inst = orig_inst, |
| | 1489 | .operand = loop_cond_br.else_block.add(l, .{ |
| | 1490 | .tag = .load, |
| | 1491 | .data = .{ .ty_op = .{ |
| | 1492 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 1493 | .operand = res_alloc_inst.toRef(), |
| | 1494 | } }, |
| | 1495 | }).toRef(), |
| | 1496 | } }, |
| | 1497 | }); |
| | 1498 | try loop_cond_br.finish(l); |
| | 1499 | } |
| | 1500 | try loop.finish(l); |
| | 1501 | } |
| | 1502 | return .{ .ty_pl = .{ |
| | 1503 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 1504 | .payload = try l.addBlockBody(res_block.body()), |
| | 1505 | } }; |
| | 1506 | } |
| | 1507 | fn scalarizeBitcastResultVectorBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 1508 | const pt = l.pt; |
| | 1509 | const zcu = pt.zcu; |
| | 1510 | |
| | 1511 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| | 1512 | const res_ty = orig_ty_op.ty.toType(); |
| | 1513 | const int_bits: u16 = @intCast(res_ty.bitSize(zcu)); |
| | 1514 | const int_ty = try pt.intType(.unsigned, int_bits); |
| | 1515 | const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits)); |
| | 1516 | const res_elem_ty = res_ty.childType(zcu); |
| | 1517 | const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu)); |
| | 1518 | const elem_int_ty = try pt.intType(.unsigned, elem_bits); |
| | 1519 | |
| | 1520 | var inst_buf: [19]Air.Inst.Index = undefined; |
| | 1521 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| | 1522 | |
| | 1523 | var res_block: Block = .init(&inst_buf); |
| | 1524 | { |
| | 1525 | const res_alloc_inst = res_block.add(l, .{ |
| | 1526 | .tag = .alloc, |
| | 1527 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, |
| | 1528 | }); |
| | 1529 | const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand); |
| | 1530 | const index_alloc_inst = res_block.add(l, .{ |
| | 1531 | .tag = .alloc, |
| | 1532 | .data = .{ .ty = .ptr_usize }, |
| | 1533 | }); |
| | 1534 | _ = res_block.add(l, .{ |
| | 1535 | .tag = .store, |
| | 1536 | .data = .{ .bin_op = .{ |
| | 1537 | .lhs = index_alloc_inst.toRef(), |
| | 1538 | .rhs = .zero_usize, |
| | 1539 | } }, |
| | 1540 | }); |
| | 1541 | |
| | 1542 | var loop: Loop = .init(l, &res_block); |
| | 1543 | loop.block = .init(res_block.stealRemainingCapacity()); |
| | 1544 | { |
| | 1545 | const cur_index_inst = loop.block.add(l, .{ |
| | 1546 | .tag = .load, |
| | 1547 | .data = .{ .ty_op = .{ |
| | 1548 | .ty = .usize_type, |
| | 1549 | .operand = index_alloc_inst.toRef(), |
| | 1550 | } }, |
| | 1551 | }); |
| | 1552 | _ = loop.block.add(l, .{ |
| | 1553 | .tag = .vector_store_elem, |
| | 1554 | .data = .{ .vector_store_elem = .{ |
| | 1555 | .vector_ptr = res_alloc_inst.toRef(), |
| | 1556 | .payload = try l.addExtra(Air.Bin, .{ |
| | 1557 | .lhs = cur_index_inst.toRef(), |
| | 1558 | .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{ |
| | 1559 | .tag = .trunc, |
| | 1560 | .data = .{ .ty_op = .{ |
| | 1561 | .ty = Air.internedToRef(elem_int_ty.toIntern()), |
| | 1562 | .operand = loop.block.add(l, .{ |
| | 1563 | .tag = .shr, |
| | 1564 | .data = .{ .bin_op = .{ |
| | 1565 | .lhs = int_ref, |
| | 1566 | .rhs = loop.block.add(l, .{ |
| | 1567 | .tag = .mul, |
| | 1568 | .data = .{ .bin_op = .{ |
| | 1569 | .lhs = loop.block.add(l, .{ |
| | 1570 | .tag = .intcast, |
| | 1571 | .data = .{ .ty_op = .{ |
| | 1572 | .ty = Air.internedToRef(shift_ty.toIntern()), |
| | 1573 | .operand = cur_index_inst.toRef(), |
| | 1574 | } }, |
| | 1575 | }).toRef(), |
| | 1576 | .rhs = try pt.intRef(shift_ty, elem_bits), |
| | 1577 | } }, |
| | 1578 | }).toRef(), |
| | 1579 | } }, |
| | 1580 | }).toRef(), |
| | 1581 | } }, |
| | 1582 | }).toRef()), |
| | 1583 | }), |
| | 1584 | } }, |
| | 1585 | }); |
| | 1586 | |
| | 1587 | var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp( |
| | 1588 | l, |
| | 1589 | .lt, |
| | 1590 | cur_index_inst.toRef(), |
| | 1591 | try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1), |
| | 1592 | .{}, |
| | 1593 | )).toRef(), &loop.block, .{}); |
| | 1594 | loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity()); |
| | 1595 | { |
| | 1596 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1597 | .tag = .store, |
| | 1598 | .data = .{ .bin_op = .{ |
| | 1599 | .lhs = index_alloc_inst.toRef(), |
| | 1600 | .rhs = loop_cond_br.then_block.add(l, .{ |
| | 1601 | .tag = .add, |
| | 1602 | .data = .{ .bin_op = .{ |
| | 1603 | .lhs = cur_index_inst.toRef(), |
| | 1604 | .rhs = .one_usize, |
| | 1605 | } }, |
| | 1606 | }).toRef(), |
| | 1607 | } }, |
| | 1608 | }); |
| | 1609 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1610 | .tag = .repeat, |
| | 1611 | .data = .{ .repeat = .{ .loop_inst = loop.inst } }, |
| | 1612 | }); |
| | 1613 | } |
| | 1614 | loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity()); |
| | 1615 | _ = loop_cond_br.else_block.add(l, .{ |
| | 1616 | .tag = .br, |
| | 1617 | .data = .{ .br = .{ |
| | 1618 | .block_inst = orig_inst, |
| | 1619 | .operand = loop_cond_br.else_block.add(l, .{ |
| | 1620 | .tag = .load, |
| | 1621 | .data = .{ .ty_op = .{ |
| | 1622 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 1623 | .operand = res_alloc_inst.toRef(), |
| | 1624 | } }, |
| | 1625 | }).toRef(), |
| | 1626 | } }, |
| | 1627 | }); |
| | 1628 | try loop_cond_br.finish(l); |
| | 1629 | } |
| | 1630 | try loop.finish(l); |
| | 1631 | } |
| | 1632 | return .{ .ty_pl = .{ |
| | 1633 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 1634 | .payload = try l.addBlockBody(res_block.body()), |
| | 1635 | } }; |
| | 1636 | } |
| | 1637 | fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 1638 | const pt = l.pt; |
| | 1639 | const zcu = pt.zcu; |
| | 1640 | |
| | 1641 | const orig = l.air_instructions.get(@intFromEnum(orig_inst)); |
| | 1642 | const res_ty = l.typeOfIndex(orig_inst); |
| | 1643 | const wrapped_res_ty = res_ty.fieldType(0, zcu); |
| | 1644 | const wrapped_res_scalar_ty = wrapped_res_ty.childType(zcu); |
| | 1645 | const res_len = wrapped_res_ty.vectorLen(zcu); |
| | 1646 | |
| | 1647 | var inst_buf: [21]Air.Inst.Index = undefined; |
| | 1648 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| | 1649 | |
| | 1650 | var res_block: Block = .init(&inst_buf); |
| | 1651 | { |
| | 1652 | const res_alloc_inst = res_block.add(l, .{ |
| | 1653 | .tag = .alloc, |
| | 1654 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, |
| | 1655 | }); |
| | 1656 | const ptr_wrapped_res_inst = res_block.add(l, .{ |
| | 1657 | .tag = .struct_field_ptr_index_0, |
| | 1658 | .data = .{ .ty_op = .{ |
| | 1659 | .ty = Air.internedToRef((try pt.singleMutPtrType(wrapped_res_ty)).toIntern()), |
| | 1660 | .operand = res_alloc_inst.toRef(), |
| | 1661 | } }, |
| | 1662 | }); |
| | 1663 | const ptr_overflow_res_inst = res_block.add(l, .{ |
| | 1664 | .tag = .struct_field_ptr_index_1, |
| | 1665 | .data = .{ .ty_op = .{ |
| | 1666 | .ty = Air.internedToRef((try pt.singleMutPtrType(res_ty.fieldType(1, zcu))).toIntern()), |
| | 1667 | .operand = res_alloc_inst.toRef(), |
| | 1668 | } }, |
| | 1669 | }); |
| | 1670 | const index_alloc_inst = res_block.add(l, .{ |
| | 1671 | .tag = .alloc, |
| | 1672 | .data = .{ .ty = .ptr_usize }, |
| | 1673 | }); |
| | 1674 | _ = res_block.add(l, .{ |
| | 1675 | .tag = .store, |
| | 1676 | .data = .{ .bin_op = .{ |
| | 1677 | .lhs = index_alloc_inst.toRef(), |
| | 1678 | .rhs = .zero_usize, |
| | 1679 | } }, |
| | 1680 | }); |
| | 1681 | |
| | 1682 | var loop: Loop = .init(l, &res_block); |
| | 1683 | loop.block = .init(res_block.stealRemainingCapacity()); |
| | 1684 | { |
| | 1685 | const cur_index_inst = loop.block.add(l, .{ |
| | 1686 | .tag = .load, |
| | 1687 | .data = .{ .ty_op = .{ |
| | 1688 | .ty = .usize_type, |
| | 1689 | .operand = index_alloc_inst.toRef(), |
| | 1690 | } }, |
| | 1691 | }); |
| | 1692 | const extra = l.extraData(Air.Bin, orig.data.ty_pl.payload).data; |
| | 1693 | const res_elem = loop.block.add(l, .{ |
| | 1694 | .tag = orig.tag, |
| | 1695 | .data = .{ .ty_pl = .{ |
| | 1696 | .ty = Air.internedToRef(try zcu.intern_pool.getTupleType(zcu.gpa, pt.tid, .{ |
| | 1697 | .types = &.{ wrapped_res_scalar_ty.toIntern(), .u1_type }, |
| | 1698 | .values = &(.{.none} ** 2), |
| | 1699 | })), |
| | 1700 | .payload = try l.addExtra(Air.Bin, .{ |
| | 1701 | .lhs = loop.block.add(l, .{ |
| | 1702 | .tag = .array_elem_val, |
| | 1703 | .data = .{ .bin_op = .{ |
| | 1704 | .lhs = extra.lhs, |
| | 1705 | .rhs = cur_index_inst.toRef(), |
| | 1706 | } }, |
| | 1707 | }).toRef(), |
| | 1708 | .rhs = loop.block.add(l, .{ |
| | 1709 | .tag = .array_elem_val, |
| | 1710 | .data = .{ .bin_op = .{ |
| | 1711 | .lhs = extra.rhs, |
| | 1712 | .rhs = cur_index_inst.toRef(), |
| | 1713 | } }, |
| | 1714 | }).toRef(), |
| | 1715 | }), |
| | 1716 | } }, |
| | 1717 | }); |
| | 1718 | _ = loop.block.add(l, .{ |
| | 1719 | .tag = .vector_store_elem, |
| | 1720 | .data = .{ .vector_store_elem = .{ |
| | 1721 | .vector_ptr = ptr_overflow_res_inst.toRef(), |
| | 1722 | .payload = try l.addExtra(Air.Bin, .{ |
| | 1723 | .lhs = cur_index_inst.toRef(), |
| | 1724 | .rhs = loop.block.add(l, .{ |
| | 1725 | .tag = .struct_field_val, |
| | 1726 | .data = .{ .ty_pl = .{ |
| | 1727 | .ty = .u1_type, |
| | 1728 | .payload = try l.addExtra(Air.StructField, .{ |
| | 1729 | .struct_operand = res_elem.toRef(), |
| | 1730 | .field_index = 1, |
| | 1731 | }), |
| | 1732 | } }, |
| | 1733 | }).toRef(), |
| | 1734 | }), |
| | 1735 | } }, |
| | 1736 | }); |
| | 1737 | _ = loop.block.add(l, .{ |
| | 1738 | .tag = .vector_store_elem, |
| | 1739 | .data = .{ .vector_store_elem = .{ |
| | 1740 | .vector_ptr = ptr_wrapped_res_inst.toRef(), |
| | 1741 | .payload = try l.addExtra(Air.Bin, .{ |
| | 1742 | .lhs = cur_index_inst.toRef(), |
| | 1743 | .rhs = loop.block.add(l, .{ |
| | 1744 | .tag = .struct_field_val, |
| | 1745 | .data = .{ .ty_pl = .{ |
| | 1746 | .ty = Air.internedToRef(wrapped_res_scalar_ty.toIntern()), |
| | 1747 | .payload = try l.addExtra(Air.StructField, .{ |
| | 1748 | .struct_operand = res_elem.toRef(), |
| | 1749 | .field_index = 0, |
| | 1750 | }), |
| | 1751 | } }, |
| | 1752 | }).toRef(), |
| | 1753 | }), |
| | 1754 | } }, |
| | 1755 | }); |
| | 1756 | |
| | 1757 | var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp( |
| | 1758 | l, |
| | 1759 | .lt, |
| | 1760 | cur_index_inst.toRef(), |
| | 1761 | try pt.intRef(.usize, res_len - 1), |
| | 1762 | .{}, |
| | 1763 | )).toRef(), &loop.block, .{}); |
| | 1764 | loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity()); |
| | 1765 | { |
| | 1766 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1767 | .tag = .store, |
| | 1768 | .data = .{ .bin_op = .{ |
| | 1769 | .lhs = index_alloc_inst.toRef(), |
| | 1770 | .rhs = loop_cond_br.then_block.add(l, .{ |
| | 1771 | .tag = .add, |
| | 1772 | .data = .{ .bin_op = .{ |
| | 1773 | .lhs = cur_index_inst.toRef(), |
| | 1774 | .rhs = .one_usize, |
| | 1775 | } }, |
| | 1776 | }).toRef(), |
| | 1777 | } }, |
| | 1778 | }); |
| | 1779 | _ = loop_cond_br.then_block.add(l, .{ |
| | 1780 | .tag = .repeat, |
| | 1781 | .data = .{ .repeat = .{ .loop_inst = loop.inst } }, |
| | 1782 | }); |
| | 1783 | } |
| | 1784 | loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity()); |
| | 1785 | _ = loop_cond_br.else_block.add(l, .{ |
| | 1786 | .tag = .br, |
| | 1787 | .data = .{ .br = .{ |
| | 1788 | .block_inst = orig_inst, |
| | 1789 | .operand = loop_cond_br.else_block.add(l, .{ |
| | 1790 | .tag = .load, |
| | 1791 | .data = .{ .ty_op = .{ |
| | 1792 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 1793 | .operand = res_alloc_inst.toRef(), |
| | 1794 | } }, |
| | 1795 | }).toRef(), |
| | 1796 | } }, |
| | 1797 | }); |
| | 1798 | try loop_cond_br.finish(l); |
| | 1799 | } |
| | 1800 | try loop.finish(l); |
| | 1801 | } |
| | 1802 | return .{ .ty_pl = .{ |
| | 1803 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 1804 | .payload = try l.addBlockBody(res_block.body()), |
| | 1805 | } }; |
| | 1806 | } |
| | 1807 | |
| | 1808 | fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 1809 | const pt = l.pt; |
| | 1810 | const zcu = pt.zcu; |
| | 1811 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| | 1812 | |
| | 1813 | const operand_ref = ty_op.operand; |
| | 1814 | const operand_ty = l.typeOf(operand_ref); |
| | 1815 | const dest_ty = ty_op.ty.toType(); |
| | 1816 | |
| | 1817 | const is_vector = operand_ty.zigTypeTag(zcu) == .vector; |
| | 1818 | const operand_scalar_ty = operand_ty.scalarType(zcu); |
| | 1819 | const dest_scalar_ty = dest_ty.scalarType(zcu); |
| | 1820 | |
| | 1821 | assert(operand_scalar_ty.zigTypeTag(zcu) == .int); |
| | 1822 | const dest_is_enum = switch (dest_scalar_ty.zigTypeTag(zcu)) { |
| | 1823 | .int => false, |
| | 1824 | .@"enum" => true, |
| | 1825 | else => unreachable, |
| | 1826 | }; |
| | 1827 | |
| | 1828 | const operand_info = operand_scalar_ty.intInfo(zcu); |
| | 1829 | const dest_info = dest_scalar_ty.intInfo(zcu); |
| | 1830 | |
| | 1831 | const have_min_check, const have_max_check = c: { |
| | 1832 | const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed); |
| | 1833 | const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed); |
| | 1834 | const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0; |
| | 1835 | const operand_allows_neg = operand_info.signedness == .signed and operand_info.bits > 0; |
| | 1836 | break :c .{ |
| | 1837 | operand_allows_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits), |
| | 1838 | dest_pos_bits < operand_pos_bits, |
| | 1839 | }; |
| | 1840 | }; |
| | 1841 | |
| | 1842 | // The worst-case scenario in terms of total instructions and total condbrs is the case where |
| | 1843 | // the result type is an exhaustive enum whose tag type is smaller than the operand type: |
| | 1844 | // |
| | 1845 | // %x = block({ |
| | 1846 | // %1 = cmp_lt(%y, @min_allowed_int) |
| | 1847 | // %2 = cmp_gt(%y, @max_allowed_int) |
| | 1848 | // %3 = bool_or(%1, %2) |
| 1271 | // %4 = cond_br(%3, { | 1849 | // %4 = cond_br(%3, { |
| 1272 | // %5 = call(@panic.invalidEnumValue, []) | 1850 | // %5 = call(@panic.invalidEnumValue, []) |
| 1273 | // %6 = unreach() | 1851 | // %6 = unreach() |
| ... | @@ -1485,6 +2063,292 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_ | ... | @@ -1485,6 +2063,292 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_ |
| 1485 | } }; | 2063 | } }; |
| 1486 | } | 2064 | } |
| 1487 | | 2065 | |
| | 2066 | fn expandBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 2067 | const pt = l.pt; |
| | 2068 | const zcu = pt.zcu; |
| | 2069 | const ip = &zcu.intern_pool; |
| | 2070 | |
| | 2071 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| | 2072 | const res_ty = orig_ty_op.ty.toType(); |
| | 2073 | const res_ty_key = ip.indexToKey(res_ty.toIntern()); |
| | 2074 | const operand_ty = l.typeOf(orig_ty_op.operand); |
| | 2075 | const operand_ty_key = ip.indexToKey(operand_ty.toIntern()); |
| | 2076 | _ = res_ty_key; |
| | 2077 | _ = operand_ty_key; |
| | 2078 | |
| | 2079 | var inst_buf: [1]Air.Inst.Index = undefined; |
| | 2080 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| | 2081 | |
| | 2082 | var res_block: Block = .init(&inst_buf); |
| | 2083 | { |
| | 2084 | _ = res_block.add(l, .{ |
| | 2085 | .tag = .br, |
| | 2086 | .data = .{ .br = .{ |
| | 2087 | .block_inst = orig_inst, |
| | 2088 | .operand = try pt.undefRef(res_ty), |
| | 2089 | } }, |
| | 2090 | }); |
| | 2091 | } |
| | 2092 | return .{ .ty_pl = .{ |
| | 2093 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 2094 | .payload = try l.addBlockBody(res_block.body()), |
| | 2095 | } }; |
| | 2096 | } |
| | 2097 | fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 2098 | const pt = l.pt; |
| | 2099 | const zcu = pt.zcu; |
| | 2100 | |
| | 2101 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| | 2102 | const res_ty = orig_ty_op.ty.toType(); |
| | 2103 | const res_int_ty = try pt.intType(.unsigned, @intCast(res_ty.bitSize(zcu))); |
| | 2104 | const ptr_ty = l.typeOf(orig_ty_op.operand); |
| | 2105 | const ptr_info = ptr_ty.ptrInfo(zcu); |
| | 2106 | // This relies on a heap of possibly invalid assumptions to work around not knowing the actual backing type. |
| | 2107 | const load_bits = 8 * ptr_info.packed_offset.host_size; |
| | 2108 | const load_ty = try pt.intType(.unsigned, load_bits); |
| | 2109 | |
| | 2110 | var inst_buf: [6]Air.Inst.Index = undefined; |
| | 2111 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| | 2112 | |
| | 2113 | var res_block: Block = .init(&inst_buf); |
| | 2114 | _ = res_block.add(l, .{ |
| | 2115 | .tag = .br, |
| | 2116 | .data = .{ .br = .{ |
| | 2117 | .block_inst = orig_inst, |
| | 2118 | .operand = res_block.addBitCast(l, res_ty, res_block.add(l, .{ |
| | 2119 | .tag = .trunc, |
| | 2120 | .data = .{ .ty_op = .{ |
| | 2121 | .ty = Air.internedToRef(res_int_ty.toIntern()), |
| | 2122 | .operand = res_block.add(l, .{ |
| | 2123 | .tag = .shr, |
| | 2124 | .data = .{ .bin_op = .{ |
| | 2125 | .lhs = res_block.add(l, .{ |
| | 2126 | .tag = .load, |
| | 2127 | .data = .{ .ty_op = .{ |
| | 2128 | .ty = Air.internedToRef(load_ty.toIntern()), |
| | 2129 | .operand = res_block.addBitCast(l, load_ptr_ty: { |
| | 2130 | var load_ptr_info = ptr_info; |
| | 2131 | load_ptr_info.child = load_ty.toIntern(); |
| | 2132 | load_ptr_info.flags.vector_index = .none; |
| | 2133 | load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 }; |
| | 2134 | break :load_ptr_ty try pt.ptrType(load_ptr_info); |
| | 2135 | }, orig_ty_op.operand), |
| | 2136 | } }, |
| | 2137 | }).toRef(), |
| | 2138 | .rhs = try pt.intRef( |
| | 2139 | try pt.intType(.unsigned, std.math.log2_int_ceil(u16, load_bits)), |
| | 2140 | ptr_info.packed_offset.bit_offset, |
| | 2141 | ), |
| | 2142 | } }, |
| | 2143 | }).toRef(), |
| | 2144 | } }, |
| | 2145 | }).toRef()), |
| | 2146 | } }, |
| | 2147 | }); |
| | 2148 | return .{ .ty_pl = .{ |
| | 2149 | .ty = Air.internedToRef(res_ty.toIntern()), |
| | 2150 | .payload = try l.addBlockBody(res_block.body()), |
| | 2151 | } }; |
| | 2152 | } |
| | 2153 | fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 2154 | const pt = l.pt; |
| | 2155 | const zcu = pt.zcu; |
| | 2156 | |
| | 2157 | const orig_bin_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].bin_op; |
| | 2158 | const ptr_ty = l.typeOf(orig_bin_op.lhs); |
| | 2159 | const ptr_info = ptr_ty.ptrInfo(zcu); |
| | 2160 | const operand_ty = l.typeOf(orig_bin_op.rhs); |
| | 2161 | const operand_bits: u16 = @intCast(operand_ty.bitSize(zcu)); |
| | 2162 | const operand_int_ty = try pt.intType(.unsigned, operand_bits); |
| | 2163 | // This relies on a heap of possibly invalid assumptions to work around not knowing the actual backing type. |
| | 2164 | const load_store_bits = 8 * ptr_info.packed_offset.host_size; |
| | 2165 | const load_store_ty = try pt.intType(.unsigned, load_store_bits); |
| | 2166 | |
| | 2167 | var inst_buf: [9]Air.Inst.Index = undefined; |
| | 2168 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| | 2169 | |
| | 2170 | var res_block: Block = .init(&inst_buf); |
| | 2171 | { |
| | 2172 | const backing_ptr_inst = res_block.add(l, .{ |
| | 2173 | .tag = .bitcast, |
| | 2174 | .data = .{ .ty_op = .{ |
| | 2175 | .ty = Air.internedToRef((load_store_ptr_ty: { |
| | 2176 | var load_ptr_info = ptr_info; |
| | 2177 | load_ptr_info.child = load_store_ty.toIntern(); |
| | 2178 | load_ptr_info.flags.vector_index = .none; |
| | 2179 | load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 }; |
| | 2180 | break :load_store_ptr_ty try pt.ptrType(load_ptr_info); |
| | 2181 | }).toIntern()), |
| | 2182 | .operand = orig_bin_op.lhs, |
| | 2183 | } }, |
| | 2184 | }); |
| | 2185 | _ = res_block.add(l, .{ |
| | 2186 | .tag = .store, |
| | 2187 | .data = .{ .bin_op = .{ |
| | 2188 | .lhs = backing_ptr_inst.toRef(), |
| | 2189 | .rhs = res_block.add(l, .{ |
| | 2190 | .tag = .bit_or, |
| | 2191 | .data = .{ .bin_op = .{ |
| | 2192 | .lhs = res_block.add(l, .{ |
| | 2193 | .tag = .bit_and, |
| | 2194 | .data = .{ .bin_op = .{ |
| | 2195 | .lhs = res_block.add(l, .{ |
| | 2196 | .tag = .load, |
| | 2197 | .data = .{ .ty_op = .{ |
| | 2198 | .ty = Air.internedToRef(load_store_ty.toIntern()), |
| | 2199 | .operand = backing_ptr_inst.toRef(), |
| | 2200 | } }, |
| | 2201 | }).toRef(), |
| | 2202 | .rhs = Air.internedToRef((keep_mask: { |
| | 2203 | const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; |
| | 2204 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| | 2205 | std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa); |
| | 2206 | const gpa = stack.get(); |
| | 2207 | |
| | 2208 | var mask_big_int: std.math.big.int.Mutable = .{ |
| | 2209 | .limbs = try gpa.alloc( |
| | 2210 | std.math.big.Limb, |
| | 2211 | std.math.big.int.calcTwosCompLimbCount(load_store_bits), |
| | 2212 | ), |
| | 2213 | .len = undefined, |
| | 2214 | .positive = undefined, |
| | 2215 | }; |
| | 2216 | defer gpa.free(mask_big_int.limbs); |
| | 2217 | mask_big_int.setTwosCompIntLimit(.max, .unsigned, operand_bits); |
| | 2218 | mask_big_int.shiftLeft(mask_big_int.toConst(), ptr_info.packed_offset.bit_offset); |
| | 2219 | mask_big_int.bitNotWrap(mask_big_int.toConst(), .unsigned, load_store_bits); |
| | 2220 | break :keep_mask try pt.intValue_big(load_store_ty, mask_big_int.toConst()); |
| | 2221 | }).toIntern()), |
| | 2222 | } }, |
| | 2223 | }).toRef(), |
| | 2224 | .rhs = res_block.add(l, .{ |
| | 2225 | .tag = .shl_exact, |
| | 2226 | .data = .{ .bin_op = .{ |
| | 2227 | .lhs = res_block.add(l, .{ |
| | 2228 | .tag = .intcast, |
| | 2229 | .data = .{ .ty_op = .{ |
| | 2230 | .ty = Air.internedToRef(load_store_ty.toIntern()), |
| | 2231 | .operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs), |
| | 2232 | } }, |
| | 2233 | }).toRef(), |
| | 2234 | .rhs = try pt.intRef( |
| | 2235 | try pt.intType(.unsigned, std.math.log2_int_ceil(u16, load_store_bits)), |
| | 2236 | ptr_info.packed_offset.bit_offset, |
| | 2237 | ), |
| | 2238 | } }, |
| | 2239 | }).toRef(), |
| | 2240 | } }, |
| | 2241 | }).toRef(), |
| | 2242 | } }, |
| | 2243 | }); |
| | 2244 | _ = res_block.add(l, .{ |
| | 2245 | .tag = .br, |
| | 2246 | .data = .{ .br = .{ |
| | 2247 | .block_inst = orig_inst, |
| | 2248 | .operand = .void_value, |
| | 2249 | } }, |
| | 2250 | }); |
| | 2251 | } |
| | 2252 | return .{ .ty_pl = .{ |
| | 2253 | .ty = .void_type, |
| | 2254 | .payload = try l.addBlockBody(res_block.body()), |
| | 2255 | } }; |
| | 2256 | } |
| | 2257 | fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 2258 | const pt = l.pt; |
| | 2259 | const zcu = pt.zcu; |
| | 2260 | |
| | 2261 | const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl; |
| | 2262 | const orig_extra = l.extraData(Air.StructField, orig_ty_pl.payload).data; |
| | 2263 | const field_ty = orig_ty_pl.ty.toType(); |
| | 2264 | const agg_ty = l.typeOf(orig_extra.struct_operand); |
| | 2265 | |
| | 2266 | var inst_buf: [5]Air.Inst.Index = undefined; |
| | 2267 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| | 2268 | |
| | 2269 | var res_block: Block = .init(&inst_buf); |
| | 2270 | { |
| | 2271 | const agg_alloc_inst = res_block.add(l, .{ |
| | 2272 | .tag = .alloc, |
| | 2273 | .data = .{ .ty = try pt.singleMutPtrType(agg_ty) }, |
| | 2274 | }); |
| | 2275 | _ = res_block.add(l, .{ |
| | 2276 | .tag = .store, |
| | 2277 | .data = .{ .bin_op = .{ |
| | 2278 | .lhs = agg_alloc_inst.toRef(), |
| | 2279 | .rhs = orig_extra.struct_operand, |
| | 2280 | } }, |
| | 2281 | }); |
| | 2282 | _ = res_block.add(l, .{ |
| | 2283 | .tag = .br, |
| | 2284 | .data = .{ .br = .{ |
| | 2285 | .block_inst = orig_inst, |
| | 2286 | .operand = res_block.add(l, .{ |
| | 2287 | .tag = .load, |
| | 2288 | .data = .{ .ty_op = .{ |
| | 2289 | .ty = Air.internedToRef(field_ty.toIntern()), |
| | 2290 | .operand = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), orig_extra.field_index)).toRef(), |
| | 2291 | } }, |
| | 2292 | }).toRef(), |
| | 2293 | } }, |
| | 2294 | }); |
| | 2295 | } |
| | 2296 | return .{ .ty_pl = .{ |
| | 2297 | .ty = Air.internedToRef(field_ty.toIntern()), |
| | 2298 | .payload = try l.addBlockBody(res_block.body()), |
| | 2299 | } }; |
| | 2300 | } |
| | 2301 | fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| | 2302 | const pt = l.pt; |
| | 2303 | const zcu = pt.zcu; |
| | 2304 | |
| | 2305 | const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl; |
| | 2306 | const field_ty = orig_ty_pl.ty.toType(); |
| | 2307 | const agg_ty = orig_ty_pl.ty.toType(); |
| | 2308 | const agg_field_count = agg_ty.structFieldCount(zcu); |
| | 2309 | |
| | 2310 | const ExpectedContents = [1 + 2 * 32 + 2]Air.Inst.Index; |
| | 2311 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| | 2312 | std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa); |
| | 2313 | const gpa = stack.get(); |
| | 2314 | |
| | 2315 | const inst_buf = try gpa.alloc(Air.Inst.Index, 1 + 2 * agg_field_count + 2); |
| | 2316 | defer gpa.free(inst_buf); |
| | 2317 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| | 2318 | |
| | 2319 | var res_block: Block = .init(inst_buf); |
| | 2320 | { |
| | 2321 | const agg_alloc_inst = res_block.add(l, .{ |
| | 2322 | .tag = .alloc, |
| | 2323 | .data = .{ .ty = try pt.singleMutPtrType(agg_ty) }, |
| | 2324 | }); |
| | 2325 | for (0..agg_field_count, orig_ty_pl.payload..) |field_index, extra_index| _ = res_block.add(l, .{ |
| | 2326 | .tag = .store, |
| | 2327 | .data = .{ .bin_op = .{ |
| | 2328 | .lhs = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), field_index)).toRef(), |
| | 2329 | .rhs = @enumFromInt(l.air_extra.items[extra_index]), |
| | 2330 | } }, |
| | 2331 | }); |
| | 2332 | _ = res_block.add(l, .{ |
| | 2333 | .tag = .br, |
| | 2334 | .data = .{ .br = .{ |
| | 2335 | .block_inst = orig_inst, |
| | 2336 | .operand = res_block.add(l, .{ |
| | 2337 | .tag = .load, |
| | 2338 | .data = .{ .ty_op = .{ |
| | 2339 | .ty = Air.internedToRef(field_ty.toIntern()), |
| | 2340 | .operand = agg_alloc_inst.toRef(), |
| | 2341 | } }, |
| | 2342 | }).toRef(), |
| | 2343 | } }, |
| | 2344 | }); |
| | 2345 | } |
| | 2346 | return .{ .ty_pl = .{ |
| | 2347 | .ty = Air.internedToRef(field_ty.toIntern()), |
| | 2348 | .payload = try l.addBlockBody(res_block.body()), |
| | 2349 | } }; |
| | 2350 | } |
| | 2351 | |
| 1488 | const Block = struct { | 2352 | const Block = struct { |
| 1489 | instructions: []Air.Inst.Index, | 2353 | instructions: []Air.Inst.Index, |
| 1490 | len: usize, | 2354 | len: usize, |
| ... | @@ -1576,6 +2440,114 @@ const Block = struct { | ... | @@ -1576,6 +2440,114 @@ const Block = struct { |
| 1576 | }); | 2440 | }); |
| 1577 | } | 2441 | } |
| 1578 | | 2442 | |
| | 2443 | /// Adds a `struct_field_ptr*` instruction to `b`. This is a fairly thin wrapper around `add` |
| | 2444 | /// that selects the optimized instruction encoding to use, although it does compute the |
| | 2445 | /// proper field pointer type. |
| | 2446 | fn addStructFieldPtr( |
| | 2447 | b: *Block, |
| | 2448 | l: *Legalize, |
| | 2449 | struct_operand: Air.Inst.Ref, |
| | 2450 | field_index: usize, |
| | 2451 | ) Error!Air.Inst.Index { |
| | 2452 | const pt = l.pt; |
| | 2453 | const zcu = pt.zcu; |
| | 2454 | |
| | 2455 | const agg_ptr_ty = l.typeOf(struct_operand); |
| | 2456 | const agg_ptr_info = agg_ptr_ty.ptrInfo(zcu); |
| | 2457 | const agg_ty: Type = .fromInterned(agg_ptr_info.child); |
| | 2458 | const agg_ptr_align = switch (agg_ptr_info.flags.alignment) { |
| | 2459 | .none => agg_ty.abiAlignment(zcu), |
| | 2460 | else => |agg_ptr_align| agg_ptr_align, |
| | 2461 | }; |
| | 2462 | const agg_layout = agg_ty.containerLayout(zcu); |
| | 2463 | const field_ty = agg_ty.fieldType(field_index, zcu); |
| | 2464 | var field_ptr_info: InternPool.Key.PtrType = .{ |
| | 2465 | .child = field_ty.toIntern(), |
| | 2466 | .flags = .{ |
| | 2467 | .is_const = agg_ptr_info.flags.is_const, |
| | 2468 | .is_volatile = agg_ptr_info.flags.is_volatile, |
| | 2469 | .address_space = agg_ptr_info.flags.address_space, |
| | 2470 | }, |
| | 2471 | }; |
| | 2472 | field_ptr_info.flags.alignment = field_ptr_align: switch (agg_layout) { |
| | 2473 | .auto => agg_ty.fieldAlignment(field_index, zcu).min(agg_ptr_align), |
| | 2474 | .@"extern" => switch (agg_ty.zigTypeTag(zcu)) { |
| | 2475 | else => unreachable, |
| | 2476 | .@"struct" => .fromLog2Units(@min( |
| | 2477 | agg_ptr_align.toLog2Units(), |
| | 2478 | @ctz(agg_ty.structFieldOffset(field_index, zcu)), |
| | 2479 | )), |
| | 2480 | .@"union" => agg_ptr_align, |
| | 2481 | }, |
| | 2482 | .@"packed" => switch (agg_ty.zigTypeTag(zcu)) { |
| | 2483 | else => unreachable, |
| | 2484 | .@"struct" => switch (agg_ty.packedStructFieldPtrInfo(agg_ptr_ty, @intCast(field_index), pt)) { |
| | 2485 | .bit_ptr => |packed_offset| { |
| | 2486 | field_ptr_info.packed_offset = packed_offset; |
| | 2487 | break :field_ptr_align agg_ptr_align; |
| | 2488 | }, |
| | 2489 | .byte_ptr => |ptr_info| ptr_info.alignment, |
| | 2490 | }, |
| | 2491 | .@"union" => { |
| | 2492 | field_ptr_info.packed_offset = .{ |
| | 2493 | .host_size = switch (agg_ptr_info.packed_offset.host_size) { |
| | 2494 | 0 => @intCast(agg_ty.abiSize(zcu)), |
| | 2495 | else => |host_size| host_size, |
| | 2496 | }, |
| | 2497 | .bit_offset = agg_ptr_info.packed_offset.bit_offset, |
| | 2498 | }; |
| | 2499 | break :field_ptr_align agg_ptr_align; |
| | 2500 | }, |
| | 2501 | }, |
| | 2502 | }; |
| | 2503 | const field_ptr_ty = try pt.ptrType(field_ptr_info); |
| | 2504 | const field_ptr_ty_ref = Air.internedToRef(field_ptr_ty.toIntern()); |
| | 2505 | return switch (field_index) { |
| | 2506 | inline 0...3 => |ct_field_index| b.add(l, .{ |
| | 2507 | .tag = switch (ct_field_index) { |
| | 2508 | 0 => .struct_field_ptr_index_0, |
| | 2509 | 1 => .struct_field_ptr_index_1, |
| | 2510 | 2 => .struct_field_ptr_index_2, |
| | 2511 | 3 => .struct_field_ptr_index_3, |
| | 2512 | else => comptime unreachable, |
| | 2513 | }, |
| | 2514 | .data = .{ .ty_op = .{ |
| | 2515 | .ty = field_ptr_ty_ref, |
| | 2516 | .operand = struct_operand, |
| | 2517 | } }, |
| | 2518 | }), |
| | 2519 | else => b.add(l, .{ |
| | 2520 | .tag = .struct_field_ptr, |
| | 2521 | .data = .{ .ty_pl = .{ |
| | 2522 | .ty = field_ptr_ty_ref, |
| | 2523 | .payload = try l.addExtra(Air.StructField, .{ |
| | 2524 | .struct_operand = struct_operand, |
| | 2525 | .field_index = @intCast(field_index), |
| | 2526 | }), |
| | 2527 | } }, |
| | 2528 | }), |
| | 2529 | }; |
| | 2530 | } |
| | 2531 | |
| | 2532 | /// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for |
| | 2533 | /// no-op casts. |
| | 2534 | fn addBitCast( |
| | 2535 | b: *Block, |
| | 2536 | l: *Legalize, |
| | 2537 | ty: Type, |
| | 2538 | operand: Air.Inst.Ref, |
| | 2539 | ) Air.Inst.Ref { |
| | 2540 | if (ty.toIntern() != l.typeOf(operand).toIntern()) return b.add(l, .{ |
| | 2541 | .tag = .bitcast, |
| | 2542 | .data = .{ .ty_op = .{ |
| | 2543 | .ty = Air.internedToRef(ty.toIntern()), |
| | 2544 | .operand = operand, |
| | 2545 | } }, |
| | 2546 | }).toRef(); |
| | 2547 | _ = b.stealCapacity(1); |
| | 2548 | return operand; |
| | 2549 | } |
| | 2550 | |
| 1579 | /// Returns the unused capacity of `b.instructions`, and shrinks `b.instructions` down to `b.len`. | 2551 | /// Returns the unused capacity of `b.instructions`, and shrinks `b.instructions` down to `b.len`. |
| 1580 | /// This is useful when you've provided a buffer big enough for all your instructions, but you are | 2552 | /// This is useful when you've provided a buffer big enough for all your instructions, but you are |
| 1581 | /// now starting a new block and some of them need to live there instead. | 2553 | /// now starting a new block and some of them need to live there instead. |
Jacob Young <jacobly@ziglang.org> 2025-06-03 00:47:33-04:00
Andrew Kelley <andrew@ziglang.org> 2025-06-03 15:04:43-04:00
src