| ... | ... | @@ -2418,7 +2418,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2418 | 2418 | } |
| 2419 | 2419 | |
| 2420 | 2420 | fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2421 | | @setEvalBranchQuota(12_300); |
| 2421 | @setEvalBranchQuota(12_400); |
| 2422 | 2422 | const pt = cg.pt; |
| 2423 | 2423 | const zcu = pt.zcu; |
| 2424 | 2424 | const ip = &zcu.intern_pool; |
| ... | ... | @@ -2486,8 +2486,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2486 | 2486 | .reduce_optimized => try cg.airReduce(inst), |
| 2487 | 2487 | .aggregate_init => try cg.airAggregateInit(inst), |
| 2488 | 2488 | .prefetch => try cg.airPrefetch(inst), |
| 2489 | | |
| 2490 | | .array_elem_val => try cg.airArrayElemVal(inst), |
| 2491 | 2489 | // zig fmt: on |
| 2492 | 2490 | |
| 2493 | 2491 | .arg => if (cg.debug_output != .none) { |
| ... | ... | @@ -15150,7 +15148,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 15150 | 15148 | }, |
| 15151 | 15149 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 15152 | 15150 | .each = .{ .once = &.{ |
| 15153 | | .{ ._, ._, .lea, .dst0p, .leaa(.src0, .add_src0_elem_size_times_src1), ._, ._ }, |
| 15151 | .{ ._, ._, .lea, .dst0p, .leaa(.src0, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 15154 | 15152 | } }, |
| 15155 | 15153 | }, .{ |
| 15156 | 15154 | .dst_constraints = .{ .{ .elem_size_is = 1 }, .any }, |
| ... | ... | @@ -15264,7 +15262,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 15264 | 15262 | }, |
| 15265 | 15263 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 15266 | 15264 | .each = .{ .once = &.{ |
| 15267 | | .{ ._, ._, .lea, .dst0p, .leaa(.src0, .sub_src0_elem_size_times_src1), ._, ._ }, |
| 15265 | .{ ._, ._, .lea, .dst0p, .leaa(.src0, .sub_src0_elem_size_mul_src1), ._, ._ }, |
| 15268 | 15266 | } }, |
| 15269 | 15267 | }, .{ |
| 15270 | 15268 | .dst_constraints = .{ .{ .elem_size_is = 1 }, .any }, |
| ... | ... | @@ -52951,6 +52949,200 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 52951 | 52949 | try ops[0].toOffset(0, cg); |
| 52952 | 52950 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 52953 | 52951 | }, |
| 52952 | .array_elem_val => if (use_old) try cg.airArrayElemVal(inst) else { |
| 52953 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| 52954 | const array_ty = cg.typeOf(bin_op.lhs); |
| 52955 | const res_ty = array_ty.elemType2(zcu); |
| 52956 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 52957 | var res: [1]Temp = undefined; |
| 52958 | cg.select(&res, &.{res_ty}, &ops, comptime &.{ .{ |
| 52959 | .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, |
| 52960 | .patterns = &.{ |
| 52961 | .{ .src = .{ .to_gpr, .imm32, .none } }, |
| 52962 | }, |
| 52963 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| 52964 | .clobbers = .{ .eflags = true }, |
| 52965 | .each = .{ .once = &.{ |
| 52966 | .{ ._, ._, .bt, .src0d, .ua(.none, .add_src1_rem_32), ._, ._ }, |
| 52967 | } }, |
| 52968 | }, .{ |
| 52969 | .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, |
| 52970 | .patterns = &.{ |
| 52971 | .{ .src = .{ .to_gpr, .to_gpr, .none } }, |
| 52972 | }, |
| 52973 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| 52974 | .clobbers = .{ .eflags = true }, |
| 52975 | .each = .{ .once = &.{ |
| 52976 | .{ ._, ._, .bt, .src0d, .src1d, ._, ._ }, |
| 52977 | } }, |
| 52978 | }, .{ |
| 52979 | .required_features = .{ .@"64bit", null, null, null }, |
| 52980 | .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, |
| 52981 | .patterns = &.{ |
| 52982 | .{ .src = .{ .to_gpr, .imm32, .none } }, |
| 52983 | }, |
| 52984 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| 52985 | .clobbers = .{ .eflags = true }, |
| 52986 | .each = .{ .once = &.{ |
| 52987 | .{ ._, ._, .bt, .src0q, .ua(.none, .add_src1_rem_64), ._, ._ }, |
| 52988 | } }, |
| 52989 | }, .{ |
| 52990 | .required_features = .{ .@"64bit", null, null, null }, |
| 52991 | .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any }, |
| 52992 | .patterns = &.{ |
| 52993 | .{ .src = .{ .to_gpr, .to_gpr, .none } }, |
| 52994 | }, |
| 52995 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| 52996 | .clobbers = .{ .eflags = true }, |
| 52997 | .each = .{ .once = &.{ |
| 52998 | .{ ._, ._, .bt, .src0q, .src1q, ._, ._ }, |
| 52999 | } }, |
| 53000 | }, .{ |
| 53001 | .src_constraints = .{ .any_bool_vec, .any, .any }, |
| 53002 | .patterns = &.{ |
| 53003 | .{ .src = .{ .to_mem, .imm32, .none } }, |
| 53004 | }, |
| 53005 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| 53006 | .clobbers = .{ .eflags = true }, |
| 53007 | .each = .{ .once = &.{ |
| 53008 | .{ ._, ._, .bt, .mema(.src0d, .add_src1_div_8_down_4), .ua(.none, .add_src1_rem_32), ._, ._ }, |
| 53009 | } }, |
| 53010 | }, .{ |
| 53011 | .src_constraints = .{ .any_bool_vec, .any, .any }, |
| 53012 | .patterns = &.{ |
| 53013 | .{ .src = .{ .to_mem, .to_gpr, .none } }, |
| 53014 | }, |
| 53015 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| 53016 | .clobbers = .{ .eflags = true }, |
| 53017 | .each = .{ .once = &.{ |
| 53018 | .{ ._, ._, .bt, .src0d, .src1d, ._, ._ }, |
| 53019 | } }, |
| 53020 | }, .{ |
| 53021 | .dst_constraints = .{ .{ .int = .byte }, .any }, |
| 53022 | .patterns = &.{ |
| 53023 | .{ .src = .{ .to_mem, .simm32, .none } }, |
| 53024 | }, |
| 53025 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53026 | .each = .{ .once = &.{ |
| 53027 | .{ ._, ._, .movzx, .dst0d, .mema(.src0b, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 53028 | } }, |
| 53029 | }, .{ |
| 53030 | .dst_constraints = .{ .{ .int = .byte }, .any }, |
| 53031 | .patterns = &.{ |
| 53032 | .{ .src = .{ .to_mem, .to_gpr, .none } }, |
| 53033 | }, |
| 53034 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53035 | .each = .{ .once = &.{ |
| 53036 | .{ ._, ._, .movzx, .dst0d, .memi(.src0b, .src1), ._, ._ }, |
| 53037 | } }, |
| 53038 | }, .{ |
| 53039 | .dst_constraints = .{ .{ .int = .word }, .any }, |
| 53040 | .patterns = &.{ |
| 53041 | .{ .src = .{ .to_mem, .simm32, .none } }, |
| 53042 | }, |
| 53043 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53044 | .each = .{ .once = &.{ |
| 53045 | .{ ._, ._, .movzx, .dst0d, .mema(.src0w, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 53046 | } }, |
| 53047 | }, .{ |
| 53048 | .dst_constraints = .{ .{ .int = .word }, .any }, |
| 53049 | .patterns = &.{ |
| 53050 | .{ .src = .{ .to_mem, .to_gpr, .none } }, |
| 53051 | }, |
| 53052 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53053 | .each = .{ .once = &.{ |
| 53054 | .{ ._, ._, .movzx, .dst0d, .memsi(.src0w, .@"2", .src1), ._, ._ }, |
| 53055 | } }, |
| 53056 | }, .{ |
| 53057 | .dst_constraints = .{ .{ .int = .dword }, .any }, |
| 53058 | .patterns = &.{ |
| 53059 | .{ .src = .{ .to_mem, .simm32, .none } }, |
| 53060 | }, |
| 53061 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53062 | .each = .{ .once = &.{ |
| 53063 | .{ ._, ._, .mov, .dst0d, .mema(.src0d, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 53064 | } }, |
| 53065 | }, .{ |
| 53066 | .dst_constraints = .{ .{ .int = .dword }, .any }, |
| 53067 | .patterns = &.{ |
| 53068 | .{ .src = .{ .to_mem, .to_gpr, .none } }, |
| 53069 | }, |
| 53070 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53071 | .each = .{ .once = &.{ |
| 53072 | .{ ._, ._, .mov, .dst0d, .memsi(.src0d, .@"4", .src1), ._, ._ }, |
| 53073 | } }, |
| 53074 | }, .{ |
| 53075 | .dst_constraints = .{ .{ .int = .qword }, .any }, |
| 53076 | .patterns = &.{ |
| 53077 | .{ .src = .{ .to_mem, .simm32, .none } }, |
| 53078 | }, |
| 53079 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53080 | .each = .{ .once = &.{ |
| 53081 | .{ ._, ._, .mov, .dst0q, .mema(.src0q, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 53082 | } }, |
| 53083 | }, .{ |
| 53084 | .required_features = .{ .@"64bit", null, null, null }, |
| 53085 | .dst_constraints = .{ .{ .int = .qword }, .any }, |
| 53086 | .patterns = &.{ |
| 53087 | .{ .src = .{ .to_mem, .to_gpr, .none } }, |
| 53088 | }, |
| 53089 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53090 | .each = .{ .once = &.{ |
| 53091 | .{ ._, ._, .mov, .dst0q, .memsi(.src0q, .@"8", .src1), ._, ._ }, |
| 53092 | } }, |
| 53093 | } }) catch |err| switch (err) { |
| 53094 | error.SelectFailed => { |
| 53095 | const elem_size = res_ty.abiSize(zcu); |
| 53096 | const base = try cg.tempAllocReg(.usize, abi.RegisterClass.gp); |
| 53097 | while (try ops[0].toBase(false, cg) or |
| 53098 | try ops[1].toRegClass(true, .general_purpose, cg)) |
| 53099 | {} |
| 53100 | const base_reg = base.tracking(cg).short.register.to64(); |
| 53101 | const rhs_reg = ops[1].tracking(cg).short.register.to64(); |
| 53102 | if (!std.math.isPowerOfTwo(elem_size)) { |
| 53103 | try cg.spillEflagsIfOccupied(); |
| 53104 | try cg.asmRegisterRegisterImmediate( |
| 53105 | .{ .i_, .mul }, |
| 53106 | rhs_reg, |
| 53107 | rhs_reg, |
| 53108 | .u(elem_size), |
| 53109 | ); |
| 53110 | try cg.asmRegisterMemory( |
| 53111 | .{ ._, .lea }, |
| 53112 | base_reg, |
| 53113 | try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), |
| 53114 | ); |
| 53115 | } else if (elem_size > 8) { |
| 53116 | try cg.spillEflagsIfOccupied(); |
| 53117 | try cg.asmRegisterImmediate( |
| 53118 | .{ ._l, .sh }, |
| 53119 | rhs_reg, |
| 53120 | .u(std.math.log2_int(u64, elem_size)), |
| 53121 | ); |
| 53122 | try cg.asmRegisterMemory( |
| 53123 | .{ ._, .lea }, |
| 53124 | base_reg, |
| 53125 | try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), |
| 53126 | ); |
| 53127 | } else try cg.asmRegisterMemory( |
| 53128 | .{ ._, .lea }, |
| 53129 | base_reg, |
| 53130 | try ops[0].tracking(cg).short.mem(cg, .{ |
| 53131 | .index = rhs_reg, |
| 53132 | .scale = .fromFactor(@intCast(elem_size)), |
| 53133 | }), |
| 53134 | ); |
| 53135 | // Hack around Sema insanity: lhs could be an arbitrarily large comptime-known array |
| 53136 | // which could easily get spilled by the upcoming `load`, which would infinite recurse |
| 53137 | // since spilling an array requires the same operation that triggered the spill. |
| 53138 | try ops[0].die(cg); |
| 53139 | ops[0] = base; |
| 53140 | res[0] = try ops[0].load(res_ty, .{}, cg); |
| 53141 | }, |
| 53142 | else => |e| return e, |
| 53143 | }; |
| 53144 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 53145 | }, |
| 52954 | 53146 | .slice_elem_val, .ptr_elem_val => |air_tag| if (use_old) switch (air_tag) { |
| 52955 | 53147 | else => unreachable, |
| 52956 | 53148 | .slice_elem_val => try cg.airSliceElemVal(inst), |
| ... | ... | @@ -52968,7 +53160,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 52968 | 53160 | }, |
| 52969 | 53161 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 52970 | 53162 | .each = .{ .once = &.{ |
| 52971 | | .{ ._, ._, .movzx, .dst0d, .leaa(.src0b, .add_src0_elem_size_times_src1), ._, ._ }, |
| 53163 | .{ ._, ._, .movzx, .dst0d, .leaa(.src0b, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 52972 | 53164 | } }, |
| 52973 | 53165 | }, .{ |
| 52974 | 53166 | .dst_constraints = .{ .{ .int = .byte }, .any }, |
| ... | ... | @@ -52986,7 +53178,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 52986 | 53178 | }, |
| 52987 | 53179 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 52988 | 53180 | .each = .{ .once = &.{ |
| 52989 | | .{ ._, ._, .movzx, .dst0d, .leaa(.src0w, .add_src0_elem_size_times_src1), ._, ._ }, |
| 53181 | .{ ._, ._, .movzx, .dst0d, .leaa(.src0w, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 52990 | 53182 | } }, |
| 52991 | 53183 | }, .{ |
| 52992 | 53184 | .dst_constraints = .{ .{ .int = .word }, .any }, |
| ... | ... | @@ -53004,7 +53196,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53004 | 53196 | }, |
| 53005 | 53197 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53006 | 53198 | .each = .{ .once = &.{ |
| 53007 | | .{ ._, ._, .mov, .dst0d, .leaa(.src0d, .add_src0_elem_size_times_src1), ._, ._ }, |
| 53199 | .{ ._, ._, .mov, .dst0d, .leaa(.src0d, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 53008 | 53200 | } }, |
| 53009 | 53201 | }, .{ |
| 53010 | 53202 | .dst_constraints = .{ .{ .int = .dword }, .any }, |
| ... | ... | @@ -53022,7 +53214,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53022 | 53214 | }, |
| 53023 | 53215 | .dst_temps = .{ .{ .rc = .general_purpose }, .unused }, |
| 53024 | 53216 | .each = .{ .once = &.{ |
| 53025 | | .{ ._, ._, .mov, .dst0q, .leaa(.src0q, .add_src0_elem_size_times_src1), ._, ._ }, |
| 53217 | .{ ._, ._, .mov, .dst0q, .leaa(.src0q, .add_src0_elem_size_mul_src1), ._, ._ }, |
| 53026 | 53218 | } }, |
| 53027 | 53219 | }, .{ |
| 53028 | 53220 | .required_features = .{ .@"64bit", null, null, null }, |
| ... | ... | @@ -53040,8 +53232,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53040 | 53232 | while (true) for (&ops) |*op| { |
| 53041 | 53233 | if (try op.toRegClass(true, .general_purpose, cg)) break; |
| 53042 | 53234 | } else break; |
| 53043 | | const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64(); |
| 53044 | | const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64(); |
| 53235 | const lhs_reg = ops[0].tracking(cg).short.register.to64(); |
| 53236 | const rhs_reg = ops[1].tracking(cg).short.register.to64(); |
| 53045 | 53237 | if (!std.math.isPowerOfTwo(elem_size)) { |
| 53046 | 53238 | try cg.spillEflagsIfOccupied(); |
| 53047 | 53239 | try cg.asmRegisterRegisterImmediate( |
| ... | ... | @@ -53052,7 +53244,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53052 | 53244 | ); |
| 53053 | 53245 | try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 53054 | 53246 | .base = .{ .reg = lhs_reg }, |
| 53055 | | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, |
| 53247 | .mod = .{ .rm = .{ .index = rhs_reg } }, |
| 53056 | 53248 | }); |
| 53057 | 53249 | } else if (elem_size > 8) { |
| 53058 | 53250 | try cg.spillEflagsIfOccupied(); |
| ... | ... | @@ -53063,12 +53255,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53063 | 53255 | ); |
| 53064 | 53256 | try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 53065 | 53257 | .base = .{ .reg = lhs_reg }, |
| 53066 | | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, |
| 53258 | .mod = .{ .rm = .{ .index = rhs_reg } }, |
| 53067 | 53259 | }); |
| 53068 | 53260 | } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 53069 | 53261 | .base = .{ .reg = lhs_reg }, |
| 53070 | 53262 | .mod = .{ .rm = .{ |
| 53071 | | .size = .qword, |
| 53072 | 53263 | .index = rhs_reg, |
| 53073 | 53264 | .scale = .fromFactor(@intCast(elem_size)), |
| 53074 | 53265 | } }, |
| ... | ... | @@ -53095,8 +53286,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53095 | 53286 | while (true) for (&ops) |*op| { |
| 53096 | 53287 | if (try op.toRegClass(true, .general_purpose, cg)) break; |
| 53097 | 53288 | } else break; |
| 53098 | | const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64(); |
| 53099 | | const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64(); |
| 53289 | const lhs_reg = ops[0].tracking(cg).short.register.to64(); |
| 53290 | const rhs_reg = ops[1].tracking(cg).short.register.to64(); |
| 53100 | 53291 | if (!std.math.isPowerOfTwo(elem_size)) { |
| 53101 | 53292 | try cg.spillEflagsIfOccupied(); |
| 53102 | 53293 | try cg.asmRegisterRegisterImmediate( |
| ... | ... | @@ -53107,7 +53298,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53107 | 53298 | ); |
| 53108 | 53299 | try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 53109 | 53300 | .base = .{ .reg = lhs_reg }, |
| 53110 | | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, |
| 53301 | .mod = .{ .rm = .{ .index = rhs_reg } }, |
| 53111 | 53302 | }); |
| 53112 | 53303 | } else if (elem_size > 8) { |
| 53113 | 53304 | try cg.spillEflagsIfOccupied(); |
| ... | ... | @@ -53118,12 +53309,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53118 | 53309 | ); |
| 53119 | 53310 | try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 53120 | 53311 | .base = .{ .reg = lhs_reg }, |
| 53121 | | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, |
| 53312 | .mod = .{ .rm = .{ .index = rhs_reg } }, |
| 53122 | 53313 | }); |
| 53123 | 53314 | } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 53124 | 53315 | .base = .{ .reg = lhs_reg }, |
| 53125 | 53316 | .mod = .{ .rm = .{ |
| 53126 | | .size = .qword, |
| 53127 | 53317 | .index = rhs_reg, |
| 53128 | 53318 | .scale = .fromFactor(@intCast(elem_size)), |
| 53129 | 53319 | } }, |
| ... | ... | @@ -75183,7 +75373,7 @@ fn airErrUnionPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 75183 | 75373 | registerAlias(dst_reg, dst_abi_size), |
| 75184 | 75374 | .{ |
| 75185 | 75375 | .base = .{ .reg = src_reg }, |
| 75186 | | .mod = .{ .rm = .{ .size = .qword, .disp = pl_off } }, |
| 75376 | .mod = .{ .rm = .{ .disp = pl_off } }, |
| 75187 | 75377 | }, |
| 75188 | 75378 | ); |
| 75189 | 75379 | break :result .{ .register = dst_reg }; |
| ... | ... | @@ -75446,7 +75636,7 @@ fn airPtrSliceLenPtr(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 75446 | 75636 | registerAlias(dst_reg, dst_abi_size), |
| 75447 | 75637 | .{ |
| 75448 | 75638 | .base = .{ .reg = src_reg }, |
| 75449 | | .mod = .{ .rm = .{ .size = .qword, .disp = 8 } }, |
| 75639 | .mod = .{ .rm = .{ .disp = 8 } }, |
| 75450 | 75640 | }, |
| 75451 | 75641 | ); |
| 75452 | 75642 | |
| ... | ... | @@ -75700,7 +75890,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 75700 | 75890 | try self.asmRegisterMemory( |
| 75701 | 75891 | .{ ._, .lea }, |
| 75702 | 75892 | addr_reg, |
| 75703 | | .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } }, |
| 75893 | .{ .base = .{ .frame = frame_index } }, |
| 75704 | 75894 | ); |
| 75705 | 75895 | }, |
| 75706 | 75896 | .load_frame => |frame_addr| try self.asmRegisterMemory( |
| ... | ... | @@ -75708,7 +75898,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 75708 | 75898 | addr_reg, |
| 75709 | 75899 | .{ |
| 75710 | 75900 | .base = .{ .frame = frame_addr.index }, |
| 75711 | | .mod = .{ .rm = .{ .size = .qword, .disp = frame_addr.off } }, |
| 75901 | .mod = .{ .rm = .{ .disp = frame_addr.off } }, |
| 75712 | 75902 | }, |
| 75713 | 75903 | ), |
| 75714 | 75904 | .memory, |
| ... | ... | @@ -76717,7 +76907,6 @@ fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 76717 | 76907 | .{ |
| 76718 | 76908 | .base = .{ .reg = dst.to64() }, |
| 76719 | 76909 | .mod = .{ .rm = .{ |
| 76720 | | .size = .qword, |
| 76721 | 76910 | .index = tmp.to64(), |
| 76722 | 76911 | .scale = .@"4", |
| 76723 | 76912 | } }, |
| ... | ... | @@ -76744,7 +76933,6 @@ fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 76744 | 76933 | .{ |
| 76745 | 76934 | .base = .{ .reg = tmp.to64() }, |
| 76746 | 76935 | .mod = .{ .rm = .{ |
| 76747 | | .size = .qword, |
| 76748 | 76936 | .index = dst.to64(), |
| 76749 | 76937 | .scale = .@"2", |
| 76750 | 76938 | } }, |
| ... | ... | @@ -85591,7 +85779,6 @@ fn genSetReg( |
| 85591 | 85779 | dst_reg.to64(), |
| 85592 | 85780 | .{ |
| 85593 | 85781 | .base = .{ .reloc = sym_off.sym_index }, |
| 85594 | | .mod = .{ .rm = .{ .size = .qword } }, |
| 85595 | 85782 | }, |
| 85596 | 85783 | ); |
| 85597 | 85784 | if (sym_off.off != 0) try self.asmRegisterMemory( |
| ... | ... | @@ -85599,10 +85786,7 @@ fn genSetReg( |
| 85599 | 85786 | dst_reg.to64(), |
| 85600 | 85787 | .{ |
| 85601 | 85788 | .base = .{ .reg = dst_reg.to64() }, |
| 85602 | | .mod = .{ .rm = .{ |
| 85603 | | .size = .qword, |
| 85604 | | .disp = sym_off.off, |
| 85605 | | } }, |
| 85789 | .mod = .{ .rm = .{ .disp = sym_off.off } }, |
| 85606 | 85790 | }, |
| 85607 | 85791 | ); |
| 85608 | 85792 | }, |
| ... | ... | @@ -85816,18 +86000,12 @@ fn genSetMem( |
| 85816 | 86000 | const src_reg = registerAlias(reg_off.reg, abi_size); |
| 85817 | 86001 | try self.asmRegisterMemory(.{ ._, .lea }, src_reg, .{ |
| 85818 | 86002 | .base = .{ .reg = src_reg }, |
| 85819 | | .mod = .{ .rm = .{ |
| 85820 | | .size = .qword, |
| 85821 | | .disp = reg_off.off, |
| 85822 | | } }, |
| 86003 | .mod = .{ .rm = .{ .disp = reg_off.off } }, |
| 85823 | 86004 | }); |
| 85824 | 86005 | try self.genSetMem(base, disp, ty, .{ .register = reg_off.reg }, opts); |
| 85825 | 86006 | return self.asmRegisterMemory(.{ ._, .lea }, src_reg, .{ |
| 85826 | 86007 | .base = .{ .reg = src_reg }, |
| 85827 | | .mod = .{ .rm = .{ |
| 85828 | | .size = .qword, |
| 85829 | | .disp = -reg_off.off, |
| 85830 | | } }, |
| 86008 | .mod = .{ .rm = .{ .disp = -reg_off.off } }, |
| 85831 | 86009 | }); |
| 85832 | 86010 | }, |
| 85833 | 86011 | else => |e| return e, |
| ... | ... | @@ -87185,10 +87363,7 @@ fn airErrorName(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 87185 | 87363 | start_reg.to64(), |
| 87186 | 87364 | .{ |
| 87187 | 87365 | .base = .{ .reg = addr_reg.to64() }, |
| 87188 | | .mod = .{ .rm = .{ |
| 87189 | | .size = .dword, |
| 87190 | | .index = start_reg.to64(), |
| 87191 | | } }, |
| 87366 | .mod = .{ .rm = .{ .index = start_reg.to64() } }, |
| 87192 | 87367 | }, |
| 87193 | 87368 | ); |
| 87194 | 87369 | try self.asmRegisterMemory( |
| ... | ... | @@ -87196,10 +87371,7 @@ fn airErrorName(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 87196 | 87371 | end_reg.to32(), |
| 87197 | 87372 | .{ |
| 87198 | 87373 | .base = .{ .reg = end_reg.to64() }, |
| 87199 | | .mod = .{ .rm = .{ |
| 87200 | | .size = .byte, |
| 87201 | | .disp = -1, |
| 87202 | | } }, |
| 87374 | .mod = .{ .rm = .{ .disp = -1 } }, |
| 87203 | 87375 | }, |
| 87204 | 87376 | ); |
| 87205 | 87377 | |
| ... | ... | @@ -89375,17 +89547,11 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 89375 | 89547 | try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{}); |
| 89376 | 89548 | if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{ |
| 89377 | 89549 | .base = .{ .reg = addr_reg }, |
| 89378 | | .mod = .{ .rm = .{ |
| 89379 | | .size = .qword, |
| 89380 | | .index = offset_reg.to64(), |
| 89381 | | } }, |
| 89550 | .mod = .{ .rm = .{ .index = offset_reg.to64() } }, |
| 89382 | 89551 | }); |
| 89383 | 89552 | try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{ |
| 89384 | 89553 | .base = .{ .reg = offset_reg.to64() }, |
| 89385 | | .mod = .{ .rm = .{ |
| 89386 | | .size = .qword, |
| 89387 | | .disp = 8, |
| 89388 | | } }, |
| 89554 | .mod = .{ .rm = .{ .disp = 8 } }, |
| 89389 | 89555 | }); |
| 89390 | 89556 | try self.genCopy(.c_uint, gp_offset, .{ .register = offset_reg }, .{}); |
| 89391 | 89557 | const done_reloc = try self.asmJmpReloc(undefined); |
| ... | ... | @@ -89394,10 +89560,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 89394 | 89560 | try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{}); |
| 89395 | 89561 | try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{ |
| 89396 | 89562 | .base = .{ .reg = addr_reg }, |
| 89397 | | .mod = .{ .rm = .{ |
| 89398 | | .size = .qword, |
| 89399 | | .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)), |
| 89400 | | } }, |
| 89563 | .mod = .{ .rm = .{ .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)) } }, |
| 89401 | 89564 | }); |
| 89402 | 89565 | try self.genCopy( |
| 89403 | 89566 | ptr_anyopaque_ty, |
| ... | ... | @@ -89423,17 +89586,11 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 89423 | 89586 | try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{}); |
| 89424 | 89587 | if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{ |
| 89425 | 89588 | .base = .{ .reg = addr_reg }, |
| 89426 | | .mod = .{ .rm = .{ |
| 89427 | | .size = .qword, |
| 89428 | | .index = offset_reg.to64(), |
| 89429 | | } }, |
| 89589 | .mod = .{ .rm = .{ .index = offset_reg.to64() } }, |
| 89430 | 89590 | }); |
| 89431 | 89591 | try self.asmRegisterMemory(.{ ._, .lea }, offset_reg, .{ |
| 89432 | 89592 | .base = .{ .reg = offset_reg.to64() }, |
| 89433 | | .mod = .{ .rm = .{ |
| 89434 | | .size = .qword, |
| 89435 | | .disp = 16, |
| 89436 | | } }, |
| 89593 | .mod = .{ .rm = .{ .disp = 16 } }, |
| 89437 | 89594 | }); |
| 89438 | 89595 | try self.genCopy(.c_uint, fp_offset, .{ .register = offset_reg }, .{}); |
| 89439 | 89596 | const done_reloc = try self.asmJmpReloc(undefined); |
| ... | ... | @@ -89442,10 +89599,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 89442 | 89599 | try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{}); |
| 89443 | 89600 | try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{ |
| 89444 | 89601 | .base = .{ .reg = addr_reg }, |
| 89445 | | .mod = .{ .rm = .{ |
| 89446 | | .size = .qword, |
| 89447 | | .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)), |
| 89448 | | } }, |
| 89602 | .mod = .{ .rm = .{ .disp = @intCast(@max(promote_ty.abiSize(zcu), 8)) } }, |
| 89449 | 89603 | }); |
| 89450 | 89604 | try self.genCopy( |
| 89451 | 89605 | ptr_anyopaque_ty, |
| ... | ... | @@ -90505,10 +90659,7 @@ const Temp = struct { |
| 90505 | 90659 | new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); |
| 90506 | 90660 | try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ |
| 90507 | 90661 | .base = .{ .reg = reg.to64() }, |
| 90508 | | .mod = .{ .rm = .{ |
| 90509 | | .size = .qword, |
| 90510 | | .disp = off, |
| 90511 | | } }, |
| 90662 | .mod = .{ .rm = .{ .disp = off } }, |
| 90512 | 90663 | }); |
| 90513 | 90664 | }, |
| 90514 | 90665 | .register_offset => |reg_off| { |
| ... | ... | @@ -90517,10 +90668,7 @@ const Temp = struct { |
| 90517 | 90668 | new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); |
| 90518 | 90669 | try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ |
| 90519 | 90670 | .base = .{ .reg = reg_off.reg.to64() }, |
| 90520 | | .mod = .{ .rm = .{ |
| 90521 | | .size = .qword, |
| 90522 | | .disp = reg_off.off + off, |
| 90523 | | } }, |
| 90671 | .mod = .{ .rm = .{ .disp = reg_off.off + off } }, |
| 90524 | 90672 | }); |
| 90525 | 90673 | }, |
| 90526 | 90674 | .lea_symbol => |sym_off| new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = .{ |
| ... | ... | @@ -90627,10 +90775,7 @@ const Temp = struct { |
| 90627 | 90775 | new_temp_index.tracking(cg).* = .init(.{ .register = new_reg }); |
| 90628 | 90776 | try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ |
| 90629 | 90777 | .base = .{ .reg = reg_off.reg.to64() }, |
| 90630 | | .mod = .{ .rm = .{ |
| 90631 | | .size = .qword, |
| 90632 | | .disp = reg_off.off + @as(u31, limb_index) * 8, |
| 90633 | | } }, |
| 90778 | .mod = .{ .rm = .{ .disp = reg_off.off + @as(u31, limb_index) * 8 } }, |
| 90634 | 90779 | }); |
| 90635 | 90780 | }, |
| 90636 | 90781 | .load_symbol => |sym_off| { |
| ... | ... | @@ -93462,13 +93607,14 @@ const Select = struct { |
| 93462 | 93607 | elem_size, |
| 93463 | 93608 | src0_elem_size, |
| 93464 | 93609 | dst0_elem_size, |
| 93465 | | src0_elem_size_times_src1, |
| 93610 | src0_elem_size_mul_src1, |
| 93611 | src1, |
| 93466 | 93612 | log2_src0_elem_size, |
| 93467 | 93613 | smin, |
| 93468 | 93614 | smax, |
| 93469 | 93615 | umax, |
| 93470 | 93616 | }, |
| 93471 | | op: enum(u2) { mul, div, rem_8_mul }, |
| 93617 | op: enum(u2) { mul, div, div_8_down, rem_8_mul }, |
| 93472 | 93618 | rhs: Memory.Scale, |
| 93473 | 93619 | |
| 93474 | 93620 | const none: Adjust = .{ .sign = .pos, .lhs = .none, .op = .mul, .rhs = .@"1" }; |
| ... | ... | @@ -93512,8 +93658,11 @@ const Select = struct { |
| 93512 | 93658 | const add_8_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"8" }; |
| 93513 | 93659 | const add_src0_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .div, .rhs = .@"8" }; |
| 93514 | 93660 | const sub_src0_elem_size: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; |
| 93515 | | const add_src0_elem_size_times_src1: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size_times_src1, .op = .mul, .rhs = .@"1" }; |
| 93516 | | const sub_src0_elem_size_times_src1: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size_times_src1, .op = .mul, .rhs = .@"1" }; |
| 93661 | const add_src0_elem_size_mul_src1: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; |
| 93662 | const sub_src0_elem_size_mul_src1: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; |
| 93663 | const add_src1_div_8_down_4: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .div_8_down, .rhs = .@"4" }; |
| 93664 | const add_src1_rem_32: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"4" }; |
| 93665 | const add_src1_rem_64: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"8" }; |
| 93517 | 93666 | const add_log2_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .log2_src0_elem_size, .op = .mul, .rhs = .@"1" }; |
| 93518 | 93667 | const add_dst0_elem_size: Adjust = .{ .sign = .pos, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; |
| 93519 | 93668 | const add_elem_limbs: Adjust = .{ .sign = .pos, .lhs = .elem_limbs, .op = .mul, .rhs = .@"1" }; |
| ... | ... | @@ -94086,8 +94235,9 @@ const Select = struct { |
| 94086 | 94235 | .elem_size => @intCast(op.base.ref.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), |
| 94087 | 94236 | .src0_elem_size => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), |
| 94088 | 94237 | .dst0_elem_size => @intCast(Select.Operand.Ref.dst0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), |
| 94089 | | .src0_elem_size_times_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * |
| 94238 | .src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * |
| 94090 | 94239 | Select.Operand.Ref.src1.valueOf(s).immediate), |
| 94240 | .src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate), |
| 94091 | 94241 | .log2_src0_elem_size => @intCast(std.math.log2(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))), |
| 94092 | 94242 | .smin => @as(SignedImm, std.math.minInt(SignedImm)) >> @truncate( |
| 94093 | 94243 | -%op.base.ref.typeOf(s).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu), |
| ... | ... | @@ -94107,6 +94257,7 @@ const Select = struct { |
| 94107 | 94257 | break :op_res op_res[0]; |
| 94108 | 94258 | }, |
| 94109 | 94259 | .div => @shrExact(lhs, rhs), |
| 94260 | .div_8_down => lhs >> 3 & @as(SignedImm, -1) << rhs, |
| 94110 | 94261 | .rem_8_mul => lhs & (@as(SignedImm, 1) << @intCast(@as(u3, 3) + rhs)) - 1, |
| 94111 | 94262 | }; |
| 94112 | 94263 | return switch (op.flags.adjust.sign) { |
Jacob Young <jacobly@ziglang.org> 2025-02-14 02:20:43-05:00
src