| ... | @@ -2437,7 +2437,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2437,7 +2437,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2437 | | 2437 | |
| 2438 | try cg.airArg(inst); | 2438 | try cg.airArg(inst); |
| 2439 | | 2439 | |
| 2440 | try cg.resetTemps(); | 2440 | try cg.resetTemps(@enumFromInt(0)); |
| 2441 | cg.checkInvariantsAfterAirInst(); | 2441 | cg.checkInvariantsAfterAirInst(); |
| 2442 | }, | 2442 | }, |
| 2443 | else => break, | 2443 | else => break, |
| ... | @@ -2477,7 +2477,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2477,7 +2477,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2477 | .shuffle => try cg.airShuffle(inst), | 2477 | .shuffle => try cg.airShuffle(inst), |
| 2478 | .reduce => try cg.airReduce(inst), | 2478 | .reduce => try cg.airReduce(inst), |
| 2479 | .reduce_optimized => try cg.airReduce(inst), | 2479 | .reduce_optimized => try cg.airReduce(inst), |
| 2480 | .aggregate_init => try cg.airAggregateInit(inst), | | |
| 2481 | // zig fmt: on | 2480 | // zig fmt: on |
| 2482 | | 2481 | |
| 2483 | .arg => if (cg.debug_output != .none) { | 2482 | .arg => if (cg.debug_output != .none) { |
| ... | @@ -80843,6 +80842,74 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -80843,6 +80842,74 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 80843 | for (ops[1..]) |op| try op.die(cg); | 80842 | for (ops[1..]) |op| try op.die(cg); |
| 80844 | try res[0].finish(inst, &.{ty_op.operand}, ops[0..1], cg); | 80843 | try res[0].finish(inst, &.{ty_op.operand}, ops[0..1], cg); |
| 80845 | }, | 80844 | }, |
| | 80845 | .aggregate_init => |air_tag| if (use_old) try cg.airAggregateInit(inst) else fallback: { |
| | 80846 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| | 80847 | const agg_ty = ty_pl.ty.toType(); |
| | 80848 | if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or |
| | 80849 | (agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst); |
| | 80850 | var res = try cg.tempAllocMem(agg_ty); |
| | 80851 | const reset_index = cg.next_temp_index; |
| | 80852 | var bt = cg.liveness.iterateBigTomb(inst); |
| | 80853 | switch (ip.indexToKey(agg_ty.toIntern())) { |
| | 80854 | inline .array_type, .vector_type => |sequence_type| { |
| | 80855 | const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..@intCast(sequence_type.len)]); |
| | 80856 | const elem_size = Type.fromInterned(sequence_type.child).abiSize(zcu); |
| | 80857 | var elem_disp: u31 = 0; |
| | 80858 | for (elems) |elem_ref| { |
| | 80859 | var elem = try cg.tempFromOperand(elem_ref, bt.feed()); |
| | 80860 | try res.write(&elem, .{ .disp = elem_disp }, cg); |
| | 80861 | try elem.die(cg); |
| | 80862 | try cg.resetTemps(reset_index); |
| | 80863 | elem_disp += @intCast(elem_size); |
| | 80864 | } |
| | 80865 | if (@hasField(@TypeOf(sequence_type), "sentinel") and sequence_type.sentinel != .none) { |
| | 80866 | var sentinel = try cg.tempFromValue(.fromInterned(sequence_type.sentinel)); |
| | 80867 | try res.write(&sentinel, .{ .disp = elem_disp }, cg); |
| | 80868 | try sentinel.die(cg); |
| | 80869 | } |
| | 80870 | }, |
| | 80871 | .struct_type => { |
| | 80872 | const loaded_struct = ip.loadStructType(agg_ty.toIntern()); |
| | 80873 | const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..loaded_struct.field_types.len]); |
| | 80874 | switch (loaded_struct.layout) { |
| | 80875 | .auto, .@"extern" => { |
| | 80876 | for (elems, 0..) |elem_ref, field_index| { |
| | 80877 | const elem_dies = bt.feed(); |
| | 80878 | if (loaded_struct.fieldIsComptime(ip, field_index)) continue; |
| | 80879 | var elem = try cg.tempFromOperand(elem_ref, elem_dies); |
| | 80880 | try res.write(&elem, .{ .disp = @intCast(loaded_struct.offsets.get(ip)[field_index]) }, cg); |
| | 80881 | try elem.die(cg); |
| | 80882 | try cg.resetTemps(reset_index); |
| | 80883 | } |
| | 80884 | }, |
| | 80885 | .@"packed" => return cg.fail("failed to select {s} {}", .{ |
| | 80886 | @tagName(air_tag), |
| | 80887 | agg_ty.fmt(pt), |
| | 80888 | }), |
| | 80889 | } |
| | 80890 | }, |
| | 80891 | .tuple_type => |tuple_type| { |
| | 80892 | const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..tuple_type.types.len]); |
| | 80893 | var elem_disp: u31 = 0; |
| | 80894 | for (elems, 0..) |elem_ref, field_index| { |
| | 80895 | const elem_dies = bt.feed(); |
| | 80896 | if (tuple_type.values.get(ip)[field_index] != .none) continue; |
| | 80897 | const field_type = Type.fromInterned(tuple_type.types.get(ip)[field_index]); |
| | 80898 | elem_disp = @intCast(field_type.abiAlignment(zcu).forward(elem_disp)); |
| | 80899 | var elem = try cg.tempFromOperand(elem_ref, elem_dies); |
| | 80900 | try res.write(&elem, .{ .disp = elem_disp }, cg); |
| | 80901 | try elem.die(cg); |
| | 80902 | try cg.resetTemps(reset_index); |
| | 80903 | elem_disp += @intCast(field_type.abiSize(zcu)); |
| | 80904 | } |
| | 80905 | }, |
| | 80906 | else => return cg.fail("failed to select {s} {}", .{ |
| | 80907 | @tagName(air_tag), |
| | 80908 | agg_ty.fmt(pt), |
| | 80909 | }), |
| | 80910 | } |
| | 80911 | try res.finish(inst, &.{}, &.{}, cg); |
| | 80912 | }, |
| 80846 | .union_init => if (use_old) try cg.airUnionInit(inst) else { | 80913 | .union_init => if (use_old) try cg.airUnionInit(inst) else { |
| 80847 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 80914 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 80848 | const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data; | 80915 | const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data; |
| ... | @@ -82199,14 +82266,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -82199,14 +82266,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 82199 | .c_va_start => try cg.airVaStart(inst), | 82266 | .c_va_start => try cg.airVaStart(inst), |
| 82200 | .work_item_id, .work_group_size, .work_group_id => unreachable, | 82267 | .work_item_id, .work_group_size, .work_group_id => unreachable, |
| 82201 | } | 82268 | } |
| 82202 | try cg.resetTemps(); | 82269 | try cg.resetTemps(@enumFromInt(0)); |
| 82203 | cg.checkInvariantsAfterAirInst(); | 82270 | cg.checkInvariantsAfterAirInst(); |
| 82204 | } | 82271 | } |
| 82205 | verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); | 82272 | verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); |
| 82206 | } | 82273 | } |
| 82207 | | 82274 | |
| 82208 | fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { | 82275 | fn genLazy(cg: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { |
| 82209 | const pt = self.pt; | 82276 | const pt = cg.pt; |
| 82210 | const zcu = pt.zcu; | 82277 | const zcu = pt.zcu; |
| 82211 | const ip = &zcu.intern_pool; | 82278 | const ip = &zcu.intern_pool; |
| 82212 | switch (ip.indexToKey(lazy_sym.ty)) { | 82279 | switch (ip.indexToKey(lazy_sym.ty)) { |
| ... | @@ -82215,97 +82282,98 @@ fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { | ... | @@ -82215,97 +82282,98 @@ fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { |
| 82215 | wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)}); | 82282 | wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)}); |
| 82216 | | 82283 | |
| 82217 | const param_regs = abi.getCAbiIntParamRegs(.auto); | 82284 | const param_regs = abi.getCAbiIntParamRegs(.auto); |
| 82218 | const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*); | 82285 | const param_locks = cg.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*); |
| 82219 | defer for (param_locks) |lock| self.register_manager.unlockReg(lock); | 82286 | defer for (param_locks) |lock| cg.register_manager.unlockReg(lock); |
| 82220 | | 82287 | |
| 82221 | const ret_mcv: MCValue = .{ .register_pair = param_regs[0..2].* }; | 82288 | const ret_mcv: MCValue = .{ .register_pair = param_regs[0..2].* }; |
| 82222 | const enum_mcv: MCValue = .{ .register = param_regs[0] }; | 82289 | var enum_temp = try cg.tempInit(enum_ty, .{ .register = param_regs[0] }); |
| 82223 | | 82290 | |
| 82224 | const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | 82291 | const data_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 82225 | const data_lock = self.register_manager.lockRegAssumeUnused(data_reg); | 82292 | const data_lock = cg.register_manager.lockRegAssumeUnused(data_reg); |
| 82226 | defer self.register_manager.unlockReg(data_lock); | 82293 | defer cg.register_manager.unlockReg(data_lock); |
| 82227 | try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = lazy_sym.ty }); | 82294 | try cg.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = lazy_sym.ty }); |
| 82228 | | 82295 | |
| 82229 | var data_off: i32 = 0; | 82296 | var data_off: i32 = 0; |
| | 82297 | const reset_index = cg.next_temp_index; |
| 82230 | const tag_names = ip.loadEnumType(lazy_sym.ty).names; | 82298 | const tag_names = ip.loadEnumType(lazy_sym.ty).names; |
| 82231 | for (0..tag_names.len) |tag_index| { | 82299 | for (0..tag_names.len) |tag_index| { |
| 82232 | var enum_temp = try self.tempInit(enum_ty, enum_mcv); | | |
| 82233 | | | |
| 82234 | const tag_name_len = tag_names.get(ip)[tag_index].length(ip); | 82300 | const tag_name_len = tag_names.get(ip)[tag_index].length(ip); |
| 82235 | var tag_temp = try self.tempFromValue(try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index))); | 82301 | var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index))); |
| 82236 | const cc_temp = enum_temp.cmpInts(.neq, &tag_temp, self) catch |err| switch (err) { | 82302 | const cc_temp = enum_temp.cmpInts(.neq, &tag_temp, cg) catch |err| switch (err) { |
| 82237 | error.SelectFailed => unreachable, | 82303 | error.SelectFailed => unreachable, |
| 82238 | else => |e| return e, | 82304 | else => |e| return e, |
| 82239 | }; | 82305 | }; |
| 82240 | try enum_temp.die(self); | 82306 | try tag_temp.die(cg); |
| 82241 | try tag_temp.die(self); | 82307 | const skip_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); |
| 82242 | const skip_reloc = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined); | 82308 | try cc_temp.die(cg); |
| 82243 | try cc_temp.die(self); | 82309 | try cg.resetTemps(reset_index); |
| 82244 | try self.resetTemps(); | | |
| 82245 | | 82310 | |
| 82246 | try self.genSetReg( | 82311 | try cg.genSetReg( |
| 82247 | ret_mcv.register_pair[0], | 82312 | ret_mcv.register_pair[0], |
| 82248 | .usize, | 82313 | .usize, |
| 82249 | .{ .register_offset = .{ .reg = data_reg, .off = data_off } }, | 82314 | .{ .register_offset = .{ .reg = data_reg, .off = data_off } }, |
| 82250 | .{}, | 82315 | .{}, |
| 82251 | ); | 82316 | ); |
| 82252 | try self.genSetReg(ret_mcv.register_pair[1], .usize, .{ .immediate = tag_name_len }, .{}); | 82317 | try cg.genSetReg(ret_mcv.register_pair[1], .usize, .{ .immediate = tag_name_len }, .{}); |
| 82253 | try self.asmOpOnly(.{ ._, .ret }); | 82318 | try cg.asmOpOnly(.{ ._, .ret }); |
| 82254 | | 82319 | |
| 82255 | self.performReloc(skip_reloc); | 82320 | cg.performReloc(skip_reloc); |
| 82256 | | 82321 | |
| 82257 | data_off += @intCast(tag_name_len + 1); | 82322 | data_off += @intCast(tag_name_len + 1); |
| 82258 | } | 82323 | } |
| | 82324 | try enum_temp.die(cg); |
| 82259 | | 82325 | |
| 82260 | try self.genSetReg(ret_mcv.register_pair[0], .usize, .{ .immediate = 0 }, .{}); | 82326 | try cg.genSetReg(ret_mcv.register_pair[0], .usize, .{ .immediate = 0 }, .{}); |
| 82261 | try self.asmOpOnly(.{ ._, .ret }); | 82327 | try cg.asmOpOnly(.{ ._, .ret }); |
| 82262 | }, | 82328 | }, |
| 82263 | .error_set_type => |error_set_type| { | 82329 | .error_set_type => |error_set_type| { |
| 82264 | const err_ty: Type = .fromInterned(lazy_sym.ty); | 82330 | const err_ty: Type = .fromInterned(lazy_sym.ty); |
| 82265 | wip_mir_log.debug("{}.@errorCast:", .{err_ty.fmt(pt)}); | 82331 | wip_mir_log.debug("{}.@errorCast:", .{err_ty.fmt(pt)}); |
| 82266 | | 82332 | |
| 82267 | const param_regs = abi.getCAbiIntParamRegs(.auto); | 82333 | const param_regs = abi.getCAbiIntParamRegs(.auto); |
| 82268 | const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*); | 82334 | const param_locks = cg.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*); |
| 82269 | defer for (param_locks) |lock| self.register_manager.unlockReg(lock); | 82335 | defer for (param_locks) |lock| cg.register_manager.unlockReg(lock); |
| 82270 | | 82336 | |
| 82271 | const ret_mcv: MCValue = .{ .register = param_regs[0] }; | 82337 | const ret_mcv: MCValue = .{ .register = param_regs[0] }; |
| 82272 | const err_mcv: MCValue = .{ .register = param_regs[0] }; | 82338 | const err_mcv: MCValue = .{ .register = param_regs[0] }; |
| | 82339 | var err_temp = try cg.tempInit(err_ty, err_mcv); |
| 82273 | | 82340 | |
| 82274 | const ExpectedContents = [32]Mir.Inst.Index; | 82341 | const ExpectedContents = [32]Mir.Inst.Index; |
| 82275 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = | 82342 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| 82276 | std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); | 82343 | std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); |
| 82277 | const allocator = stack.get(); | 82344 | const allocator = stack.get(); |
| 82278 | | 82345 | |
| 82279 | const relocs = try allocator.alloc(Mir.Inst.Index, error_set_type.names.len); | 82346 | const relocs = try allocator.alloc(Mir.Inst.Index, error_set_type.names.len); |
| 82280 | defer allocator.free(relocs); | 82347 | defer allocator.free(relocs); |
| 82281 | | 82348 | |
| | 82349 | const reset_index = cg.next_temp_index; |
| 82282 | for (0.., relocs) |tag_index, *reloc| { | 82350 | for (0.., relocs) |tag_index, *reloc| { |
| 82283 | var err_temp = try self.tempInit(err_ty, err_mcv); | 82351 | var tag_temp = try cg.tempInit(.anyerror, .{ |
| 82284 | | | |
| 82285 | var tag_temp = try self.tempInit(.anyerror, .{ | | |
| 82286 | .immediate = ip.getErrorValueIfExists(error_set_type.names.get(ip)[tag_index]).?, | 82352 | .immediate = ip.getErrorValueIfExists(error_set_type.names.get(ip)[tag_index]).?, |
| 82287 | }); | 82353 | }); |
| 82288 | const cc_temp = err_temp.cmpInts(.eq, &tag_temp, self) catch |err| switch (err) { | 82354 | const cc_temp = err_temp.cmpInts(.eq, &tag_temp, cg) catch |err| switch (err) { |
| 82289 | error.SelectFailed => unreachable, | 82355 | error.SelectFailed => unreachable, |
| 82290 | else => |e| return e, | 82356 | else => |e| return e, |
| 82291 | }; | 82357 | }; |
| 82292 | try err_temp.die(self); | 82358 | try tag_temp.die(cg); |
| 82293 | try tag_temp.die(self); | 82359 | reloc.* = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); |
| 82294 | reloc.* = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined); | 82360 | try cc_temp.die(cg); |
| 82295 | try cc_temp.die(self); | 82361 | try cg.resetTemps(reset_index); |
| 82296 | try self.resetTemps(); | | |
| 82297 | } | 82362 | } |
| | 82363 | try err_temp.die(cg); |
| 82298 | | 82364 | |
| 82299 | try self.genCopy(.usize, ret_mcv, .{ .immediate = 0 }, .{}); | 82365 | try cg.genCopy(.usize, ret_mcv, .{ .immediate = 0 }, .{}); |
| 82300 | for (relocs) |reloc| self.performReloc(reloc); | 82366 | for (relocs) |reloc| cg.performReloc(reloc); |
| 82301 | assert(ret_mcv.register == err_mcv.register); | 82367 | assert(ret_mcv.register == err_mcv.register); |
| 82302 | try self.asmOpOnly(.{ ._, .ret }); | 82368 | try cg.asmOpOnly(.{ ._, .ret }); |
| 82303 | }, | 82369 | }, |
| 82304 | else => return self.fail( | 82370 | else => return cg.fail( |
| 82305 | "TODO implement {s} for {}", | 82371 | "TODO implement {s} for {}", |
| 82306 | .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) }, | 82372 | .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) }, |
| 82307 | ), | 82373 | ), |
| 82308 | } | 82374 | } |
| | 82375 | try cg.resetTemps(@enumFromInt(0)); |
| | 82376 | cg.checkInvariantsAfterAirInst(); |
| 82309 | } | 82377 | } |
| 82310 | | 82378 | |
| 82311 | fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void { | 82379 | fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void { |
| ... | @@ -93621,17 +93689,17 @@ fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index | ... | @@ -93621,17 +93689,17 @@ fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index |
| 93621 | } | 93689 | } |
| 93622 | | 93690 | |
| 93623 | fn lowerSwitchBr( | 93691 | fn lowerSwitchBr( |
| 93624 | self: *CodeGen, | 93692 | cg: *CodeGen, |
| 93625 | inst: Air.Inst.Index, | 93693 | inst: Air.Inst.Index, |
| 93626 | switch_br: Air.UnwrappedSwitch, | 93694 | switch_br: Air.UnwrappedSwitch, |
| 93627 | condition: MCValue, | 93695 | condition: MCValue, |
| 93628 | condition_dies: bool, | 93696 | condition_dies: bool, |
| 93629 | is_loop: bool, | 93697 | is_loop: bool, |
| 93630 | ) !void { | 93698 | ) !void { |
| 93631 | const zcu = self.pt.zcu; | 93699 | const zcu = cg.pt.zcu; |
| 93632 | const condition_ty = self.typeOf(switch_br.operand); | 93700 | const condition_ty = cg.typeOf(switch_br.operand); |
| 93633 | const condition_int_info = self.intInfo(condition_ty).?; | 93701 | const condition_int_info = cg.intInfo(condition_ty).?; |
| 93634 | const condition_int_ty = try self.pt.intType(condition_int_info.signedness, condition_int_info.bits); | 93702 | const condition_int_ty = try cg.pt.intType(condition_int_info.signedness, condition_int_info.bits); |
| 93635 | | 93703 | |
| 93636 | const ExpectedContents = extern struct { | 93704 | const ExpectedContents = extern struct { |
| 93637 | liveness_deaths: [1 << 8 | 1]Air.Inst.Index, | 93705 | liveness_deaths: [1 << 8 | 1]Air.Inst.Index, |
| ... | @@ -93639,15 +93707,15 @@ fn lowerSwitchBr( | ... | @@ -93639,15 +93707,15 @@ fn lowerSwitchBr( |
| 93639 | relocs: [1 << 6]Mir.Inst.Index, | 93707 | relocs: [1 << 6]Mir.Inst.Index, |
| 93640 | }; | 93708 | }; |
| 93641 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = | 93709 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| 93642 | std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); | 93710 | std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa); |
| 93643 | const allocator = stack.get(); | 93711 | const allocator = stack.get(); |
| 93644 | | 93712 | |
| 93645 | const state = try self.saveState(); | 93713 | const state = try cg.saveState(); |
| 93646 | | 93714 | |
| 93647 | const liveness = try self.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1); | 93715 | const liveness = try cg.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1); |
| 93648 | defer allocator.free(liveness.deaths); | 93716 | defer allocator.free(liveness.deaths); |
| 93649 | | 93717 | |
| 93650 | if (!self.mod.pic and self.target.ofmt == .elf) table: { | 93718 | if (!cg.mod.pic and cg.target.ofmt == .elf) table: { |
| 93651 | var prong_items: u32 = 0; | 93719 | var prong_items: u32 = 0; |
| 93652 | var min: ?Value = null; | 93720 | var min: ?Value = null; |
| 93653 | var max: ?Value = null; | 93721 | var max: ?Value = null; |
| ... | @@ -93690,41 +93758,41 @@ fn lowerSwitchBr( | ... | @@ -93690,41 +93758,41 @@ fn lowerSwitchBr( |
| 93690 | if (prong_items < table_len >> 2) break :table; // no more than 75% waste | 93758 | if (prong_items < table_len >> 2) break :table; // no more than 75% waste |
| 93691 | | 93759 | |
| 93692 | const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: { | 93760 | const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: { |
| 93693 | const condition_index = try self.allocTempRegOrMem(condition_ty, true); | 93761 | const condition_index = try cg.allocTempRegOrMem(condition_ty, true); |
| 93694 | try self.genCopy(condition_ty, condition_index, condition, .{}); | 93762 | try cg.genCopy(condition_ty, condition_index, condition, .{}); |
| 93695 | break :condition_index condition_index; | 93763 | break :condition_index condition_index; |
| 93696 | }; | 93764 | }; |
| 93697 | try self.spillEflagsIfOccupied(); | 93765 | try cg.spillEflagsIfOccupied(); |
| 93698 | if (min.?.orderAgainstZero(zcu).compare(.neq)) try self.genBinOpMir( | 93766 | if (min.?.orderAgainstZero(zcu).compare(.neq)) try cg.genBinOpMir( |
| 93699 | .{ ._, .sub }, | 93767 | .{ ._, .sub }, |
| 93700 | condition_ty, | 93768 | condition_ty, |
| 93701 | condition_index, | 93769 | condition_index, |
| 93702 | .{ .air_ref = Air.internedToRef(min.?.toIntern()) }, | 93770 | .{ .air_ref = Air.internedToRef(min.?.toIntern()) }, |
| 93703 | ); | 93771 | ); |
| 93704 | const else_reloc = if (switch_br.else_body_len > 0) else_reloc: { | 93772 | const else_reloc = if (switch_br.else_body_len > 0) else_reloc: { |
| 93705 | var cond_temp = try self.tempInit(condition_ty, condition_index); | 93773 | var cond_temp = try cg.tempInit(condition_ty, condition_index); |
| 93706 | var table_max_temp = try self.tempFromValue(try self.pt.intValue(condition_int_ty, table_len - 1)); | 93774 | var table_max_temp = try cg.tempFromValue(try cg.pt.intValue(condition_int_ty, table_len - 1)); |
| 93707 | const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) { | 93775 | const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, cg) catch |err| switch (err) { |
| 93708 | error.SelectFailed => unreachable, | 93776 | error.SelectFailed => unreachable, |
| 93709 | else => |e| return e, | 93777 | else => |e| return e, |
| 93710 | }; | 93778 | }; |
| 93711 | try cond_temp.die(self); | 93779 | try cond_temp.die(cg); |
| 93712 | try table_max_temp.die(self); | 93780 | try table_max_temp.die(cg); |
| 93713 | const else_reloc = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined); | 93781 | const else_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined); |
| 93714 | try cc_temp.die(self); | 93782 | try cc_temp.die(cg); |
| 93715 | break :else_reloc else_reloc; | 93783 | break :else_reloc else_reloc; |
| 93716 | } else undefined; | 93784 | } else undefined; |
| 93717 | const table_start: u31 = @intCast(self.mir_table.items.len); | 93785 | const table_start: u31 = @intCast(cg.mir_table.items.len); |
| 93718 | { | 93786 | { |
| 93719 | const condition_index_reg = if (condition_index.isRegister()) | 93787 | const condition_index_reg = if (condition_index.isRegister()) |
| 93720 | condition_index.getReg().? | 93788 | condition_index.getReg().? |
| 93721 | else | 93789 | else |
| 93722 | try self.copyToTmpRegister(.usize, condition_index); | 93790 | try cg.copyToTmpRegister(.usize, condition_index); |
| 93723 | const condition_index_lock = self.register_manager.lockReg(condition_index_reg); | 93791 | const condition_index_lock = cg.register_manager.lockReg(condition_index_reg); |
| 93724 | defer if (condition_index_lock) |lock| self.register_manager.unlockReg(lock); | 93792 | defer if (condition_index_lock) |lock| cg.register_manager.unlockReg(lock); |
| 93725 | try self.truncateRegister(condition_ty, condition_index_reg); | 93793 | try cg.truncateRegister(condition_ty, condition_index_reg); |
| 93726 | const ptr_size = @divExact(self.target.ptrBitWidth(), 8); | 93794 | const ptr_size = @divExact(cg.target.ptrBitWidth(), 8); |
| 93727 | try self.asmMemory(.{ ._mp, .j }, .{ | 93795 | try cg.asmMemory(.{ ._mp, .j }, .{ |
| 93728 | .base = .table, | 93796 | .base = .table, |
| 93729 | .mod = .{ .rm = .{ | 93797 | .mod = .{ .rm = .{ |
| 93730 | .size = .ptr, | 93798 | .size = .ptr, |
| ... | @@ -93735,32 +93803,32 @@ fn lowerSwitchBr( | ... | @@ -93735,32 +93803,32 @@ fn lowerSwitchBr( |
| 93735 | }); | 93803 | }); |
| 93736 | } | 93804 | } |
| 93737 | const else_reloc_marker: u32 = 0; | 93805 | const else_reloc_marker: u32 = 0; |
| 93738 | assert(self.mir_instructions.len > else_reloc_marker); | 93806 | assert(cg.mir_instructions.len > else_reloc_marker); |
| 93739 | try self.mir_table.appendNTimes(self.gpa, else_reloc_marker, table_len); | 93807 | try cg.mir_table.appendNTimes(cg.gpa, else_reloc_marker, table_len); |
| 93740 | if (is_loop) try self.loop_switches.putNoClobber(self.gpa, inst, .{ | 93808 | if (is_loop) try cg.loop_switches.putNoClobber(cg.gpa, inst, .{ |
| 93741 | .start = table_start, | 93809 | .start = table_start, |
| 93742 | .len = table_len, | 93810 | .len = table_len, |
| 93743 | .min = min.?, | 93811 | .min = min.?, |
| 93744 | .else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable", | 93812 | .else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable", |
| 93745 | }); | 93813 | }); |
| 93746 | defer if (is_loop) { | 93814 | defer if (is_loop) { |
| 93747 | var loop_switch_data = self.loop_switches.fetchRemove(inst).?.value; | 93815 | var loop_switch_data = cg.loop_switches.fetchRemove(inst).?.value; |
| 93748 | switch (loop_switch_data.else_relocs) { | 93816 | switch (loop_switch_data.else_relocs) { |
| 93749 | .@"unreachable", .backward => {}, | 93817 | .@"unreachable", .backward => {}, |
| 93750 | .forward => |*else_relocs| else_relocs.deinit(self.gpa), | 93818 | .forward => |*else_relocs| else_relocs.deinit(cg.gpa), |
| 93751 | } | 93819 | } |
| 93752 | }; | 93820 | }; |
| 93753 | var cases_it = switch_br.iterateCases(); | 93821 | var cases_it = switch_br.iterateCases(); |
| 93754 | while (cases_it.next()) |case| { | 93822 | while (cases_it.next()) |case| { |
| 93755 | { | 93823 | { |
| 93756 | const table = self.mir_table.items[table_start..][0..table_len]; | 93824 | const table = cg.mir_table.items[table_start..][0..table_len]; |
| 93757 | for (case.items) |item| { | 93825 | for (case.items) |item| { |
| 93758 | const val = Value.fromInterned(item.toInterned().?); | 93826 | const val = Value.fromInterned(item.toInterned().?); |
| 93759 | var val_space: Value.BigIntSpace = undefined; | 93827 | var val_space: Value.BigIntSpace = undefined; |
| 93760 | const val_bigint = val.toBigInt(&val_space, zcu); | 93828 | const val_bigint = val.toBigInt(&val_space, zcu); |
| 93761 | var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; | 93829 | var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; |
| 93762 | index_bigint.sub(val_bigint, min_bigint); | 93830 | index_bigint.sub(val_bigint, min_bigint); |
| 93763 | table[index_bigint.toConst().to(u10) catch unreachable] = @intCast(self.mir_instructions.len); | 93831 | table[index_bigint.toConst().to(u10) catch unreachable] = @intCast(cg.mir_instructions.len); |
| 93764 | } | 93832 | } |
| 93765 | for (case.ranges) |range| { | 93833 | for (case.ranges) |range| { |
| 93766 | var low_space: Value.BigIntSpace = undefined; | 93834 | var low_space: Value.BigIntSpace = undefined; |
| ... | @@ -93772,14 +93840,14 @@ fn lowerSwitchBr( | ... | @@ -93772,14 +93840,14 @@ fn lowerSwitchBr( |
| 93772 | const start = index_bigint.toConst().to(u10) catch unreachable; | 93840 | const start = index_bigint.toConst().to(u10) catch unreachable; |
| 93773 | index_bigint.sub(high_bigint, min_bigint); | 93841 | index_bigint.sub(high_bigint, min_bigint); |
| 93774 | const end = @as(u11, index_bigint.toConst().to(u10) catch unreachable) + 1; | 93842 | const end = @as(u11, index_bigint.toConst().to(u10) catch unreachable) + 1; |
| 93775 | @memset(table[start..end], @intCast(self.mir_instructions.len)); | 93843 | @memset(table[start..end], @intCast(cg.mir_instructions.len)); |
| 93776 | } | 93844 | } |
| 93777 | } | 93845 | } |
| 93778 | | 93846 | |
| 93779 | for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand); | 93847 | for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand); |
| 93780 | | 93848 | |
| 93781 | try self.genBodyBlock(case.body); | 93849 | try cg.genBodyBlock(case.body); |
| 93782 | try self.restoreState(state, &.{}, .{ | 93850 | try cg.restoreState(state, &.{}, .{ |
| 93783 | .emit_instructions = false, | 93851 | .emit_instructions = false, |
| 93784 | .update_tracking = true, | 93852 | .update_tracking = true, |
| 93785 | .resurrect = true, | 93853 | .resurrect = true, |
| ... | @@ -93790,21 +93858,21 @@ fn lowerSwitchBr( | ... | @@ -93790,21 +93858,21 @@ fn lowerSwitchBr( |
| 93790 | const else_body = cases_it.elseBody(); | 93858 | const else_body = cases_it.elseBody(); |
| 93791 | | 93859 | |
| 93792 | const else_deaths = liveness.deaths.len - 1; | 93860 | const else_deaths = liveness.deaths.len - 1; |
| 93793 | for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand); | 93861 | for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand); |
| 93794 | | 93862 | |
| 93795 | self.performReloc(else_reloc); | 93863 | cg.performReloc(else_reloc); |
| 93796 | if (is_loop) { | 93864 | if (is_loop) { |
| 93797 | const loop_switch_data = self.loop_switches.getPtr(inst).?; | 93865 | const loop_switch_data = cg.loop_switches.getPtr(inst).?; |
| 93798 | for (loop_switch_data.else_relocs.forward.items) |reloc| self.performReloc(reloc); | 93866 | for (loop_switch_data.else_relocs.forward.items) |reloc| cg.performReloc(reloc); |
| 93799 | loop_switch_data.else_relocs.forward.deinit(self.gpa); | 93867 | loop_switch_data.else_relocs.forward.deinit(cg.gpa); |
| 93800 | loop_switch_data.else_relocs = .{ .backward = @intCast(self.mir_instructions.len) }; | 93868 | loop_switch_data.else_relocs = .{ .backward = @intCast(cg.mir_instructions.len) }; |
| 93801 | } | 93869 | } |
| 93802 | for (self.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) { | 93870 | for (cg.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) { |
| 93803 | entry.* = @intCast(self.mir_instructions.len); | 93871 | entry.* = @intCast(cg.mir_instructions.len); |
| 93804 | }; | 93872 | }; |
| 93805 | | 93873 | |
| 93806 | try self.genBodyBlock(else_body); | 93874 | try cg.genBodyBlock(else_body); |
| 93807 | try self.restoreState(state, &.{}, .{ | 93875 | try cg.restoreState(state, &.{}, .{ |
| 93808 | .emit_instructions = false, | 93876 | .emit_instructions = false, |
| 93809 | .update_tracking = true, | 93877 | .update_tracking = true, |
| 93810 | .resurrect = true, | 93878 | .resurrect = true, |
| ... | @@ -93819,9 +93887,12 @@ fn lowerSwitchBr( | ... | @@ -93819,9 +93887,12 @@ fn lowerSwitchBr( |
| 93819 | const relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len); | 93887 | const relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len); |
| 93820 | defer allocator.free(relocs); | 93888 | defer allocator.free(relocs); |
| 93821 | | 93889 | |
| 93822 | try self.spillEflagsIfOccupied(); | 93890 | var cond_temp = try cg.tempInit(condition_ty, condition); |
| | 93891 | const reset_index = cg.next_temp_index; |
| | 93892 | |
| | 93893 | try cg.spillEflagsIfOccupied(); |
| 93823 | for (case.items, relocs[0..case.items.len]) |item, *reloc| { | 93894 | for (case.items, relocs[0..case.items.len]) |item, *reloc| { |
| 93824 | const item_mcv = try self.resolveInst(item); | 93895 | const item_mcv = try cg.resolveInst(item); |
| 93825 | const cc: Condition = switch (condition) { | 93896 | const cc: Condition = switch (condition) { |
| 93826 | .eflags => |cc| switch (item_mcv.immediate) { | 93897 | .eflags => |cc| switch (item_mcv.immediate) { |
| 93827 | 0 => cc.negate(), | 93898 | 0 => cc.negate(), |
| ... | @@ -93829,27 +93900,24 @@ fn lowerSwitchBr( | ... | @@ -93829,27 +93900,24 @@ fn lowerSwitchBr( |
| 93829 | else => unreachable, | 93900 | else => unreachable, |
| 93830 | }, | 93901 | }, |
| 93831 | else => cc: { | 93902 | else => cc: { |
| 93832 | var cond_temp = try self.tempInit(condition_ty, condition); | 93903 | var item_temp = try cg.tempInit(condition_ty, item_mcv); |
| 93833 | var item_temp = try self.tempInit(condition_ty, item_mcv); | 93904 | const cc_temp = cond_temp.cmpInts(.eq, &item_temp, cg) catch |err| switch (err) { |
| 93834 | const cc_temp = cond_temp.cmpInts(.eq, &item_temp, self) catch |err| switch (err) { | | |
| 93835 | error.SelectFailed => unreachable, | 93905 | error.SelectFailed => unreachable, |
| 93836 | else => |e| return e, | 93906 | else => |e| return e, |
| 93837 | }; | 93907 | }; |
| 93838 | try cond_temp.die(self); | 93908 | try item_temp.die(cg); |
| 93839 | try item_temp.die(self); | 93909 | const cc = cc_temp.tracking(cg).short.eflags; |
| 93840 | const cc = cc_temp.tracking(self).short.eflags; | 93910 | try cc_temp.die(cg); |
| 93841 | try cc_temp.die(self); | 93911 | try cg.resetTemps(reset_index); |
| 93842 | try self.resetTemps(); | | |
| 93843 | break :cc cc; | 93912 | break :cc cc; |
| 93844 | }, | 93913 | }, |
| 93845 | }; | 93914 | }; |
| 93846 | reloc.* = try self.asmJccReloc(cc, undefined); | 93915 | reloc.* = try cg.asmJccReloc(cc, undefined); |
| 93847 | } | 93916 | } |
| 93848 | | 93917 | |
| 93849 | for (case.ranges, relocs[case.items.len..]) |range, *reloc| { | 93918 | for (case.ranges, relocs[case.items.len..]) |range, *reloc| { |
| 93850 | var cond_temp = try self.tempInit(condition_ty, condition); | 93919 | const min_mcv = try cg.resolveInst(range[0]); |
| 93851 | const min_mcv = try self.resolveInst(range[0]); | 93920 | const max_mcv = try cg.resolveInst(range[1]); |
| 93852 | const max_mcv = try self.resolveInst(range[1]); | | |
| 93853 | // `null` means always false. | 93921 | // `null` means always false. |
| 93854 | const lt_min = cc: switch (condition) { | 93922 | const lt_min = cc: switch (condition) { |
| 93855 | .eflags => |cc| switch (min_mcv.immediate) { | 93923 | .eflags => |cc| switch (min_mcv.immediate) { |
| ... | @@ -93858,19 +93926,19 @@ fn lowerSwitchBr( | ... | @@ -93858,19 +93926,19 @@ fn lowerSwitchBr( |
| 93858 | else => unreachable, | 93926 | else => unreachable, |
| 93859 | }, | 93927 | }, |
| 93860 | else => { | 93928 | else => { |
| 93861 | var min_temp = try self.tempInit(condition_ty, min_mcv); | 93929 | var min_temp = try cg.tempInit(condition_ty, min_mcv); |
| 93862 | const cc_temp = cond_temp.cmpInts(.lt, &min_temp, self) catch |err| switch (err) { | 93930 | const cc_temp = cond_temp.cmpInts(.lt, &min_temp, cg) catch |err| switch (err) { |
| 93863 | error.SelectFailed => unreachable, | 93931 | error.SelectFailed => unreachable, |
| 93864 | else => |e| return e, | 93932 | else => |e| return e, |
| 93865 | }; | 93933 | }; |
| 93866 | try min_temp.die(self); | 93934 | try min_temp.die(cg); |
| 93867 | const cc = cc_temp.tracking(self).short.eflags; | 93935 | const cc = cc_temp.tracking(cg).short.eflags; |
| 93868 | try cc_temp.die(self); | 93936 | try cc_temp.die(cg); |
| 93869 | break :cc cc; | 93937 | break :cc cc; |
| 93870 | }, | 93938 | }, |
| 93871 | }; | 93939 | }; |
| 93872 | const lt_min_reloc = if (lt_min) |cc| r: { | 93940 | const lt_min_reloc = if (lt_min) |cc| r: { |
| 93873 | break :r try self.asmJccReloc(cc, undefined); | 93941 | break :r try cg.asmJccReloc(cc, undefined); |
| 93874 | } else null; | 93942 | } else null; |
| 93875 | // `null` means always true. | 93943 | // `null` means always true. |
| 93876 | const lte_max = switch (condition) { | 93944 | const lte_max = switch (condition) { |
| ... | @@ -93880,38 +93948,41 @@ fn lowerSwitchBr( | ... | @@ -93880,38 +93948,41 @@ fn lowerSwitchBr( |
| 93880 | else => unreachable, | 93948 | else => unreachable, |
| 93881 | }, | 93949 | }, |
| 93882 | else => cc: { | 93950 | else => cc: { |
| 93883 | var max_temp = try self.tempInit(condition_ty, max_mcv); | 93951 | var max_temp = try cg.tempInit(condition_ty, max_mcv); |
| 93884 | const cc_temp = cond_temp.cmpInts(.lte, &max_temp, self) catch |err| switch (err) { | 93952 | const cc_temp = cond_temp.cmpInts(.lte, &max_temp, cg) catch |err| switch (err) { |
| 93885 | error.SelectFailed => unreachable, | 93953 | error.SelectFailed => unreachable, |
| 93886 | else => |e| return e, | 93954 | else => |e| return e, |
| 93887 | }; | 93955 | }; |
| 93888 | try max_temp.die(self); | 93956 | try max_temp.die(cg); |
| 93889 | const cc = cc_temp.tracking(self).short.eflags; | 93957 | const cc = cc_temp.tracking(cg).short.eflags; |
| 93890 | try cc_temp.die(self); | 93958 | try cc_temp.die(cg); |
| 93891 | break :cc cc; | 93959 | break :cc cc; |
| 93892 | }, | 93960 | }, |
| 93893 | }; | 93961 | }; |
| 93894 | try cond_temp.die(self); | 93962 | try cg.resetTemps(reset_index); |
| 93895 | try self.resetTemps(); | | |
| 93896 | // "Success" case is in `reloc`.... | 93963 | // "Success" case is in `reloc`.... |
| 93897 | if (lte_max) |cc| { | 93964 | if (lte_max) |cc| { |
| 93898 | reloc.* = try self.asmJccReloc(cc, undefined); | 93965 | reloc.* = try cg.asmJccReloc(cc, undefined); |
| 93899 | } else { | 93966 | } else { |
| 93900 | reloc.* = try self.asmJmpReloc(undefined); | 93967 | reloc.* = try cg.asmJmpReloc(undefined); |
| 93901 | } | 93968 | } |
| 93902 | // ...and "fail" case falls through to next checks. | 93969 | // ...and "fail" case falls through to next checks. |
| 93903 | if (lt_min_reloc) |r| self.performReloc(r); | 93970 | if (lt_min_reloc) |r| cg.performReloc(r); |
| 93904 | } | 93971 | } |
| 93905 | | 93972 | |
| | 93973 | try cond_temp.die(cg); |
| | 93974 | try cg.resetTemps(@enumFromInt(0)); |
| | 93975 | cg.checkInvariantsAfterAirInst(); |
| | 93976 | |
| 93906 | // The jump to skip this case if the conditions all failed. | 93977 | // The jump to skip this case if the conditions all failed. |
| 93907 | const skip_case_reloc = try self.asmJmpReloc(undefined); | 93978 | const skip_case_reloc = try cg.asmJmpReloc(undefined); |
| 93908 | | 93979 | |
| 93909 | for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand); | 93980 | for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand); |
| 93910 | | 93981 | |
| 93911 | // Relocate all success cases to the body we're about to generate. | 93982 | // Relocate all success cases to the body we're about to generate. |
| 93912 | for (relocs) |reloc| self.performReloc(reloc); | 93983 | for (relocs) |reloc| cg.performReloc(reloc); |
| 93913 | try self.genBodyBlock(case.body); | 93984 | try cg.genBodyBlock(case.body); |
| 93914 | try self.restoreState(state, &.{}, .{ | 93985 | try cg.restoreState(state, &.{}, .{ |
| 93915 | .emit_instructions = false, | 93986 | .emit_instructions = false, |
| 93916 | .update_tracking = true, | 93987 | .update_tracking = true, |
| 93917 | .resurrect = true, | 93988 | .resurrect = true, |
| ... | @@ -93919,16 +93990,16 @@ fn lowerSwitchBr( | ... | @@ -93919,16 +93990,16 @@ fn lowerSwitchBr( |
| 93919 | }); | 93990 | }); |
| 93920 | | 93991 | |
| 93921 | // Relocate the "skip" branch to fall through to the next case. | 93992 | // Relocate the "skip" branch to fall through to the next case. |
| 93922 | self.performReloc(skip_case_reloc); | 93993 | cg.performReloc(skip_case_reloc); |
| 93923 | } | 93994 | } |
| 93924 | if (switch_br.else_body_len > 0) { | 93995 | if (switch_br.else_body_len > 0) { |
| 93925 | const else_body = cases_it.elseBody(); | 93996 | const else_body = cases_it.elseBody(); |
| 93926 | | 93997 | |
| 93927 | const else_deaths = liveness.deaths.len - 1; | 93998 | const else_deaths = liveness.deaths.len - 1; |
| 93928 | for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand); | 93999 | for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand); |
| 93929 | | 94000 | |
| 93930 | try self.genBodyBlock(else_body); | 94001 | try cg.genBodyBlock(else_body); |
| 93931 | try self.restoreState(state, &.{}, .{ | 94002 | try cg.restoreState(state, &.{}, .{ |
| 93932 | .emit_instructions = false, | 94003 | .emit_instructions = false, |
| 93933 | .update_tracking = true, | 94004 | .update_tracking = true, |
| 93934 | .resurrect = true, | 94005 | .resurrect = true, |
| ... | @@ -101339,6 +101410,7 @@ const Temp = struct { | ... | @@ -101339,6 +101410,7 @@ const Temp = struct { |
| 101339 | const val_mcv = val.tracking(cg).short; | 101410 | const val_mcv = val.tracking(cg).short; |
| 101340 | switch (val_mcv) { | 101411 | switch (val_mcv) { |
| 101341 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), | 101412 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| | 101413 | .none => {}, |
| 101342 | .undef => if (opts.safe) { | 101414 | .undef => if (opts.safe) { |
| 101343 | var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp)); | 101415 | var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp)); |
| 101344 | var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa }); | 101416 | var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa }); |
| ... | @@ -105786,9 +105858,9 @@ const Temp = struct { | ... | @@ -105786,9 +105858,9 @@ const Temp = struct { |
| 105786 | }; | 105858 | }; |
| 105787 | }; | 105859 | }; |
| 105788 | | 105860 | |
| 105789 | fn resetTemps(cg: *CodeGen) InnerError!void { | 105861 | fn resetTemps(cg: *CodeGen, from_index: Temp.Index) InnerError!void { |
| 105790 | var any_valid = false; | 105862 | var any_valid = false; |
| 105791 | for (0..@intFromEnum(cg.next_temp_index)) |temp_index| { | 105863 | for (@intFromEnum(from_index)..@intFromEnum(cg.next_temp_index)) |temp_index| { |
| 105792 | const temp: Temp.Index = @enumFromInt(temp_index); | 105864 | const temp: Temp.Index = @enumFromInt(temp_index); |
| 105793 | if (temp.isValid(cg)) { | 105865 | if (temp.isValid(cg)) { |
| 105794 | any_valid = true; | 105866 | any_valid = true; |
| ... | @@ -105800,7 +105872,7 @@ fn resetTemps(cg: *CodeGen) InnerError!void { | ... | @@ -105800,7 +105872,7 @@ fn resetTemps(cg: *CodeGen) InnerError!void { |
| 105800 | cg.temp_type[temp_index] = undefined; | 105872 | cg.temp_type[temp_index] = undefined; |
| 105801 | } | 105873 | } |
| 105802 | if (any_valid) return cg.fail("failed to kill all temps", .{}); | 105874 | if (any_valid) return cg.fail("failed to kill all temps", .{}); |
| 105803 | cg.next_temp_index = @enumFromInt(0); | 105875 | cg.next_temp_index = from_index; |
| 105804 | } | 105876 | } |
| 105805 | | 105877 | |
| 105806 | fn reuseTemp( | 105878 | fn reuseTemp( |
| ... | @@ -105889,70 +105961,75 @@ fn tempMemFromValue(cg: *CodeGen, value: Value) InnerError!Temp { | ... | @@ -105889,70 +105961,75 @@ fn tempMemFromValue(cg: *CodeGen, value: Value) InnerError!Temp { |
| 105889 | return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerUav(value)); | 105961 | return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerUav(value)); |
| 105890 | } | 105962 | } |
| 105891 | | 105963 | |
| 105892 | fn tempFromOperand( | 105964 | fn tempFromOperand(cg: *CodeGen, op_ref: Air.Inst.Ref, op_dies: bool) InnerError!Temp { |
| 105893 | cg: *CodeGen, | | |
| 105894 | inst: Air.Inst.Index, | | |
| 105895 | op_index: Liveness.OperandInt, | | |
| 105896 | op_ref: Air.Inst.Ref, | | |
| 105897 | ignore_death: bool, | | |
| 105898 | ) InnerError!Temp { | | |
| 105899 | const zcu = cg.pt.zcu; | 105965 | const zcu = cg.pt.zcu; |
| 105900 | const ip = &zcu.intern_pool; | 105966 | const ip = &zcu.intern_pool; |
| 105901 | | 105967 | |
| 105902 | if (ignore_death or !cg.liveness.operandDies(inst, op_index)) { | 105968 | if (op_dies) { |
| 105903 | if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; | 105969 | const temp_index = cg.next_temp_index; |
| 105904 | const val = op_ref.toInterned().?; | 105970 | const temp: Temp = .{ .index = temp_index.toIndex() }; |
| 105905 | const gop = try cg.const_tracking.getOrPut(cg.gpa, val); | 105971 | const op_inst = op_ref.toIndex().?; |
| 105906 | if (!gop.found_existing) gop.value_ptr.* = .init(init: { | 105972 | const tracking = cg.getResolvedInstValue(op_inst); |
| 105907 | const const_mcv = try cg.genTypedValue(.fromInterned(val)); | 105973 | temp_index.tracking(cg).* = tracking.*; |
| 105908 | switch (const_mcv) { | 105974 | if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? }; |
| 105909 | .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) { | 105975 | cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref); |
| 105910 | .elf, .macho => { | 105976 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| 105911 | if (cg.mod.pic) { | 105977 | return temp; |
| 105912 | try cg.spillRegisters(&.{ .rdi, .rax }); | 105978 | } |
| 105913 | } else { | 105979 | |
| 105914 | try cg.spillRegisters(&.{.rax}); | 105980 | if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; |
| 105915 | } | 105981 | const val = op_ref.toInterned().?; |
| 105916 | const frame_index = try cg.allocFrameIndex(.init(.{ | 105982 | const gop = try cg.const_tracking.getOrPut(cg.gpa, val); |
| 105917 | .size = 8, | 105983 | if (!gop.found_existing) gop.value_ptr.* = .init(init: { |
| 105918 | .alignment = .@"8", | 105984 | const const_mcv = try cg.genTypedValue(.fromInterned(val)); |
| 105919 | })); | 105985 | switch (const_mcv) { |
| 105920 | try cg.genSetMem( | 105986 | .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) { |
| 105921 | .{ .frame = frame_index }, | 105987 | .elf, .macho => { |
| 105922 | 0, | 105988 | if (cg.mod.pic) { |
| 105923 | .usize, | 105989 | try cg.spillRegisters(&.{ .rdi, .rax }); |
| 105924 | .{ .lea_symbol = .{ .sym_index = tlv_sym } }, | 105990 | } else { |
| 105925 | .{}, | 105991 | try cg.spillRegisters(&.{.rax}); |
| 105926 | ); | 105992 | } |
| 105927 | break :init .{ .load_frame = .{ .index = frame_index } }; | 105993 | const frame_index = try cg.allocFrameIndex(.init(.{ |
| 105928 | }, | 105994 | .size = 8, |
| 105929 | else => break :init const_mcv, | 105995 | .alignment = .@"8", |
| | 105996 | })); |
| | 105997 | try cg.genSetMem( |
| | 105998 | .{ .frame = frame_index }, |
| | 105999 | 0, |
| | 106000 | .usize, |
| | 106001 | .{ .lea_symbol = .{ .sym_index = tlv_sym } }, |
| | 106002 | .{}, |
| | 106003 | ); |
| | 106004 | break :init .{ .load_frame = .{ .index = frame_index } }; |
| 105930 | }, | 106005 | }, |
| 105931 | else => break :init const_mcv, | 106006 | else => break :init const_mcv, |
| 105932 | } | 106007 | }, |
| 105933 | }); | 106008 | else => break :init const_mcv, |
| 105934 | return cg.tempInit(.fromInterned(ip.typeOf(val)), gop.value_ptr.short); | 106009 | } |
| 105935 | } | 106010 | }); |
| | 106011 | return cg.tempInit(.fromInterned(ip.typeOf(val)), gop.value_ptr.short); |
| | 106012 | } |
| 105936 | | 106013 | |
| 105937 | const temp_index = cg.next_temp_index; | 106014 | fn tempsFromOperandsInner( |
| 105938 | const temp: Temp = .{ .index = temp_index.toIndex() }; | 106015 | cg: *CodeGen, |
| 105939 | const op_inst = op_ref.toIndex().?; | 106016 | inst: Air.Inst.Index, |
| 105940 | const tracking = cg.getResolvedInstValue(op_inst); | 106017 | op_temps: []Temp, |
| 105941 | temp_index.tracking(cg).* = tracking.*; | 106018 | op_refs: []const Air.Inst.Ref, |
| 105942 | if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? }; | 106019 | ) InnerError!void { |
| 105943 | cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref); | 106020 | for (op_temps, 0.., op_refs) |*op_temp, op_index, op_ref| op_temp.* = try cg.tempFromOperand(op_ref, for (op_refs[0..op_index]) |prev_op_ref| { |
| 105944 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); | 106021 | if (op_ref == prev_op_ref) break false; |
| 105945 | return temp; | 106022 | } else cg.liveness.operandDies(inst, @intCast(op_index))); |
| 105946 | } | 106023 | } |
| 105947 | | 106024 | |
| 105948 | inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) InnerError![op_refs.len]Temp { | 106025 | inline fn tempsFromOperands( |
| 105949 | var temps: [op_refs.len]Temp = undefined; | 106026 | cg: *CodeGen, |
| 105950 | inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| { | 106027 | inst: Air.Inst.Index, |
| 105951 | temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| { | 106028 | op_refs: anytype, |
| 105952 | if (op_ref == op_refs[prev_op_index]) break true; | 106029 | ) InnerError![op_refs.len]Temp { |
| 105953 | } else false); | 106030 | var op_temps: [op_refs.len]Temp = undefined; |
| 105954 | } | 106031 | try cg.tempsFromOperandsInner(inst, &op_temps, &op_refs); |
| 105955 | return temps; | 106032 | return op_temps; |
| 105956 | } | 106033 | } |
| 105957 | | 106034 | |
| 105958 | const Operand = union(enum) { | 106035 | const Operand = union(enum) { |
Jacob Young <jacobly@ziglang.org> 2025-02-25 13:38:31-05:00
src