| ... | ... | @@ -995,7 +995,6 @@ fn addInst(func: *Func, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index { |
| 995 | 995 | .pseudo_dbg_prologue_end, |
| 996 | 996 | .pseudo_dbg_line_column, |
| 997 | 997 | .pseudo_dbg_epilogue_begin, |
| 998 | | .pseudo_mv, |
| 999 | 998 | .pseudo_dead, |
| 1000 | 999 | => false, |
| 1001 | 1000 | }) wip_mir_log.debug("{}", .{func.fmtWipMir(result_index)}); |
| ... | ... | @@ -2445,6 +2444,7 @@ fn genBinOp( |
| 2445 | 2444 | .Vector => { |
| 2446 | 2445 | const mir_tag: Mir.Inst.Tag = switch (tag) { |
| 2447 | 2446 | .add => .vaddvv, |
| 2447 | .sub => .vsubvv, |
| 2448 | 2448 | else => return func.fail("TODO: genBinOp {s} Vector", .{@tagName(tag)}), |
| 2449 | 2449 | }; |
| 2450 | 2450 | |
| ... | ... | @@ -2454,7 +2454,6 @@ fn genBinOp( |
| 2454 | 2454 | const elem_size = lhs_ty.childType(zcu).bitSize(pt); |
| 2455 | 2455 | |
| 2456 | 2456 | try func.setVl(.zero, num_elem, .{ |
| 2457 | | .vlmul = .mf2, |
| 2458 | 2457 | .vsew = switch (elem_size) { |
| 2459 | 2458 | 8 => .@"8", |
| 2460 | 2459 | 16 => .@"16", |
| ... | ... | @@ -2462,6 +2461,7 @@ fn genBinOp( |
| 2462 | 2461 | 64 => .@"64", |
| 2463 | 2462 | else => unreachable, |
| 2464 | 2463 | }, |
| 2464 | .vlmul = .m1, |
| 2465 | 2465 | .vma = true, |
| 2466 | 2466 | .vta = true, |
| 2467 | 2467 | }); |
| ... | ... | @@ -2472,8 +2472,8 @@ fn genBinOp( |
| 2472 | 2472 | .data = .{ |
| 2473 | 2473 | .r_type = .{ |
| 2474 | 2474 | .rd = dst_reg, |
| 2475 | | .rs1 = lhs_reg, |
| 2476 | | .rs2 = rhs_reg, |
| 2475 | .rs1 = rhs_reg, |
| 2476 | .rs2 = lhs_reg, |
| 2477 | 2477 | }, |
| 2478 | 2478 | }, |
| 2479 | 2479 | }); |
| ... | ... | @@ -3576,20 +3576,54 @@ fn airArrayElemVal(func: *Func, inst: Air.Inst.Index) !void { |
| 3576 | 3576 | else => try func.genSetReg(Type.usize, addr_reg, array_mcv.address()), |
| 3577 | 3577 | } |
| 3578 | 3578 | |
| 3579 | const dst_mcv = try func.allocRegOrMem(result_ty, inst, false); |
| 3580 | |
| 3581 | if (array_ty.isVector(zcu)) { |
| 3582 | // we need to load the vector, vslidedown to get the element we want |
| 3583 | // and store that element at in a load frame. |
| 3584 | |
| 3585 | const src_reg, const src_lock = try func.allocReg(.vector); |
| 3586 | defer func.register_manager.unlockReg(src_lock); |
| 3587 | |
| 3588 | // load the vector into a temporary register |
| 3589 | try func.genCopy(array_ty, .{ .register = src_reg }, .{ .indirect = .{ .reg = addr_reg } }); |
| 3590 | |
| 3591 | // we need to construct a 1xbitSize vector because of how lane splitting works in RISC-V |
| 3592 | const single_ty = try pt.vectorType(.{ .child = elem_ty.toIntern(), .len = 1 }); |
| 3593 | |
| 3594 | // we can do a shortcut here where we don't need a vslicedown |
| 3595 | // and can just copy to the frame index. |
| 3596 | if (!(index_mcv == .immediate and index_mcv.immediate == 0)) { |
| 3597 | const index_reg = try func.copyToTmpRegister(Type.usize, index_mcv); |
| 3598 | |
| 3599 | _ = try func.addInst(.{ |
| 3600 | .tag = .vslidedownvx, |
| 3601 | .ops = .rrr, |
| 3602 | .data = .{ .r_type = .{ |
| 3603 | .rd = src_reg, |
| 3604 | .rs1 = index_reg, |
| 3605 | .rs2 = src_reg, |
| 3606 | } }, |
| 3607 | }); |
| 3608 | } |
| 3609 | |
| 3610 | try func.genCopy(single_ty, dst_mcv, .{ .register = src_reg }); |
| 3611 | break :result dst_mcv; |
| 3612 | } |
| 3613 | |
| 3579 | 3614 | const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_abi_size); |
| 3580 | 3615 | const offset_lock = func.register_manager.lockRegAssumeUnused(offset_reg); |
| 3581 | 3616 | defer func.register_manager.unlockReg(offset_lock); |
| 3582 | | |
| 3583 | | const dst_mcv = try func.allocRegOrMem(result_ty, inst, false); |
| 3584 | 3617 | _ = try func.addInst(.{ |
| 3585 | 3618 | .tag = .add, |
| 3586 | 3619 | .ops = .rrr, |
| 3587 | 3620 | .data = .{ .r_type = .{ |
| 3588 | 3621 | .rd = addr_reg, |
| 3589 | | .rs1 = offset_reg, |
| 3590 | | .rs2 = addr_reg, |
| 3622 | .rs1 = addr_reg, |
| 3623 | .rs2 = offset_reg, |
| 3591 | 3624 | } }, |
| 3592 | 3625 | }); |
| 3626 | |
| 3593 | 3627 | try func.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); |
| 3594 | 3628 | break :result dst_mcv; |
| 3595 | 3629 | }; |
| ... | ... | @@ -5965,6 +5999,27 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 5965 | 5999 | if (src_reg.id() == reg.id()) |
| 5966 | 6000 | return; |
| 5967 | 6001 | |
| 6002 | // there is no instruction for loading the contents of a vector register |
| 6003 | // into an integer register, however we can cheat a bit by setting the element |
| 6004 | // size to the total size of the vector, and vmv.x.s will work then |
| 6005 | if (src_reg.class() == .vector) { |
| 6006 | try func.setVl(.zero, 0, .{ |
| 6007 | .vsew = switch (ty.totalVectorBits(pt)) { |
| 6008 | 8 => .@"8", |
| 6009 | 16 => .@"16", |
| 6010 | 32 => .@"32", |
| 6011 | 64 => .@"64", |
| 6012 | else => |vec_bits| return func.fail("TODO: genSetReg vec -> {s} bits {d}", .{ |
| 6013 | @tagName(reg.class()), |
| 6014 | vec_bits, |
| 6015 | }), |
| 6016 | }, |
| 6017 | .vlmul = .m1, |
| 6018 | .vta = true, |
| 6019 | .vma = true, |
| 6020 | }); |
| 6021 | } |
| 6022 | |
| 5968 | 6023 | // mv reg, src_reg |
| 5969 | 6024 | _ = try func.addInst(.{ |
| 5970 | 6025 | .tag = .pseudo, |
| ... | ... | @@ -5978,57 +6033,28 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 5978 | 6033 | .register_pair => return func.fail("genSetReg should we allow reg -> reg_pair?", .{}), |
| 5979 | 6034 | .load_frame => |frame| { |
| 5980 | 6035 | if (reg.class() == .vector) { |
| 5981 | | if (abi_size > 8) |
| 5982 | | return func.fail("TODO: genSetReg vectors > 8", .{}); |
| 5983 | | |
| 5984 | | const temp_reg = try func.register_manager.allocReg(null, abi.Registers.Integer.temporary); |
| 5985 | | const temp_lock = func.register_manager.lockRegAssumeUnused(temp_reg); |
| 5986 | | defer func.register_manager.unlockReg(temp_lock); |
| 5987 | | |
| 5988 | | try func.setVl(.zero, 1, .{ |
| 5989 | | .vsew = switch (abi_size) { |
| 5990 | | 1 => .@"8", |
| 5991 | | 2 => .@"16", |
| 5992 | | 4 => .@"32", |
| 5993 | | 8 => .@"64", |
| 5994 | | else => unreachable, |
| 5995 | | }, |
| 5996 | | .vlmul = .m1, |
| 5997 | | .vma = true, |
| 5998 | | .vta = true, |
| 5999 | | }); |
| 6000 | | |
| 6001 | | try func.genCopy(ty, .{ .register = temp_reg }, .{ .load_frame = frame }); |
| 6036 | const addr_reg, const addr_lock = try func.allocReg(.int); |
| 6037 | defer func.register_manager.unlockReg(addr_lock); |
| 6002 | 6038 | |
| 6039 | try func.genCopy(ty, .{ .register = addr_reg }, src_mcv.address()); |
| 6040 | try func.genCopy(ty, .{ .register = reg }, .{ .indirect = .{ .reg = addr_reg } }); |
| 6041 | } else { |
| 6003 | 6042 | _ = try func.addInst(.{ |
| 6004 | 6043 | .tag = .pseudo, |
| 6005 | | .ops = .pseudo_mv, |
| 6006 | | .data = .{ |
| 6007 | | .rr = .{ |
| 6008 | | .rd = reg, |
| 6009 | | .rs = temp_reg, |
| 6044 | .ops = .pseudo_load_rm, |
| 6045 | .data = .{ .rm = .{ |
| 6046 | .r = reg, |
| 6047 | .m = .{ |
| 6048 | .base = .{ .frame = frame.index }, |
| 6049 | .mod = .{ |
| 6050 | .size = func.memSize(ty), |
| 6051 | .unsigned = ty.isUnsignedInt(zcu), |
| 6052 | .disp = frame.off, |
| 6053 | }, |
| 6010 | 6054 | }, |
| 6011 | | }, |
| 6055 | } }, |
| 6012 | 6056 | }); |
| 6013 | | |
| 6014 | | return; |
| 6015 | 6057 | } |
| 6016 | | |
| 6017 | | _ = try func.addInst(.{ |
| 6018 | | .tag = .pseudo, |
| 6019 | | .ops = .pseudo_load_rm, |
| 6020 | | .data = .{ .rm = .{ |
| 6021 | | .r = reg, |
| 6022 | | .m = .{ |
| 6023 | | .base = .{ .frame = frame.index }, |
| 6024 | | .mod = .{ |
| 6025 | | .size = func.memSize(ty), |
| 6026 | | .unsigned = ty.isUnsignedInt(zcu), |
| 6027 | | .disp = frame.off, |
| 6028 | | }, |
| 6029 | | }, |
| 6030 | | } }, |
| 6031 | | }); |
| 6032 | 6058 | }, |
| 6033 | 6059 | .memory => |addr| { |
| 6034 | 6060 | try func.genSetReg(ty, reg, .{ .immediate = addr }); |
| ... | ... | @@ -6072,20 +6098,64 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 6072 | 6098 | }); |
| 6073 | 6099 | }, |
| 6074 | 6100 | .indirect => |reg_off| { |
| 6075 | | const float_class = dst_reg_class == .float; |
| 6101 | const load_tag: Mir.Inst.Tag = switch (reg.class()) { |
| 6102 | .float => switch (abi_size) { |
| 6103 | 1 => unreachable, // Zig does not support 8-bit floats |
| 6104 | 2 => return func.fail("TODO: genSetReg indirect 16-bit float", .{}), |
| 6105 | 4 => .flw, |
| 6106 | 8 => .fld, |
| 6107 | else => return std.debug.panic("TODO: genSetReg for float size {d}", .{abi_size}), |
| 6108 | }, |
| 6109 | .int => switch (abi_size) { |
| 6110 | 1 => .lb, |
| 6111 | 2 => .lh, |
| 6112 | 4 => .lw, |
| 6113 | 8 => .ld, |
| 6114 | else => return std.debug.panic("TODO: genSetReg for int size {d}", .{abi_size}), |
| 6115 | }, |
| 6116 | .vector => { |
| 6117 | assert(reg_off.off == 0); |
| 6118 | |
| 6119 | // There is no vector instruction for loading with an offset to a base register, |
| 6120 | // so we need to get an offset register containing the address of the vector first |
| 6121 | // and load from it. |
| 6122 | const len: u5 = math.cast(u5, ty.vectorLen(zcu)) orelse { |
| 6123 | return func.fail("TODO: genSetReg load_frame -> vec reg, vector length doesn't fit into imm avl", .{}); |
| 6124 | }; |
| 6125 | const elem_ty = ty.childType(zcu); |
| 6126 | const elem_size = elem_ty.abiSize(pt); |
| 6076 | 6127 | |
| 6077 | | const load_tag: Mir.Inst.Tag = switch (abi_size) { |
| 6078 | | 1 => if (float_class) |
| 6079 | | unreachable // Zig does not support 8-bit floats |
| 6080 | | else |
| 6081 | | .lb, |
| 6082 | | 2 => if (float_class) |
| 6083 | | return func.fail("TODO: genSetReg indirect 16-bit float", .{}) |
| 6084 | | else |
| 6085 | | .lh, |
| 6086 | | 4 => if (float_class) .flw else .lw, |
| 6087 | | 8 => if (float_class) .fld else .ld, |
| 6088 | | else => return std.debug.panic("TODO: genSetReg for size {d}", .{abi_size}), |
| 6128 | try func.setVl(.zero, len, .{ |
| 6129 | .vsew = switch (elem_size) { |
| 6130 | 1 => .@"8", |
| 6131 | 2 => .@"16", |
| 6132 | 4 => .@"32", |
| 6133 | 8 => .@"64", |
| 6134 | else => unreachable, |
| 6135 | }, |
| 6136 | .vlmul = .m1, |
| 6137 | .vma = true, |
| 6138 | .vta = true, |
| 6139 | }); |
| 6140 | |
| 6141 | _ = try func.addInst(.{ |
| 6142 | .tag = .pseudo, |
| 6143 | .ops = .pseudo_load_rm, |
| 6144 | .data = .{ .rm = .{ |
| 6145 | .r = reg, |
| 6146 | .m = .{ |
| 6147 | .base = .{ .reg = reg_off.reg }, |
| 6148 | .mod = .{ |
| 6149 | .size = func.memSize(elem_ty), |
| 6150 | .unsigned = false, |
| 6151 | .disp = 0, |
| 6152 | }, |
| 6153 | }, |
| 6154 | } }, |
| 6155 | }); |
| 6156 | |
| 6157 | return; |
| 6158 | }, |
| 6089 | 6159 | }; |
| 6090 | 6160 | |
| 6091 | 6161 | _ = try func.addInst(.{ |
| ... | ... | @@ -6100,7 +6170,6 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 6100 | 6170 | }, |
| 6101 | 6171 | .lea_symbol => |sym_off| { |
| 6102 | 6172 | assert(sym_off.off == 0); |
| 6103 | | |
| 6104 | 6173 | const atom_index = try func.symbolIndex(); |
| 6105 | 6174 | |
| 6106 | 6175 | _ = try func.addInst(.{ |
| ... | ... | @@ -6166,12 +6235,12 @@ fn genSetMem( |
| 6166 | 6235 | => switch (abi_size) { |
| 6167 | 6236 | 0 => {}, |
| 6168 | 6237 | 1, 2, 4, 8 => { |
| 6169 | | // no matter what type, it should use an integer register |
| 6170 | | const src_reg = try func.copyToTmpRegister(ty, src_mcv); |
| 6171 | | const src_lock = func.register_manager.lockRegAssumeUnused(src_reg); |
| 6238 | const reg = try func.register_manager.allocReg(null, abi.Registers.Integer.temporary); |
| 6239 | const src_lock = func.register_manager.lockRegAssumeUnused(reg); |
| 6172 | 6240 | defer func.register_manager.unlockReg(src_lock); |
| 6173 | 6241 | |
| 6174 | | try func.genSetMem(base, disp, ty, .{ .register = src_reg }); |
| 6242 | try func.genSetReg(ty, reg, src_mcv); |
| 6243 | try func.genSetMem(base, disp, ty, .{ .register = reg }); |
| 6175 | 6244 | }, |
| 6176 | 6245 | else => try func.genInlineMemcpy( |
| 6177 | 6246 | dst_ptr_mcv, |
| ... | ... | @@ -6180,6 +6249,46 @@ fn genSetMem( |
| 6180 | 6249 | ), |
| 6181 | 6250 | }, |
| 6182 | 6251 | .register => |reg| { |
| 6252 | if (reg.class() == .vector) { |
| 6253 | const addr_reg = try func.copyToTmpRegister(Type.usize, dst_ptr_mcv); |
| 6254 | |
| 6255 | const num_elem: u5 = math.cast(u5, ty.vectorLen(pt.zcu)) orelse { |
| 6256 | return func.fail("TODO: genBinOp use vsetvli for larger avl sizes", .{}); |
| 6257 | }; |
| 6258 | const elem_size = ty.childType(pt.zcu).bitSize(pt); |
| 6259 | |
| 6260 | try func.setVl(.zero, num_elem, .{ |
| 6261 | .vsew = switch (elem_size) { |
| 6262 | 8 => .@"8", |
| 6263 | 16 => .@"16", |
| 6264 | 32 => .@"32", |
| 6265 | 64 => .@"64", |
| 6266 | else => unreachable, |
| 6267 | }, |
| 6268 | .vlmul = .m1, |
| 6269 | .vma = true, |
| 6270 | .vta = true, |
| 6271 | }); |
| 6272 | |
| 6273 | _ = try func.addInst(.{ |
| 6274 | .tag = .pseudo, |
| 6275 | .ops = .pseudo_store_rm, |
| 6276 | .data = .{ .rm = .{ |
| 6277 | .r = reg, |
| 6278 | .m = .{ |
| 6279 | .base = .{ .reg = addr_reg }, |
| 6280 | .mod = .{ |
| 6281 | .disp = 0, |
| 6282 | .size = func.memSize(ty.childType(pt.zcu)), |
| 6283 | .unsigned = false, |
| 6284 | }, |
| 6285 | }, |
| 6286 | } }, |
| 6287 | }); |
| 6288 | |
| 6289 | return; |
| 6290 | } |
| 6291 | |
| 6183 | 6292 | const mem_size = switch (base) { |
| 6184 | 6293 | .frame => |base_fi| mem_size: { |
| 6185 | 6294 | assert(disp >= 0); |