| ... | ... | @@ -88,8 +88,9 @@ exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .{}, |
| 88 | 88 | /// across each runtime branch upon joining. |
| 89 | 89 | branch_stack: *std.ArrayList(Branch), |
| 90 | 90 | |
| 91 | | // The current bit length of vector registers. |
| 92 | | vec_len: u32, |
| 91 | // Currently set vector properties, null means they haven't been set yet in the function. |
| 92 | avl: ?u64, |
| 93 | vtype: ?bits.VType, |
| 93 | 94 | |
| 94 | 95 | // Key is the block instruction |
| 95 | 96 | blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{}, |
| ... | ... | @@ -751,7 +752,8 @@ pub fn generate( |
| 751 | 752 | .end_di_line = func.rbrace_line, |
| 752 | 753 | .end_di_column = func.rbrace_column, |
| 753 | 754 | .scope_generation = 0, |
| 754 | | .vec_len = 16 * 8, // TODO: set this per cpu |
| 755 | .avl = null, |
| 756 | .vtype = null, |
| 755 | 757 | }; |
| 756 | 758 | defer { |
| 757 | 759 | function.frame_allocs.deinit(gpa); |
| ... | ... | @@ -1064,8 +1066,17 @@ fn getCsr(func: *Func, csr: CSR) !Register { |
| 1064 | 1066 | return dst_reg; |
| 1065 | 1067 | } |
| 1066 | 1068 | |
| 1067 | | fn setVl(func: *Func, dst_reg: Register, avl: u5, options: bits.VType) !void { |
| 1069 | fn setVl(func: *Func, dst_reg: Register, avl: u64, options: bits.VType) !void { |
| 1070 | if (func.avl == avl) if (func.vtype) |vtype| { |
| 1071 | // it's already set, we don't need to do anything |
| 1072 | if (@as(u8, @bitCast(vtype)) == @as(u8, @bitCast(options))) return; |
| 1073 | }; |
| 1074 | |
| 1075 | func.avl = avl; |
| 1076 | func.vtype = options; |
| 1077 | |
| 1068 | 1078 | if (avl == 0) { |
| 1079 | // the caller means to do "vsetvli zero, zero ..." which keeps the avl to whatever it was before |
| 1069 | 1080 | const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options)); |
| 1070 | 1081 | _ = try func.addInst(.{ |
| 1071 | 1082 | .tag = .vsetvli, |
| ... | ... | @@ -1077,18 +1088,33 @@ fn setVl(func: *Func, dst_reg: Register, avl: u5, options: bits.VType) !void { |
| 1077 | 1088 | } }, |
| 1078 | 1089 | }); |
| 1079 | 1090 | } else { |
| 1080 | | const options_int: u12 = (~@as(u12, 0) << 10) | @as(u8, @bitCast(options)); |
| 1081 | | _ = try func.addInst(.{ |
| 1082 | | .tag = .vsetivli, |
| 1083 | | .ops = .rri, |
| 1084 | | .data = .{ |
| 1085 | | .i_type = .{ |
| 1091 | // if the avl can fit into u5 we can use vsetivli otherwise use vsetvli |
| 1092 | if (avl <= std.math.maxInt(u5)) { |
| 1093 | const options_int: u12 = (~@as(u12, 0) << 10) | @as(u8, @bitCast(options)); |
| 1094 | _ = try func.addInst(.{ |
| 1095 | .tag = .vsetivli, |
| 1096 | .ops = .rri, |
| 1097 | .data = .{ |
| 1098 | .i_type = .{ |
| 1099 | .rd = dst_reg, |
| 1100 | .rs1 = @enumFromInt(avl), |
| 1101 | .imm12 = Immediate.u(options_int), |
| 1102 | }, |
| 1103 | }, |
| 1104 | }); |
| 1105 | } else { |
| 1106 | const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options)); |
| 1107 | const temp_reg = try func.copyToTmpRegister(Type.usize, .{ .immediate = avl }); |
| 1108 | _ = try func.addInst(.{ |
| 1109 | .tag = .vsetvli, |
| 1110 | .ops = .rri, |
| 1111 | .data = .{ .i_type = .{ |
| 1086 | 1112 | .rd = dst_reg, |
| 1087 | | .rs1 = @enumFromInt(avl), |
| 1113 | .rs1 = temp_reg, |
| 1088 | 1114 | .imm12 = Immediate.u(options_int), |
| 1089 | | }, |
| 1090 | | }, |
| 1091 | | }); |
| 1115 | } }, |
| 1116 | }); |
| 1117 | } |
| 1092 | 1118 | } |
| 1093 | 1119 | } |
| 1094 | 1120 | |
| ... | ... | @@ -1939,7 +1965,7 @@ fn allocRegOrMem(func: *Func, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool |
| 1939 | 1965 | const bit_size = elem_ty.bitSize(pt); |
| 1940 | 1966 | const min_size: u64 = switch (elem_ty.zigTypeTag(pt.zcu)) { |
| 1941 | 1967 | .Float => if (func.hasFeature(.d)) 64 else 32, |
| 1942 | | .Vector => func.vec_len, |
| 1968 | .Vector => 256, // TODO: calculate it from avl * vsew |
| 1943 | 1969 | else => 64, |
| 1944 | 1970 | }; |
| 1945 | 1971 | |
| ... | ... | @@ -2293,7 +2319,11 @@ fn binOp( |
| 2293 | 2319 | return func.fail("binOp libcall runtime-float ops", .{}); |
| 2294 | 2320 | } |
| 2295 | 2321 | |
| 2296 | | if (lhs_ty.bitSize(pt) > 64) return func.fail("TODO: binOp >= 64 bits", .{}); |
| 2322 | // don't have support for certain sizes of addition |
| 2323 | switch (lhs_ty.zigTypeTag(pt.zcu)) { |
| 2324 | .Vector => {}, // works differently and fails in a different place |
| 2325 | else => if (lhs_ty.bitSize(pt) > 64) return func.fail("TODO: binOp >= 64 bits", .{}), |
| 2326 | } |
| 2297 | 2327 | |
| 2298 | 2328 | const lhs_mcv = try func.resolveInst(lhs_air); |
| 2299 | 2329 | const rhs_mcv = try func.resolveInst(rhs_air); |
| ... | ... | @@ -2442,17 +2472,25 @@ fn genBinOp( |
| 2442 | 2472 | }); |
| 2443 | 2473 | }, |
| 2444 | 2474 | .Vector => { |
| 2475 | const num_elem = lhs_ty.vectorLen(zcu); |
| 2476 | const elem_size = lhs_ty.childType(zcu).bitSize(pt); |
| 2477 | |
| 2478 | const child_ty = lhs_ty.childType(zcu); |
| 2479 | |
| 2445 | 2480 | const mir_tag: Mir.Inst.Tag = switch (tag) { |
| 2446 | | .add => .vaddvv, |
| 2447 | | .sub => .vsubvv, |
| 2481 | .add => switch (child_ty.zigTypeTag(zcu)) { |
| 2482 | .Int => .vaddvv, |
| 2483 | .Float => .vfaddvv, |
| 2484 | else => unreachable, |
| 2485 | }, |
| 2486 | .sub => switch (child_ty.zigTypeTag(zcu)) { |
| 2487 | .Int => .vsubvv, |
| 2488 | .Float => .vfsubvv, |
| 2489 | else => unreachable, |
| 2490 | }, |
| 2448 | 2491 | else => return func.fail("TODO: genBinOp {s} Vector", .{@tagName(tag)}), |
| 2449 | 2492 | }; |
| 2450 | 2493 | |
| 2451 | | const num_elem: u5 = math.cast(u5, lhs_ty.vectorLen(zcu)) orelse { |
| 2452 | | return func.fail("TODO: genBinOp use vsetvli for larger avl sizes", .{}); |
| 2453 | | }; |
| 2454 | | const elem_size = lhs_ty.childType(zcu).bitSize(pt); |
| 2455 | | |
| 2456 | 2494 | try func.setVl(.zero, num_elem, .{ |
| 2457 | 2495 | .vsew = switch (elem_size) { |
| 2458 | 2496 | 8 => .@"8", |
| ... | ... | @@ -2761,78 +2799,55 @@ fn airAddWithOverflow(func: *Func, inst: Air.Inst.Index) !void { |
| 2761 | 2799 | const extra = func.air.extraData(Air.Bin, ty_pl.payload).data; |
| 2762 | 2800 | |
| 2763 | 2801 | const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { |
| 2764 | | const lhs_ty = func.typeOf(extra.lhs); |
| 2765 | | |
| 2766 | | const int_info = lhs_ty.intInfo(zcu); |
| 2767 | | |
| 2768 | | const tuple_ty = func.typeOfIndex(inst); |
| 2769 | | const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false); |
| 2770 | | const offset = result_mcv.load_frame; |
| 2802 | const ty = func.typeOf(extra.lhs); |
| 2803 | switch (ty.zigTypeTag(zcu)) { |
| 2804 | .Vector => return func.fail("TODO implement add with overflow for Vector type", .{}), |
| 2805 | .Int => { |
| 2806 | const int_info = ty.intInfo(zcu); |
| 2771 | 2807 | |
| 2772 | | if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) { |
| 2773 | | const add_result = try func.binOp(null, .add, extra.lhs, extra.rhs); |
| 2774 | | const add_result_reg = try func.copyToTmpRegister(lhs_ty, add_result); |
| 2775 | | const add_result_reg_lock = func.register_manager.lockRegAssumeUnused(add_result_reg); |
| 2776 | | defer func.register_manager.unlockReg(add_result_reg_lock); |
| 2808 | const tuple_ty = func.typeOfIndex(inst); |
| 2809 | const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false); |
| 2810 | const offset = result_mcv.load_frame; |
| 2777 | 2811 | |
| 2778 | | const shift_amount: u6 = @intCast(Type.usize.bitSize(pt) - int_info.bits); |
| 2812 | if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) { |
| 2813 | const add_result = try func.binOp(null, .add, extra.lhs, extra.rhs); |
| 2779 | 2814 | |
| 2780 | | const shift_reg, const shift_lock = try func.allocReg(.int); |
| 2781 | | defer func.register_manager.unlockReg(shift_lock); |
| 2815 | const add_result_reg = try func.copyToTmpRegister(ty, add_result); |
| 2816 | const add_result_reg_lock = func.register_manager.lockRegAssumeUnused(add_result_reg); |
| 2817 | defer func.register_manager.unlockReg(add_result_reg_lock); |
| 2782 | 2818 | |
| 2783 | | _ = try func.addInst(.{ |
| 2784 | | .tag = .slli, |
| 2785 | | .ops = .rri, |
| 2786 | | .data = .{ |
| 2787 | | .i_type = .{ |
| 2788 | | .rd = shift_reg, |
| 2789 | | .rs1 = add_result_reg, |
| 2790 | | .imm12 = Immediate.u(shift_amount), |
| 2791 | | }, |
| 2792 | | }, |
| 2793 | | }); |
| 2794 | | |
| 2795 | | _ = try func.addInst(.{ |
| 2796 | | .tag = if (int_info.signedness == .unsigned) .srli else .srai, |
| 2797 | | .ops = .rri, |
| 2798 | | .data = .{ |
| 2799 | | .i_type = .{ |
| 2800 | | .rd = shift_reg, |
| 2801 | | .rs1 = shift_reg, |
| 2802 | | .imm12 = Immediate.u(shift_amount), |
| 2803 | | }, |
| 2804 | | }, |
| 2805 | | }); |
| 2806 | | |
| 2807 | | try func.genSetMem( |
| 2808 | | .{ .frame = offset.index }, |
| 2809 | | offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, pt))), |
| 2810 | | lhs_ty, |
| 2811 | | add_result, |
| 2812 | | ); |
| 2819 | try func.genSetMem( |
| 2820 | .{ .frame = offset.index }, |
| 2821 | offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, pt))), |
| 2822 | ty, |
| 2823 | add_result, |
| 2824 | ); |
| 2813 | 2825 | |
| 2814 | | const overflow_reg, const overflow_lock = try func.allocReg(.int); |
| 2815 | | defer func.register_manager.unlockReg(overflow_lock); |
| 2826 | const overflow_reg, const overflow_lock = try func.allocReg(.int); |
| 2827 | defer func.register_manager.unlockReg(overflow_lock); |
| 2816 | 2828 | |
| 2817 | | try func.genBinOp( |
| 2818 | | .cmp_neq, |
| 2819 | | .{ .register = shift_reg }, |
| 2820 | | lhs_ty, |
| 2821 | | .{ .register = add_result_reg }, |
| 2822 | | lhs_ty, |
| 2823 | | overflow_reg, |
| 2824 | | ); |
| 2829 | try func.genBinOp( |
| 2830 | .cmp_neq, |
| 2831 | .{ .register = add_result_reg }, |
| 2832 | ty, |
| 2833 | .{ .register = add_result_reg }, |
| 2834 | ty, |
| 2835 | overflow_reg, |
| 2836 | ); |
| 2825 | 2837 | |
| 2826 | | try func.genSetMem( |
| 2827 | | .{ .frame = offset.index }, |
| 2828 | | offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, pt))), |
| 2829 | | Type.u1, |
| 2830 | | .{ .register = overflow_reg }, |
| 2831 | | ); |
| 2838 | try func.genSetMem( |
| 2839 | .{ .frame = offset.index }, |
| 2840 | offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, pt))), |
| 2841 | Type.u1, |
| 2842 | .{ .register = overflow_reg }, |
| 2843 | ); |
| 2832 | 2844 | |
| 2833 | | break :result result_mcv; |
| 2834 | | } else { |
| 2835 | | return func.fail("TODO: less than 8 bit or non-pow 2 addition", .{}); |
| 2845 | break :result result_mcv; |
| 2846 | } else { |
| 2847 | return func.fail("TODO: less than 8 bit or non-pow 2 addition", .{}); |
| 2848 | } |
| 2849 | }, |
| 2850 | else => unreachable, |
| 2836 | 2851 | } |
| 2837 | 2852 | }; |
| 2838 | 2853 | |
| ... | ... | @@ -5519,8 +5534,6 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { |
| 5519 | 5534 | const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra_i..][0..extra.data.inputs_len]); |
| 5520 | 5535 | extra_i += inputs.len; |
| 5521 | 5536 | |
| 5522 | | log.debug("airAsm input: {any}", .{inputs}); |
| 5523 | | |
| 5524 | 5537 | const dead = !is_volatile and func.liveness.isUnused(inst); |
| 5525 | 5538 | const result: MCValue = if (dead) .unreach else result: { |
| 5526 | 5539 | if (outputs.len > 1) { |
| ... | ... | @@ -5897,7 +5910,7 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 5897 | 5910 | const max_size: u32 = switch (reg.class()) { |
| 5898 | 5911 | .int => 64, |
| 5899 | 5912 | .float => if (func.hasFeature(.d)) 64 else 32, |
| 5900 | | .vector => func.vec_len, |
| 5913 | .vector => 64, // TODO: calculate it from avl * vsew |
| 5901 | 5914 | }; |
| 5902 | 5915 | if (abi_size > max_size) return std.debug.panic("tried to set reg with size {}", .{abi_size}); |
| 5903 | 5916 | const dst_reg_class = reg.class(); |
| ... | ... | @@ -6033,6 +6046,8 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 6033 | 6046 | .register_pair => return func.fail("genSetReg should we allow reg -> reg_pair?", .{}), |
| 6034 | 6047 | .load_frame => |frame| { |
| 6035 | 6048 | if (reg.class() == .vector) { |
| 6049 | // vectors don't support an offset memory load so we need to put the true |
| 6050 | // address into a register before loading from it. |
| 6036 | 6051 | const addr_reg, const addr_lock = try func.allocReg(.int); |
| 6037 | 6052 | defer func.register_manager.unlockReg(addr_lock); |
| 6038 | 6053 | |
| ... | ... | @@ -6073,28 +6088,30 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 6073 | 6088 | _ = try func.addInst(.{ |
| 6074 | 6089 | .tag = .pseudo, |
| 6075 | 6090 | .ops = .pseudo_lea_rm, |
| 6076 | | .data = .{ .rm = .{ |
| 6077 | | .r = reg, |
| 6078 | | .m = switch (src_mcv) { |
| 6079 | | .register_offset => |reg_off| .{ |
| 6080 | | .base = .{ .reg = reg_off.reg }, |
| 6081 | | .mod = .{ |
| 6082 | | .size = func.memSize(ty), |
| 6083 | | .disp = reg_off.off, |
| 6084 | | .unsigned = false, |
| 6091 | .data = .{ |
| 6092 | .rm = .{ |
| 6093 | .r = reg, |
| 6094 | .m = switch (src_mcv) { |
| 6095 | .register_offset => |reg_off| .{ |
| 6096 | .base = .{ .reg = reg_off.reg }, |
| 6097 | .mod = .{ |
| 6098 | .size = .byte, // the size doesn't matter |
| 6099 | .disp = reg_off.off, |
| 6100 | .unsigned = false, |
| 6101 | }, |
| 6085 | 6102 | }, |
| 6086 | | }, |
| 6087 | | .lea_frame => |frame| .{ |
| 6088 | | .base = .{ .frame = frame.index }, |
| 6089 | | .mod = .{ |
| 6090 | | .size = func.memSize(ty), |
| 6091 | | .disp = frame.off, |
| 6092 | | .unsigned = false, |
| 6103 | .lea_frame => |frame| .{ |
| 6104 | .base = .{ .frame = frame.index }, |
| 6105 | .mod = .{ |
| 6106 | .size = .byte, // the size doesn't matter |
| 6107 | .disp = frame.off, |
| 6108 | .unsigned = false, |
| 6109 | }, |
| 6093 | 6110 | }, |
| 6111 | else => unreachable, |
| 6094 | 6112 | }, |
| 6095 | | else => unreachable, |
| 6096 | 6113 | }, |
| 6097 | | } }, |
| 6114 | }, |
| 6098 | 6115 | }); |
| 6099 | 6116 | }, |
| 6100 | 6117 | .indirect => |reg_off| { |
| ... | ... | @@ -6119,9 +6136,7 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 6119 | 6136 | // There is no vector instruction for loading with an offset to a base register, |
| 6120 | 6137 | // so we need to get an offset register containing the address of the vector first |
| 6121 | 6138 | // and load from it. |
| 6122 | | const len: u5 = math.cast(u5, ty.vectorLen(zcu)) orelse { |
| 6123 | | return func.fail("TODO: genSetReg load_frame -> vec reg, vector length doesn't fit into imm avl", .{}); |
| 6124 | | }; |
| 6139 | const len = ty.vectorLen(zcu); |
| 6125 | 6140 | const elem_ty = ty.childType(zcu); |
| 6126 | 6141 | const elem_size = elem_ty.abiSize(pt); |
| 6127 | 6142 | |
| ... | ... | @@ -6202,6 +6217,8 @@ fn genSetMem( |
| 6202 | 6217 | src_mcv: MCValue, |
| 6203 | 6218 | ) InnerError!void { |
| 6204 | 6219 | const pt = func.pt; |
| 6220 | const zcu = pt.zcu; |
| 6221 | |
| 6205 | 6222 | const abi_size: u32 = @intCast(ty.abiSize(pt)); |
| 6206 | 6223 | const dst_ptr_mcv: MCValue = switch (base) { |
| 6207 | 6224 | .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, |
| ... | ... | @@ -6252,10 +6269,8 @@ fn genSetMem( |
| 6252 | 6269 | if (reg.class() == .vector) { |
| 6253 | 6270 | const addr_reg = try func.copyToTmpRegister(Type.usize, dst_ptr_mcv); |
| 6254 | 6271 | |
| 6255 | | const num_elem: u5 = math.cast(u5, ty.vectorLen(pt.zcu)) orelse { |
| 6256 | | return func.fail("TODO: genBinOp use vsetvli for larger avl sizes", .{}); |
| 6257 | | }; |
| 6258 | | const elem_size = ty.childType(pt.zcu).bitSize(pt); |
| 6272 | const num_elem = ty.vectorLen(zcu); |
| 6273 | const elem_size = ty.childType(zcu).bitSize(pt); |
| 6259 | 6274 | |
| 6260 | 6275 | try func.setVl(.zero, num_elem, .{ |
| 6261 | 6276 | .vsew = switch (elem_size) { |
| ... | ... | @@ -6279,7 +6294,7 @@ fn genSetMem( |
| 6279 | 6294 | .base = .{ .reg = addr_reg }, |
| 6280 | 6295 | .mod = .{ |
| 6281 | 6296 | .disp = 0, |
| 6282 | | .size = func.memSize(ty.childType(pt.zcu)), |
| 6297 | .size = func.memSize(ty.childType(zcu)), |
| 6283 | 6298 | .unsigned = false, |
| 6284 | 6299 | }, |
| 6285 | 6300 | }, |