authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-05-11 08:36:33-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-05-11 08:36:33-07:00
log5569e6b49d9b421d35e3175df36eb9fe7e4e8084
tree4622b826b87d84de1abc3598de27317d92f21041
parentc857959372ec87e3988c2548876d818d92df5f9a
parent2e6a6d7564901009aace5fef99e647959ad5bb90
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15639 from jacobly0/signed-mod

llvm/cbe: fix signed `@mod`/`@divFloor` computations

4 files changed, 64 insertions(+), 48 deletions(-)

lib/zig.h+17-20
...@@ -487,14 +487,14 @@ typedef ptrdiff_t intptr_t;...@@ -487,14 +487,14 @@ typedef ptrdiff_t intptr_t;
487 zig_basic_operator(uint##w##_t, div_floor_u##w, /) \487 zig_basic_operator(uint##w##_t, div_floor_u##w, /) \
488\488\
489 static inline int##w##_t zig_div_floor_i##w(int##w##_t lhs, int##w##_t rhs) { \489 static inline int##w##_t zig_div_floor_i##w(int##w##_t lhs, int##w##_t rhs) { \
490 return lhs / rhs - (((lhs ^ rhs) & (lhs % rhs)) < INT##w##_C(0)); \490 return lhs / rhs + (lhs % rhs != INT##w##_C(0) ? zig_shr_i##w(lhs ^ rhs, UINT8_C(w) - UINT8_C(1)) : INT##w##_C(0)); \
491 } \491 } \
492\492\
493 zig_basic_operator(uint##w##_t, mod_u##w, %) \493 zig_basic_operator(uint##w##_t, mod_u##w, %) \
494\494\
495 static inline int##w##_t zig_mod_i##w(int##w##_t lhs, int##w##_t rhs) { \495 static inline int##w##_t zig_mod_i##w(int##w##_t lhs, int##w##_t rhs) { \
496 int##w##_t rem = lhs % rhs; \496 int##w##_t rem = lhs % rhs; \
497 return rem + (((lhs ^ rhs) & rem) < INT##w##_C(0) ? rhs : INT##w##_C(0)); \497 return rem + (rem != INT##w##_C(0) ? rhs & zig_shr_i##w(lhs ^ rhs, UINT8_C(w) - UINT8_C(1)) : INT##w##_C(0)); \
498 } \498 } \
499\499\
500 static inline uint##w##_t zig_shlw_u##w(uint##w##_t lhs, uint8_t rhs, uint8_t bits) { \500 static inline uint##w##_t zig_shlw_u##w(uint##w##_t lhs, uint8_t rhs, uint8_t bits) { \
...@@ -1078,7 +1078,7 @@ static inline int64_t zig_bit_reverse_i64(int64_t val, uint8_t bits) {...@@ -1078,7 +1078,7 @@ static inline int64_t zig_bit_reverse_i64(int64_t val, uint8_t bits) {
1078 uint##w##_t temp = val - ((val >> 1) & (UINT##w##_MAX / 3)); \1078 uint##w##_t temp = val - ((val >> 1) & (UINT##w##_MAX / 3)); \
1079 temp = (temp & (UINT##w##_MAX / 5)) + ((temp >> 2) & (UINT##w##_MAX / 5)); \1079 temp = (temp & (UINT##w##_MAX / 5)) + ((temp >> 2) & (UINT##w##_MAX / 5)); \
1080 temp = (temp + (temp >> 4)) & (UINT##w##_MAX / 17); \1080 temp = (temp + (temp >> 4)) & (UINT##w##_MAX / 17); \
1081 return temp * (UINT##w##_MAX / 255) >> (w - 8); \1081 return temp * (UINT##w##_MAX / 255) >> (UINT8_C(w) - UINT8_C(8)); \
1082 } \1082 } \
1083\1083\
1084 zig_builtin_popcount_common(w)1084 zig_builtin_popcount_common(w)
...@@ -1298,15 +1298,6 @@ static inline zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) {...@@ -1298,15 +1298,6 @@ static inline zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) {
1298 return lhs % rhs;1298 return lhs % rhs;
1299}1299}
13001300
1301static inline zig_i128 zig_div_floor_i128(zig_i128 lhs, zig_i128 rhs) {
1302 return zig_div_trunc_i128(lhs, rhs) - (((lhs ^ rhs) & zig_rem_i128(lhs, rhs)) < zig_make_i128(0, 0));
1303}
1304
1305static inline zig_i128 zig_mod_i128(zig_i128 lhs, zig_i128 rhs) {
1306 zig_i128 rem = zig_rem_i128(lhs, rhs);
1307 return rem + (((lhs ^ rhs) & rem) < zig_make_i128(0, 0) ? rhs : zig_make_i128(0, 0));
1308}
1309
1310#else /* zig_has_int128 */1301#else /* zig_has_int128 */
13111302
1312static inline zig_u128 zig_not_u128(zig_u128 val, uint8_t bits) {1303static inline zig_u128 zig_not_u128(zig_u128 val, uint8_t bits) {
...@@ -1394,20 +1385,26 @@ static zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) {...@@ -1394,20 +1385,26 @@ static zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) {
1394 return __modti3(lhs, rhs);1385 return __modti3(lhs, rhs);
1395}1386}
13961387
1397static inline zig_i128 zig_mod_i128(zig_i128 lhs, zig_i128 rhs) {1388#endif /* zig_has_int128 */
1398 zig_i128 rem = zig_rem_i128(lhs, rhs);1389
1399 return zig_add_i128(rem, ((lhs.hi ^ rhs.hi) & rem.hi) < INT64_C(0) ? rhs : zig_make_i128(0, 0));1390#define zig_div_floor_u128 zig_div_trunc_u128
1400}
14011391
1402static inline zig_i128 zig_div_floor_i128(zig_i128 lhs, zig_i128 rhs) {1392static inline zig_i128 zig_div_floor_i128(zig_i128 lhs, zig_i128 rhs) {
1403 return zig_sub_i128(zig_div_trunc_i128(lhs, rhs), zig_make_i128(0, zig_cmp_i128(zig_and_i128(zig_xor_i128(lhs, rhs), zig_rem_i128(lhs, rhs)), zig_make_i128(0, 0)) < INT32_C(0)));1393 zig_i128 rem = zig_rem_i128(lhs, rhs);
1394 int64_t mask = zig_or_u64((uint64_t)zig_hi_i128(rem), zig_lo_i128(rem)) != UINT64_C(0)
1395 ? zig_shr_i64(zig_xor_i64(zig_hi_i128(lhs), zig_hi_i128(rhs)), UINT8_C(63)) : INT64_C(0);
1396 return zig_add_i128(zig_div_trunc_i128(lhs, rhs), zig_make_i128(mask, (uint64_t)mask));
1404}1397}
14051398
1406#endif /* zig_has_int128 */
1407
1408#define zig_div_floor_u128 zig_div_trunc_u128
1409#define zig_mod_u128 zig_rem_u1281399#define zig_mod_u128 zig_rem_u128
14101400
1401static inline zig_i128 zig_mod_i128(zig_i128 lhs, zig_i128 rhs) {
1402 zig_i128 rem = zig_rem_i128(lhs, rhs);
1403 int64_t mask = zig_or_u64((uint64_t)zig_hi_i128(rem), zig_lo_i128(rem)) != UINT64_C(0)
1404 ? zig_shr_i64(zig_xor_i64(zig_hi_i128(lhs), zig_hi_i128(rhs)), UINT8_C(63)) : INT64_C(0);
1405 return zig_add_i128(rem, zig_and_i128(rhs, zig_make_i128(mask, (uint64_t)mask)));
1406}
1407
1411static inline zig_u128 zig_min_u128(zig_u128 lhs, zig_u128 rhs) {1408static inline zig_u128 zig_min_u128(zig_u128 lhs, zig_u128 rhs) {
1412 return zig_cmp_u128(lhs, rhs) < INT32_C(0) ? lhs : rhs;1409 return zig_cmp_u128(lhs, rhs) < INT32_C(0) ? lhs : rhs;
1413}1410}
src/codegen/c.zig+3-10
...@@ -6508,23 +6508,16 @@ fn airSplat(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -6508,23 +6508,16 @@ fn airSplat(f: *Function, inst: Air.Inst.Index) !CValue {
65086508
6509 const inst_ty = f.air.typeOfIndex(inst);6509 const inst_ty = f.air.typeOfIndex(inst);
6510 const inst_scalar_ty = inst_ty.scalarType();6510 const inst_scalar_ty = inst_ty.scalarType();
6511 const inst_scalar_cty = try f.typeToIndex(inst_scalar_ty, .complete);
6512 const need_memcpy = f.indexToCType(inst_scalar_cty).tag() == .array;
65136511
6514 const writer = f.object.writer();6512 const writer = f.object.writer();
6515 const local = try f.allocLocal(inst, inst_ty);6513 const local = try f.allocLocal(inst, inst_ty);
6516 const v = try Vectorize.start(f, inst, writer, inst_ty);6514 const v = try Vectorize.start(f, inst, writer, inst_ty);
6517 if (need_memcpy) try writer.writeAll("memcpy(&");6515 const a = try Assignment.init(f, inst_scalar_ty);
6518 try f.writeCValue(writer, local, .Other);6516 try f.writeCValue(writer, local, .Other);
6519 try v.elem(f, writer);6517 try v.elem(f, writer);
6520 try writer.writeAll(if (need_memcpy) ", &" else " = ");6518 try a.assign(f, writer);
6521 try f.writeCValue(writer, operand, .Other);6519 try f.writeCValue(writer, operand, .Other);
6522 if (need_memcpy) {6520 try a.end(f, writer);
6523 try writer.writeAll(", sizeof(");
6524 try f.renderCType(writer, inst_scalar_cty);
6525 try writer.writeAll("))");
6526 }
6527 try writer.writeAll(";\n");
6528 try v.end(f, inst, writer);6521 try v.end(f, inst, writer);
65296522
6530 return local;6523 return local;
src/codegen/llvm.zig+41-18
...@@ -7215,20 +7215,28 @@ pub const FuncGen = struct {...@@ -7215,20 +7215,28 @@ pub const FuncGen = struct {
7215 return self.buildFloatOp(.floor, inst_ty, 1, .{result});7215 return self.buildFloatOp(.floor, inst_ty, 1, .{result});
7216 }7216 }
7217 if (scalar_ty.isSignedInt()) {7217 if (scalar_ty.isSignedInt()) {
7218 // const d = @divTrunc(a, b);7218 const target = self.dg.module.getTarget();
7219 // const r = @rem(a, b);7219 const inst_llvm_ty = try self.dg.lowerType(inst_ty);
7220 // return if (r == 0) d else d - ((a < 0) ^ (b < 0));7220 const scalar_bit_size_minus_one = scalar_ty.bitSize(target) - 1;
7221 const result_llvm_ty = try self.dg.lowerType(inst_ty);7221 const bit_size_minus_one = if (inst_ty.zigTypeTag() == .Vector) const_vector: {
7222 const zero = result_llvm_ty.constNull();7222 const vec_len = inst_ty.vectorLen();
7223 const div_trunc = self.builder.buildSDiv(lhs, rhs, "");7223 const scalar_llvm_ty = try self.dg.lowerType(scalar_ty);
7224
7225 const shifts = try self.gpa.alloc(*llvm.Value, vec_len);
7226 defer self.gpa.free(shifts);
7227
7228 @memset(shifts, scalar_llvm_ty.constInt(scalar_bit_size_minus_one, .False));
7229 break :const_vector llvm.constVector(shifts.ptr, vec_len);
7230 } else inst_llvm_ty.constInt(scalar_bit_size_minus_one, .False);
7231
7232 const div = self.builder.buildSDiv(lhs, rhs, "");
7224 const rem = self.builder.buildSRem(lhs, rhs, "");7233 const rem = self.builder.buildSRem(lhs, rhs, "");
7225 const rem_eq_0 = self.builder.buildICmp(.EQ, rem, zero, "");7234 const div_sign = self.builder.buildXor(lhs, rhs, "");
7226 const a_lt_0 = self.builder.buildICmp(.SLT, lhs, zero, "");7235 const div_sign_mask = self.builder.buildAShr(div_sign, bit_size_minus_one, "");
7227 const b_lt_0 = self.builder.buildICmp(.SLT, rhs, zero, "");7236 const zero = inst_llvm_ty.constNull();
7228 const a_b_xor = self.builder.buildXor(a_lt_0, b_lt_0, "");7237 const rem_nonzero = self.builder.buildICmp(.NE, rem, zero, "");
7229 const a_b_xor_ext = self.builder.buildZExt(a_b_xor, div_trunc.typeOf(), "");7238 const correction = self.builder.buildSelect(rem_nonzero, div_sign_mask, zero, "");
7230 const d_sub_xor = self.builder.buildSub(div_trunc, a_b_xor_ext, "");7239 return self.builder.buildNSWAdd(div, correction, "");
7231 return self.builder.buildSelect(rem_eq_0, div_trunc, d_sub_xor, "");
7232 }7240 }
7233 return self.builder.buildUDiv(lhs, rhs, "");7241 return self.builder.buildUDiv(lhs, rhs, "");
7234 }7242 }
...@@ -7280,12 +7288,27 @@ pub const FuncGen = struct {...@@ -7280,12 +7288,27 @@ pub const FuncGen = struct {
7280 return self.builder.buildSelect(ltz, c, a, "");7288 return self.builder.buildSelect(ltz, c, a, "");
7281 }7289 }
7282 if (scalar_ty.isSignedInt()) {7290 if (scalar_ty.isSignedInt()) {
7283 const a = self.builder.buildSRem(lhs, rhs, "");7291 const target = self.dg.module.getTarget();
7284 const b = self.builder.buildNSWAdd(a, rhs, "");7292 const scalar_bit_size_minus_one = scalar_ty.bitSize(target) - 1;
7285 const c = self.builder.buildSRem(b, rhs, "");7293 const bit_size_minus_one = if (inst_ty.zigTypeTag() == .Vector) const_vector: {
7294 const vec_len = inst_ty.vectorLen();
7295 const scalar_llvm_ty = try self.dg.lowerType(scalar_ty);
7296
7297 const shifts = try self.gpa.alloc(*llvm.Value, vec_len);
7298 defer self.gpa.free(shifts);
7299
7300 @memset(shifts, scalar_llvm_ty.constInt(scalar_bit_size_minus_one, .False));
7301 break :const_vector llvm.constVector(shifts.ptr, vec_len);
7302 } else inst_llvm_ty.constInt(scalar_bit_size_minus_one, .False);
7303
7304 const rem = self.builder.buildSRem(lhs, rhs, "");
7305 const div_sign = self.builder.buildXor(lhs, rhs, "");
7306 const div_sign_mask = self.builder.buildAShr(div_sign, bit_size_minus_one, "");
7307 const rhs_masked = self.builder.buildAnd(rhs, div_sign_mask, "");
7286 const zero = inst_llvm_ty.constNull();7308 const zero = inst_llvm_ty.constNull();
7287 const ltz = self.builder.buildICmp(.SLT, lhs, zero, "");7309 const rem_nonzero = self.builder.buildICmp(.NE, rem, zero, "");
7288 return self.builder.buildSelect(ltz, c, a, "");7310 const correction = self.builder.buildSelect(rem_nonzero, rhs_masked, zero, "");
7311 return self.builder.buildNSWAdd(rem, correction, "");
7289 }7312 }
7290 return self.builder.buildURem(lhs, rhs, "");7313 return self.builder.buildURem(lhs, rhs, "");
7291 }7314 }
test/behavior/math.zig+3
...@@ -449,6 +449,9 @@ fn testDivision() !void {...@@ -449,6 +449,9 @@ fn testDivision() !void {
449 try expect(mod(i32, 10, 12) == 10);449 try expect(mod(i32, 10, 12) == 10);
450 try expect(mod(i32, -14, 12) == 10);450 try expect(mod(i32, -14, 12) == 10);
451 try expect(mod(i32, -2, 12) == 10);451 try expect(mod(i32, -2, 12) == 10);
452 try expect(mod(i32, 10, -12) == -2);
453 try expect(mod(i32, -14, -12) == -2);
454 try expect(mod(i32, -2, -12) == -2);
452455
453 comptime {456 comptime {
454 try expect(457 try expect(