authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2022-02-05 20:36:48+13:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-02-06 21:39:34-05:00
log53e6c719efe5073307ee58436b17ee80f574b984
treea29757c8ae4e1fc4ad294646a97e0540fed904d3
parent545aa790a430bd2c8390435ee52f5fbe147f6c54

std/math: optimize division with divisors less than a half-limb

This adds a new path which avoids using compiler_rt generated div udivmod instructions in the case that a divisor is less than half the max usize value. Two half-limb divisions are performed instead which ensures that non-emulated division instructions are actually used. This does not improve the udivmod code which should still be reviewed independently of this issue. Notably this improves the performance of the toString implementation of non-power-of-two bases considerably. Division performance is improved ~1000% based on some coarse testing. The following test code is used to provide a rough comparison between the old vs. new method. ``` const std = @import("std"); const Managed = std.math.big.int.Managed; const allocator = std.heap.c_allocator; fn fib(a: *Managed, n: usize) !void { var b = try Managed.initSet(allocator, 1); defer b.deinit(); var c = try Managed.init(allocator); defer c.deinit(); var i: usize = 0; while (i < n) : (i += 1) { try c.add(a.toConst(), b.toConst()); a.swap(&b); b.swap(&c); } } pub fn main() !void { var a = try Managed.initSet(allocator, 0); defer a.deinit(); try fib(&a, 1_000_000); // Note: Next two lines (and printed digit count) omitted on no-print version. const as = try a.toString(allocator, 10, .lower); defer allocator.free(as); std.debug.print("fib: digit count: {}, limb count: {}\n", .{ as.len, a.limbs.len }); } ``` ``` ==> time.no-print <== limb count: 10849 ________________________________________________________ Executed in 10.60 secs fish external usr time 10.44 secs 0.00 millis 10.44 secs sys time 0.02 secs 1.12 millis 0.02 secs ==> time.old <== fib: digit count: 208988, limb count: 10849 ________________________________________________________ Executed in 22.78 secs fish external usr time 22.43 secs 1.01 millis 22.43 secs sys time 0.03 secs 0.13 millis 0.03 secs ==> time.optimized <== fib: digit count: 208988, limb count: 10849 ________________________________________________________ Executed in 11.59 secs fish external usr time 11.56 secs 1.03 millis 11.56 secs sys time 0.03 secs 0.12 millis 0.03 secs ``` Perf data for non-optimized and optimized, verifying no udivmod is generated by the new code. ``` $ perf report -i perf.data.old --stdio - Total Lost Samples: 0 - - Samples: 90K of event 'cycles:u' - Event count (approx.): 71603695208 - - Overhead Command Shared Object Symbol - ........ ....... ................ ........................................... - 52.97% t t [.] compiler_rt.udivmod.udivmod 45.97% t t [.] std.math.big.int.Mutable.addCarry 0.83% t t [.] main 0.08% t libc-2.33.so [.] __memmove_avx_unaligned_erms 0.08% t t [.] __udivti3 0.03% t [unknown] [k] 0xffffffff9a0010a7 0.02% t t [.] std.math.big.int.Managed.ensureCapacity 0.01% t libc-2.33.so [.] _int_malloc 0.00% t libc-2.33.so [.] __malloc_usable_size 0.00% t libc-2.33.so [.] _int_free 0.00% t t [.] 0x0000000000004a80 0.00% t t [.] std.heap.CAllocator.resize 0.00% t libc-2.33.so [.] _mid_memalign 0.00% t libc-2.33.so [.] sysmalloc 0.00% t libc-2.33.so [.] __posix_memalign 0.00% t t [.] std.heap.CAllocator.alloc 0.00% t ld-2.33.so [.] do_lookup_x $ perf report -i perf.data.optimized --stdio - Total Lost Samples: 0 - - Samples: 46K of event 'cycles:u' - Event count (approx.): 36790112336 - - Overhead Command Shared Object Symbol - ........ ....... ................ ........................................... - 79.98% t t [.] std.math.big.int.Mutable.addCarry 15.14% t t [.] main 4.58% t t [.] std.math.big.int.Managed.ensureCapacity 0.21% t libc-2.33.so [.] __memmove_avx_unaligned_erms 0.05% t [unknown] [k] 0xffffffff9a0010a7 0.02% t libc-2.33.so [.] _int_malloc 0.01% t t [.] std.heap.CAllocator.alloc 0.01% t libc-2.33.so [.] __malloc_usable_size 0.00% t libc-2.33.so [.] systrim.constprop.0 0.00% t libc-2.33.so [.] _mid_memalign 0.00% t t [.] 0x0000000000000c7d 0.00% t libc-2.33.so [.] malloc 0.00% t ld-2.33.so [.] check_match ``` Closes #10630.

3 files changed, 74 insertions(+), 4 deletions(-)

lib/std/math/big.zig+1
...@@ -7,6 +7,7 @@ pub const Limb = usize;...@@ -7,6 +7,7 @@ pub const Limb = usize;
7const limb_info = @typeInfo(Limb).Int;7const limb_info = @typeInfo(Limb).Int;
8pub const SignedLimb = std.meta.Int(.signed, limb_info.bits);8pub const SignedLimb = std.meta.Int(.signed, limb_info.bits);
9pub const DoubleLimb = std.meta.Int(.unsigned, 2 * limb_info.bits);9pub const DoubleLimb = std.meta.Int(.unsigned, 2 * limb_info.bits);
10pub const HalfLimb = std.meta.Int(.unsigned, limb_info.bits / 2);
10pub const SignedDoubleLimb = std.meta.Int(.signed, 2 * limb_info.bits);11pub const SignedDoubleLimb = std.meta.Int(.signed, 2 * limb_info.bits);
11pub const Log2Limb = std.math.Log2Int(Limb);12pub const Log2Limb = std.math.Log2Int(Limb);
1213
lib/std/math/big/int.zig+38-2
...@@ -2,6 +2,8 @@ const std = @import("../../std.zig");...@@ -2,6 +2,8 @@ const std = @import("../../std.zig");
2const math = std.math;2const math = std.math;
3const Limb = std.math.big.Limb;3const Limb = std.math.big.Limb;
4const limb_bits = @typeInfo(Limb).Int.bits;4const limb_bits = @typeInfo(Limb).Int.bits;
5const HalfLimb = std.math.big.HalfLimb;
6const half_limb_bits = @typeInfo(HalfLimb).Int.bits;
5const DoubleLimb = std.math.big.DoubleLimb;7const DoubleLimb = std.math.big.DoubleLimb;
6const SignedDoubleLimb = std.math.big.SignedDoubleLimb;8const SignedDoubleLimb = std.math.big.SignedDoubleLimb;
7const Log2Limb = std.math.big.Log2Limb;9const Log2Limb = std.math.big.Log2Limb;
...@@ -1335,7 +1337,16 @@ pub const Mutable = struct {...@@ -1335,7 +1337,16 @@ pub const Mutable = struct {
1335 const xy_trailing = math.min(x_trailing, y_trailing);1337 const xy_trailing = math.min(x_trailing, y_trailing);
13361338
1337 if (y.len - xy_trailing == 1) {1339 if (y.len - xy_trailing == 1) {
1338 lldiv1(q.limbs, &r.limbs[0], x.limbs[xy_trailing..x.len], y.limbs[y.len - 1]);1340 const divisor = y.limbs[y.len - 1];
1341
1342 // Optimization for small divisor. By using a half limb we can avoid requiring DoubleLimb
1343 // divisions in the hot code path. This may often require compiler_rt software-emulation.
1344 if (divisor < maxInt(HalfLimb)) {
1345 lldiv0p5(q.limbs, &r.limbs[0], x.limbs[xy_trailing..x.len], @intCast(HalfLimb, divisor));
1346 } else {
1347 lldiv1(q.limbs, &r.limbs[0], x.limbs[xy_trailing..x.len], divisor);
1348 }
1349
1339 q.normalize(x.len - xy_trailing);1350 q.normalize(x.len - xy_trailing);
1340 q.positive = q_positive;1351 q.positive = q_positive;
13411352
...@@ -1939,7 +1950,8 @@ pub const Const = struct {...@@ -1939,7 +1950,8 @@ pub const Const = struct {
1939 }1950 }
1940 } else {1951 } else {
1941 // Non power-of-two: batch divisions per word size.1952 // Non power-of-two: batch divisions per word size.
1942 const digits_per_limb = math.log(Limb, base, maxInt(Limb));1953 // We use a HalfLimb here so the division uses the faster lldiv0p5 over lldiv1 codepath.
1954 const digits_per_limb = math.log(HalfLimb, base, maxInt(HalfLimb));
1943 var limb_base: Limb = 1;1955 var limb_base: Limb = 1;
1944 var j: usize = 0;1956 var j: usize = 0;
1945 while (j < digits_per_limb) : (j += 1) {1957 while (j < digits_per_limb) : (j += 1) {
...@@ -3208,6 +3220,30 @@ fn lldiv1(quo: []Limb, rem: *Limb, a: []const Limb, b: Limb) void {...@@ -3208,6 +3220,30 @@ fn lldiv1(quo: []Limb, rem: *Limb, a: []const Limb, b: Limb) void {
3208 }3220 }
3209}3221}
32103222
3223fn lldiv0p5(quo: []Limb, rem: *Limb, a: []const Limb, b: HalfLimb) void {
3224 @setRuntimeSafety(debug_safety);
3225 assert(a.len > 1 or a[0] >= b);
3226 assert(quo.len >= a.len);
3227
3228 rem.* = 0;
3229 for (a) |_, ri| {
3230 const i = a.len - ri - 1;
3231 const ai_high = a[i] >> half_limb_bits;
3232 const ai_low = a[i] & ((1 << half_limb_bits) - 1);
3233
3234 // Split the division into two divisions acting on half a limb each. Carry remainder.
3235 const ai_high_with_carry = (rem.* << half_limb_bits) | ai_high;
3236 const ai_high_quo = ai_high_with_carry / b;
3237 rem.* = ai_high_with_carry % b;
3238
3239 const ai_low_with_carry = (rem.* << half_limb_bits) | ai_low;
3240 const ai_low_quo = ai_low_with_carry / b;
3241 rem.* = ai_low_with_carry % b;
3242
3243 quo[i] = (ai_high_quo << half_limb_bits) | ai_low_quo;
3244 }
3245}
3246
3211fn llshl(r: []Limb, a: []const Limb, shift: usize) void {3247fn llshl(r: []Limb, a: []const Limb, shift: usize) void {
3212 @setRuntimeSafety(debug_safety);3248 @setRuntimeSafety(debug_safety);
3213 assert(a.len >= 1);3249 assert(a.len >= 1);
lib/std/math/big/int_test.zig+35-2
...@@ -1064,7 +1064,7 @@ test "big.int mulWrap large" {...@@ -1064,7 +1064,7 @@ test "big.int mulWrap large" {
1064 try testing.expect(b.eq(c));1064 try testing.expect(b.eq(c));
1065}1065}
10661066
1067test "big.int div single-single no rem" {1067test "big.int div single-half no rem" {
1068 var a = try Managed.initSet(testing.allocator, 50);1068 var a = try Managed.initSet(testing.allocator, 50);
1069 defer a.deinit();1069 defer a.deinit();
1070 var b = try Managed.initSet(testing.allocator, 5);1070 var b = try Managed.initSet(testing.allocator, 5);
...@@ -1080,7 +1080,7 @@ test "big.int div single-single no rem" {...@@ -1080,7 +1080,7 @@ test "big.int div single-single no rem" {
1080 try testing.expect((try r.to(u32)) == 0);1080 try testing.expect((try r.to(u32)) == 0);
1081}1081}
10821082
1083test "big.int div single-single with rem" {1083test "big.int div single-half with rem" {
1084 var a = try Managed.initSet(testing.allocator, 49);1084 var a = try Managed.initSet(testing.allocator, 49);
1085 defer a.deinit();1085 defer a.deinit();
1086 var b = try Managed.initSet(testing.allocator, 5);1086 var b = try Managed.initSet(testing.allocator, 5);
...@@ -1096,6 +1096,39 @@ test "big.int div single-single with rem" {...@@ -1096,6 +1096,39 @@ test "big.int div single-single with rem" {
1096 try testing.expect((try r.to(u32)) == 4);1096 try testing.expect((try r.to(u32)) == 4);
1097}1097}
10981098
1099test "big.int div single-single no rem" {
1100 // assumes usize is <= 64 bits.
1101 var a = try Managed.initSet(testing.allocator, 1 << 52);
1102 defer a.deinit();
1103 var b = try Managed.initSet(testing.allocator, 1 << 35);
1104 defer b.deinit();
1105
1106 var q = try Managed.init(testing.allocator);
1107 defer q.deinit();
1108 var r = try Managed.init(testing.allocator);
1109 defer r.deinit();
1110 try Managed.divTrunc(&q, &r, a.toConst(), b.toConst());
1111
1112 try testing.expect((try q.to(u32)) == 131072);
1113 try testing.expect((try r.to(u32)) == 0);
1114}
1115
1116test "big.int div single-single with rem" {
1117 var a = try Managed.initSet(testing.allocator, (1 << 52) | (1 << 33));
1118 defer a.deinit();
1119 var b = try Managed.initSet(testing.allocator, (1 << 35));
1120 defer b.deinit();
1121
1122 var q = try Managed.init(testing.allocator);
1123 defer q.deinit();
1124 var r = try Managed.init(testing.allocator);
1125 defer r.deinit();
1126 try Managed.divTrunc(&q, &r, a.toConst(), b.toConst());
1127
1128 try testing.expect((try q.to(u64)) == 131072);
1129 try testing.expect((try r.to(u64)) == 8589934592);
1130}
1131
1099test "big.int div multi-single no rem" {1132test "big.int div multi-single no rem" {
1100 const op1 = 0xffffeeeeddddcccc;1133 const op1 = 0xffffeeeeddddcccc;
1101 const op2 = 34;1134 const op2 = 34;