authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2022-02-05 20:36:48+13:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-02-06 21:39:34-05:00
log53e6c719efe5073307ee58436b17ee80f574b984
treea29757c8ae4e1fc4ad294646a97e0540fed904d3
parent545aa790a430bd2c8390435ee52f5fbe147f6c54

std/math: optimize division with divisors less than a half-limb

This adds a new path which avoids using compiler_rt generated div udivmod instructions in the case that a divisor is less than half the max usize value. Two half-limb divisions are performed instead which ensures that non-emulated division instructions are actually used. This does not improve the udivmod code which should still be reviewed independently of this issue. Notably this improves the performance of the toString implementation of non-power-of-two bases considerably. Division performance is improved ~1000% based on some coarse testing. The following test code is used to provide a rough comparison between the old vs. new method. ``` const std = @import("std"); const Managed = std.math.big.int.Managed; const allocator = std.heap.c_allocator; fn fib(a: *Managed, n: usize) !void { var b = try Managed.initSet(allocator, 1); defer b.deinit(); var c = try Managed.init(allocator); defer c.deinit(); var i: usize = 0; while (i < n) : (i += 1) { try c.add(a.toConst(), b.toConst()); a.swap(&b); b.swap(&c); } } pub fn main() !void { var a = try Managed.initSet(allocator, 0); defer a.deinit(); try fib(&a, 1_000_000); // Note: Next two lines (and printed digit count) omitted on no-print version. const as = try a.toString(allocator, 10, .lower); defer allocator.free(as); std.debug.print("fib: digit count: {}, limb count: {}\n", .{ as.len, a.limbs.len }); } ``` ``` ==> time.no-print <== limb count: 10849 ________________________________________________________ Executed in 10.60 secs fish external usr time 10.44 secs 0.00 millis 10.44 secs sys time 0.02 secs 1.12 millis 0.02 secs ==> time.old <== fib: digit count: 208988, limb count: 10849 ________________________________________________________ Executed in 22.78 secs fish external usr time 22.43 secs 1.01 millis 22.43 secs sys time 0.03 secs 0.13 millis 0.03 secs ==> time.optimized <== fib: digit count: 208988, limb count: 10849 ________________________________________________________ Executed in 11.59 secs fish external usr time 11.56 secs 1.03 millis 11.56 secs sys time 0.03 secs 0.12 millis 0.03 secs ``` Perf data for non-optimized and optimized, verifying no udivmod is generated by the new code. ``` $ perf report -i perf.data.old --stdio - Total Lost Samples: 0 - - Samples: 90K of event 'cycles:u' - Event count (approx.): 71603695208 - - Overhead Command Shared Object Symbol - ........ ....... ................ ........................................... - 52.97% t t [.] compiler_rt.udivmod.udivmod 45.97% t t [.] std.math.big.int.Mutable.addCarry 0.83% t t [.] main 0.08% t libc-2.33.so [.] __memmove_avx_unaligned_erms 0.08% t t [.] __udivti3 0.03% t [unknown] [k] 0xffffffff9a0010a7 0.02% t t [.] std.math.big.int.Managed.ensureCapacity 0.01% t libc-2.33.so [.] _int_malloc 0.00% t libc-2.33.so [.] __malloc_usable_size 0.00% t libc-2.33.so [.] _int_free 0.00% t t [.] 0x0000000000004a80 0.00% t t [.] std.heap.CAllocator.resize 0.00% t libc-2.33.so [.] _mid_memalign 0.00% t libc-2.33.so [.] sysmalloc 0.00% t libc-2.33.so [.] __posix_memalign 0.00% t t [.] std.heap.CAllocator.alloc 0.00% t ld-2.33.so [.] do_lookup_x $ perf report -i perf.data.optimized --stdio - Total Lost Samples: 0 - - Samples: 46K of event 'cycles:u' - Event count (approx.): 36790112336 - - Overhead Command Shared Object Symbol - ........ ....... ................ ........................................... - 79.98% t t [.] std.math.big.int.Mutable.addCarry 15.14% t t [.] main 4.58% t t [.] std.math.big.int.Managed.ensureCapacity 0.21% t libc-2.33.so [.] __memmove_avx_unaligned_erms 0.05% t [unknown] [k] 0xffffffff9a0010a7 0.02% t libc-2.33.so [.] _int_malloc 0.01% t t [.] std.heap.CAllocator.alloc 0.01% t libc-2.33.so [.] __malloc_usable_size 0.00% t libc-2.33.so [.] systrim.constprop.0 0.00% t libc-2.33.so [.] _mid_memalign 0.00% t t [.] 0x0000000000000c7d 0.00% t libc-2.33.so [.] malloc 0.00% t ld-2.33.so [.] check_match ``` Closes #10630.

3 files changed, 74 insertions(+), 4 deletions(-)

lib/std/math/big.zig+1
......@@ -7,6 +7,7 @@ pub const Limb = usize;
77const limb_info = @typeInfo(Limb).Int;
88pub const SignedLimb = std.meta.Int(.signed, limb_info.bits);
99pub const DoubleLimb = std.meta.Int(.unsigned, 2 * limb_info.bits);
10pub const HalfLimb = std.meta.Int(.unsigned, limb_info.bits / 2);
1011pub const SignedDoubleLimb = std.meta.Int(.signed, 2 * limb_info.bits);
1112pub const Log2Limb = std.math.Log2Int(Limb);
1213
lib/std/math/big/int.zig+38-2
......@@ -2,6 +2,8 @@ const std = @import("../../std.zig");
22const math = std.math;
33const Limb = std.math.big.Limb;
44const limb_bits = @typeInfo(Limb).Int.bits;
5const HalfLimb = std.math.big.HalfLimb;
6const half_limb_bits = @typeInfo(HalfLimb).Int.bits;
57const DoubleLimb = std.math.big.DoubleLimb;
68const SignedDoubleLimb = std.math.big.SignedDoubleLimb;
79const Log2Limb = std.math.big.Log2Limb;
......@@ -1335,7 +1337,16 @@ pub const Mutable = struct {
13351337 const xy_trailing = math.min(x_trailing, y_trailing);
13361338
13371339 if (y.len - xy_trailing == 1) {
1338 lldiv1(q.limbs, &r.limbs[0], x.limbs[xy_trailing..x.len], y.limbs[y.len - 1]);
1340 const divisor = y.limbs[y.len - 1];
1341
1342 // Optimization for small divisor. By using a half limb we can avoid requiring DoubleLimb
1343 // divisions in the hot code path. This may often require compiler_rt software-emulation.
1344 if (divisor < maxInt(HalfLimb)) {
1345 lldiv0p5(q.limbs, &r.limbs[0], x.limbs[xy_trailing..x.len], @intCast(HalfLimb, divisor));
1346 } else {
1347 lldiv1(q.limbs, &r.limbs[0], x.limbs[xy_trailing..x.len], divisor);
1348 }
1349
13391350 q.normalize(x.len - xy_trailing);
13401351 q.positive = q_positive;
13411352
......@@ -1939,7 +1950,8 @@ pub const Const = struct {
19391950 }
19401951 } else {
19411952 // Non power-of-two: batch divisions per word size.
1942 const digits_per_limb = math.log(Limb, base, maxInt(Limb));
1953 // We use a HalfLimb here so the division uses the faster lldiv0p5 over lldiv1 codepath.
1954 const digits_per_limb = math.log(HalfLimb, base, maxInt(HalfLimb));
19431955 var limb_base: Limb = 1;
19441956 var j: usize = 0;
19451957 while (j < digits_per_limb) : (j += 1) {
......@@ -3208,6 +3220,30 @@ fn lldiv1(quo: []Limb, rem: *Limb, a: []const Limb, b: Limb) void {
32083220 }
32093221}
32103222
3223fn lldiv0p5(quo: []Limb, rem: *Limb, a: []const Limb, b: HalfLimb) void {
3224 @setRuntimeSafety(debug_safety);
3225 assert(a.len > 1 or a[0] >= b);
3226 assert(quo.len >= a.len);
3227
3228 rem.* = 0;
3229 for (a) |_, ri| {
3230 const i = a.len - ri - 1;
3231 const ai_high = a[i] >> half_limb_bits;
3232 const ai_low = a[i] & ((1 << half_limb_bits) - 1);
3233
3234 // Split the division into two divisions acting on half a limb each. Carry remainder.
3235 const ai_high_with_carry = (rem.* << half_limb_bits) | ai_high;
3236 const ai_high_quo = ai_high_with_carry / b;
3237 rem.* = ai_high_with_carry % b;
3238
3239 const ai_low_with_carry = (rem.* << half_limb_bits) | ai_low;
3240 const ai_low_quo = ai_low_with_carry / b;
3241 rem.* = ai_low_with_carry % b;
3242
3243 quo[i] = (ai_high_quo << half_limb_bits) | ai_low_quo;
3244 }
3245}
3246
32113247fn llshl(r: []Limb, a: []const Limb, shift: usize) void {
32123248 @setRuntimeSafety(debug_safety);
32133249 assert(a.len >= 1);
lib/std/math/big/int_test.zig+35-2
......@@ -1064,7 +1064,7 @@ test "big.int mulWrap large" {
10641064 try testing.expect(b.eq(c));
10651065}
10661066
1067test "big.int div single-single no rem" {
1067test "big.int div single-half no rem" {
10681068 var a = try Managed.initSet(testing.allocator, 50);
10691069 defer a.deinit();
10701070 var b = try Managed.initSet(testing.allocator, 5);
......@@ -1080,7 +1080,7 @@ test "big.int div single-single no rem" {
10801080 try testing.expect((try r.to(u32)) == 0);
10811081}
10821082
1083test "big.int div single-single with rem" {
1083test "big.int div single-half with rem" {
10841084 var a = try Managed.initSet(testing.allocator, 49);
10851085 defer a.deinit();
10861086 var b = try Managed.initSet(testing.allocator, 5);
......@@ -1096,6 +1096,39 @@ test "big.int div single-single with rem" {
10961096 try testing.expect((try r.to(u32)) == 4);
10971097}
10981098
1099test "big.int div single-single no rem" {
1100 // assumes usize is <= 64 bits.
1101 var a = try Managed.initSet(testing.allocator, 1 << 52);
1102 defer a.deinit();
1103 var b = try Managed.initSet(testing.allocator, 1 << 35);
1104 defer b.deinit();
1105
1106 var q = try Managed.init(testing.allocator);
1107 defer q.deinit();
1108 var r = try Managed.init(testing.allocator);
1109 defer r.deinit();
1110 try Managed.divTrunc(&q, &r, a.toConst(), b.toConst());
1111
1112 try testing.expect((try q.to(u32)) == 131072);
1113 try testing.expect((try r.to(u32)) == 0);
1114}
1115
1116test "big.int div single-single with rem" {
1117 var a = try Managed.initSet(testing.allocator, (1 << 52) | (1 << 33));
1118 defer a.deinit();
1119 var b = try Managed.initSet(testing.allocator, (1 << 35));
1120 defer b.deinit();
1121
1122 var q = try Managed.init(testing.allocator);
1123 defer q.deinit();
1124 var r = try Managed.init(testing.allocator);
1125 defer r.deinit();
1126 try Managed.divTrunc(&q, &r, a.toConst(), b.toConst());
1127
1128 try testing.expect((try q.to(u64)) == 131072);
1129 try testing.expect((try r.to(u64)) == 8589934592);
1130}
1131
10991132test "big.int div multi-single no rem" {
11001133 const op1 = 0xffffeeeeddddcccc;
11011134 const op2 = 34;