authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2023-04-13 18:20:17+12:00
committergravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2023-06-17 13:50:04+12:00
log947dd36341bdef0cb359a008257e7f1638f60594
treee472f129f9bfa370ea9e35dff7dbf66f4df94624
parent0f5aff34414bcb024443540fe905039f3783803a

optimize udivmod

See https://reviews.llvm.org/D81809 for upstream description. In summary this is ~10x improvement for small divisors and similar performance for equal divisors. Closes #13523.

1 files changed, 136 insertions(+), 180 deletions(-)

lib/compiler_rt/udivmod.zig+136-180
...@@ -1,201 +1,157 @@...@@ -1,201 +1,157 @@
1const std = @import("std");
1const builtin = @import("builtin");2const builtin = @import("builtin");
2const is_test = builtin.is_test;3const is_test = builtin.is_test;
3const native_endian = builtin.cpu.arch.endian();4const Log2Int = std.math.Log2Int;
4const std = @import("std");
55
6const low = switch (native_endian) {6const lo = switch (builtin.cpu.arch.endian()) {
7 .Big => 1,7 .Big => 1,
8 .Little => 0,8 .Little => 0,
9};9};
10const high = 1 - low;10const hi = 1 - lo;
1111
12pub fn udivmod(comptime DoubleInt: type, a: DoubleInt, b: DoubleInt, maybe_rem: ?*DoubleInt) DoubleInt {12fn HalfInt(comptime T: type) type {
13 std.debug.assert(@typeInfo(T) == .Int);
14 std.debug.assert(@bitSizeOf(T) % 2 == 0);
15 return std.meta.Int(.unsigned, @bitSizeOf(T) / 2);
16}
17
18// Performs division of a double-word specified in its single-word components. Most commonly used
19// for computing u128 bit divisions in terms of 64-bit integers.
20//
21// q = U / v
22// r = U % v
23// where U = (u1 | u0)
24fn divwide_generic(comptime T: type, _u1: T, _u0: T, v_: T, r: *T) T {
13 @setRuntimeSafety(is_test);25 @setRuntimeSafety(is_test);
26 var v = v_;
1427
15 const double_int_bits = @typeInfo(DoubleInt).Int.bits;28 const b = @as(T, 1) << (@bitSizeOf(T) / 2);
16 const single_int_bits = @divExact(double_int_bits, 2);29 var un64: T = undefined;
17 const SingleInt = std.meta.Int(.unsigned, single_int_bits);30 var un10: T = undefined;
18 const SignedDoubleInt = std.meta.Int(.signed, double_int_bits);31
19 const Log2SingleInt = std.math.Log2Int(SingleInt);32 const s = @intCast(Log2Int(T), @clz(v));
2033 if (s > 0) {
21 const n = @bitCast([2]SingleInt, a);34 // Normalize divisor
22 const d = @bitCast([2]SingleInt, b);35 v <<= s;
23 var q: [2]SingleInt = undefined;36 un64 = (_u1 << s) | (_u0 >> @intCast(Log2Int(T), (@bitSizeOf(T) - @intCast(T, s))));
24 var r: [2]SingleInt = undefined;37 un10 = _u0 << s;
25 var sr: c_uint = undefined;38 } else {
26 // special cases, X is unknown, K != 039 // Avoid undefined behavior of (u0 >> @bitSizeOf(T))
27 if (n[high] == 0) {40 un64 = _u1;
28 if (d[high] == 0) {41 un10 = _u0;
29 // 0 X42 }
30 // ---43
31 // 0 X44 // Break divisor up into two 32-bit digits
32 if (maybe_rem) |rem| {45 const vn1 = v >> (@bitSizeOf(T) / 2);
33 rem.* = n[low] % d[low];46 const vn0 = v & std.math.maxInt(HalfInt(T));
34 }47
35 return n[low] / d[low];48 // Break right half of dividend into two digits
36 }49 const un1 = un10 >> (@bitSizeOf(T) / 2);
37 // 0 X50 const un0 = un10 & std.math.maxInt(HalfInt(T));
38 // ---51
39 // K X52 // Compute the first quotient digit, q1
53 var q1 = un64 / vn1;
54 var rhat = un64 -% q1 *% vn1;
55
56 // q1 has at most error 2. No more than 2 iterations
57 while (q1 >= b or q1 * vn0 > b * rhat + un1) {
58 q1 -= 1;
59 rhat += vn1;
60 if (rhat >= b) break;
61 }
62
63 var un21 = un64 *% b +% un1 -% q1 *% v;
64
65 // Compute the second quotient digit
66 var q0 = un21 / vn1;
67 rhat = un21 -% q0 *% vn1;
68
69 // q0 has at most error 2. No more than 2 iterations.
70 while (q0 >= b or q0 * vn0 > b * rhat + un0) {
71 q0 -= 1;
72 rhat += vn1;
73 if (rhat >= b) break;
74 }
75
76 r.* = (un21 *% b +% un0 -% q0 *% v) >> s;
77 return q1 *% b +% q0;
78}
79
80fn divwide(comptime T: type, _u1: T, _u0: T, v: T, r: *T) T {
81 @setRuntimeSafety(is_test);
82 if (T == u64 and builtin.target.cpu.arch == .x86_64) {
83 var rem: T = undefined;
84 const quo = asm (
85 \\divq %[v]
86 : [_] "={rax}" (-> T),
87 [_] "={rdx}" (rem),
88 : [v] "r" (v),
89 [_] "{rax}" (_u0),
90 [_] "{rdx}" (_u1),
91 );
92 r.* = rem;
93 return quo;
94 } else {
95 return divwide_generic(T, _u1, _u0, v, r);
96 }
97}
98
99// return q = a / b, *r = a % b
100pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
101 @setRuntimeSafety(is_test);
102 const HalfT = HalfInt(T);
103 const SignedT = std.meta.Int(.signed, @bitSizeOf(T));
104
105 if (b_ > a_) {
40 if (maybe_rem) |rem| {106 if (maybe_rem) |rem| {
41 rem.* = n[low];107 rem.* = a_;
42 }108 }
43 return 0;109 return 0;
44 }110 }
45 // n[high] != 0111
46 if (d[low] == 0) {112 var a = @bitCast([2]HalfT, a_);
47 if (d[high] == 0) {113 var b = @bitCast([2]HalfT, b_);
48 // K X114 var q: [2]HalfT = undefined;
49 // ---115 var r: [2]HalfT = undefined;
50 // 0 0116
51 if (maybe_rem) |rem| {117 // When the divisor fits in 64 bits, we can use an optimized path
52 rem.* = n[high] % d[low];118 if (b[hi] == 0) {
53 }119 r[hi] = 0;
54 return n[high] / d[low];120 if (a[hi] < b[lo]) {
55 }121 // The result fits in 64 bits
56 // d[high] != 0122 q[hi] = 0;
57 if (n[low] == 0) {123 q[lo] = divwide(HalfT, a[hi], a[lo], b[lo], &r[lo]);
58 // K 0
59 // ---
60 // K 0
61 if (maybe_rem) |rem| {
62 r[high] = n[high] % d[high];
63 r[low] = 0;
64 rem.* = @bitCast(DoubleInt, r);
65 }
66 return n[high] / d[high];
67 }
68 // K K
69 // ---
70 // K 0
71 if ((d[high] & (d[high] - 1)) == 0) {
72 // d is a power of 2
73 if (maybe_rem) |rem| {
74 r[low] = n[low];
75 r[high] = n[high] & (d[high] - 1);
76 rem.* = @bitCast(DoubleInt, r);
77 }
78 return n[high] >> @intCast(Log2SingleInt, @ctz(d[high]));
79 }
80 // K K
81 // ---
82 // K 0
83 sr = @bitCast(c_uint, @as(c_int, @clz(d[high])) - @as(c_int, @clz(n[high])));
84 // 0 <= sr <= single_int_bits - 2 or sr large
85 if (sr > single_int_bits - 2) {
86 if (maybe_rem) |rem| {
87 rem.* = a;
88 }
89 return 0;
90 }
91 sr += 1;
92 // 1 <= sr <= single_int_bits - 1
93 // q.all = a << (double_int_bits - sr);
94 q[low] = 0;
95 q[high] = n[low] << @intCast(Log2SingleInt, single_int_bits - sr);
96 // r.all = a >> sr;
97 r[high] = n[high] >> @intCast(Log2SingleInt, sr);
98 r[low] = (n[high] << @intCast(Log2SingleInt, single_int_bits - sr)) | (n[low] >> @intCast(Log2SingleInt, sr));
99 } else {
100 // d[low] != 0
101 if (d[high] == 0) {
102 // K X
103 // ---
104 // 0 K
105 if ((d[low] & (d[low] - 1)) == 0) {
106 // d is a power of 2
107 if (maybe_rem) |rem| {
108 rem.* = n[low] & (d[low] - 1);
109 }
110 if (d[low] == 1) {
111 return a;
112 }
113 sr = @ctz(d[low]);
114 q[high] = n[high] >> @intCast(Log2SingleInt, sr);
115 q[low] = (n[high] << @intCast(Log2SingleInt, single_int_bits - sr)) | (n[low] >> @intCast(Log2SingleInt, sr));
116 return @bitCast(DoubleInt, q);
117 }
118 // K X
119 // ---
120 // 0 K
121 sr = 1 + single_int_bits + @as(c_uint, @clz(d[low])) - @as(c_uint, @clz(n[high]));
122 // 2 <= sr <= double_int_bits - 1
123 // q.all = a << (double_int_bits - sr);
124 // r.all = a >> sr;
125 if (sr == single_int_bits) {
126 q[low] = 0;
127 q[high] = n[low];
128 r[high] = 0;
129 r[low] = n[high];
130 } else if (sr < single_int_bits) {
131 // 2 <= sr <= single_int_bits - 1
132 q[low] = 0;
133 q[high] = n[low] << @intCast(Log2SingleInt, single_int_bits - sr);
134 r[high] = n[high] >> @intCast(Log2SingleInt, sr);
135 r[low] = (n[high] << @intCast(Log2SingleInt, single_int_bits - sr)) | (n[low] >> @intCast(Log2SingleInt, sr));
136 } else {
137 // single_int_bits + 1 <= sr <= double_int_bits - 1
138 q[low] = n[low] << @intCast(Log2SingleInt, double_int_bits - sr);
139 q[high] = (n[high] << @intCast(Log2SingleInt, double_int_bits - sr)) | (n[low] >> @intCast(Log2SingleInt, sr - single_int_bits));
140 r[high] = 0;
141 r[low] = n[high] >> @intCast(Log2SingleInt, sr - single_int_bits);
142 }
143 } else {124 } else {
144 // K X125 // First, divide with the high part to get the remainder. After that a_hi < b_lo.
145 // ---126 q[hi] = a[hi] / b[lo];
146 // K K127 q[lo] = divwide(HalfT, a[hi] % b[lo], a[lo], b[lo], &r[lo]);
147 sr = @bitCast(c_uint, @as(c_int, @clz(d[high])) - @as(c_int, @clz(n[high])));
148 // 0 <= sr <= single_int_bits - 1 or sr large
149 if (sr > single_int_bits - 1) {
150 if (maybe_rem) |rem| {
151 rem.* = a;
152 }
153 return 0;
154 }
155 sr += 1;
156 // 1 <= sr <= single_int_bits
157 // q.all = a << (double_int_bits - sr);
158 // r.all = a >> sr;
159 q[low] = 0;
160 if (sr == single_int_bits) {
161 q[high] = n[low];
162 r[high] = 0;
163 r[low] = n[high];
164 } else {
165 r[high] = n[high] >> @intCast(Log2SingleInt, sr);
166 r[low] = (n[high] << @intCast(Log2SingleInt, single_int_bits - sr)) | (n[low] >> @intCast(Log2SingleInt, sr));
167 q[high] = n[low] << @intCast(Log2SingleInt, single_int_bits - sr);
168 }
169 }128 }
129 if (maybe_rem) |rem| {
130 rem.* = @bitCast(T, r);
131 }
132 return @bitCast(T, q);
170 }133 }
171 // Not a special case134
172 // q and r are initialized with:135 // 0 <= shift <= 63
173 // q.all = a << (double_int_bits - sr);136 var shift: Log2Int(T) = @clz(b[hi]) - @clz(a[hi]);
174 // r.all = a >> sr;137 var af = @bitCast(T, a);
175 // 1 <= sr <= double_int_bits - 1138 var bf = @bitCast(T, b) << shift;
176 var carry: u32 = 0;139 q = @bitCast([2]HalfT, @as(T, 0));
177 var r_all: DoubleInt = undefined;140
178 while (sr > 0) : (sr -= 1) {141 for (0..shift + 1) |_| {
179 // r:q = ((r:q) << 1) | carry142 q[lo] <<= 1;
180 r[high] = (r[high] << 1) | (r[low] >> (single_int_bits - 1));143 // Branchless version of:
181 r[low] = (r[low] << 1) | (q[high] >> (single_int_bits - 1));144 // if (a >= b) {
182 q[high] = (q[high] << 1) | (q[low] >> (single_int_bits - 1));145 // a -= b;
183 q[low] = (q[low] << 1) | carry;146 // q[lo] |= 1;
184 // carry = 0;
185 // if (r.all >= b)
186 // {
187 // r.all -= b;
188 // carry = 1;
189 // }147 // }
190 r_all = @bitCast(DoubleInt, r);148 const s = @bitCast(SignedT, bf -% af -% 1) >> (@bitSizeOf(T) - 1);
191 const s: SignedDoubleInt = @bitCast(SignedDoubleInt, b -% r_all -% 1) >> (double_int_bits - 1);149 q[lo] |= @intCast(HalfT, s & 1);
192 carry = @intCast(u32, s & 1);150 af -= bf & @bitCast(T, s);
193 r_all -= b & @bitCast(DoubleInt, s);151 bf >>= 1;
194 r = @bitCast([2]SingleInt, r_all);
195 }152 }
196 const q_all = (@bitCast(DoubleInt, q) << 1) | carry;
197 if (maybe_rem) |rem| {153 if (maybe_rem) |rem| {
198 rem.* = r_all;154 rem.* = @bitCast(T, af);
199 }155 }
200 return q_all;156 return @bitCast(T, q);
201}157}