| ... | ... | @@ -1919,7 +1919,7 @@ static inline zig_i128 zig_bit_reverse_i128(zig_i128 val, uint8_t bits) { |
| 1919 | 1919 | |
| 1920 | 1920 | /* ========================== Big Integer Support =========================== */ |
| 1921 | 1921 | |
| 1922 | | static inline uint16_t zig_big_bytes(uint16_t bits) { |
| 1922 | static inline uint16_t zig_int_bytes(uint16_t bits) { |
| 1923 | 1923 | uint16_t bytes = (bits + CHAR_BIT - 1) / CHAR_BIT; |
| 1924 | 1924 | uint16_t alignment = 16; |
| 1925 | 1925 | while (alignment / 2 >= bytes) alignment /= 2; |
| ... | ... | @@ -1931,7 +1931,7 @@ static inline int32_t zig_cmp_big(const void *lhs, const void *rhs, bool is_sign |
| 1931 | 1931 | const uint8_t *rhs_bytes = rhs; |
| 1932 | 1932 | uint16_t byte_offset = 0; |
| 1933 | 1933 | bool do_signed = is_signed; |
| 1934 | | uint16_t remaining_bytes = zig_big_bytes(bits); |
| 1934 | uint16_t remaining_bytes = zig_int_bytes(bits); |
| 1935 | 1935 | |
| 1936 | 1936 | #if zig_little_endian |
| 1937 | 1937 | byte_offset = remaining_bytes; |
| ... | ... | @@ -1965,7 +1965,7 @@ static inline int32_t zig_cmp_big(const void *lhs, const void *rhs, bool is_sign |
| 1965 | 1965 | remaining_bytes -= 128 / CHAR_BIT; |
| 1966 | 1966 | |
| 1967 | 1967 | #if zig_big_endian |
| 1968 | | byte_offset -= 128 / CHAR_BIT; |
| 1968 | byte_offset += 128 / CHAR_BIT; |
| 1969 | 1969 | #endif |
| 1970 | 1970 | } |
| 1971 | 1971 | |
| ... | ... | @@ -1994,7 +1994,7 @@ static inline int32_t zig_cmp_big(const void *lhs, const void *rhs, bool is_sign |
| 1994 | 1994 | remaining_bytes -= 64 / CHAR_BIT; |
| 1995 | 1995 | |
| 1996 | 1996 | #if zig_big_endian |
| 1997 | | byte_offset -= 64 / CHAR_BIT; |
| 1997 | byte_offset += 64 / CHAR_BIT; |
| 1998 | 1998 | #endif |
| 1999 | 1999 | } |
| 2000 | 2000 | |
| ... | ... | @@ -2023,7 +2023,7 @@ static inline int32_t zig_cmp_big(const void *lhs, const void *rhs, bool is_sign |
| 2023 | 2023 | remaining_bytes -= 32 / CHAR_BIT; |
| 2024 | 2024 | |
| 2025 | 2025 | #if zig_big_endian |
| 2026 | | byte_offset -= 32 / CHAR_BIT; |
| 2026 | byte_offset += 32 / CHAR_BIT; |
| 2027 | 2027 | #endif |
| 2028 | 2028 | } |
| 2029 | 2029 | |
| ... | ... | @@ -2052,7 +2052,7 @@ static inline int32_t zig_cmp_big(const void *lhs, const void *rhs, bool is_sign |
| 2052 | 2052 | remaining_bytes -= 16 / CHAR_BIT; |
| 2053 | 2053 | |
| 2054 | 2054 | #if zig_big_endian |
| 2055 | | byte_offset -= 16 / CHAR_BIT; |
| 2055 | byte_offset += 16 / CHAR_BIT; |
| 2056 | 2056 | #endif |
| 2057 | 2057 | } |
| 2058 | 2058 | |
| ... | ... | @@ -2081,13 +2081,368 @@ static inline int32_t zig_cmp_big(const void *lhs, const void *rhs, bool is_sign |
| 2081 | 2081 | remaining_bytes -= 8 / CHAR_BIT; |
| 2082 | 2082 | |
| 2083 | 2083 | #if zig_big_endian |
| 2084 | | byte_offset -= 8 / CHAR_BIT; |
| 2084 | byte_offset += 8 / CHAR_BIT; |
| 2085 | 2085 | #endif |
| 2086 | 2086 | } |
| 2087 | 2087 | |
| 2088 | 2088 | return 0; |
| 2089 | 2089 | } |
| 2090 | 2090 | |
| 2091 | static inline uint16_t zig_clz_big(const void *val, bool is_signed, uint16_t bits) { |
| 2092 | const uint8_t *val_bytes = val; |
| 2093 | uint16_t byte_offset = 0; |
| 2094 | uint16_t remaining_bytes = zig_int_bytes(bits); |
| 2095 | uint16_t skip_bits = remaining_bytes * 8 - bits; |
| 2096 | uint16_t total_lz = 0; |
| 2097 | uint16_t limb_lz; |
| 2098 | (void)is_signed; |
| 2099 | |
| 2100 | #if zig_little_endian |
| 2101 | byte_offset = remaining_bytes; |
| 2102 | #endif |
| 2103 | |
| 2104 | while (remaining_bytes >= 128 / CHAR_BIT) { |
| 2105 | #if zig_little_endian |
| 2106 | byte_offset -= 128 / CHAR_BIT; |
| 2107 | #endif |
| 2108 | |
| 2109 | { |
| 2110 | zig_u128 val_limb; |
| 2111 | |
| 2112 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2113 | limb_lz = zig_clz_u128(val_limb, 128 - skip_bits); |
| 2114 | } |
| 2115 | |
| 2116 | total_lz += limb_lz; |
| 2117 | if (limb_lz < 128 - skip_bits) return total_lz; |
| 2118 | skip_bits = 0; |
| 2119 | remaining_bytes -= 128 / CHAR_BIT; |
| 2120 | |
| 2121 | #if zig_big_endian |
| 2122 | byte_offset += 128 / CHAR_BIT; |
| 2123 | #endif |
| 2124 | } |
| 2125 | |
| 2126 | while (remaining_bytes >= 64 / CHAR_BIT) { |
| 2127 | #if zig_little_endian |
| 2128 | byte_offset -= 64 / CHAR_BIT; |
| 2129 | #endif |
| 2130 | |
| 2131 | { |
| 2132 | uint64_t val_limb; |
| 2133 | |
| 2134 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2135 | limb_lz = zig_clz_u64(val_limb, 64 - skip_bits); |
| 2136 | } |
| 2137 | |
| 2138 | total_lz += limb_lz; |
| 2139 | if (limb_lz < 64 - skip_bits) return total_lz; |
| 2140 | skip_bits = 0; |
| 2141 | remaining_bytes -= 64 / CHAR_BIT; |
| 2142 | |
| 2143 | #if zig_big_endian |
| 2144 | byte_offset += 64 / CHAR_BIT; |
| 2145 | #endif |
| 2146 | } |
| 2147 | |
| 2148 | while (remaining_bytes >= 32 / CHAR_BIT) { |
| 2149 | #if zig_little_endian |
| 2150 | byte_offset -= 32 / CHAR_BIT; |
| 2151 | #endif |
| 2152 | |
| 2153 | { |
| 2154 | uint32_t val_limb; |
| 2155 | |
| 2156 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2157 | limb_lz = zig_clz_u32(val_limb, 32 - skip_bits); |
| 2158 | } |
| 2159 | |
| 2160 | total_lz += limb_lz; |
| 2161 | if (limb_lz < 32 - skip_bits) return total_lz; |
| 2162 | skip_bits = 0; |
| 2163 | remaining_bytes -= 32 / CHAR_BIT; |
| 2164 | |
| 2165 | #if zig_big_endian |
| 2166 | byte_offset += 32 / CHAR_BIT; |
| 2167 | #endif |
| 2168 | } |
| 2169 | |
| 2170 | while (remaining_bytes >= 16 / CHAR_BIT) { |
| 2171 | #if zig_little_endian |
| 2172 | byte_offset -= 16 / CHAR_BIT; |
| 2173 | #endif |
| 2174 | |
| 2175 | { |
| 2176 | uint16_t val_limb; |
| 2177 | |
| 2178 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2179 | limb_lz = zig_clz_u16(val_limb, 16 - skip_bits); |
| 2180 | } |
| 2181 | |
| 2182 | total_lz += limb_lz; |
| 2183 | if (limb_lz < 16 - skip_bits) return total_lz; |
| 2184 | skip_bits = 0; |
| 2185 | remaining_bytes -= 16 / CHAR_BIT; |
| 2186 | |
| 2187 | #if zig_big_endian |
| 2188 | byte_offset += 16 / CHAR_BIT; |
| 2189 | #endif |
| 2190 | } |
| 2191 | |
| 2192 | while (remaining_bytes >= 8 / CHAR_BIT) { |
| 2193 | #if zig_little_endian |
| 2194 | byte_offset -= 8 / CHAR_BIT; |
| 2195 | #endif |
| 2196 | |
| 2197 | { |
| 2198 | uint8_t val_limb; |
| 2199 | |
| 2200 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2201 | limb_lz = zig_clz_u8(val_limb, 8 - skip_bits); |
| 2202 | } |
| 2203 | |
| 2204 | total_lz += limb_lz; |
| 2205 | if (limb_lz < 8 - skip_bits) return total_lz; |
| 2206 | skip_bits = 0; |
| 2207 | remaining_bytes -= 8 / CHAR_BIT; |
| 2208 | |
| 2209 | #if zig_big_endian |
| 2210 | byte_offset += 8 / CHAR_BIT; |
| 2211 | #endif |
| 2212 | } |
| 2213 | |
| 2214 | return total_lz; |
| 2215 | } |
| 2216 | |
| 2217 | static inline uint16_t zig_ctz_big(const void *val, bool is_signed, uint16_t bits) { |
| 2218 | const uint8_t *val_bytes = val; |
| 2219 | uint16_t byte_offset = 0; |
| 2220 | uint16_t remaining_bytes = zig_int_bytes(bits); |
| 2221 | uint16_t total_tz = 0; |
| 2222 | uint16_t limb_tz; |
| 2223 | (void)is_signed; |
| 2224 | |
| 2225 | #if zig_big_endian |
| 2226 | byte_offset = remaining_bytes; |
| 2227 | #endif |
| 2228 | |
| 2229 | while (remaining_bytes >= 128 / CHAR_BIT) { |
| 2230 | #if zig_big_endian |
| 2231 | byte_offset -= 128 / CHAR_BIT; |
| 2232 | #endif |
| 2233 | |
| 2234 | { |
| 2235 | zig_u128 val_limb; |
| 2236 | |
| 2237 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2238 | limb_tz = zig_ctz_u128(val_limb, 128); |
| 2239 | } |
| 2240 | |
| 2241 | total_tz += limb_tz; |
| 2242 | if (limb_tz < 128) return total_tz; |
| 2243 | remaining_bytes -= 128 / CHAR_BIT; |
| 2244 | |
| 2245 | #if zig_little_endian |
| 2246 | byte_offset += 128 / CHAR_BIT; |
| 2247 | #endif |
| 2248 | } |
| 2249 | |
| 2250 | while (remaining_bytes >= 64 / CHAR_BIT) { |
| 2251 | #if zig_big_endian |
| 2252 | byte_offset -= 64 / CHAR_BIT; |
| 2253 | #endif |
| 2254 | |
| 2255 | { |
| 2256 | uint64_t val_limb; |
| 2257 | |
| 2258 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2259 | limb_tz = zig_ctz_u64(val_limb, 64); |
| 2260 | } |
| 2261 | |
| 2262 | total_tz += limb_tz; |
| 2263 | if (limb_tz < 64) return total_tz; |
| 2264 | remaining_bytes -= 64 / CHAR_BIT; |
| 2265 | |
| 2266 | #if zig_little_endian |
| 2267 | byte_offset += 64 / CHAR_BIT; |
| 2268 | #endif |
| 2269 | } |
| 2270 | |
| 2271 | while (remaining_bytes >= 32 / CHAR_BIT) { |
| 2272 | #if zig_big_endian |
| 2273 | byte_offset -= 32 / CHAR_BIT; |
| 2274 | #endif |
| 2275 | |
| 2276 | { |
| 2277 | uint32_t val_limb; |
| 2278 | |
| 2279 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2280 | limb_tz = zig_ctz_u32(val_limb, 32); |
| 2281 | } |
| 2282 | |
| 2283 | total_tz += limb_tz; |
| 2284 | if (limb_tz < 32) return total_tz; |
| 2285 | remaining_bytes -= 32 / CHAR_BIT; |
| 2286 | |
| 2287 | #if zig_little_endian |
| 2288 | byte_offset += 32 / CHAR_BIT; |
| 2289 | #endif |
| 2290 | } |
| 2291 | |
| 2292 | while (remaining_bytes >= 16 / CHAR_BIT) { |
| 2293 | #if zig_big_endian |
| 2294 | byte_offset -= 16 / CHAR_BIT; |
| 2295 | #endif |
| 2296 | |
| 2297 | { |
| 2298 | uint16_t val_limb; |
| 2299 | |
| 2300 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2301 | limb_tz = zig_ctz_u16(val_limb, 16); |
| 2302 | } |
| 2303 | |
| 2304 | total_tz += limb_tz; |
| 2305 | if (limb_tz < 16) return total_tz; |
| 2306 | remaining_bytes -= 16 / CHAR_BIT; |
| 2307 | |
| 2308 | #if zig_little_endian |
| 2309 | byte_offset += 16 / CHAR_BIT; |
| 2310 | #endif |
| 2311 | } |
| 2312 | |
| 2313 | while (remaining_bytes >= 8 / CHAR_BIT) { |
| 2314 | #if zig_big_endian |
| 2315 | byte_offset -= 8 / CHAR_BIT; |
| 2316 | #endif |
| 2317 | |
| 2318 | { |
| 2319 | uint8_t val_limb; |
| 2320 | |
| 2321 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2322 | limb_tz = zig_ctz_u8(val_limb, 8); |
| 2323 | } |
| 2324 | |
| 2325 | total_tz += limb_tz; |
| 2326 | if (limb_tz < 8) return total_tz; |
| 2327 | remaining_bytes -= 8 / CHAR_BIT; |
| 2328 | |
| 2329 | #if zig_little_endian |
| 2330 | byte_offset += 8 / CHAR_BIT; |
| 2331 | #endif |
| 2332 | } |
| 2333 | |
| 2334 | return total_tz; |
| 2335 | } |
| 2336 | |
| 2337 | static inline uint16_t zig_popcount_big(const void *val, bool is_signed, uint16_t bits) { |
| 2338 | const uint8_t *val_bytes = val; |
| 2339 | uint16_t byte_offset = 0; |
| 2340 | uint16_t remaining_bytes = zig_int_bytes(bits); |
| 2341 | uint16_t total_pc = 0; |
| 2342 | (void)is_signed; |
| 2343 | |
| 2344 | #if zig_big_endian |
| 2345 | byte_offset = remaining_bytes; |
| 2346 | #endif |
| 2347 | |
| 2348 | while (remaining_bytes >= 128 / CHAR_BIT) { |
| 2349 | #if zig_big_endian |
| 2350 | byte_offset -= 128 / CHAR_BIT; |
| 2351 | #endif |
| 2352 | |
| 2353 | { |
| 2354 | zig_u128 val_limb; |
| 2355 | |
| 2356 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2357 | total_pc += zig_popcount_u128(val_limb, 128); |
| 2358 | } |
| 2359 | |
| 2360 | remaining_bytes -= 128 / CHAR_BIT; |
| 2361 | |
| 2362 | #if zig_little_endian |
| 2363 | byte_offset += 128 / CHAR_BIT; |
| 2364 | #endif |
| 2365 | } |
| 2366 | |
| 2367 | while (remaining_bytes >= 64 / CHAR_BIT) { |
| 2368 | #if zig_big_endian |
| 2369 | byte_offset -= 64 / CHAR_BIT; |
| 2370 | #endif |
| 2371 | |
| 2372 | { |
| 2373 | uint64_t val_limb; |
| 2374 | |
| 2375 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2376 | total_pc += zig_popcount_u64(val_limb, 64); |
| 2377 | } |
| 2378 | |
| 2379 | remaining_bytes -= 64 / CHAR_BIT; |
| 2380 | |
| 2381 | #if zig_little_endian |
| 2382 | byte_offset += 64 / CHAR_BIT; |
| 2383 | #endif |
| 2384 | } |
| 2385 | |
| 2386 | while (remaining_bytes >= 32 / CHAR_BIT) { |
| 2387 | #if zig_big_endian |
| 2388 | byte_offset -= 32 / CHAR_BIT; |
| 2389 | #endif |
| 2390 | |
| 2391 | { |
| 2392 | uint32_t val_limb; |
| 2393 | |
| 2394 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2395 | total_pc += zig_popcount_u32(val_limb, 32); |
| 2396 | } |
| 2397 | |
| 2398 | remaining_bytes -= 32 / CHAR_BIT; |
| 2399 | |
| 2400 | #if zig_little_endian |
| 2401 | byte_offset += 32 / CHAR_BIT; |
| 2402 | #endif |
| 2403 | } |
| 2404 | |
| 2405 | while (remaining_bytes >= 16 / CHAR_BIT) { |
| 2406 | #if zig_big_endian |
| 2407 | byte_offset -= 16 / CHAR_BIT; |
| 2408 | #endif |
| 2409 | |
| 2410 | { |
| 2411 | uint16_t val_limb; |
| 2412 | |
| 2413 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2414 | total_pc = zig_popcount_u16(val_limb, 16); |
| 2415 | } |
| 2416 | |
| 2417 | remaining_bytes -= 16 / CHAR_BIT; |
| 2418 | |
| 2419 | #if zig_little_endian |
| 2420 | byte_offset += 16 / CHAR_BIT; |
| 2421 | #endif |
| 2422 | } |
| 2423 | |
| 2424 | while (remaining_bytes >= 8 / CHAR_BIT) { |
| 2425 | #if zig_big_endian |
| 2426 | byte_offset -= 8 / CHAR_BIT; |
| 2427 | #endif |
| 2428 | |
| 2429 | { |
| 2430 | uint8_t val_limb; |
| 2431 | |
| 2432 | memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); |
| 2433 | total_pc = zig_popcount_u8(val_limb, 8); |
| 2434 | } |
| 2435 | |
| 2436 | remaining_bytes -= 8 / CHAR_BIT; |
| 2437 | |
| 2438 | #if zig_little_endian |
| 2439 | byte_offset += 8 / CHAR_BIT; |
| 2440 | #endif |
| 2441 | } |
| 2442 | |
| 2443 | return total_pc; |
| 2444 | } |
| 2445 | |
| 2091 | 2446 | /* ========================= Floating Point Support ========================= */ |
| 2092 | 2447 | |
| 2093 | 2448 | #if _MSC_VER |
| ... | ... | @@ -2742,7 +3097,7 @@ zig_msvc_atomics_128op(u128, max) |
| 2742 | 3097 | uint32_t index = 0; \ |
| 2743 | 3098 | const uint8_t *lhs_ptr = lhs; \ |
| 2744 | 3099 | const uint8_t *rhs_ptr = rhs; \ |
| 2745 | | uint16_t elem_bytes = zig_big_bytes(elem_bits); \ |
| 3100 | uint16_t elem_bytes = zig_int_bytes(elem_bits); \ |
| 2746 | 3101 | \ |
| 2747 | 3102 | while (index < len) { \ |
| 2748 | 3103 | result[index] = zig_cmp_big(lhs_ptr, rhs_ptr, is_signed, elem_bits) operator 0; \ |
| ... | ... | @@ -2758,6 +3113,57 @@ zig_cmp_vec(le, <=) |
| 2758 | 3113 | zig_cmp_vec(gt, > ) |
| 2759 | 3114 | zig_cmp_vec(ge, >=) |
| 2760 | 3115 | |
| 3116 | static inline void zig_clz_vec(void *result, const void *val, uint32_t len, bool is_signed, uint16_t elem_bits) { |
| 3117 | uint32_t index = 0; |
| 3118 | const uint8_t *val_ptr = val; |
| 3119 | uint16_t elem_bytes = zig_int_bytes(elem_bits); |
| 3120 | |
| 3121 | while (index < len) { |
| 3122 | uint16_t lz = zig_clz_big(val_ptr, is_signed, elem_bits); |
| 3123 | if (elem_bits <= 128) { |
| 3124 | ((uint8_t *)result)[index] = (uint8_t)lz; |
| 3125 | } else { |
| 3126 | ((uint16_t *)result)[index] = lz; |
| 3127 | } |
| 3128 | val_ptr += elem_bytes; |
| 3129 | index += 1; |
| 3130 | } |
| 3131 | } |
| 3132 | |
| 3133 | static inline void zig_ctz_vec(void *result, const void *val, uint32_t len, bool is_signed, uint16_t elem_bits) { |
| 3134 | uint32_t index = 0; |
| 3135 | const uint8_t *val_ptr = val; |
| 3136 | uint16_t elem_bytes = zig_int_bytes(elem_bits); |
| 3137 | |
| 3138 | while (index < len) { |
| 3139 | uint16_t tz = zig_ctz_big(val_ptr, is_signed, elem_bits); |
| 3140 | if (elem_bits <= 128) { |
| 3141 | ((uint8_t *)result)[index] = (uint8_t)tz; |
| 3142 | } else { |
| 3143 | ((uint16_t *)result)[index] = tz; |
| 3144 | } |
| 3145 | val_ptr += elem_bytes; |
| 3146 | index += 1; |
| 3147 | } |
| 3148 | } |
| 3149 | |
| 3150 | static inline void zig_popcount_vec(void *result, const void *val, uint32_t len, bool is_signed, uint16_t elem_bits) { |
| 3151 | uint32_t index = 0; |
| 3152 | const uint8_t *val_ptr = val; |
| 3153 | uint16_t elem_bytes = zig_int_bytes(elem_bits); |
| 3154 | |
| 3155 | while (index < len) { |
| 3156 | uint16_t pc = zig_popcount_big(val_ptr, is_signed, elem_bits); |
| 3157 | if (elem_bits <= 128) { |
| 3158 | ((uint8_t *)result)[index] = (uint8_t)pc; |
| 3159 | } else { |
| 3160 | ((uint16_t *)result)[index] = pc; |
| 3161 | } |
| 3162 | val_ptr += elem_bytes; |
| 3163 | index += 1; |
| 3164 | } |
| 3165 | } |
| 3166 | |
| 2761 | 3167 | /* ======================== Special Case Intrinsics ========================= */ |
| 2762 | 3168 | |
| 2763 | 3169 | #if (_MSC_VER && _M_X64) || defined(__x86_64__) |