authorgravatar for daniele.cocca@gmail.comDaniele Cocca <daniele.cocca@gmail.com> 2022-03-13 00:43:51+00:00
committergravatar for daniele.cocca@gmail.comDaniele Cocca <daniele.cocca@gmail.com> 2022-03-13 09:59:15+00:00
logd0277a3d17f738036a951a44f367839597dcaee4
tree5415e8fcfc445b567fb2c0ba2e2f1b187f46d021
parent87744a7ea9a2449764a110da4210d7750e3938ee

CBE: implement popCount, byteSwap, bitReverse for ints <= 128 bits

This folds the airCountZeroes() code from 226fcd7c709ec664c5d883042cf7beb3026f66cb back into airBuiltinCall(), since most of these builtins happen to require the same arguments and can be unified under a common function signature.

6 files changed, 107 insertions(+), 28 deletions(-)

src/codegen/c.zig+9-22
...@@ -1711,8 +1711,8 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO...@@ -1711,8 +1711,8 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO
1711 .memcpy => try airMemcpy(f, inst),1711 .memcpy => try airMemcpy(f, inst),
1712 .set_union_tag => try airSetUnionTag(f, inst),1712 .set_union_tag => try airSetUnionTag(f, inst),
1713 .get_union_tag => try airGetUnionTag(f, inst),1713 .get_union_tag => try airGetUnionTag(f, inst),
1714 .clz => try airCountZeroes(f, inst, "clz"),1714 .clz => try airBuiltinCall(f, inst, "clz"),
1715 .ctz => try airCountZeroes(f, inst, "ctz"),1715 .ctz => try airBuiltinCall(f, inst, "ctz"),
1716 .popcount => try airBuiltinCall(f, inst, "popcount"),1716 .popcount => try airBuiltinCall(f, inst, "popcount"),
1717 .byte_swap => try airBuiltinCall(f, inst, "byte_swap"),1717 .byte_swap => try airBuiltinCall(f, inst, "byte_swap"),
1718 .bit_reverse => try airBuiltinCall(f, inst, "bit_reverse"),1718 .bit_reverse => try airBuiltinCall(f, inst, "bit_reverse"),
...@@ -3352,23 +3352,6 @@ fn airPtrToInt(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3352,23 +3352,6 @@ fn airPtrToInt(f: *Function, inst: Air.Inst.Index) !CValue {
3352fn airBuiltinCall(f: *Function, inst: Air.Inst.Index, fn_name: [*:0]const u8) !CValue {3352fn airBuiltinCall(f: *Function, inst: Air.Inst.Index, fn_name: [*:0]const u8) !CValue {
3353 if (f.liveness.isUnused(inst)) return CValue.none;3353 if (f.liveness.isUnused(inst)) return CValue.none;
33543354
3355 const inst_ty = f.air.typeOfIndex(inst);
3356 const local = try f.allocLocal(inst_ty, .Const);
3357 const ty_op = f.air.instructions.items(.data)[inst].ty_op;
3358 const writer = f.object.writer();
3359 const operand = try f.resolveInst(ty_op.operand);
3360
3361 // TODO implement the function in zig.h and call it here
3362
3363 try writer.print(" = {s}(", .{fn_name});
3364 try f.writeCValue(writer, operand);
3365 try writer.writeAll(");\n");
3366 return local;
3367}
3368
3369fn airCountZeroes(f: *Function, inst: Air.Inst.Index, fn_name: [*:0]const u8) !CValue {
3370 if (f.liveness.isUnused(inst)) return CValue.none;
3371
3372 const inst_ty = f.air.typeOfIndex(inst);3355 const inst_ty = f.air.typeOfIndex(inst);
3373 const local = try f.allocLocal(inst_ty, .Const);3356 const local = try f.allocLocal(inst_ty, .Const);
3374 const operand = f.air.instructions.items(.data)[inst].ty_op.operand;3357 const operand = f.air.instructions.items(.data)[inst].ty_op.operand;
...@@ -3376,13 +3359,17 @@ fn airCountZeroes(f: *Function, inst: Air.Inst.Index, fn_name: [*:0]const u8) !C...@@ -3376,13 +3359,17 @@ fn airCountZeroes(f: *Function, inst: Air.Inst.Index, fn_name: [*:0]const u8) !C
3376 const target = f.object.dg.module.getTarget();3359 const target = f.object.dg.module.getTarget();
3377 const writer = f.object.writer();3360 const writer = f.object.writer();
33783361
3379 const zig_bits = operand_ty.intInfo(target).bits;3362 const int_info = operand_ty.intInfo(target);
3380 _ = toCIntBits(zig_bits) orelse3363 _ = toCIntBits(int_info.bits) orelse
3381 return f.fail("TODO: C backend: implement integer types larger than 128 bits", .{});3364 return f.fail("TODO: C backend: implement integer types larger than 128 bits", .{});
3365 const signed_type = switch (int_info.signedness) {
3366 .signed => "true",
3367 .unsigned => "false",
3368 };
33823369
3383 try writer.print(" = zig_{s}(", .{fn_name});3370 try writer.print(" = zig_{s}(", .{fn_name});
3384 try f.writeCValue(writer, try f.resolveInst(operand));3371 try f.writeCValue(writer, try f.resolveInst(operand));
3385 try writer.print(", {d});\n", .{zig_bits});3372 try writer.print(", {d}, {s});\n", .{ int_info.bits, signed_type });
3386 return local;3373 return local;
3387}3374}
33883375
src/link/C/zig.h+94-2
...@@ -502,8 +502,20 @@ zig_shl_sat_s(int, int, ((sizeof(int )) * CHAR_BIT - 1))...@@ -502,8 +502,20 @@ zig_shl_sat_s(int, int, ((sizeof(int )) * CHAR_BIT - 1))
502zig_shl_sat_s(long, long, ((sizeof(long )) * CHAR_BIT - 1))502zig_shl_sat_s(long, long, ((sizeof(long )) * CHAR_BIT - 1))
503503
504#define zig_bitsizeof(T) (CHAR_BIT * sizeof(T))504#define zig_bitsizeof(T) (CHAR_BIT * sizeof(T))
505#define zig_bit_mask(T, bit_width) \
506 ((bit_width) == 128 \
507 ? ((T)(((uint128_t) 0xffffffffffffffff) << 64 | \
508 (uint128_t) 0xffffffffffffffff)) \
509 : (((T)1 << (T)(bit_width)) - 1))
510
511static inline uint128_t zig_sign_extend(uint128_t value, uint128_t zig_type_bit_width) {
512 const uint128_t m = (uint128_t)1 << (zig_type_bit_width - 1);
513 return (value ^ m) - m;
514}
515
516static inline int zig_clz(unsigned long long value, uint8_t zig_type_bit_width, bool signed_type) {
517 (void)signed_type; // unused
505518
506static inline int zig_clz(uint64_t value, uint8_t zig_type_bit_width) {
507 if (value == 0) return zig_type_bit_width;519 if (value == 0) return zig_type_bit_width;
508 if (zig_type_bit_width <= zig_bitsizeof(unsigned int))520 if (zig_type_bit_width <= zig_bitsizeof(unsigned int))
509 return (__builtin_clz(value) - zig_bitsizeof(unsigned int) + zig_type_bit_width);521 return (__builtin_clz(value) - zig_bitsizeof(unsigned int) + zig_type_bit_width);
...@@ -512,9 +524,89 @@ static inline int zig_clz(uint64_t value, uint8_t zig_type_bit_width) {...@@ -512,9 +524,89 @@ static inline int zig_clz(uint64_t value, uint8_t zig_type_bit_width) {
512 return (__builtin_clzll(value) - zig_bitsizeof(unsigned long long) + zig_type_bit_width);524 return (__builtin_clzll(value) - zig_bitsizeof(unsigned long long) + zig_type_bit_width);
513}525}
514526
515static inline int zig_ctz(uint64_t value, uint8_t zig_type_bit_width) {527static inline int zig_ctz(unsigned long long value, uint8_t zig_type_bit_width, bool signed_type) {
528 (void)signed_type; // unused
529
516 if (value == 0) return zig_type_bit_width;530 if (value == 0) return zig_type_bit_width;
517 if (zig_type_bit_width <= zig_bitsizeof(unsigned int)) return __builtin_ctz(value);531 if (zig_type_bit_width <= zig_bitsizeof(unsigned int)) return __builtin_ctz(value);
518 if (zig_type_bit_width <= zig_bitsizeof(unsigned long)) return __builtin_ctzl(value);532 if (zig_type_bit_width <= zig_bitsizeof(unsigned long)) return __builtin_ctzl(value);
519 return __builtin_ctzll(value);533 return __builtin_ctzll(value);
520}534}
535
536static inline int zig_popcount(unsigned long long value, uint8_t zig_type_bit_width, bool signed_type) {
537 (void)signed_type; // unused
538
539 const unsigned long long mask = zig_bit_mask(unsigned long long, zig_type_bit_width);
540 if (zig_type_bit_width <= zig_bitsizeof(unsigned int))
541 return __builtin_popcount(value & mask);
542 if (zig_type_bit_width <= zig_bitsizeof(unsigned long))
543 return __builtin_popcountl(value & mask);
544 return __builtin_popcountll(value & mask);
545}
546
547static inline uint128_t zig_byte_swap(uint128_t value, uint8_t zig_type_bit_width, bool signed_type) {
548 if (zig_type_bit_width <= 8) return value;
549 if (zig_type_bit_width <= 16) return __builtin_bswap16(value);
550 if (zig_type_bit_width <= 32) {
551 const uint32_t swapped = __builtin_bswap32(value) >> (32 - zig_type_bit_width);
552 return (signed_type ? zig_sign_extend(swapped, zig_type_bit_width) : swapped);
553 }
554 if (zig_type_bit_width <= 64) {
555 const uint64_t swapped = __builtin_bswap64(value) >> (64 - zig_type_bit_width);
556 return (signed_type ? zig_sign_extend(swapped, zig_type_bit_width) : swapped);
557 }
558 const uint128_t mask = zig_bit_mask(uint128_t, zig_type_bit_width);
559 const uint128_t hi = __builtin_bswap64((uint64_t)(value >> 64));
560 const uint128_t lo = __builtin_bswap64((uint64_t)value);
561 const uint128_t swapped = (((lo << 64 | hi) >> (128 - zig_type_bit_width))) & mask;
562 return (signed_type ? zig_sign_extend(swapped, zig_type_bit_width) : swapped);
563}
564
565static inline uint128_t zig_bit_reverse(uint128_t value, uint8_t zig_type_bit_width, bool signed_type) {
566 static const uint128_t reverse_lut[256] = {
567 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, 0x10, 0x90, 0x50, 0xd0,
568 0x30, 0xb0, 0x70, 0xf0, 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
569 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, 0x04, 0x84, 0x44, 0xc4,
570 0x24, 0xa4, 0x64, 0xe4, 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
571 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, 0x1c, 0x9c, 0x5c, 0xdc,
572 0x3c, 0xbc, 0x7c, 0xfc, 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
573 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, 0x0a, 0x8a, 0x4a, 0xca,
574 0x2a, 0xaa, 0x6a, 0xea, 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
575 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, 0x16, 0x96, 0x56, 0xd6,
576 0x36, 0xb6, 0x76, 0xf6, 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
577 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, 0x01, 0x81, 0x41, 0xc1,
578 0x21, 0xa1, 0x61, 0xe1, 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
579 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, 0x19, 0x99, 0x59, 0xd9,
580 0x39, 0xb9, 0x79, 0xf9, 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
581 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, 0x0d, 0x8d, 0x4d, 0xcd,
582 0x2d, 0xad, 0x6d, 0xed, 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
583 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, 0x13, 0x93, 0x53, 0xd3,
584 0x33, 0xb3, 0x73, 0xf3, 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
585 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, 0x07, 0x87, 0x47, 0xc7,
586 0x27, 0xa7, 0x67, 0xe7, 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
587 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, 0x1f, 0x9f, 0x5f, 0xdf,
588 0x3f, 0xbf, 0x7f, 0xff
589 };
590 const uint128_t mask = zig_bit_mask(uint128_t, zig_type_bit_width);
591 const uint128_t swapped = zig_byte_swap(value, zig_type_bit_width, signed_type);
592 const uint128_t reversed = (
593 (reverse_lut[(swapped >> 0x78) & 0xff] << 0x78) |
594 (reverse_lut[(swapped >> 0x70) & 0xff] << 0x70) |
595 (reverse_lut[(swapped >> 0x68) & 0xff] << 0x68) |
596 (reverse_lut[(swapped >> 0x60) & 0xff] << 0x60) |
597 (reverse_lut[(swapped >> 0x58) & 0xff] << 0x58) |
598 (reverse_lut[(swapped >> 0x50) & 0xff] << 0x50) |
599 (reverse_lut[(swapped >> 0x48) & 0xff] << 0x48) |
600 (reverse_lut[(swapped >> 0x40) & 0xff] << 0x40) |
601 (reverse_lut[(swapped >> 0x38) & 0xff] << 0x38) |
602 (reverse_lut[(swapped >> 0x30) & 0xff] << 0x30) |
603 (reverse_lut[(swapped >> 0x28) & 0xff] << 0x28) |
604 (reverse_lut[(swapped >> 0x20) & 0xff] << 0x20) |
605 (reverse_lut[(swapped >> 0x18) & 0xff] << 0x18) |
606 (reverse_lut[(swapped >> 0x10) & 0xff] << 0x10) |
607 (reverse_lut[(swapped >> 0x08) & 0xff] << 0x08) |
608 (reverse_lut[(swapped >> 0x00) & 0xff] << 0x00));
609 const uint128_t result =
610 (reversed >> (zig_type_bit_width < 8 ? 8 - zig_type_bit_width : 0)) & mask;
611 return (signed_type ? zig_sign_extend(result, zig_type_bit_width) : result);
612}
test/behavior.zig+1-1
...@@ -139,6 +139,7 @@ test {...@@ -139,6 +139,7 @@ test {
139 _ = @import("behavior/eval.zig");139 _ = @import("behavior/eval.zig");
140 _ = @import("behavior/export_self_referential_type_info.zig");140 _ = @import("behavior/export_self_referential_type_info.zig");
141 _ = @import("behavior/int128.zig");141 _ = @import("behavior/int128.zig");
142 _ = @import("behavior/popcount.zig");
142 _ = @import("behavior/translate_c_macros.zig");143 _ = @import("behavior/translate_c_macros.zig");
143 _ = @import("behavior/union_with_members.zig");144 _ = @import("behavior/union_with_members.zig");
144145
...@@ -147,7 +148,6 @@ test {...@@ -147,7 +148,6 @@ test {
147 _ = @import("behavior/atomics.zig");148 _ = @import("behavior/atomics.zig");
148 _ = @import("behavior/export.zig");149 _ = @import("behavior/export.zig");
149 _ = @import("behavior/maximum_minimum.zig");150 _ = @import("behavior/maximum_minimum.zig");
150 _ = @import("behavior/popcount.zig");
151 _ = @import("behavior/saturating_arithmetic.zig");151 _ = @import("behavior/saturating_arithmetic.zig");
152 _ = @import("behavior/widening.zig");152 _ = @import("behavior/widening.zig");
153 _ = @import("behavior/bugs/2114.zig");153 _ = @import("behavior/bugs/2114.zig");
test/behavior/bitreverse.zig-2
...@@ -5,7 +5,6 @@ const minInt = std.math.minInt;...@@ -5,7 +5,6 @@ const minInt = std.math.minInt;
55
6test "@bitReverse large exotic integer" {6test "@bitReverse large exotic integer" {
7 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;7 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
8 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
9 // Currently failing on stage1 for big-endian targets8 // Currently failing on stage1 for big-endian targets
10 if (builtin.zig_backend == .stage1) return error.SkipZigTest;9 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
1110
...@@ -14,7 +13,6 @@ test "@bitReverse large exotic integer" {...@@ -14,7 +13,6 @@ test "@bitReverse large exotic integer" {
1413
15test "@bitReverse" {14test "@bitReverse" {
16 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;15 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
17 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
18 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;16 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
19 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;17 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
20 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;18 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
test/behavior/byteswap.zig-1
...@@ -4,7 +4,6 @@ const expect = std.testing.expect;...@@ -4,7 +4,6 @@ const expect = std.testing.expect;
44
5test "@byteSwap integers" {5test "@byteSwap integers" {
6 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;6 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
7 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
8 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;7 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
9 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;8 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
10 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;9 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
test/behavior/popcount.zig+3
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1const builtin = @import("builtin");
1const std = @import("std");2const std = @import("std");
2const expect = std.testing.expect;3const expect = std.testing.expect;
3const expectEqual = std.testing.expectEqual;4const expectEqual = std.testing.expectEqual;
...@@ -45,6 +46,8 @@ fn testPopCountIntegers() !void {...@@ -45,6 +46,8 @@ fn testPopCountIntegers() !void {
45}46}
4647
47test "@popCount vectors" {48test "@popCount vectors" {
49 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
50
48 comptime try testPopCountVectors();51 comptime try testPopCountVectors();
49 try testPopCountVectors();52 try testPopCountVectors();
50}53}