authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-11-19 14:58:11-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-11-19 14:58:11-08:00
log473cb1fd74d6d478bb3d5fda4707ce3f6e6e5bf6
treec481901be2833d5178bd2f92ce5ce546c054cb30
parentd526b0ffb08445fb8da1263bc833deea2474149c
parent60638f0c82374714b55de094a5f3cec4d05e9e9b
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6390 from LemonBoy/reboot-3970

std.fmt meets UTF-8

3 files changed, 220 insertions(+), 61 deletions(-)

lib/std/fmt.zig+74-19
...@@ -7,6 +7,7 @@ const std = @import("std.zig");...@@ -7,6 +7,7 @@ const std = @import("std.zig");
7const math = std.math;7const math = std.math;
8const assert = std.debug.assert;8const assert = std.debug.assert;
9const mem = std.mem;9const mem = std.mem;
10const unicode = std.unicode;
10const builtin = @import("builtin");11const builtin = @import("builtin");
11const errol = @import("fmt/errol.zig");12const errol = @import("fmt/errol.zig");
12const lossyCast = std.math.lossyCast;13const lossyCast = std.math.lossyCast;
...@@ -76,6 +77,7 @@ fn peekIsAlign(comptime fmt: []const u8) bool {...@@ -76,6 +77,7 @@ fn peekIsAlign(comptime fmt: []const u8) bool {
76/// - `b`: output integer value in binary notation77/// - `b`: output integer value in binary notation
77/// - `o`: output integer value in octal notation78/// - `o`: output integer value in octal notation
78/// - `c`: output integer as an ASCII character. Integer type must have 8 bits at max.79/// - `c`: output integer as an ASCII character. Integer type must have 8 bits at max.
80/// - `u`: output integer as an UTF-8 sequence. Integer type must have 21 bits at max.
79/// - `*`: output the address of the value instead of the value itself.81/// - `*`: output the address of the value instead of the value itself.
80///82///
81/// If a formatted user type contains a function of the type83/// If a formatted user type contains a function of the type
...@@ -555,6 +557,12 @@ pub fn formatIntValue(...@@ -555,6 +557,12 @@ pub fn formatIntValue(
555 } else {557 } else {
556 @compileError("Cannot escape character with more than 8 bits");558 @compileError("Cannot escape character with more than 8 bits");
557 }559 }
560 } else if (comptime std.mem.eql(u8, fmt, "u")) {
561 if (@typeInfo(@TypeOf(int_value)).Int.bits <= 21) {
562 return formatUnicodeCodepoint(@as(u21, int_value), options, writer);
563 } else {
564 @compileError("Cannot print integer that is larger than 21 bits as an UTF-8 sequence");
565 }
558 } else if (comptime std.mem.eql(u8, fmt, "b")) {566 } else if (comptime std.mem.eql(u8, fmt, "b")) {
559 radix = 2;567 radix = 2;
560 uppercase = false;568 uppercase = false;
...@@ -641,30 +649,54 @@ pub fn formatAsciiChar(...@@ -641,30 +649,54 @@ pub fn formatAsciiChar(
641 return writer.writeAll(@as(*const [1]u8, &c));649 return writer.writeAll(@as(*const [1]u8, &c));
642}650}
643651
652pub fn formatUnicodeCodepoint(
653 c: u21,
654 options: FormatOptions,
655 writer: anytype,
656) !void {
657 var buf: [4]u8 = undefined;
658 const len = std.unicode.utf8Encode(c, &buf) catch |err| switch (err) {
659 error.Utf8CannotEncodeSurrogateHalf, error.CodepointTooLarge => {
660 // In case of error output the replacement char U+FFFD
661 return formatBuf(&[_]u8{ 0xef, 0xbf, 0xbd }, options, writer);
662 },
663 };
664 return formatBuf(buf[0..len], options, writer);
665}
666
644pub fn formatBuf(667pub fn formatBuf(
645 buf: []const u8,668 buf: []const u8,
646 options: FormatOptions,669 options: FormatOptions,
647 writer: anytype,670 writer: anytype,
648) !void {671) !void {
649 const width = options.width orelse buf.len;672 if (options.width) |min_width| {
650 const padding = if (width > buf.len) (width - buf.len) else 0;673 // In case of error assume the buffer content is ASCII-encoded
651674 const width = unicode.utf8CountCodepoints(buf) catch |_| buf.len;
652 switch (options.alignment) {675 const padding = if (width < min_width) min_width - width else 0;
653 .Left => {676
654 try writer.writeAll(buf);677 if (padding == 0)
655 try writer.writeByteNTimes(options.fill, padding);678 return writer.writeAll(buf);
656 },679
657 .Center => {680 switch (options.alignment) {
658 const left_padding = padding / 2;681 .Left => {
659 const right_padding = (padding + 1) / 2;682 try writer.writeAll(buf);
660 try writer.writeByteNTimes(options.fill, left_padding);683 try writer.writeByteNTimes(options.fill, padding);
661 try writer.writeAll(buf);684 },
662 try writer.writeByteNTimes(options.fill, right_padding);685 .Center => {
663 },686 const left_padding = padding / 2;
664 .Right => {687 const right_padding = (padding + 1) / 2;
665 try writer.writeByteNTimes(options.fill, padding);688 try writer.writeByteNTimes(options.fill, left_padding);
666 try writer.writeAll(buf);689 try writer.writeAll(buf);
667 },690 try writer.writeByteNTimes(options.fill, right_padding);
691 },
692 .Right => {
693 try writer.writeByteNTimes(options.fill, padding);
694 try writer.writeAll(buf);
695 },
696 }
697 } else {
698 // Fast path, avoid counting the number of codepoints
699 try writer.writeAll(buf);
668 }700 }
669}701}
670702
...@@ -1385,6 +1417,22 @@ test "int.specifier" {...@@ -1385,6 +1417,22 @@ test "int.specifier" {
1385 const value: u16 = 0o1234;1417 const value: u16 = 0o1234;
1386 try testFmt("u16: 0o1234\n", "u16: 0o{o}\n", .{value});1418 try testFmt("u16: 0o1234\n", "u16: 0o{o}\n", .{value});
1387 }1419 }
1420 {
1421 const value: u8 = 'a';
1422 try testFmt("UTF-8: a\n", "UTF-8: {u}\n", .{value});
1423 }
1424 {
1425 const value: u21 = 0x1F310;
1426 try testFmt("UTF-8: 🌐\n", "UTF-8: {u}\n", .{value});
1427 }
1428 {
1429 const value: u21 = 0xD800;
1430 try testFmt("UTF-8: �\n", "UTF-8: {u}\n", .{value});
1431 }
1432 {
1433 const value: u21 = 0x110001;
1434 try testFmt("UTF-8: �\n", "UTF-8: {u}\n", .{value});
1435 }
1388}1436}
13891437
1390test "int.padded" {1438test "int.padded" {
...@@ -1400,6 +1448,10 @@ test "int.padded" {...@@ -1400,6 +1448,10 @@ test "int.padded" {
1400 try testFmt("i16: '-12345'", "i16: '{:4}'", .{@as(i16, -12345)});1448 try testFmt("i16: '-12345'", "i16: '{:4}'", .{@as(i16, -12345)});
1401 try testFmt("i16: '+12345'", "i16: '{:4}'", .{@as(i16, 12345)});1449 try testFmt("i16: '+12345'", "i16: '{:4}'", .{@as(i16, 12345)});
1402 try testFmt("u16: '12345'", "u16: '{:4}'", .{@as(u16, 12345)});1450 try testFmt("u16: '12345'", "u16: '{:4}'", .{@as(u16, 12345)});
1451
1452 try testFmt("UTF-8: 'ü '", "UTF-8: '{u:<4}'", .{'ü'});
1453 try testFmt("UTF-8: ' ü'", "UTF-8: '{u:>4}'", .{'ü'});
1454 try testFmt("UTF-8: ' ü '", "UTF-8: '{u:^4}'", .{'ü'});
1403}1455}
14041456
1405test "buffer" {1457test "buffer" {
...@@ -1929,6 +1981,9 @@ test "padding" {...@@ -1929,6 +1981,9 @@ test "padding" {
1929 try testFmt("==================Filled", "{:=>24}", .{"Filled"});1981 try testFmt("==================Filled", "{:=>24}", .{"Filled"});
1930 try testFmt(" Centered ", "{:^24}", .{"Centered"});1982 try testFmt(" Centered ", "{:^24}", .{"Centered"});
1931 try testFmt("-", "{:-^1}", .{""});1983 try testFmt("-", "{:-^1}", .{""});
1984 try testFmt("==crêpe===", "{:=^10}", .{"crêpe"});
1985 try testFmt("=====crêpe", "{:=>10}", .{"crêpe"});
1986 try testFmt("crêpe=====", "{:=<10}", .{"crêpe"});
1932}1987}
19331988
1934test "decimal float padding" {1989test "decimal float padding" {
lib/std/unicode.zig+78-6
...@@ -23,11 +23,12 @@ pub fn utf8CodepointSequenceLength(c: u21) !u3 {...@@ -23,11 +23,12 @@ pub fn utf8CodepointSequenceLength(c: u21) !u3 {
23/// returns a number 1-4 indicating the total length of the codepoint in bytes.23/// returns a number 1-4 indicating the total length of the codepoint in bytes.
24/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.24/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.
25pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {25pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
26 return switch (@clz(u8, ~first_byte)) {26 // The switch is optimized much better than a "smart" approach using @clz
27 0 => 1,27 return switch (first_byte) {
28 2 => 2,28 0b0000_0000 ... 0b0111_1111 => 1,
29 3 => 3,29 0b1100_0000 ... 0b1101_1111 => 2,
30 4 => 4,30 0b1110_0000 ... 0b1110_1111 => 3,
31 0b1111_0000 ... 0b1111_0111 => 4,
31 else => error.Utf8InvalidStartByte,32 else => error.Utf8InvalidStartByte,
32 };33 };
33}34}
...@@ -153,6 +154,50 @@ pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {...@@ -153,6 +154,50 @@ pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
153 return value;154 return value;
154}155}
155156
157/// Returns true if the given unicode codepoint can be encoded in UTF-8.
158pub fn utf8ValidCodepoint(value: u21) bool {
159 return switch (value) {
160 0xD800 ... 0xDFFF => false, // Surrogates range
161 0x110000 ... 0x1FFFFF => false, // Above the maximum codepoint value
162 else => true,
163 };
164}
165
166/// Returns the length of a supplied UTF-8 string literal in terms of unicode
167/// codepoints.
168/// Asserts that the data is valid UTF-8.
169pub fn utf8CountCodepoints(s: []const u8) !usize {
170 var len: usize = 0;
171
172 const N = @sizeOf(usize);
173 const MASK = 0x80 * (std.math.maxInt(usize) / 0xff);
174
175 var i: usize = 0;
176 while (i < s.len) {
177 // Fast path for ASCII sequences
178 while (i + N <= s.len) : (i += N) {
179 const v = mem.readIntNative(usize, s[i..][0..N]);
180 if (v & MASK != 0) break;
181 len += N;
182 }
183
184 if (i < s.len) {
185 const n = try utf8ByteSequenceLength(s[i]);
186 if (i + n > s.len) return error.TruncatedInput;
187
188 switch (n) {
189 1 => {}, // ASCII, no validation needed
190 else => _ = try utf8Decode(s[i .. i + n]),
191 }
192
193 i += n;
194 len += 1;
195 }
196 }
197
198 return len;
199}
200
156pub fn utf8ValidateSlice(s: []const u8) bool {201pub fn utf8ValidateSlice(s: []const u8) bool {
157 var i: usize = 0;202 var i: usize = 0;
158 while (i < s.len) {203 while (i < s.len) {
...@@ -687,7 +732,6 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le...@@ -687,7 +732,6 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le
687 }732 }
688}733}
689734
690/// Returns length of a supplied UTF-8 string literal. Asserts that the data is valid UTF-8.
691fn calcUtf16LeLen(utf8: []const u8) usize {735fn calcUtf16LeLen(utf8: []const u8) usize {
692 var src_i: usize = 0;736 var src_i: usize = 0;
693 var dest_len: usize = 0;737 var dest_len: usize = 0;
...@@ -757,3 +801,31 @@ test "utf8ToUtf16LeStringLiteral" {...@@ -757,3 +801,31 @@ test "utf8ToUtf16LeStringLiteral" {
757 testing.expect(utf16[2] == 0);801 testing.expect(utf16[2] == 0);
758 }802 }
759}803}
804
805fn testUtf8CountCodepoints() !void {
806 testing.expectEqual(@as(usize, 10), try utf8CountCodepoints("abcdefghij"));
807 testing.expectEqual(@as(usize, 10), try utf8CountCodepoints("äåéëþüúíóö"));
808 testing.expectEqual(@as(usize, 5), try utf8CountCodepoints("こんにちは"));
809 // testing.expectError(error.Utf8EncodesSurrogateHalf, utf8CountCodepoints("\xED\xA0\x80"));
810}
811
812test "utf8 count codepoints" {
813 try testUtf8CountCodepoints();
814 comptime testUtf8CountCodepoints() catch unreachable;
815}
816
817fn testUtf8ValidCodepoint() !void {
818 testing.expect(utf8ValidCodepoint('e'));
819 testing.expect(utf8ValidCodepoint('ë'));
820 testing.expect(utf8ValidCodepoint('は'));
821 testing.expect(utf8ValidCodepoint(0xe000));
822 testing.expect(utf8ValidCodepoint(0x10ffff));
823 testing.expect(!utf8ValidCodepoint(0xd800));
824 testing.expect(!utf8ValidCodepoint(0xdfff));
825 testing.expect(!utf8ValidCodepoint(0x110000));
826}
827
828test "utf8 valid codepoint" {
829 try testUtf8ValidCodepoint();
830 comptime testUtf8ValidCodepoint() catch unreachable;
831}
lib/std/unicode/throughput_test.zig+68-36
...@@ -3,47 +3,79 @@...@@ -3,47 +3,79 @@
3// This file is part of [zig](https://ziglang.org/), which is MIT licensed.3// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4// The MIT license requires this copyright notice to be included in all copies4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.5// and substantial portions of the software.
6const builtin = @import("builtin");
7const std = @import("std");6const std = @import("std");
7const builtin = std.builtin;
8const time = std.time;
9const unicode = std.unicode;
10
11const Timer = time.Timer;
12
13const N = 1_000_000;
14
15const KiB = 1024;
16const MiB = 1024 * KiB;
17const GiB = 1024 * MiB;
18
19const ResultCount = struct {
20 count: usize,
21 throughput: u64,
22};
23
24fn benchmarkCodepointCount(buf: []const u8) !ResultCount {
25 var timer = try Timer.start();
26
27 const bytes = N * buf.len;
28
29 const start = timer.lap();
30 var i: usize = 0;
31 var r: usize = undefined;
32 while (i < N) : (i += 1) {
33 r = try @call(
34 .{ .modifier = .never_inline },
35 std.unicode.utf8CountCodepoints,
36 .{buf},
37 );
38 }
39 const end = timer.read();
40
41 const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s;
42 const throughput = @floatToInt(u64, @intToFloat(f64, bytes) / elapsed_s);
43
44 return ResultCount{ .count = r, .throughput = throughput };
45}
846
9pub fn main() !void {47pub fn main() !void {
10 const stdout = std.io.getStdOut().outStream();48 const stdout = std.io.getStdOut().outStream();
1149
12 const args = try std.process.argsAlloc(std.heap.page_allocator);50 const args = try std.process.argsAlloc(std.heap.page_allocator);
1351
14 // Warm up runs52 try stdout.print("short ASCII strings\n", .{});
15 var buffer0: [32767]u16 align(4096) = undefined;53 {
16 _ = try std.unicode.utf8ToUtf16Le(&buffer0, args[1]);54 const result = try benchmarkCodepointCount("abc");
17 _ = try std.unicode.utf8ToUtf16Le_better(&buffer0, args[1]);55 try stdout.print(" count: {:5} MiB/s [{d}]\n", .{ result.throughput / (1 * MiB), result.count });
1856 }
19 @fence(.SeqCst);57
20 var timer = try std.time.Timer.start();58 try stdout.print("short Unicode strings\n", .{});
21 @fence(.SeqCst);59 {
2260 const result = try benchmarkCodepointCount("ŌŌŌ");
23 var buffer1: [32767]u16 align(4096) = undefined;61 try stdout.print(" count: {:5} MiB/s [{d}]\n", .{ result.throughput / (1 * MiB), result.count });
24 _ = try std.unicode.utf8ToUtf16Le(&buffer1, args[1]);62 }
2563
26 @fence(.SeqCst);64 try stdout.print("pure ASCII strings\n", .{});
27 const elapsed_ns_orig = timer.lap();65 {
28 @fence(.SeqCst);66 const result = try benchmarkCodepointCount("hello" ** 16);
2967 try stdout.print(" count: {:5} MiB/s [{d}]\n", .{ result.throughput / (1 * MiB), result.count });
30 var buffer2: [32767]u16 align(4096) = undefined;68 }
31 _ = try std.unicode.utf8ToUtf16Le_better(&buffer2, args[1]);69
3270 try stdout.print("pure Unicode strings\n", .{});
33 @fence(.SeqCst);71 {
34 const elapsed_ns_better = timer.lap();72 const result = try benchmarkCodepointCount("こんにちは" ** 16);
35 @fence(.SeqCst);73 try stdout.print(" count: {:5} MiB/s [{d}]\n", .{ result.throughput / (1 * MiB), result.count });
3674 }
37 std.debug.warn("original utf8ToUtf16Le: elapsed: {} ns ({} ms)\n", .{75
38 elapsed_ns_orig, elapsed_ns_orig / 1000000,76 try stdout.print("mixed ASCII/Unicode strings\n", .{});
39 });77 {
40 std.debug.warn("new utf8ToUtf16Le: elapsed: {} ns ({} ms)\n", .{78 const result = try benchmarkCodepointCount("Hyvää huomenta" ** 16);
41 elapsed_ns_better, elapsed_ns_better / 1000000,79 try stdout.print(" count: {:5} MiB/s [{d}]\n", .{ result.throughput / (1 * MiB), result.count });
42 });80 }
43 asm volatile ("nop"
44 :
45 : [a] "r" (&buffer1),
46 [b] "r" (&buffer2)
47 : "memory"
48 );
49}81}