| ... | ... | @@ -57,11 +57,12 @@ pub fn utf8Encode(c: u32, out: []u8) !u3 { |
| 57 | 57 | return length; |
| 58 | 58 | } |
| 59 | 59 | |
| 60 | const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error; |
| 60 | 61 | /// Decodes the UTF-8 codepoint encoded in the given slice of bytes. |
| 61 | 62 | /// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable. |
| 62 | 63 | /// If you already know the length at comptime, you can call one of |
| 63 | 64 | /// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function. |
| 64 | | pub fn utf8Decode(bytes: []const u8) !u32 { |
| 65 | pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u32 { |
| 65 | 66 | return switch (bytes.len) { |
| 66 | 67 | 1 => u32(bytes[0]), |
| 67 | 68 | 2 => utf8Decode2(bytes), |
| ... | ... | @@ -71,7 +72,11 @@ pub fn utf8Decode(bytes: []const u8) !u32 { |
| 71 | 72 | }; |
| 72 | 73 | } |
| 73 | 74 | |
| 74 | | pub fn utf8Decode2(bytes: []const u8) !u32 { |
| 75 | const Utf8Decode2Error = error{ |
| 76 | Utf8ExpectedContinuation, |
| 77 | Utf8OverlongEncoding, |
| 78 | }; |
| 79 | pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u32 { |
| 75 | 80 | debug.assert(bytes.len == 2); |
| 76 | 81 | debug.assert(bytes[0] & 0b11100000 == 0b11000000); |
| 77 | 82 | var value: u32 = bytes[0] & 0b00011111; |
| ... | ... | @@ -85,7 +90,12 @@ pub fn utf8Decode2(bytes: []const u8) !u32 { |
| 85 | 90 | return value; |
| 86 | 91 | } |
| 87 | 92 | |
| 88 | | pub fn utf8Decode3(bytes: []const u8) !u32 { |
| 93 | const Utf8Decode3Error = error{ |
| 94 | Utf8ExpectedContinuation, |
| 95 | Utf8OverlongEncoding, |
| 96 | Utf8EncodesSurrogateHalf, |
| 97 | }; |
| 98 | pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u32 { |
| 89 | 99 | debug.assert(bytes.len == 3); |
| 90 | 100 | debug.assert(bytes[0] & 0b11110000 == 0b11100000); |
| 91 | 101 | var value: u32 = bytes[0] & 0b00001111; |
| ... | ... | @@ -104,7 +114,12 @@ pub fn utf8Decode3(bytes: []const u8) !u32 { |
| 104 | 114 | return value; |
| 105 | 115 | } |
| 106 | 116 | |
| 107 | | pub fn utf8Decode4(bytes: []const u8) !u32 { |
| 117 | const Utf8Decode4Error = error{ |
| 118 | Utf8ExpectedContinuation, |
| 119 | Utf8OverlongEncoding, |
| 120 | Utf8CodepointTooLarge, |
| 121 | }; |
| 122 | pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u32 { |
| 108 | 123 | debug.assert(bytes.len == 4); |
| 109 | 124 | debug.assert(bytes[0] & 0b11111000 == 0b11110000); |
| 110 | 125 | var value: u32 = bytes[0] & 0b00000111; |
| ... | ... | @@ -206,19 +221,21 @@ const Utf8Iterator = struct { |
| 206 | 221 | pub fn nextCodepoint(it: &Utf8Iterator) ?u32 { |
| 207 | 222 | const slice = it.nextCodepointSlice() ?? return null; |
| 208 | 223 | |
| 209 | | const r = switch (slice.len) { |
| 210 | | 1 => u32(slice[0]), |
| 211 | | 2 => utf8Decode2(slice), |
| 212 | | 3 => utf8Decode3(slice), |
| 213 | | 4 => utf8Decode4(slice), |
| 224 | switch (slice.len) { |
| 225 | 1 => return u32(slice[0]), |
| 226 | 2 => return utf8Decode2(slice) catch unreachable, |
| 227 | 3 => return utf8Decode3(slice) catch unreachable, |
| 228 | 4 => return utf8Decode4(slice) catch unreachable, |
| 214 | 229 | else => unreachable, |
| 215 | | }; |
| 216 | | |
| 217 | | return r catch unreachable; |
| 230 | } |
| 218 | 231 | } |
| 219 | 232 | }; |
| 220 | 233 | |
| 221 | 234 | test "utf8 encode" { |
| 235 | comptime testUtf8Encode() catch unreachable; |
| 236 | try testUtf8Encode(); |
| 237 | } |
| 238 | fn testUtf8Encode() !void { |
| 222 | 239 | // A few taken from wikipedia a few taken elsewhere |
| 223 | 240 | var array: [4]u8 = undefined; |
| 224 | 241 | debug.assert((try utf8Encode(try utf8Decode("€"), array[0..])) == 3); |
| ... | ... | @@ -241,6 +258,10 @@ test "utf8 encode" { |
| 241 | 258 | } |
| 242 | 259 | |
| 243 | 260 | test "utf8 encode error" { |
| 261 | comptime testUtf8EncodeError(); |
| 262 | testUtf8EncodeError(); |
| 263 | } |
| 264 | fn testUtf8EncodeError() void { |
| 244 | 265 | var array: [4]u8 = undefined; |
| 245 | 266 | testErrorEncode(0xd800, array[0..], error.Utf8CannotEncodeSurrogateHalf); |
| 246 | 267 | testErrorEncode(0xdfff, array[0..], error.Utf8CannotEncodeSurrogateHalf); |
| ... | ... | @@ -257,6 +278,10 @@ fn testErrorEncode(codePoint: u32, array: []u8, expectedErr: error) void { |
| 257 | 278 | } |
| 258 | 279 | |
| 259 | 280 | test "utf8 iterator on ascii" { |
| 281 | comptime testUtf8IteratorOnAscii(); |
| 282 | testUtf8IteratorOnAscii(); |
| 283 | } |
| 284 | fn testUtf8IteratorOnAscii() void { |
| 260 | 285 | const s = Utf8View.initComptime("abc"); |
| 261 | 286 | |
| 262 | 287 | var it1 = s.iterator(); |
| ... | ... | @@ -273,6 +298,10 @@ test "utf8 iterator on ascii" { |
| 273 | 298 | } |
| 274 | 299 | |
| 275 | 300 | test "utf8 view bad" { |
| 301 | comptime testUtf8ViewBad(); |
| 302 | testUtf8ViewBad(); |
| 303 | } |
| 304 | fn testUtf8ViewBad() void { |
| 276 | 305 | // Compile-time error. |
| 277 | 306 | // const s3 = Utf8View.initComptime("\xfe\xf2"); |
| 278 | 307 | |
| ... | ... | @@ -281,6 +310,10 @@ test "utf8 view bad" { |
| 281 | 310 | } |
| 282 | 311 | |
| 283 | 312 | test "utf8 view ok" { |
| 313 | comptime testUtf8ViewOk(); |
| 314 | testUtf8ViewOk(); |
| 315 | } |
| 316 | fn testUtf8ViewOk() void { |
| 284 | 317 | const s = Utf8View.initComptime("東京市"); |
| 285 | 318 | |
| 286 | 319 | var it1 = s.iterator(); |
| ... | ... | @@ -297,6 +330,10 @@ test "utf8 view ok" { |
| 297 | 330 | } |
| 298 | 331 | |
| 299 | 332 | test "bad utf8 slice" { |
| 333 | comptime testBadUtf8Slice(); |
| 334 | testBadUtf8Slice(); |
| 335 | } |
| 336 | fn testBadUtf8Slice() void { |
| 300 | 337 | debug.assert(utf8ValidateSlice("abc")); |
| 301 | 338 | debug.assert(!utf8ValidateSlice("abc\xc0")); |
| 302 | 339 | debug.assert(!utf8ValidateSlice("abc\xc0abc")); |
| ... | ... | @@ -304,6 +341,10 @@ test "bad utf8 slice" { |
| 304 | 341 | } |
| 305 | 342 | |
| 306 | 343 | test "valid utf8" { |
| 344 | comptime testValidUtf8(); |
| 345 | testValidUtf8(); |
| 346 | } |
| 347 | fn testValidUtf8() void { |
| 307 | 348 | testValid("\x00", 0x0); |
| 308 | 349 | testValid("\x20", 0x20); |
| 309 | 350 | testValid("\x7f", 0x7f); |
| ... | ... | @@ -319,6 +360,10 @@ test "valid utf8" { |
| 319 | 360 | } |
| 320 | 361 | |
| 321 | 362 | test "invalid utf8 continuation bytes" { |
| 363 | comptime testInvalidUtf8ContinuationBytes(); |
| 364 | testInvalidUtf8ContinuationBytes(); |
| 365 | } |
| 366 | fn testInvalidUtf8ContinuationBytes() void { |
| 322 | 367 | // unexpected continuation |
| 323 | 368 | testError("\x80", error.Utf8InvalidStartByte); |
| 324 | 369 | testError("\xbf", error.Utf8InvalidStartByte); |
| ... | ... | @@ -347,6 +392,10 @@ test "invalid utf8 continuation bytes" { |
| 347 | 392 | } |
| 348 | 393 | |
| 349 | 394 | test "overlong utf8 codepoint" { |
| 395 | comptime testOverlongUtf8Codepoint(); |
| 396 | testOverlongUtf8Codepoint(); |
| 397 | } |
| 398 | fn testOverlongUtf8Codepoint() void { |
| 350 | 399 | testError("\xc0\x80", error.Utf8OverlongEncoding); |
| 351 | 400 | testError("\xc1\xbf", error.Utf8OverlongEncoding); |
| 352 | 401 | testError("\xe0\x80\x80", error.Utf8OverlongEncoding); |
| ... | ... | @@ -356,6 +405,10 @@ test "overlong utf8 codepoint" { |
| 356 | 405 | } |
| 357 | 406 | |
| 358 | 407 | test "misc invalid utf8" { |
| 408 | comptime testMiscInvalidUtf8(); |
| 409 | testMiscInvalidUtf8(); |
| 410 | } |
| 411 | fn testMiscInvalidUtf8() void { |
| 359 | 412 | // codepoint out of bounds |
| 360 | 413 | testError("\xf4\x90\x80\x80", error.Utf8CodepointTooLarge); |
| 361 | 414 | testError("\xf7\xbf\xbf\xbf", error.Utf8CodepointTooLarge); |
Josh Wolfe <thejoshwolfe@gmail.com> 2018-04-29 17:38:41-04:00
std