authorgravatar for r00ster91@proton.meWooster <r00ster91@proton.me> 2022-08-12 14:28:22+02:00
committergravatar for r00ster91@proton.meWooster <r00ster91@proton.me> 2022-08-16 00:20:19+02:00
log5490688d658973982d111ad5ad52bc497ef15d84
treefd6d4df15f52a48e786e5366f83e036996432f85
parente3b3eab840d6fc055cdf12cde6d9077cb59b4022

refactor: use std.ascii functions

1 files changed, 3 insertions(+), 4 deletions(-)

lib/std/zig/tokenizer.zig+3-4
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const mem = std.mem;
32
4pub const Token = struct {3pub const Token = struct {
5 tag: Tag,4 tag: Tag,
...@@ -350,7 +349,7 @@ pub const Tokenizer = struct {...@@ -350,7 +349,7 @@ pub const Tokenizer = struct {
350349
351 pub fn init(buffer: [:0]const u8) Tokenizer {350 pub fn init(buffer: [:0]const u8) Tokenizer {
352 // Skip the UTF-8 BOM if present351 // Skip the UTF-8 BOM if present
353 const src_start = if (mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else @as(usize, 0);352 const src_start: usize = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0;
354 return Tokenizer{353 return Tokenizer{
355 .buffer = buffer,354 .buffer = buffer,
356 .index = src_start,355 .index = src_start,
...@@ -1433,8 +1432,8 @@ pub const Tokenizer = struct {...@@ -1433,8 +1432,8 @@ pub const Tokenizer = struct {
14331432
1434 fn getInvalidCharacterLength(self: *Tokenizer) u3 {1433 fn getInvalidCharacterLength(self: *Tokenizer) u3 {
1435 const c0 = self.buffer[self.index];1434 const c0 = self.buffer[self.index];
1436 if (c0 < 0x80) {1435 if (std.ascii.isASCII(c0)) {
1437 if (c0 < 0x20 or c0 == 0x7f) {1436 if (std.ascii.isCntrl(c0)) {
1438 // ascii control codes are never allowed1437 // ascii control codes are never allowed
1439 // (note that \n was checked before we got here)1438 // (note that \n was checked before we got here)
1440 return 1;1439 return 1;