authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2019-09-11 11:46:51+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-09-11 15:20:18-04:00
logf36b8fd7b2c89d46dba95eca05b60487638dd2a0
treebd9f2164f99d43a0ce8a3b1e01e7ac9592718f78
parent0eddee449d8a09a999b352d769b98379865e8dbc

Recognize & skip the UTF-8 BOM

2 files changed, 17 insertions(+), 3 deletions(-)

src/tokenizer.cpp+7-2
......@@ -407,9 +407,14 @@ void tokenize(Buf *buf, Tokenization *out) {
407407 t.buf = buf;
408408
409409 out->line_offsets = allocate<ZigList<size_t>>(1);
410
411410 out->line_offsets->append(0);
412 for (t.pos = 0; t.pos < buf_len(t.buf); t.pos += 1) {
411
412 // Skip the UTF-8 BOM if present
413 if (buf_starts_with_mem(buf, "\xEF\xBB\xBF", 3)) {
414 t.pos += 3;
415 }
416
417 for (; t.pos < buf_len(t.buf); t.pos += 1) {
413418 uint8_t c = buf_ptr(t.buf)[t.pos];
414419 switch (t.state) {
415420 case TokenizeStateError:
std/zig/tokenizer.zig+10-1
......@@ -222,9 +222,11 @@ pub const Tokenizer = struct {
222222 },
223223 };
224224 } else {
225 // Skip the UTF-8 BOM if present
226 const src_start = if (mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else usize(0);
225227 return Tokenizer{
226228 .buffer = buffer,
227 .index = 0,
229 .index = src_start,
228230 .pending_invalid_token = null,
229231 };
230232 }
......@@ -1455,6 +1457,13 @@ test "tokenizer - line comment followed by identifier" {
14551457 });
14561458}
14571459
1460test "tokenizer - UTF-8 BOM is recognized and skipped" {
1461 testTokenize("\xEF\xBB\xBFa;\n", [_]Token.Id{
1462 Token.Id.Identifier,
1463 Token.Id.Semicolon,
1464 });
1465}
1466
14581467fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
14591468 var tokenizer = Tokenizer.init(source);
14601469 for (expected_tokens) |expected_token_id| {