| author |  LemonBoy <thatlemon@gmail.com> 2019-09-11 11:46:51+02:00 |
| committer |  Andrew Kelley <andrew@ziglang.org> 2019-09-11 15:20:18-04:00 |
| log | f36b8fd7b2c89d46dba95eca05b60487638dd2a0 |
| tree | bd9f2164f99d43a0ce8a3b1e01e7ac9592718f78 |
| parent | 0eddee449d8a09a999b352d769b98379865e8dbc |
Recognize & skip the UTF-8 BOM
2 files changed, 17 insertions(+), 3 deletions(-)
std
src/tokenizer.cpp+7-2
| ... | @@ -407,9 +407,14 @@ void tokenize(Buf *buf, Tokenization *out) { | ... | @@ -407,9 +407,14 @@ void tokenize(Buf *buf, Tokenization *out) { |
| 407 | t.buf = buf; | 407 | t.buf = buf; |
| 408 | 408 | ||
| 409 | out->line_offsets = allocate<ZigList<size_t>>(1); | 409 | out->line_offsets = allocate<ZigList<size_t>>(1); |
| 410 | |||
| 411 | out->line_offsets->append(0); | 410 | out->line_offsets->append(0); |
| 412 | for (t.pos = 0; t.pos < buf_len(t.buf); t.pos += 1) { | 411 | |
| 412 | // Skip the UTF-8 BOM if present | ||
| 413 | if (buf_starts_with_mem(buf, "\xEF\xBB\xBF", 3)) { | ||
| 414 | t.pos += 3; | ||
| 415 | } | ||
| 416 | |||
| 417 | for (; t.pos < buf_len(t.buf); t.pos += 1) { | ||
| 413 | uint8_t c = buf_ptr(t.buf)[t.pos]; | 418 | uint8_t c = buf_ptr(t.buf)[t.pos]; |
| 414 | switch (t.state) { | 419 | switch (t.state) { |
| 415 | case TokenizeStateError: | 420 | case TokenizeStateError: |
std/zig/tokenizer.zig+10-1
| ... | @@ -222,9 +222,11 @@ pub const Tokenizer = struct { | ... | @@ -222,9 +222,11 @@ pub const Tokenizer = struct { |
| 222 | }, | 222 | }, |
| 223 | }; | 223 | }; |
| 224 | } else { | 224 | } else { |
| 225 | // Skip the UTF-8 BOM if present | ||
| 226 | const src_start = if (mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else usize(0); | ||
| 225 | return Tokenizer{ | 227 | return Tokenizer{ |
| 226 | .buffer = buffer, | 228 | .buffer = buffer, |
| 227 | .index = 0, | 229 | .index = src_start, |
| 228 | .pending_invalid_token = null, | 230 | .pending_invalid_token = null, |
| 229 | }; | 231 | }; |
| 230 | } | 232 | } |
| ... | @@ -1455,6 +1457,13 @@ test "tokenizer - line comment followed by identifier" { | ... | @@ -1455,6 +1457,13 @@ test "tokenizer - line comment followed by identifier" { |
| 1455 | }); | 1457 | }); |
| 1456 | } | 1458 | } |
| 1457 | 1459 | ||
| 1460 | test "tokenizer - UTF-8 BOM is recognized and skipped" { | ||
| 1461 | testTokenize("\xEF\xBB\xBFa;\n", [_]Token.Id{ | ||
| 1462 | Token.Id.Identifier, | ||
| 1463 | Token.Id.Semicolon, | ||
| 1464 | }); | ||
| 1465 | } | ||
| 1466 | |||
| 1458 | fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { | 1467 | fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { |
| 1459 | var tokenizer = Tokenizer.init(source); | 1468 | var tokenizer = Tokenizer.init(source); |
| 1460 | for (expected_tokens) |expected_token_id| { | 1469 | for (expected_tokens) |expected_token_id| { |