| ... | @@ -338,26 +338,13 @@ pub const Tokenizer = struct { | ... | @@ -338,26 +338,13 @@ pub const Tokenizer = struct { |
| 338 | } | 338 | } |
| 339 | | 339 | |
| 340 | pub fn init(buffer: []const u8) Tokenizer { | 340 | pub fn init(buffer: []const u8) Tokenizer { |
| 341 | if (mem.startsWith(u8, buffer, "#!")) { | 341 | // Skip the UTF-8 BOM if present |
| 342 | const src_start = if (mem.indexOfScalar(u8, buffer, '\n')) |i| i + 1 else buffer.len; | 342 | const src_start = if (mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else @as(usize, 0); |
| 343 | return Tokenizer{ | 343 | return Tokenizer{ |
| 344 | .buffer = buffer, | 344 | .buffer = buffer, |
| 345 | .index = src_start, | 345 | .index = src_start, |
| 346 | .pending_invalid_token = Token{ | 346 | .pending_invalid_token = null, |
| 347 | .id = Token.Id.ShebangLine, | 347 | }; |
| 348 | .start = 0, | | |
| 349 | .end = src_start, | | |
| 350 | }, | | |
| 351 | }; | | |
| 352 | } else { | | |
| 353 | // Skip the UTF-8 BOM if present | | |
| 354 | const src_start = if (mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else @as(usize, 0); | | |
| 355 | return Tokenizer{ | | |
| 356 | .buffer = buffer, | | |
| 357 | .index = src_start, | | |
| 358 | .pending_invalid_token = null, | | |
| 359 | }; | | |
| 360 | } | | |
| 361 | } | 348 | } |
| 362 | | 349 | |
| 363 | const State = enum { | 350 | const State = enum { |
Andrew Kelley <andrew@ziglang.org> 2019-11-20 20:18:19-05:00
Commit is signed but in an unrecognized format.
lib