| ... | ... | @@ -338,26 +338,13 @@ pub const Tokenizer = struct { |
| 338 | 338 | } |
| 339 | 339 | |
| 340 | 340 | pub fn init(buffer: []const u8) Tokenizer { |
| 341 | | if (mem.startsWith(u8, buffer, "#!")) { |
| 342 | | const src_start = if (mem.indexOfScalar(u8, buffer, '\n')) |i| i + 1 else buffer.len; |
| 343 | | return Tokenizer{ |
| 344 | | .buffer = buffer, |
| 345 | | .index = src_start, |
| 346 | | .pending_invalid_token = Token{ |
| 347 | | .id = Token.Id.ShebangLine, |
| 348 | | .start = 0, |
| 349 | | .end = src_start, |
| 350 | | }, |
| 351 | | }; |
| 352 | | } else { |
| 353 | | // Skip the UTF-8 BOM if present |
| 354 | | const src_start = if (mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else @as(usize, 0); |
| 355 | | return Tokenizer{ |
| 356 | | .buffer = buffer, |
| 357 | | .index = src_start, |
| 358 | | .pending_invalid_token = null, |
| 359 | | }; |
| 360 | | } |
| 341 | // Skip the UTF-8 BOM if present |
| 342 | const src_start = if (mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else @as(usize, 0); |
| 343 | return Tokenizer{ |
| 344 | .buffer = buffer, |
| 345 | .index = src_start, |
| 346 | .pending_invalid_token = null, |
| 347 | }; |
| 361 | 348 | } |
| 362 | 349 | |
| 363 | 350 | const State = enum { |
Andrew Kelley <andrew@ziglang.org> 2019-11-20 20:18:19-05:00
Commit is signed but in an unrecognized format.
lib