doc/langref.html.in           |  4 +--
 lib/std/zig/tokenizer.zig     | 41 +++++++++++++++++++++----
 src/tokenizer.cpp             | 51 +++++++++++++++++++++++++-------
 test/stage1/behavior/misc.zig |  4 +++
 4 files changed, 81 insertions(+), 19 deletions(-)