doc/langref.html.in           |  13 ++--
 src/tokenizer.cpp             | 115 +++++++++++++++++---------------
 std/zig/parser_test.zig       |   2 +-
 std/zig/tokenizer.zig         | 113 +++++++++++++++++++++++++++----
 test/compile_errors.zig       |  18 +++++
 test/stage1/behavior/misc.zig |   4 +-
 6 files changed, 186 insertions(+), 79 deletions(-)