Commit d0b11af2bd445d10383049d2d7574c19a95c9006 | zig

author	Andrew Kelley <andrew@ziglang.org> 2016-08-01 23:11:31-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2016-08-04 22:52:38-07:00
log	d0b11af2bd445d10383049d2d7574c19a95c9006
tree	c841cdb1773668133c16d3a8be389630608ad523
parent	0450b73e3e4a488a2a20f2b8933443dcbe5fd5e3

new multiline string syntax

This patch also moves a bunch of the parser code into the tokenizer.

Closes #162.

15 files changed, 802 insertions(+), 1093 deletions(-)

doc
- langref.md
- vim
  - syntax
    zig.vim
src
test
- run_tests.cpp
- self_hosted.zig

doc/langref.md+65-36

...	@@ -7,27 +7,27 @@ Root = many(TopLevelDecl) "EOF"	...	@@ -7,27 +7,27 @@ Root = many(TopLevelDecl) "EOF"
7		7
8	TopLevelDecl = many(Directive) option(VisibleMod) (FnDef \| ExternDecl \| ContainerDecl \| GlobalVarDecl \| ErrorValueDecl \| TypeDecl \| UseDecl)	8	TopLevelDecl = many(Directive) option(VisibleMod) (FnDef \| ExternDecl \| ContainerDecl \| GlobalVarDecl \| ErrorValueDecl \| TypeDecl \| UseDecl)
9		9
10	TypeDecl = "type" "Symbol" "=" TypeExpr ";"	10	TypeDecl = "type" Symbol "=" TypeExpr ";"
11		11
12	ErrorValueDecl = "error" "Symbol" ";"	12	ErrorValueDecl = "error" Symbol ";"
13		13
14	GlobalVarDecl = VariableDeclaration ";"	14	GlobalVarDecl = VariableDeclaration ";"
15		15
16	VariableDeclaration = ("var" \| "const") "Symbol" option(":" TypeExpr) "=" Expression	16	VariableDeclaration = ("var" \| "const") Symbol option(":" TypeExpr) "=" Expression
17		17
18	ContainerDecl = ("struct" \| "enum" \| "union") "Symbol" option(ParamDeclList) "{" many(StructMember) "}"	18	ContainerDecl = ("struct" \| "enum" \| "union") Symbol option(ParamDeclList) "{" many(StructMember) "}"
19		19
20	StructMember = many(Directive) option(VisibleMod) (StructField \| FnDef \| GlobalVarDecl \| ContainerDecl)	20	StructMember = many(Directive) option(VisibleMod) (StructField \| FnDef \| GlobalVarDecl \| ContainerDecl)
21		21
22	StructField = "Symbol" option(":" Expression) ",")	22	StructField = Symbol option(":" Expression) ",")
23		23
24	UseDecl = "use" Expression ";"	24	UseDecl = "use" Expression ";"
25		25
26	ExternDecl = "extern" (FnProto \| VariableDeclaration) ";"	26	ExternDecl = "extern" (FnProto \| VariableDeclaration) ";"
27		27
28	FnProto = "fn" option("Symbol") ParamDeclList option("->" TypeExpr)	28	FnProto = "fn" option(Symbol) ParamDeclList option("->" TypeExpr)
29		29
30	Directive = "#" "Symbol" "(" Expression ")"	30	Directive = "#" Symbol "(" Expression ")"
31		31
32	VisibleMod = "pub" \| "export"	32	VisibleMod = "pub" \| "export"
33		33
...	@@ -35,13 +35,13 @@ FnDef = option("inline" \| "extern") FnProto Block	...	@@ -35,13 +35,13 @@ FnDef = option("inline" \| "extern") FnProto Block
35		35
36	ParamDeclList = "(" list(ParamDecl, ",") ")"	36	ParamDeclList = "(" list(ParamDecl, ",") ")"
37		37
38	ParamDecl = option("noalias" \| "inline") option("Symbol" ":") TypeExpr \| "..."	38	ParamDecl = option("noalias" \| "inline") option(Symbol ":") TypeExpr \| "..."
39		39
40	Block = "{" list(option(Statement), ";") "}"	40	Block = "{" list(option(Statement), ";") "}"
41		41
42	Statement = Label \| VariableDeclaration ";" \| Defer ";" \| NonBlockExpression ";" \| BlockExpression	42	Statement = Label \| VariableDeclaration ";" \| Defer ";" \| NonBlockExpression ";" \| BlockExpression
43		43
44	Label = "Symbol" ":"	44	Label = Symbol ":"
45		45
46	Expression = BlockExpression \| NonBlockExpression	46	Expression = BlockExpression \| NonBlockExpression
47		47
...	@@ -49,23 +49,23 @@ TypeExpr = PrefixOpExpression	...	@@ -49,23 +49,23 @@ TypeExpr = PrefixOpExpression
49		49
50	NonBlockExpression = ReturnExpression \| AssignmentExpression	50	NonBlockExpression = ReturnExpression \| AssignmentExpression
51		51
52	AsmExpression = "asm" option("volatile") "(" "String" option(AsmOutput) ")"	52	AsmExpression = "asm" option("volatile") "(" String option(AsmOutput) ")"
53		53
54	AsmOutput = ":" list(AsmOutputItem, ",") option(AsmInput)	54	AsmOutput = ":" list(AsmOutputItem, ",") option(AsmInput)
55		55
56	AsmInput = ":" list(AsmInputItem, ",") option(AsmClobbers)	56	AsmInput = ":" list(AsmInputItem, ",") option(AsmClobbers)
57		57
58	AsmOutputItem = "[" "Symbol" "]" "String" "(" ("Symbol" \| "->" TypeExpr) ")"	58	AsmOutputItem = "[" Symbol "]" String "(" (Symbol \| "->" TypeExpr) ")"
59		59
60	AsmInputItem = "[" "Symbol" "]" "String" "(" Expression ")"	60	AsmInputItem = "[" Symbol "]" String "(" Expression ")"
61		61
62	AsmClobbers= ":" list("String", ",")	62	AsmClobbers= ":" list(String, ",")
63		63
64	UnwrapExpression = BoolOrExpression (UnwrapMaybe \| UnwrapError) \| BoolOrExpression	64	UnwrapExpression = BoolOrExpression (UnwrapMaybe \| UnwrapError) \| BoolOrExpression
65		65
66	UnwrapMaybe = "??" Expression	66	UnwrapMaybe = "??" Expression
67		67
68	UnwrapError = "%%" option("\|" "Symbol" "\|") Expression	68	UnwrapError = "%%" option("\|" Symbol "\|") Expression
69		69
70	AssignmentExpression = UnwrapExpression AssignmentOperator UnwrapExpression \| UnwrapExpression	70	AssignmentExpression = UnwrapExpression AssignmentOperator UnwrapExpression \| UnwrapExpression
71		71
...	@@ -75,13 +75,13 @@ BlockExpression = IfExpression \| Block \| WhileExpression \| ForExpression \| Switc	...	@@ -75,13 +75,13 @@ BlockExpression = IfExpression \| Block \| WhileExpression \| ForExpression \| Switc
75		75
76	SwitchExpression = "switch" "(" Expression ")" "{" many(SwitchProng) "}"	76	SwitchExpression = "switch" "(" Expression ")" "{" many(SwitchProng) "}"
77		77
78	SwitchProng = (list(SwitchItem, ",") \| "else") "=>" option("\|" "Symbol" "\|") Expression ","	78	SwitchProng = (list(SwitchItem, ",") \| "else") "=>" option("\|" Symbol "\|") Expression ","
79		79
80	SwitchItem = Expression \| (Expression "..." Expression)	80	SwitchItem = Expression \| (Expression "..." Expression)
81		81
82	WhileExpression = "while" "(" Expression option(";" Expression) ")" Expression	82	WhileExpression = "while" "(" Expression option(";" Expression) ")" Expression
83		83
84	ForExpression = "for" "(" Expression ")" option("\|" option("*") "Symbol" option("," "Symbol") "\|") Expression	84	ForExpression = "for" "(" Expression ")" option("\|" option("*") Symbol option("," Symbol) "\|") Expression
85		85
86	BoolOrExpression = BoolAndExpression "\|\|" BoolOrExpression \| BoolAndExpression	86	BoolOrExpression = BoolAndExpression "\|\|" BoolOrExpression \| BoolAndExpression
87		87
...	@@ -93,7 +93,7 @@ IfExpression = IfVarExpression \| IfBoolExpression	...	@@ -93,7 +93,7 @@ IfExpression = IfVarExpression \| IfBoolExpression
93		93
94	IfBoolExpression = "if" "(" Expression ")" Expression option(Else)	94	IfBoolExpression = "if" "(" Expression ")" Expression option(Else)
95		95
96	IfVarExpression = "if" "(" ("const" \| "var") option("*") "Symbol" option(":" TypeExpr) "?=" Expression ")" Expression Option(Else)	96	IfVarExpression = "if" "(" ("const" \| "var") option("*") Symbol option(":" TypeExpr) "?=" Expression ")" Expression Option(Else)
97		97
98	Else = "else" Expression	98	Else = "else" Expression
99		99
...	@@ -127,7 +127,7 @@ PrefixOpExpression = PrefixOp PrefixOpExpression \| SuffixOpExpression	...	@@ -127,7 +127,7 @@ PrefixOpExpression = PrefixOp PrefixOpExpression \| SuffixOpExpression
127		127
128	SuffixOpExpression = PrimaryExpression option(FnCallExpression \| ArrayAccessExpression \| FieldAccessExpression \| SliceExpression)	128	SuffixOpExpression = PrimaryExpression option(FnCallExpression \| ArrayAccessExpression \| FieldAccessExpression \| SliceExpression)
129		129
130	FieldAccessExpression = "." "Symbol"	130	FieldAccessExpression = "." Symbol
131		131
132	FnCallExpression = "(" list(Expression, ",") ")"	132	FnCallExpression = "(" list(Expression, ",") ")"
133		133
...	@@ -139,15 +139,15 @@ ContainerInitExpression = "{" ContainerInitBody "}"	...	@@ -139,15 +139,15 @@ ContainerInitExpression = "{" ContainerInitBody "}"
139		139
140	ContainerInitBody = list(StructLiteralField, ",") \| list(Expression, ",")	140	ContainerInitBody = list(StructLiteralField, ",") \| list(Expression, ",")
141		141
142	StructLiteralField = "." "Symbol" "=" Expression	142	StructLiteralField = "." Symbol "=" Expression
143		143
144	PrefixOp = "!" \| "-" \| "~" \| "*" \| ("&" option("const")) \| "?" \| "%" \| "%%" \| "??" \| "-%"	144	PrefixOp = "!" \| "-" \| "~" \| "*" \| ("&" option("const")) \| "?" \| "%" \| "%%" \| "??" \| "-%"
145		145
146	PrimaryExpression = "Number" \| "String" \| "CharLiteral" \| KeywordLiteral \| GroupedExpression \| GotoExpression \| BlockExpression \| "Symbol" \| ("@" "Symbol" FnCallExpression) \| ArrayType \| (option("extern") FnProto) \| AsmExpression \| ("error" "." "Symbol")	146	PrimaryExpression = Number \| String \| CharLiteral \| KeywordLiteral \| GroupedExpression \| GotoExpression \| BlockExpression \| Symbol \| ("@" Symbol FnCallExpression) \| ArrayType \| (option("extern") FnProto) \| AsmExpression \| ("error" "." Symbol)
147		147
148	ArrayType = "[" option(Expression) "]" option("const") TypeExpr	148	ArrayType = "[" option(Expression) "]" option("const") TypeExpr
149		149
150	GotoExpression = "goto" "Symbol"	150	GotoExpression = "goto" Symbol
151		151
152	GroupedExpression = "(" Expression ")"	152	GroupedExpression = "(" Expression ")"
153		153
...	@@ -265,14 +265,13 @@ from codegen.	...	@@ -265,14 +265,13 @@ from codegen.
265	### Literals	265	### Literals
266		266
267	#### Character and String Literals	267	#### Character and String Literals
		268
268	```	269	```
269	Literal Example Characters Escapes Null Term Type	270	Literal Example Characters Escapes Null Term Type
270		271
271	Byte 'H' All ASCII Byte No u8	272	Byte 'H' All ASCII Byte No u8
272	UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8	273	UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8
273	UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8	274	UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8
274	UTF-8 Raw String r"X(hello)X" All Unicode None No [5]u8
275	UTF-8 Raw C String rc"X(hello)X" All Unicode None Yes &const u8
276	```	275	```
277		276
278	### Escapes	277	### Escapes
...	@@ -291,26 +290,56 @@ UTF-8 Raw C String rc"X(hello)X" All Unicode None Yes &const	...	@@ -291,26 +290,56 @@ UTF-8 Raw C String rc"X(hello)X" All Unicode None Yes &const
291		290
292	Note that the maximum valid Unicode point is 0x10ffff.	291	Note that the maximum valid Unicode point is 0x10ffff.
293		292
294	##### Raw Strings	293	##### Multiline String Literals
295		294
296	Raw string literals have no escapes and can span across multiple lines. To	295	Multiline string literals have no escapes and can span across multiple lines.
297	start a raw string, use 'r"' or 'rc"' followed by unique bytes followed by '('.	296	To start a multiline string literal, use the `\\` token. Just like a comment,
298	To end a raw string, use ')' followed by the same unique bytes, followed by '"'.	297	the string literal goes until the end of the line. The end of the line is not
		298	included in the string literal.
299		299
		300	However, if the next line begins with `\\` then a newline is appended and
		301	the string literal continues.
300		302
301	#### Numeric Literals	303	Example:
302		304
		305	```zig
		306	const hello_world_in_c =
		307	\\#include <stdio.h>
		308	\\
		309	\\int main(int argc, char **argv) {
		310	\\ printf("hello world\n");
		311	\\ return 0;
		312	\\}
		313	;
303	```	314	```
304	Number literals Example Exponentiation	315
305		316	For a multiline C string literal, prepend `c` to each `\\`. Example:
306	Decimal integer 98222 N/A	317
307	Hex integer 0xff N/A	318	```zig
308	Octal integer 0o77 N/A	319	const c_string_literal =
309	Binary integer 0b11110000 N/A	320	c\\#include <stdio.h>
310	Floating-point 123.0E+77 Optional	321	c\\
311	Hex floating point TODO TODO	322	c\\int main(int argc, char **argv) {
		323	c\\ printf("hello world\n");
		324	c\\ return 0;
		325	c\\}
		326	;
312	```	327	```
313		328
		329	In this example the variable `c_string_literal` has type `&const char` and
		330	has a terminating null byte.
		331
		332	#### Number Literals
		333
		334	Number literals \| Example \| Exponentiation
		335	--------------------\|-------------\|--------------
		336	Decimal integer \| 98222 \| N/A
		337	Hex integer \| 0xff \| N/A
		338	Octal integer \| 0o77 \| N/A
		339	Binary integer \| 0b11110000 \| N/A
		340	Floating point \| 123.0E+77 \| Optional
		341	Hex floating point \| 0x103.70p-5 \| Optional
		342
314	### Identifiers	343	### Identifiers
315		344
316	TODO	345	TODO

doc/vim/syntax/zig.vim+16-18

 @@ -1,11 +1,12 @@
 " Vim syntax file
 " Language: Zig
 " Maintainer: Andrew Kelley
-" Latest Revision: 28 July 2016
+" Latest Revision: 03 August 2016
 if exists("b:current_syntax")
   finish
 endif
+let b:current_syntax = "zig"
 syn keyword zigStorage const var extern export pub noalias inline noinline
 syn keyword zigStructure struct enum union
 @@ -24,33 +25,30 @@ syn keyword zigBoolean true false
 syn match zigOperator display "\%(+%\?\|-%\?\|/\|*%\?\|=\|\^\|&\|?\||\|!\|>\|<\|%\|<<%\?\|>>\|&&\|||\)=\?"
 syn match zigArrowCharacter display "->"
-syn match zigDecNumber display "\<[0-9][0-9_]*\%([iu]\%(size\|8\|16\|32\|64\)\)\="
+syn match zigDecNumber display "\<[0-9]*\%(.[0-9]\+\)\=\%([eE][+-]\?[0-9]\+\)\="
-syn match zigHexNumber display "\<0x[a-fA-F0-9_]\+\%([iu]\%(size\|8\|16\|32\|64\)\)\="
+syn match zigHexNumber display "\<0x[a-fA-F0-9]\+\%(.[a-fA-F0-9]\+\%([pP][+-]\?[0-9]\+\)\?\)\="
-syn match zigOctNumber display "\<0o[0-7_]\+\%([iu]\%(size\|8\|16\|32\|64\)\)\="
+syn match zigOctNumber display "\<0o[0-7]\+"
-syn match zigBinNumber display "\<0b[01_]\+\%([iu]\%(size\|8\|16\|32\|64\)\)\="
+syn match zigBinNumber display "\<0b[01]\+\%(.[01]\+\%([eE][+-]\?[0-9]\+\)\?\)\="
 syn match zigCharacterInvalid display contained /b\?'\zs[\n\r\t']\ze'/
 syn match zigCharacterInvalidUnicode display contained /b'\zs[^[:cntrl:][:graph:][:alnum:][:space:]]\ze'/
 syn match zigCharacter /b'\([^\\]\|\\\(.\|x\x\{2}\)\)'/ contains=zigEscape,zigEscapeError,zigCharacterInvalid,zigCharacterInvalidUnicode
-syn match zigCharacter /'\([^\\]\|\\\(.\|x\x\{2}\|u\x\{4}\|U\x\{8}\|u{\x\{1,6}}\)\)'/ contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigCharacterInvalid
+syn match zigCharacter /'\([^\\]\|\\\(.\|x\x\{2}\|u\x\{4}\|U\x\{6}\)\)'/ contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigCharacterInvalid
-syn match zigShebang /\%^#![^[].*/
 syn region zigCommentLine start="//" end="$" contains=zigTodo,@Spell
 syn region zigCommentLineDoc start="//\%(//\@!\|!\)" end="$" contains=zigTodo,@Spell
+" TODO match only the first '\\' within the zigMultilineString as zigMultilineStringPrefix
+syn match zigMultilineStringPrefix display contained /c\?\\\\/
+syn region zigMultilineString start="c\?\\\\" end="$" contains=zigMultilineStringPrefix
 syn keyword zigTodo contained TODO XXX
 syn match     zigEscapeError   display contained /\\./
-syn match     zigEscape        display contained /\\\([nrt0\\'"]\|x\x\{2}\)/
+syn match     zigEscape        display contained /\\\([nrt\\'"]\|x\x\{2}\)/
-syn match     zigEscapeUnicode display contained /\\\(u\x\{4}\|U\x\{8}\)/
+syn match     zigEscapeUnicode display contained /\\\(u\x\{4}\|U\x\{6}\)/
-syn match     zigEscapeUnicode display contained /\\u{\x\{1,6}}/
+syn region    zigString      start=+c\?"+ skip=+\\\\\|\\"+ end=+"+ oneline contains=zigEscape,zigEscapeUnicode,zigEscapeError,@Spell
-syn match     zigStringContinuation display contained /\\\n\s*/
-syn region    zigString      start=+c\?"+ skip=+\\\\\|\\"+ end=+"+ oneline contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigStringContinuation,@Spell
-syn region    zigString      start='r"\z([^)]*\)(' end=')\z1"' contains=@Spell
-let b:current_syntax = "zig"
 hi def link zigDecNumber zigNumber
 hi def link zigHexNumber zigNumber
 @@ -59,12 +57,12 @@ hi def link zigBinNumber zigNumber
 hi def link zigKeyword Keyword
 hi def link zigType Type
-hi def link zigShebang Comment
 hi def link zigCommentLine Comment
 hi def link zigCommentLineDoc SpecialComment
 hi def link zigTodo Todo
-hi def link zigStringContinuation Special
 hi def link zigString String
+hi def link zigMultilineString String
+hi def link zigMultilineStringPrefix Comment
 hi def link zigCharacterInvalid Error
 hi def link zigCharacterInvalidUnicode zigCharacterInvalid
 hi def link zigCharacter Character

src/all_types.hpp+21-32

...	@@ -194,7 +194,7 @@ struct AstNodeRoot {	...	@@ -194,7 +194,7 @@ struct AstNodeRoot {
194		194
195	struct AstNodeFnProto {	195	struct AstNodeFnProto {
196	TopLevelDecl top_level_decl;	196	TopLevelDecl top_level_decl;
197	Buf name;	197	Buf *name;
198	ZigList<AstNode *> params;	198	ZigList<AstNode *> params;
199	AstNode *return_type;	199	AstNode *return_type;
200	bool is_var_args;	200	bool is_var_args;
...	@@ -229,7 +229,7 @@ struct AstNodeFnDecl {	...	@@ -229,7 +229,7 @@ struct AstNodeFnDecl {
229	};	229	};
230		230
231	struct AstNodeParamDecl {	231	struct AstNodeParamDecl {
232	Buf name;	232	Buf *name;
233	AstNode *type;	233	AstNode *type;
234	bool is_noalias;	234	bool is_noalias;
235	bool is_inline;	235	bool is_inline;
...	@@ -279,7 +279,7 @@ struct AstNodeDefer {	...	@@ -279,7 +279,7 @@ struct AstNodeDefer {
279		279
280	struct AstNodeVariableDeclaration {	280	struct AstNodeVariableDeclaration {
281	TopLevelDecl top_level_decl;	281	TopLevelDecl top_level_decl;
282	Buf symbol;	282	Buf *symbol;
283	bool is_const;	283	bool is_const;
284	bool is_extern;	284	bool is_extern;
285	// one or both of type and expr will be non null	285	// one or both of type and expr will be non null
...	@@ -293,7 +293,7 @@ struct AstNodeVariableDeclaration {	...	@@ -293,7 +293,7 @@ struct AstNodeVariableDeclaration {
293		293
294	struct AstNodeTypeDecl {	294	struct AstNodeTypeDecl {
295	TopLevelDecl top_level_decl;	295	TopLevelDecl top_level_decl;
296	Buf symbol;	296	Buf *symbol;
297	AstNode *child_type;	297	AstNode *child_type;
298		298
299	// populated by semantic analyzer	299	// populated by semantic analyzer
...	@@ -305,7 +305,7 @@ struct AstNodeTypeDecl {	...	@@ -305,7 +305,7 @@ struct AstNodeTypeDecl {
305		305
306	struct AstNodeErrorValueDecl {	306	struct AstNodeErrorValueDecl {
307	TopLevelDecl top_level_decl;	307	TopLevelDecl top_level_decl;
308	Buf name;	308	Buf *name;
309		309
310	// populated by semantic analyzer	310	// populated by semantic analyzer
311	ErrorTableEntry *err;	311	ErrorTableEntry *err;
...	@@ -434,7 +434,7 @@ struct AstNodeSliceExpr {	...	@@ -434,7 +434,7 @@ struct AstNodeSliceExpr {
434		434
435	struct AstNodeFieldAccessExpr {	435	struct AstNodeFieldAccessExpr {
436	AstNode *struct_expr;	436	AstNode *struct_expr;
437	Buf field_name;	437	Buf *field_name;
438		438
439	// populated by semantic analyzer	439	// populated by semantic analyzer
440	TypeStructField *type_struct_field;	440	TypeStructField *type_struct_field;
...	@@ -448,7 +448,7 @@ struct AstNodeFieldAccessExpr {	...	@@ -448,7 +448,7 @@ struct AstNodeFieldAccessExpr {
448	};	448	};
449		449
450	struct AstNodeDirective {	450	struct AstNodeDirective {
451	Buf name;	451	Buf *name;
452	AstNode *expr;	452	AstNode *expr;
453	};	453	};
454		454
...	@@ -555,7 +555,7 @@ struct AstNodeSwitchRange {	...	@@ -555,7 +555,7 @@ struct AstNodeSwitchRange {
555	};	555	};
556		556
557	struct AstNodeLabel {	557	struct AstNodeLabel {
558	Buf name;	558	Buf *name;
559		559
560	// populated by semantic analyzer	560	// populated by semantic analyzer
561	Expr resolved_expr;	561	Expr resolved_expr;
...	@@ -563,7 +563,7 @@ struct AstNodeLabel {	...	@@ -563,7 +563,7 @@ struct AstNodeLabel {
563	};	563	};
564		564
565	struct AstNodeGoto {	565	struct AstNodeGoto {
566	Buf name;	566	Buf *name;
567		567
568	// populated by semantic analyzer	568	// populated by semantic analyzer
569	Expr resolved_expr;	569	Expr resolved_expr;
...	@@ -571,9 +571,9 @@ struct AstNodeGoto {	...	@@ -571,9 +571,9 @@ struct AstNodeGoto {
571	};	571	};
572		572
573	struct AsmOutput {	573	struct AsmOutput {
574	Buf asm_symbolic_name;	574	Buf *asm_symbolic_name;
575	Buf constraint;	575	Buf *constraint;
576	Buf variable_name;	576	Buf *variable_name;
577	AstNode *return_type; // null unless "=r" and return	577	AstNode *return_type; // null unless "=r" and return
578		578
579	// populated by semantic analyzer	579	// populated by semantic analyzer
...	@@ -581,8 +581,8 @@ struct AsmOutput {	...	@@ -581,8 +581,8 @@ struct AsmOutput {
581	};	581	};
582		582
583	struct AsmInput {	583	struct AsmInput {
584	Buf asm_symbolic_name;	584	Buf *asm_symbolic_name;
585	Buf constraint;	585	Buf *constraint;
586	AstNode *expr;	586	AstNode *expr;
587	};	587	};
588		588
...	@@ -593,8 +593,7 @@ struct SrcPos {	...	@@ -593,8 +593,7 @@ struct SrcPos {
593		593
594	struct AstNodeAsmExpr {	594	struct AstNodeAsmExpr {
595	bool is_volatile;	595	bool is_volatile;
596	Buf asm_template;	596	Buf *asm_template;
597	ZigList<SrcPos> offset_map;
598	ZigList<AsmToken> token_list;	597	ZigList<AsmToken> token_list;
599	ZigList<AsmOutput*> output_list;	598	ZigList<AsmOutput*> output_list;
600	ZigList<AsmInput*> input_list;	599	ZigList<AsmInput*> input_list;
...	@@ -613,7 +612,7 @@ enum ContainerKind {	...	@@ -613,7 +612,7 @@ enum ContainerKind {
613		612
614	struct AstNodeStructDecl {	613	struct AstNodeStructDecl {
615	TopLevelDecl top_level_decl;	614	TopLevelDecl top_level_decl;
616	Buf name;	615	Buf *name;
617	ContainerKind kind;	616	ContainerKind kind;
618	ZigList<AstNode *> generic_params;	617	ZigList<AstNode *> generic_params;
619	bool generic_params_is_var_args; // always an error but it can happen from parsing	618	bool generic_params_is_var_args; // always an error but it can happen from parsing
...	@@ -629,12 +628,12 @@ struct AstNodeStructDecl {	...	@@ -629,12 +628,12 @@ struct AstNodeStructDecl {
629		628
630	struct AstNodeStructField {	629	struct AstNodeStructField {
631	TopLevelDecl top_level_decl;	630	TopLevelDecl top_level_decl;
632	Buf name;	631	Buf *name;
633	AstNode *type;	632	AstNode *type;
634	};	633	};
635		634
636	struct AstNodeStringLiteral {	635	struct AstNodeStringLiteral {
637	Buf buf;	636	Buf *buf;
638	bool c;	637	bool c;
639		638
640	// populated by semantic analyzer:	639	// populated by semantic analyzer:
...	@@ -648,29 +647,19 @@ struct AstNodeCharLiteral {	...	@@ -648,29 +647,19 @@ struct AstNodeCharLiteral {
648	Expr resolved_expr;	647	Expr resolved_expr;
649	};	648	};
650		649
651	enum NumLit {
652	NumLitFloat,
653	NumLitUInt,
654	};
655
656	struct AstNodeNumberLiteral {	650	struct AstNodeNumberLiteral {
657	NumLit kind;	651	BigNum *bignum;
658		652
659	// overflow is true if when parsing the number, we discovered it would not	653	// overflow is true if when parsing the number, we discovered it would not
660	// fit without losing data in a uint64_t or double	654	// fit without losing data in a uint64_t or double
661	bool overflow;	655	bool overflow;
662		656
663	union {
664	uint64_t x_uint;
665	double x_float;
666	} data;
667
668	// populated by semantic analyzer	657	// populated by semantic analyzer
669	Expr resolved_expr;	658	Expr resolved_expr;
670	};	659	};
671		660
672	struct AstNodeStructValueField {	661	struct AstNodeStructValueField {
673	Buf name;	662	Buf *name;
674	AstNode *expr;	663	AstNode *expr;
675		664
676	// populated by semantic analyzer	665	// populated by semantic analyzer
...	@@ -706,7 +695,7 @@ struct AstNodeUndefinedLiteral {	...	@@ -706,7 +695,7 @@ struct AstNodeUndefinedLiteral {
706	};	695	};
707		696
708	struct AstNodeSymbolExpr {	697	struct AstNodeSymbolExpr {
709	Buf symbol;	698	Buf *symbol;
710		699
711	// populated by semantic analyzer	700	// populated by semantic analyzer
712	Expr resolved_expr;	701	Expr resolved_expr;

src/analyze.cpp+58-58

src/ast_render.cpp+62-22

 @@ -78,6 +78,24 @@ static const char *visib_mod_string(VisibMod mod) {
     zig_unreachable();
 }
+static const char *return_string(ReturnKind kind) {
+    switch (kind) {
+        case ReturnKindUnconditional: return "return";
+        case ReturnKindError: return "%return";
+        case ReturnKindMaybe: return "?return";
+    }
+    zig_unreachable();
+}
+static const char *defer_string(ReturnKind kind) {
+    switch (kind) {
+        case ReturnKindUnconditional: return "defer";
+        case ReturnKindError: return "%defer";
+        case ReturnKindMaybe: return "?defer";
+    }
+    zig_unreachable();
+}
 static const char *extern_string(bool is_extern) {
     return is_extern ? "extern " : "";
 }
 @@ -243,7 +261,7 @@ static bool is_node_void(AstNode *node) {
     if (node->type == NodeTypeSymbol) {
         if (node->data.symbol_expr.override_type_entry) {
             return node->data.symbol_expr.override_type_entry->id == TypeTableEntryIdVoid;
-        } else if (buf_eql_str(&node->data.symbol_expr.symbol, "void")) {
+        } else if (buf_eql_str(node->data.symbol_expr.symbol, "void")) {
             return true;
         }
     }
 @@ -260,7 +278,12 @@ static bool is_digit(uint8_t c) {
 }
 static bool is_printable(uint8_t c) {
-    return is_alpha_under(c) || is_digit(c) || c == ' ';
+    static const uint8_t printables[] =
+        " abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.~`!@#$%^&*()_-+=\\{}[];'\"?/<>,";
+    for (size_t i = 0; i < array_length(printables); i += 1) {
+        if (c == printables[i]) return true;
+    }
+    return false;
 }
 static void string_literal_escape(Buf *source, Buf *dest) {
 @@ -353,18 +376,18 @@ static void render_node(AstRender *ar, AstNode *node) {
                 const char *extern_str = extern_string(node->data.fn_proto.is_extern);
                 const char *inline_str = inline_string(node->data.fn_proto.is_inline);
                 fprintf(ar->f, "%s%s%sfn ", pub_str, inline_str, extern_str);
-                print_symbol(ar, &node->data.fn_proto.name);
+                print_symbol(ar, node->data.fn_proto.name);
                 fprintf(ar->f, "(");
                 int arg_count = node->data.fn_proto.params.length;
                 bool is_var_args = node->data.fn_proto.is_var_args;
                 for (int arg_i = 0; arg_i < arg_count; arg_i += 1) {
                     AstNode *param_decl = node->data.fn_proto.params.at(arg_i);
                     assert(param_decl->type == NodeTypeParamDecl);
-                    if (buf_len(&param_decl->data.param_decl.name) > 0) {
+                    if (buf_len(param_decl->data.param_decl.name) > 0) {
                         const char *noalias_str = param_decl->data.param_decl.is_noalias ? "noalias " : "";
                         const char *inline_str = param_decl->data.param_decl.is_inline ? "inline  " : "";
                         fprintf(ar->f, "%s%s", noalias_str, inline_str);
-                        print_symbol(ar, &param_decl->data.param_decl.name);
+                        print_symbol(ar, param_decl->data.param_decl.name);
                         fprintf(ar->f, ": ");
                     }
                     render_node(ar, param_decl->data.param_decl.type);
 @@ -417,21 +440,31 @@ static void render_node(AstRender *ar, AstNode *node) {
             fprintf(ar->f, "}");
             break;
         case NodeTypeDirective:
-            fprintf(ar->f, "#%s(",  buf_ptr(&node->data.directive.name));
+            fprintf(ar->f, "#%s(",  buf_ptr(node->data.directive.name));
             render_node(ar, node->data.directive.expr);
             fprintf(ar->f, ")\n");
             break;
         case NodeTypeReturnExpr:
-            zig_panic("TODO");
+            {
+                const char *return_str = return_string(node->data.return_expr.kind);
+                fprintf(ar->f, "%s ", return_str);
+                render_node(ar, node->data.return_expr.expr);
+                break;
+            }
         case NodeTypeDefer:
-            zig_panic("TODO");
+            {
+                const char *defer_str = defer_string(node->data.defer.kind);
+                fprintf(ar->f, "%s ", defer_str);
+                render_node(ar, node->data.return_expr.expr);
+                break;
+            }
         case NodeTypeVariableDeclaration:
             {
                 const char *pub_str = visib_mod_string(node->data.variable_declaration.top_level_decl.visib_mod);
                 const char *extern_str = extern_string(node->data.variable_declaration.is_extern);
                 const char *const_or_var = const_or_var_string(node->data.variable_declaration.is_const);
                 fprintf(ar->f, "%s%s%s ", pub_str, extern_str, const_or_var);
-                print_symbol(ar, &node->data.variable_declaration.symbol);
+                print_symbol(ar, node->data.variable_declaration.symbol);
                 if (node->data.variable_declaration.type) {
                     fprintf(ar->f, ": ");
 @@ -446,7 +479,7 @@ static void render_node(AstRender *ar, AstNode *node) {
         case NodeTypeTypeDecl:
             {
                 const char *pub_str = visib_mod_string(node->data.type_decl.top_level_decl.visib_mod);
-                const char *var_name = buf_ptr(&node->data.type_decl.symbol);
+                const char *var_name = buf_ptr(node->data.type_decl.symbol);
                 fprintf(ar->f, "%stype %s = ", pub_str, var_name);
                 render_node(ar, node->data.type_decl.child_type);
                 break;
 @@ -463,12 +496,15 @@ static void render_node(AstRender *ar, AstNode *node) {
         case NodeTypeUnwrapErrorExpr:
             zig_panic("TODO");
         case NodeTypeNumberLiteral:
-            switch (node->data.number_literal.kind) {
+            switch (node->data.number_literal.bignum->kind) {
-                case NumLitUInt:
+                case BigNumKindInt:
-                    fprintf(ar->f, "%" PRIu64, node->data.number_literal.data.x_uint);
+                    {
+                        const char *negative_str = node->data.number_literal.bignum->is_negative ? "-" : "";
+                        fprintf(ar->f, "%s%llu", negative_str, node->data.number_literal.bignum->data.x_uint);
+                    }
                     break;
-                case NumLitFloat:
+                case BigNumKindFloat:
-                    fprintf(ar->f, "%f", node->data.number_literal.data.x_float);
+                    fprintf(ar->f, "%f", node->data.number_literal.bignum->data.x_float);
                     break;
             }
             break;
 @@ -478,7 +514,7 @@ static void render_node(AstRender *ar, AstNode *node) {
                     fprintf(ar->f, "c");
                 }
                 Buf tmp_buf = BUF_INIT;
-                string_literal_escape(&node->data.string_literal.buf, &tmp_buf);
+                string_literal_escape(node->data.string_literal.buf, &tmp_buf);
                 fprintf(ar->f, "\"%s\"", buf_ptr(&tmp_buf));
             }
             break;
 @@ -498,7 +534,7 @@ static void render_node(AstRender *ar, AstNode *node) {
                 if (override_type) {
                     fprintf(ar->f, "%s", buf_ptr(&override_type->name));
                 } else {
-                    fprintf(ar->f, "%s", buf_ptr(&node->data.symbol_expr.symbol));
+                    print_symbol(ar, node->data.symbol_expr.symbol);
                 }
             }
             break;
 @@ -513,10 +549,14 @@ static void render_node(AstRender *ar, AstNode *node) {
         case NodeTypeFnCallExpr:
             if (node->data.fn_call_expr.is_builtin) {
                 fprintf(ar->f, "@");
+            } else {
+                fprintf(ar->f, "(");
             }
-            fprintf(ar->f, "(");
             render_node(ar, node->data.fn_call_expr.fn_ref_expr);
-            fprintf(ar->f, ")(");
+            if (!node->data.fn_call_expr.is_builtin) {
+                fprintf(ar->f, ")");
+            }
+            fprintf(ar->f, "(");
             for (int i = 0; i < node->data.fn_call_expr.params.length; i += 1) {
                 AstNode *param = node->data.fn_call_expr.params.at(i);
                 if (i != 0) {
 @@ -537,7 +577,7 @@ static void render_node(AstRender *ar, AstNode *node) {
         case NodeTypeFieldAccessExpr:
             {
                 AstNode *lhs = node->data.field_access_expr.struct_expr;
-                Buf *rhs = &node->data.field_access_expr.field_name;
+                Buf *rhs = node->data.field_access_expr.field_name;
                 render_node(ar, lhs);
                 fprintf(ar->f, ".");
                 print_symbol(ar, rhs);
 @@ -577,7 +617,7 @@ static void render_node(AstRender *ar, AstNode *node) {
             zig_panic("TODO");
         case NodeTypeContainerDecl:
             {
-                const char *struct_name = buf_ptr(&node->data.struct_decl.name);
+                const char *struct_name = buf_ptr(node->data.struct_decl.name);
                 const char *pub_str = visib_mod_string(node->data.struct_decl.top_level_decl.visib_mod);
                 const char *container_str = container_string(node->data.struct_decl.kind);
                 fprintf(ar->f, "%s%s %s {\n", pub_str, container_str, struct_name);
 @@ -586,7 +626,7 @@ static void render_node(AstRender *ar, AstNode *node) {
                     AstNode *field_node = node->data.struct_decl.fields.at(field_i);
                     assert(field_node->type == NodeTypeStructField);
                     print_indent(ar);
-                    print_symbol(ar, &field_node->data.struct_field.name);
+                    print_symbol(ar, field_node->data.struct_field.name);
                     if (!is_node_void(field_node->data.struct_field.type)) {
                         fprintf(ar->f, ": ");
                         render_node(ar, field_node->data.struct_field.type);

src/bignum.cpp+17

...	@@ -6,6 +6,7 @@	...	@@ -6,6 +6,7 @@
6	*/	6	*/
7		7
8	#include "bignum.hpp"	8	#include "bignum.hpp"
		9	#include "buffer.hpp"
9		10
10	#include <assert.h>	11	#include <assert.h>
11	#include <math.h>	12	#include <math.h>
...	@@ -41,6 +42,10 @@ void bignum_init_signed(BigNum *dest, int64_t x) {	...	@@ -41,6 +42,10 @@ void bignum_init_signed(BigNum *dest, int64_t x) {
41	}	42	}
42	}	43	}
43		44
		45	void bignum_init_bignum(BigNum dest, BigNum src) {
		46	memcpy(dest, src, sizeof(BigNum));
		47	}
		48
44	bool bignum_fits_in_bits(BigNum *bn, int bit_count, bool is_signed) {	49	bool bignum_fits_in_bits(BigNum *bn, int bit_count, bool is_signed) {
45	assert(bn->kind == BigNumKindInt);	50	assert(bn->kind == BigNumKindInt);
46		51
...	@@ -343,3 +348,15 @@ bool bignum_cmp_gte(BigNum op1, BigNum op2) {	...	@@ -343,3 +348,15 @@ bool bignum_cmp_gte(BigNum op1, BigNum op2) {
343	return true;	348	return true;
344	}	349	}
345	}	350	}
		351
		352	bool bignum_increment_by_scalar(BigNum *bignum, uint64_t scalar) {
		353	assert(bignum->kind == BigNumKindInt);
		354	assert(!bignum->is_negative);
		355	return __builtin_uaddll_overflow(bignum->data.x_uint, scalar, &bignum->data.x_uint);
		356	}
		357
		358	bool bignum_multiply_by_scalar(BigNum *bignum, uint64_t scalar) {
		359	assert(bignum->kind == BigNumKindInt);
		360	assert(!bignum->is_negative);
		361	return __builtin_umulll_overflow(bignum->data.x_uint, scalar, &bignum->data.x_uint);
		362	}

src/bignum.hpp+10-1

...	@@ -5,7 +5,8 @@	...	@@ -5,7 +5,8 @@
5	* See http://opensource.org/licenses/MIT	5	* See http://opensource.org/licenses/MIT
6	*/	6	*/
7		7
8	#include "buffer.hpp"	8	#ifndef ZIG_BIGNUM_HPP
		9	#define ZIG_BIGNUM_HPP
9		10
10	#include <stdint.h>	11	#include <stdint.h>
11		12
...	@@ -26,6 +27,7 @@ struct BigNum {	...	@@ -26,6 +27,7 @@ struct BigNum {
26	void bignum_init_float(BigNum *dest, double x);	27	void bignum_init_float(BigNum *dest, double x);
27	void bignum_init_unsigned(BigNum *dest, uint64_t x);	28	void bignum_init_unsigned(BigNum *dest, uint64_t x);
28	void bignum_init_signed(BigNum *dest, int64_t x);	29	void bignum_init_signed(BigNum *dest, int64_t x);
		30	void bignum_init_bignum(BigNum dest, BigNum src);
29		31
30	bool bignum_fits_in_bits(BigNum *bn, int bit_count, bool is_signed);	32	bool bignum_fits_in_bits(BigNum *bn, int bit_count, bool is_signed);
31	uint64_t bignum_to_twos_complement(BigNum *bn);	33	uint64_t bignum_to_twos_complement(BigNum *bn);
...	@@ -57,4 +59,11 @@ bool bignum_cmp_gt(BigNum op1, BigNum op2);	...	@@ -57,4 +59,11 @@ bool bignum_cmp_gt(BigNum op1, BigNum op2);
57	bool bignum_cmp_lte(BigNum op1, BigNum op2);	59	bool bignum_cmp_lte(BigNum op1, BigNum op2);
58	bool bignum_cmp_gte(BigNum op1, BigNum op2);	60	bool bignum_cmp_gte(BigNum op1, BigNum op2);
59		61
		62	// helper functions
		63	bool bignum_increment_by_scalar(BigNum *bignum, uint64_t scalar);
		64	bool bignum_multiply_by_scalar(BigNum *bignum, uint64_t scalar);
		65
		66	struct Buf;
60	Buf bignum_to_buf(BigNum bn);	67	Buf bignum_to_buf(BigNum bn);
		68
		69	#endif

src/codegen.cpp+14-14

src/eval.cpp+7-13

...	@@ -427,7 +427,7 @@ static EvalVar find_var(EvalFn ef, Buf *name) {	...	@@ -427,7 +427,7 @@ static EvalVar find_var(EvalFn ef, Buf *name) {
427	static bool eval_symbol_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {	427	static bool eval_symbol_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {
428	assert(node->type == NodeTypeSymbol);	428	assert(node->type == NodeTypeSymbol);
429		429
430	Buf *name = &node->data.symbol_expr.symbol;	430	Buf *name = node->data.symbol_expr.symbol;
431	EvalVar *var = find_var(ef, name);	431	EvalVar *var = find_var(ef, name);
432	assert(var);	432	assert(var);
433		433
...	@@ -924,7 +924,7 @@ static bool eval_field_access_expr(EvalFn ef, AstNode node, ConstExprValue *ou	...	@@ -924,7 +924,7 @@ static bool eval_field_access_expr(EvalFn ef, AstNode node, ConstExprValue *ou
924	TypeTableEntry *struct_type = get_resolved_expr(struct_expr)->type_entry;	924	TypeTableEntry *struct_type = get_resolved_expr(struct_expr)->type_entry;
925		925
926	if (struct_type->id == TypeTableEntryIdArray) {	926	if (struct_type->id == TypeTableEntryIdArray) {
927	Buf *name = &node->data.field_access_expr.field_name;	927	Buf *name = node->data.field_access_expr.field_name;
928	assert(buf_eql_str(name, "len"));	928	assert(buf_eql_str(name, "len"));
929	zig_panic("TODO");	929	zig_panic("TODO");
930	} else if (struct_type->id == TypeTableEntryIdStruct \|\| (struct_type->id == TypeTableEntryIdPointer &&	930	} else if (struct_type->id == TypeTableEntryIdStruct \|\| (struct_type->id == TypeTableEntryIdPointer &&
...	@@ -971,7 +971,7 @@ static bool eval_for_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {	...	@@ -971,7 +971,7 @@ static bool eval_for_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {
971	if (eval_expr(ef, array_node, &array_val)) return true;	971	if (eval_expr(ef, array_node, &array_val)) return true;
972		972
973	assert(elem_node->type == NodeTypeSymbol);	973	assert(elem_node->type == NodeTypeSymbol);
974	Buf *elem_var_name = &elem_node->data.symbol_expr.symbol;	974	Buf *elem_var_name = elem_node->data.symbol_expr.symbol;
975		975
976	if (node->data.for_expr.elem_is_ptr) {	976	if (node->data.for_expr.elem_is_ptr) {
977	zig_panic("TODO");	977	zig_panic("TODO");
...	@@ -980,7 +980,7 @@ static bool eval_for_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {	...	@@ -980,7 +980,7 @@ static bool eval_for_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {
980	Buf *index_var_name = nullptr;	980	Buf *index_var_name = nullptr;
981	if (index_node) {	981	if (index_node) {
982	assert(index_node->type == NodeTypeSymbol);	982	assert(index_node->type == NodeTypeSymbol);
983	index_var_name = &index_node->data.symbol_expr.symbol;	983	index_var_name = index_node->data.symbol_expr.symbol;
984	}	984	}
985		985
986	uint64_t it_index = 0;	986	uint64_t it_index = 0;
...	@@ -1164,7 +1164,7 @@ static bool eval_var_decl_expr(EvalFn ef, AstNode node, ConstExprValue *out_va	...	@@ -1164,7 +1164,7 @@ static bool eval_var_decl_expr(EvalFn ef, AstNode node, ConstExprValue *out_va
1164		1164
1165	my_scope->vars.add_one();	1165	my_scope->vars.add_one();
1166	EvalVar *var = &my_scope->vars.last();	1166	EvalVar *var = &my_scope->vars.last();
1167	var->name = &node->data.variable_declaration.symbol;	1167	var->name = node->data.variable_declaration.symbol;
1168		1168
1169	if (eval_expr(ef, node->data.variable_declaration.expr, &var->value)) return true;	1169	if (eval_expr(ef, node->data.variable_declaration.expr, &var->value)) return true;
1170		1170
...	@@ -1178,13 +1178,7 @@ static bool eval_number_literal_expr(EvalFn ef, AstNode node, ConstExprValue *	...	@@ -1178,13 +1178,7 @@ static bool eval_number_literal_expr(EvalFn ef, AstNode node, ConstExprValue *
1178	assert(!node->data.number_literal.overflow);	1178	assert(!node->data.number_literal.overflow);
1179		1179
1180	out_val->ok = true;	1180	out_val->ok = true;
1181	if (node->data.number_literal.kind == NumLitUInt) {	1181	bignum_init_bignum(&out_val->data.x_bignum, node->data.number_literal.bignum);
1182	bignum_init_unsigned(&out_val->data.x_bignum, node->data.number_literal.data.x_uint);
1183	} else if (node->data.number_literal.kind == NumLitFloat) {
1184	bignum_init_float(&out_val->data.x_bignum, node->data.number_literal.data.x_float);
1185	} else {
1186	zig_unreachable();
1187	}
1188		1182
1189	return false;	1183	return false;
1190	}	1184	}
...	@@ -1339,7 +1333,7 @@ static bool eval_fn_args(EvalFnRoot efr, FnTableEntry fn, ConstExprValue *args	...	@@ -1339,7 +1333,7 @@ static bool eval_fn_args(EvalFnRoot efr, FnTableEntry fn, ConstExprValue *args
1339		1333
1340	root_scope->vars.add_one();	1334	root_scope->vars.add_one();
1341	EvalVar *eval_var = &root_scope->vars.last();	1335	EvalVar *eval_var = &root_scope->vars.last();
1342	eval_var->name = &decl_param_node->data.param_decl.name;	1336	eval_var->name = decl_param_node->data.param_decl.name;
1343	eval_var->value = *src_const_val;	1337	eval_var->value = *src_const_val;
1344	}	1338	}
1345		1339

src/parseh.cpp+23-23

src/parser.cpp+57-540

...	@@ -21,6 +21,9 @@ struct ParseContext {	...	@@ -21,6 +21,9 @@ struct ParseContext {
21	ImportTableEntry *owner;	21	ImportTableEntry *owner;
22	ErrColor err_color;	22	ErrColor err_color;
23	uint32_t *next_node_index;	23	uint32_t *next_node_index;
		24	// These buffers are used freqently so we preallocate them once here.
		25	Buf *void_buf;
		26	Buf *empty_buf;
24	};	27	};
25		28
26	__attribute__ ((format (printf, 4, 5)))	29	__attribute__ ((format (printf, 4, 5)))
...	@@ -29,7 +32,9 @@ static void ast_asm_error(ParseContext pc, AstNode node, int offset, const cha	...	@@ -29,7 +32,9 @@ static void ast_asm_error(ParseContext pc, AstNode node, int offset, const cha
29	assert(node->type == NodeTypeAsmExpr);	32	assert(node->type == NodeTypeAsmExpr);
30		33
31		34
32	SrcPos pos = node->data.asm_expr.offset_map.at(offset);	35	// TODO calculate or otherwise keep track of originating line/column number for strings
		36	//SrcPos pos = node->data.asm_expr.offset_map.at(offset);
		37	SrcPos pos = { node->line, node->column };
33		38
34	va_list ap;	39	va_list ap;
35	va_start(ap, format);	40	va_start(ap, format);
...	@@ -83,12 +88,12 @@ static AstNode ast_create_node(ParseContext pc, NodeType type, Token *first_to	...	@@ -83,12 +88,12 @@ static AstNode ast_create_node(ParseContext pc, NodeType type, Token *first_to
83		88
84	static AstNode ast_create_void_type_node(ParseContext pc, Token *token) {	89	static AstNode ast_create_void_type_node(ParseContext pc, Token *token) {
85	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);	90	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);
86	buf_init_from_str(&node->data.symbol_expr.symbol, "void");	91	node->data.symbol_expr.symbol = pc->void_buf;
87	return node;	92	return node;
88	}	93	}
89		94
90	static void parse_asm_template(ParseContext pc, AstNode node) {	95	static void parse_asm_template(ParseContext pc, AstNode node) {
91	Buf *asm_template = &node->data.asm_expr.asm_template;	96	Buf *asm_template = node->data.asm_expr.asm_template;
92		97
93	enum State {	98	enum State {
94	StateStart,	99	StateStart,
...	@@ -170,514 +175,29 @@ static void parse_asm_template(ParseContext pc, AstNode node) {	...	@@ -170,514 +175,29 @@ static void parse_asm_template(ParseContext pc, AstNode node) {
170	}	175	}
171	}	176	}
172		177
173	static uint8_t parse_char_literal(ParseContext pc, Token token) {	178	static Buf token_buf(Token token) {
174	// skip the single quotes at beginning and end	179	assert(token->id == TokenIdStringLiteral \|\| token->id == TokenIdSymbol);
175	// convert escape sequences	180	return &token->data.str_lit.str;
176	bool escape = false;
177	int return_count = 0;
178	uint8_t return_value;
179	for (int i = token->start_pos + 1; i < token->end_pos - 1; i += 1) {
180	uint8_t c = ((uint8_t)buf_ptr(pc->buf) + i);
181	if (escape) {
182	switch (c) {
183	case '\\':
184	return_value = '\\';
185	return_count += 1;
186	break;
187	case 'r':
188	return_value = '\r';
189	return_count += 1;
190	break;
191	case 'n':
192	return_value = '\n';
193	return_count += 1;
194	break;
195	case 't':
196	return_value = '\t';
197	return_count += 1;
198	break;
199	case '\'':
200	return_value = '\'';
201	return_count += 1;
202	break;
203	default:
204	ast_error(pc, token, "invalid escape character");
205	}
206	escape = false;
207	} else if (c == '\\') {
208	escape = true;
209	} else {
210	return_value = c;
211	return_count += 1;
212	}
213	}
214	if (return_count == 0) {
215	ast_error(pc, token, "character literal too short");
216	} else if (return_count > 1) {
217	ast_error(pc, token, "character literal too long");
218	}
219	return return_value;
220	}
221
222	static uint32_t get_hex_digit(uint8_t c) {
223	switch (c) {
224	case '0': return 0;
225	case '1': return 1;
226	case '2': return 2;
227	case '3': return 3;
228	case '4': return 4;
229	case '5': return 5;
230	case '6': return 6;
231	case '7': return 7;
232	case '8': return 8;
233	case '9': return 9;
234
235	case 'a':
236	case 'A':
237	return 10;
238	case 'b':
239	case 'B':
240	return 11;
241	case 'c':
242	case 'C':
243	return 12;
244	case 'd':
245	case 'D':
246	return 13;
247	case 'e':
248	case 'E':
249	return 14;
250	case 'f':
251	case 'F':
252	return 15;
253	default:
254	return UINT32_MAX;
255	}
256	}
257
258	static void parse_string_literal(ParseContext pc, Token token, Buf buf, bool out_c_str,
259	ZigList<SrcPos> *offset_map)
260	{
261	if (token->raw_string_start > 0) {
262	uint8_t c1 = ((uint8_t)buf_ptr(pc->buf) + token->start_pos);
263	uint8_t c2 = ((uint8_t)buf_ptr(pc->buf) + token->start_pos + 1);
264	assert(c1 == 'r');
265	if (out_c_str) {
266	*out_c_str = (c2 == 'c');
267	}
268	const char *str = buf_ptr(pc->buf) + token->raw_string_start;
269	buf_init_from_mem(buf, str, token->raw_string_end - token->raw_string_start);
270	if (offset_map) {
271	SrcPos pos = {token->start_line, token->start_column};
272	for (int i = token->start_pos; i < token->raw_string_start; i += 1) {
273	uint8_t c = buf_ptr(pc->buf)[i];
274	if (c == '\n') {
275	pos.line += 1;
276	pos.column = 0;
277	} else {
278	pos.column += 1;
279	}
280	}
281	for (int i = token->raw_string_start; i < token->raw_string_end; i += 1) {
282	offset_map->append(pos);
283
284	uint8_t c = buf_ptr(pc->buf)[i];
285	if (c == '\n') {
286	pos.line += 1;
287	pos.column = 0;
288	} else {
289	pos.column += 1;
290	}
291	}
292	}
293	return;
294	}
295
296	// skip the double quotes at beginning and end
297	// convert escape sequences
298	// detect c string literal
299
300	enum State {
301	StatePre,
302	StateSkipQuot,
303	StateStart,
304	StateEscape,
305	StateHex1,
306	StateHex2,
307	StateUnicode,
308	};
309
310	buf_resize(buf, 0);
311
312	int unicode_index;
313	int unicode_end;
314
315	State state = StatePre;
316	SrcPos pos = {token->start_line, token->start_column};
317	uint32_t hex_value = 0;
318	for (int i = token->start_pos; i < token->end_pos - 1; i += 1) {
319	uint8_t c = ((uint8_t)buf_ptr(pc->buf) + i);
320
321	switch (state) {
322	case StatePre:
323	switch (c) {
324	case '@':
325	state = StateSkipQuot;
326	break;
327	case 'c':
328	if (out_c_str) {
329	*out_c_str = true;
330	} else {
331	ast_error(pc, token, "C string literal not allowed here");
332	}
333	state = StateSkipQuot;
334	break;
335	case '"':
336	state = StateStart;
337	break;
338	default:
339	ast_error(pc, token, "invalid string character");
340	}
341	break;
342	case StateSkipQuot:
343	state = StateStart;
344	break;
345	case StateStart:
346	if (c == '\\') {
347	state = StateEscape;
348	} else {
349	buf_append_char(buf, c);
350	if (offset_map) offset_map->append(pos);
351	}
352	break;
353	case StateEscape:
354	switch (c) {
355	case '\\':
356	buf_append_char(buf, '\\');
357	if (offset_map) offset_map->append(pos);
358	state = StateStart;
359	break;
360	case 'r':
361	buf_append_char(buf, '\r');
362	if (offset_map) offset_map->append(pos);
363	state = StateStart;
364	break;
365	case 'n':
366	buf_append_char(buf, '\n');
367	if (offset_map) offset_map->append(pos);
368	state = StateStart;
369	break;
370	case 't':
371	buf_append_char(buf, '\t');
372	if (offset_map) offset_map->append(pos);
373	state = StateStart;
374	break;
375	case '"':
376	buf_append_char(buf, '"');
377	if (offset_map) offset_map->append(pos);
378	state = StateStart;
379	break;
380	case '\'':
381	buf_append_char(buf, '\'');
382	if (offset_map) offset_map->append(pos);
383	state = StateStart;
384	break;
385	case 'x':
386	state = StateHex1;
387	break;
388	case 'u':
389	state = StateUnicode;
390	unicode_index = 0;
391	unicode_end = 4;
392	hex_value = 0;
393	break;
394	case 'U':
395	state = StateUnicode;
396	unicode_index = 0;
397	unicode_end = 6;
398	hex_value = 0;
399	break;
400	default:
401	ast_error(pc, token, "invalid escape character");
402	}
403	break;
404	case StateHex1:
405	{
406	uint32_t hex_digit = get_hex_digit(c);
407	if (hex_digit == UINT32_MAX) {
408	ast_error(pc, token, "invalid hex digit: '%c'", c);
409	}
410	hex_value = hex_digit * 16;
411	state = StateHex2;
412	break;
413	}
414	case StateHex2:
415	{
416	uint32_t hex_digit = get_hex_digit(c);
417	if (hex_digit == UINT32_MAX) {
418	ast_error(pc, token, "invalid hex digit: '%c'", c);
419	}
420	hex_value += hex_digit;
421	assert(hex_value >= 0 && hex_value <= 255);
422	buf_append_char(buf, hex_value);
423	state = StateStart;
424	break;
425	}
426	case StateUnicode:
427	{
428	uint32_t hex_digit = get_hex_digit(c);
429	if (hex_digit == UINT32_MAX) {
430	ast_error(pc, token, "invalid hex digit: '%c'", c);
431	}
432	hex_value *= 16;
433	hex_value += hex_digit;
434	unicode_index += 1;
435	if (unicode_index >= unicode_end) {
436	if (hex_value <= 0x7f) {
437	// 00000000 00000000 00000000 0xxxxxxx
438	buf_append_char(buf, hex_value);
439	} else if (hex_value <= 0x7ff) {
440	// 00000000 00000000 00000xxx xx000000
441	buf_append_char(buf, (unsigned char)(0xc0 \| (hex_value >> 6)));
442	// 00000000 00000000 00000000 00xxxxxx
443	buf_append_char(buf, (unsigned char)(0x80 \| (hex_value & 0x3f)));
444	} else if (hex_value <= 0xffff) {
445	// 00000000 00000000 xxxx0000 00000000
446	buf_append_char(buf, (unsigned char)(0xe0 \| (hex_value >> 12)));
447	// 00000000 00000000 0000xxxx xx000000
448	buf_append_char(buf, (unsigned char)(0x80 \| ((hex_value >> 6) & 0x3f)));
449	// 00000000 00000000 00000000 00xxxxxx
450	buf_append_char(buf, (unsigned char)(0x80 \| (hex_value & 0x3f)));
451	} else if (hex_value <= 0x10ffff) {
452	// 00000000 000xxx00 00000000 00000000
453	buf_append_char(buf, (unsigned char)(0xf0 \| (hex_value >> 18)));
454	// 00000000 000000xx xxxx0000 00000000
455	buf_append_char(buf, (unsigned char)(0x80 \| ((hex_value >> 12) & 0x3f)));
456	// 00000000 00000000 0000xxxx xx000000
457	buf_append_char(buf, (unsigned char)(0x80 \| ((hex_value >> 6) & 0x3f)));
458	// 00000000 00000000 00000000 00xxxxxx
459	buf_append_char(buf, (unsigned char)(0x80 \| (hex_value & 0x3f)));
460	} else {
461	ast_error(pc, token, "unicode value out of range: %x", hex_value);
462	}
463	state = StateStart;
464	}
465	break;
466	}
467	}
468	if (c == '\n') {
469	pos.line += 1;
470	pos.column = 0;
471	} else {
472	pos.column += 1;
473	}
474	}
475	assert(state == StateStart);
476	if (offset_map) offset_map->append(pos);
477	}	181	}
478		182
479	static void ast_buf_from_token(ParseContext pc, Token token, Buf *buf) {	183	static BigNum token_bignum(Token token) {
480	uint8_t first_char = (uint8_t )buf_ptr(pc->buf) + token->start_pos;	184	assert(token->id == TokenIdNumberLiteral);
481	bool at_sign = *first_char == '@';	185	return &token->data.num_lit.bignum;
482	if (at_sign) {
483	parse_string_literal(pc, token, buf, nullptr, nullptr);
484	} else {
485	buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
486	}
487	}	186	}
488		187
489		188	static uint8_t token_char_lit(Token *token) {
490	static unsigned long long parse_int_digits(ParseContext *pc, int digits_start, int digits_end, int radix,	189	assert(token->id == TokenIdCharLiteral);
491	int skip_index, bool *overflow)	190	return token->data.char_lit.c;
492	{
493	unsigned long long x = 0;
494
495	for (int i = digits_start; i < digits_end; i++) {
496	if (i == skip_index)
497	continue;
498	uint8_t c = ((uint8_t)buf_ptr(pc->buf) + i);
499	unsigned long long digit = get_digit_value(c);
500
501	// x *= radix;
502	if (__builtin_umulll_overflow(x, radix, &x)) {
503	*overflow = true;
504	return 0;
505	}
506
507	// x += digit
508	if (__builtin_uaddll_overflow(x, digit, &x)) {
509	*overflow = true;
510	return 0;
511	}
512	}
513	return x;
514	}	191	}
515		192
516	static void parse_number_literal(ParseContext pc, Token token, AstNodeNumberLiteral *num_lit) {	193	static void ast_buf_from_token(ParseContext pc, Token token, Buf *buf) {
517	assert(token->id == TokenIdNumberLiteral);	194	if (token->id == TokenIdSymbol) {
518		195	buf_init_from_buf(buf, token_buf(token));
519	int whole_number_start = token->start_pos;
520	if (token->radix != 10) {
521	// skip the "0x"
522	whole_number_start += 2;
523	}
524
525	int whole_number_end = token->decimal_point_pos;
526	if (whole_number_end <= whole_number_start) {
527	// TODO: error for empty whole number part
528	num_lit->overflow = true;
529	return;
530	}
531
532	if (token->decimal_point_pos == token->end_pos) {
533	// integer
534	unsigned long long whole_number = parse_int_digits(pc, whole_number_start, whole_number_end,
535	token->radix, -1, &num_lit->overflow);
536	if (num_lit->overflow) return;
537
538	num_lit->data.x_uint = whole_number;
539	num_lit->kind = NumLitUInt;
540	} else {	196	} else {
541	// float	197	buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
542
543	if (token->radix == 10) {
544	// use a third-party base-10 float parser
545	char *str_begin = buf_ptr(pc->buf) + whole_number_start;
546	char *str_end;
547	errno = 0;
548	double x = strtod(str_begin, &str_end);
549	if (errno) {
550	// TODO: forward error to user
551	num_lit->overflow = true;
552	return;
553	}
554	assert(str_end == buf_ptr(pc->buf) + token->end_pos);
555	num_lit->data.x_float = x;
556	num_lit->kind = NumLitFloat;
557	return;
558	}
559
560	if (token->decimal_point_pos < token->exponent_marker_pos) {
561	// fraction
562	int fraction_start = token->decimal_point_pos + 1;
563	int fraction_end = token->exponent_marker_pos;
564	if (fraction_end <= fraction_start) {
565	// TODO: error for empty fraction part
566	num_lit->overflow = true;
567	return;
568	}
569	}
570
571	// trim leading and trailing zeros in the significand digit sequence
572	int significand_start = whole_number_start;
573	for (; significand_start < token->exponent_marker_pos; significand_start++) {
574	if (significand_start == token->decimal_point_pos)
575	continue;
576	uint8_t c = ((uint8_t)buf_ptr(pc->buf) + significand_start);
577	if (c != '0')
578	break;
579	}
580	int significand_end = token->exponent_marker_pos;
581	for (; significand_end - 1 > significand_start; significand_end--) {
582	if (significand_end - 1 <= token->decimal_point_pos) {
583	significand_end = token->decimal_point_pos;
584	break;
585	}
586	uint8_t c = ((uint8_t)buf_ptr(pc->buf) + significand_end - 1);
587	if (c != '0')
588	break;
589	}
590
591	unsigned long long significand_as_int = parse_int_digits(pc, significand_start, significand_end,
592	token->radix, token->decimal_point_pos, &num_lit->overflow);
593	if (num_lit->overflow) return;
594
595	int exponent_in_bin_or_dec = 0;
596	if (significand_end > token->decimal_point_pos) {
597	exponent_in_bin_or_dec = token->decimal_point_pos + 1 - significand_end;
598	if (token->radix == 2) {
599	// already good
600	} else if (token->radix == 8) {
601	exponent_in_bin_or_dec *= 3;
602	} else if (token->radix == 10) {
603	// already good
604	} else if (token->radix == 16) {
605	exponent_in_bin_or_dec *= 4;
606	} else zig_unreachable();
607	}
608
609	if (token->exponent_marker_pos < token->end_pos) {
610	// exponent
611	int exponent_start = token->exponent_marker_pos + 1;
612	int exponent_end = token->end_pos;
613	if (exponent_end <= exponent_start) {
614	// TODO: error for empty exponent part
615	num_lit->overflow = true;
616	return;
617	}
618	bool is_exponent_negative = false;
619	uint8_t c = ((uint8_t)buf_ptr(pc->buf) + exponent_start);
620	if (c == '+') {
621	exponent_start += 1;
622	} else if (c == '-') {
623	exponent_start += 1;
624	is_exponent_negative = true;
625	}
626
627	if (exponent_end <= exponent_start) {
628	// TODO: error for empty exponent part
629	num_lit->overflow = true;
630	return;
631	}
632
633	unsigned long long specified_exponent = parse_int_digits(pc, exponent_start, exponent_end,
634	10, -1, &num_lit->overflow);
635	// TODO: this check is a little silly
636	if (specified_exponent >= LLONG_MAX) {
637	num_lit->overflow = true;
638	return;
639	}
640
641	if (is_exponent_negative) {
642	exponent_in_bin_or_dec -= specified_exponent;
643	} else {
644	exponent_in_bin_or_dec += specified_exponent;
645	}
646	}
647
648	uint64_t significand_bits;
649	uint64_t exponent_bits;
650	if (significand_as_int != 0) {
651	// normalize the significand
652	if (token->radix == 10) {
653	zig_panic("TODO: decimal floats");
654	} else {
655	int significand_magnitude_in_bin = __builtin_clzll(1) - __builtin_clzll(significand_as_int);
656	exponent_in_bin_or_dec += significand_magnitude_in_bin;
657	if (!(-1023 <= exponent_in_bin_or_dec && exponent_in_bin_or_dec < 1023)) {
658	num_lit->overflow = true;
659	return;
660	}
661
662	// this should chop off exactly one 1 bit from the top.
663	significand_bits = ((uint64_t)significand_as_int << (52 - significand_magnitude_in_bin)) & 0xfffffffffffffULL;
664	exponent_bits = exponent_in_bin_or_dec + 1023;
665	}
666	} else {
667	// 0 is all 0's
668	significand_bits = 0;
669	exponent_bits = 0;
670	}
671
672	uint64_t double_bits = (exponent_bits << 52) \| significand_bits;
673	double x = (double )&double_bits;
674
675	num_lit->data.x_float = x;
676	num_lit->kind = NumLitFloat;
677	}	198	}
678	}	199	}
679		200
680
681	__attribute__ ((noreturn))	201	__attribute__ ((noreturn))
682	static void ast_invalid_token_error(ParseContext pc, Token token) {	202	static void ast_invalid_token_error(ParseContext pc, Token token) {
683	Buf token_value = BUF_INIT;	203	Buf token_value = BUF_INIT;
...	@@ -723,7 +243,7 @@ static AstNode ast_parse_directive(ParseContext pc, int *token_index) {	...	@@ -723,7 +243,7 @@ static AstNode ast_parse_directive(ParseContext pc, int *token_index) {
723		243
724	Token *name_symbol = ast_eat_token(pc, token_index, TokenIdSymbol);	244	Token *name_symbol = ast_eat_token(pc, token_index, TokenIdSymbol);
725		245
726	ast_buf_from_token(pc, name_symbol, &node->data.directive.name);	246	node->data.directive.name = token_buf(name_symbol);
727		247
728	node->data.directive.expr = ast_parse_grouped_expr(pc, token_index, true);	248	node->data.directive.expr = ast_parse_grouped_expr(pc, token_index, true);
729		249
...	@@ -769,12 +289,12 @@ static AstNode ast_parse_param_decl(ParseContext pc, int *token_index) {	...	@@ -769,12 +289,12 @@ static AstNode ast_parse_param_decl(ParseContext pc, int *token_index) {
769	token = &pc->tokens->at(*token_index);	289	token = &pc->tokens->at(*token_index);
770	}	290	}
771		291
772	buf_resize(&node->data.param_decl.name, 0);	292	node->data.param_decl.name = pc->empty_buf;
773		293
774	if (token->id == TokenIdSymbol) {	294	if (token->id == TokenIdSymbol) {
775	Token next_token = &pc->tokens->at(token_index + 1);	295	Token next_token = &pc->tokens->at(token_index + 1);
776	if (next_token->id == TokenIdColon) {	296	if (next_token->id == TokenIdColon) {
777	ast_buf_from_token(pc, token, &node->data.param_decl.name);	297	node->data.param_decl.name = token_buf(token);
778	*token_index += 2;	298	*token_index += 2;
779	}	299	}
780	}	300	}
...	@@ -915,8 +435,8 @@ static void ast_parse_asm_input_item(ParseContext pc, int token_index, AstNode	...	@@ -915,8 +435,8 @@ static void ast_parse_asm_input_item(ParseContext pc, int token_index, AstNode
915	ast_eat_token(pc, token_index, TokenIdRParen);	435	ast_eat_token(pc, token_index, TokenIdRParen);
916		436
917	AsmInput *asm_input = allocate<AsmInput>(1);	437	AsmInput *asm_input = allocate<AsmInput>(1);
918	ast_buf_from_token(pc, alias, &asm_input->asm_symbolic_name);	438	asm_input->asm_symbolic_name = token_buf(alias);
919	parse_string_literal(pc, constraint, &asm_input->constraint, nullptr, nullptr);	439	asm_input->constraint = token_buf(constraint);
920	asm_input->expr = expr_node;	440	asm_input->expr = expr_node;
921	node->data.asm_expr.input_list.append(asm_input);	441	node->data.asm_expr.input_list.append(asm_input);
922	}	442	}
...	@@ -938,7 +458,7 @@ static void ast_parse_asm_output_item(ParseContext pc, int token_index, AstNod	...	@@ -938,7 +458,7 @@ static void ast_parse_asm_output_item(ParseContext pc, int token_index, AstNod
938	Token token = &pc->tokens->at(token_index);	458	Token token = &pc->tokens->at(token_index);
939	*token_index += 1;	459	*token_index += 1;
940	if (token->id == TokenIdSymbol) {	460	if (token->id == TokenIdSymbol) {
941	ast_buf_from_token(pc, token, &asm_output->variable_name);	461	asm_output->variable_name = token_buf(token);
942	} else if (token->id == TokenIdArrow) {	462	} else if (token->id == TokenIdArrow) {
943	asm_output->return_type = ast_parse_prefix_op_expr(pc, token_index, true);	463	asm_output->return_type = ast_parse_prefix_op_expr(pc, token_index, true);
944	} else {	464	} else {
...	@@ -947,8 +467,8 @@ static void ast_parse_asm_output_item(ParseContext pc, int token_index, AstNod	...	@@ -947,8 +467,8 @@ static void ast_parse_asm_output_item(ParseContext pc, int token_index, AstNod
947		467
948	ast_eat_token(pc, token_index, TokenIdRParen);	468	ast_eat_token(pc, token_index, TokenIdRParen);
949		469
950	ast_buf_from_token(pc, alias, &asm_output->asm_symbolic_name);	470	asm_output->asm_symbolic_name = token_buf(alias);
951	parse_string_literal(pc, constraint, &asm_output->constraint, nullptr, nullptr);	471	asm_output->constraint = token_buf(constraint);
952	node->data.asm_expr.output_list.append(asm_output);	472	node->data.asm_expr.output_list.append(asm_output);
953	}	473	}
954		474
...	@@ -968,8 +488,7 @@ static void ast_parse_asm_clobbers(ParseContext pc, int token_index, AstNode *	...	@@ -968,8 +488,7 @@ static void ast_parse_asm_clobbers(ParseContext pc, int token_index, AstNode *
968	ast_expect_token(pc, string_tok, TokenIdStringLiteral);	488	ast_expect_token(pc, string_tok, TokenIdStringLiteral);
969	*token_index += 1;	489	*token_index += 1;
970		490
971	Buf *clobber_buf = buf_alloc();	491	Buf *clobber_buf = token_buf(string_tok);
972	parse_string_literal(pc, string_tok, clobber_buf, nullptr, nullptr);
973	node->data.asm_expr.clobber_list.append(clobber_buf);	492	node->data.asm_expr.clobber_list.append(clobber_buf);
974		493
975	Token comma = &pc->tokens->at(token_index);	494	Token comma = &pc->tokens->at(token_index);
...	@@ -1072,19 +591,14 @@ static AstNode ast_parse_asm_expr(ParseContext pc, int *token_index, bool mand	...	@@ -1072,19 +591,14 @@ static AstNode ast_parse_asm_expr(ParseContext pc, int *token_index, bool mand
1072	ast_expect_token(pc, lparen_tok, TokenIdLParen);	591	ast_expect_token(pc, lparen_tok, TokenIdLParen);
1073	*token_index += 1;	592	*token_index += 1;
1074		593
1075	Token template_tok = &pc->tokens->at(token_index);	594	Token *template_tok = ast_eat_token(pc, token_index, TokenIdStringLiteral);
1076	ast_expect_token(pc, template_tok, TokenIdStringLiteral);
1077	*token_index += 1;
1078		595
1079	parse_string_literal(pc, template_tok, &node->data.asm_expr.asm_template, nullptr,	596	node->data.asm_expr.asm_template = token_buf(template_tok);
1080	&node->data.asm_expr.offset_map);
1081	parse_asm_template(pc, node);	597	parse_asm_template(pc, node);
1082		598
1083	ast_parse_asm_output(pc, token_index, node);	599	ast_parse_asm_output(pc, token_index, node);
1084		600
1085	Token rparen_tok = &pc->tokens->at(token_index);	601	ast_eat_token(pc, token_index, TokenIdRParen);
1086	ast_expect_token(pc, rparen_tok, TokenIdRParen);
1087	*token_index += 1;
1088		602
1089	normalize_parent_ptrs(node);	603	normalize_parent_ptrs(node);
1090	return node;	604	return node;
...	@@ -1099,17 +613,19 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool	...	@@ -1099,17 +613,19 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool
1099		613
1100	if (token->id == TokenIdNumberLiteral) {	614	if (token->id == TokenIdNumberLiteral) {
1101	AstNode *node = ast_create_node(pc, NodeTypeNumberLiteral, token);	615	AstNode *node = ast_create_node(pc, NodeTypeNumberLiteral, token);
1102	parse_number_literal(pc, token, &node->data.number_literal);	616	node->data.number_literal.bignum = token_bignum(token);
		617	node->data.number_literal.overflow = token->data.num_lit.overflow;
1103	*token_index += 1;	618	*token_index += 1;
1104	return node;	619	return node;
1105	} else if (token->id == TokenIdStringLiteral) {	620	} else if (token->id == TokenIdStringLiteral) {
1106	AstNode *node = ast_create_node(pc, NodeTypeStringLiteral, token);	621	AstNode *node = ast_create_node(pc, NodeTypeStringLiteral, token);
1107	parse_string_literal(pc, token, &node->data.string_literal.buf, &node->data.string_literal.c, nullptr);	622	node->data.string_literal.buf = token_buf(token);
		623	node->data.string_literal.c = token->data.str_lit.is_c_str;
1108	*token_index += 1;	624	*token_index += 1;
1109	return node;	625	return node;
1110	} else if (token->id == TokenIdCharLiteral) {	626	} else if (token->id == TokenIdCharLiteral) {
1111	AstNode *node = ast_create_node(pc, NodeTypeCharLiteral, token);	627	AstNode *node = ast_create_node(pc, NodeTypeCharLiteral, token);
1112	node->data.char_literal.value = parse_char_literal(pc, token);	628	node->data.char_literal.value = token_char_lit(token);
1113	*token_index += 1;	629	*token_index += 1;
1114	return node;	630	return node;
1115	} else if (token->id == TokenIdKeywordTrue) {	631	} else if (token->id == TokenIdKeywordTrue) {
...	@@ -1155,7 +671,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool	...	@@ -1155,7 +671,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool
1155	*token_index += 1;	671	*token_index += 1;
1156	Token *name_tok = ast_eat_token(pc, token_index, TokenIdSymbol);	672	Token *name_tok = ast_eat_token(pc, token_index, TokenIdSymbol);
1157	AstNode *name_node = ast_create_node(pc, NodeTypeSymbol, name_tok);	673	AstNode *name_node = ast_create_node(pc, NodeTypeSymbol, name_tok);
1158	ast_buf_from_token(pc, name_tok, &name_node->data.symbol_expr.symbol);	674	name_node->data.symbol_expr.symbol = token_buf(name_tok);
1159		675
1160	AstNode *node = ast_create_node(pc, NodeTypeFnCallExpr, token);	676	AstNode *node = ast_create_node(pc, NodeTypeFnCallExpr, token);
1161	node->data.fn_call_expr.fn_ref_expr = name_node;	677	node->data.fn_call_expr.fn_ref_expr = name_node;
...	@@ -1168,7 +684,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool	...	@@ -1168,7 +684,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool
1168	} else if (token->id == TokenIdSymbol) {	684	} else if (token->id == TokenIdSymbol) {
1169	*token_index += 1;	685	*token_index += 1;
1170	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);	686	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);
1171	ast_buf_from_token(pc, token, &node->data.symbol_expr.symbol);	687	node->data.symbol_expr.symbol = token_buf(token);
1172	return node;	688	return node;
1173	} else if (token->id == TokenIdKeywordGoto) {	689	} else if (token->id == TokenIdKeywordGoto) {
1174	AstNode *node = ast_create_node(pc, NodeTypeGoto, token);	690	AstNode *node = ast_create_node(pc, NodeTypeGoto, token);
...	@@ -1178,7 +694,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool	...	@@ -1178,7 +694,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool
1178	*token_index += 1;	694	*token_index += 1;
1179	ast_expect_token(pc, dest_symbol, TokenIdSymbol);	695	ast_expect_token(pc, dest_symbol, TokenIdSymbol);
1180		696
1181	ast_buf_from_token(pc, dest_symbol, &node->data.goto_expr.name);	697	node->data.goto_expr.name = token_buf(dest_symbol);
1182	return node;	698	return node;
1183	}	699	}
1184		700
...	@@ -1243,7 +759,7 @@ static AstNode ast_parse_curly_suffix_expr(ParseContext pc, int *token_index,	...	@@ -1243,7 +759,7 @@ static AstNode ast_parse_curly_suffix_expr(ParseContext pc, int *token_index,
1243		759
1244	AstNode *field_node = ast_create_node(pc, NodeTypeStructValueField, token);	760	AstNode *field_node = ast_create_node(pc, NodeTypeStructValueField, token);
1245		761
1246	ast_buf_from_token(pc, field_name_tok, &field_node->data.struct_val_field.name);	762	field_node->data.struct_val_field.name = token_buf(field_name_tok);
1247	field_node->data.struct_val_field.expr = ast_parse_expression(pc, token_index, true);	763	field_node->data.struct_val_field.expr = ast_parse_expression(pc, token_index, true);
1248		764
1249	normalize_parent_ptrs(field_node);	765	normalize_parent_ptrs(field_node);
...	@@ -1370,7 +886,7 @@ static AstNode ast_parse_suffix_op_expr(ParseContext pc, int *token_index, boo	...	@@ -1370,7 +886,7 @@ static AstNode ast_parse_suffix_op_expr(ParseContext pc, int *token_index, boo
1370		886
1371	AstNode *node = ast_create_node(pc, NodeTypeFieldAccessExpr, first_token);	887	AstNode *node = ast_create_node(pc, NodeTypeFieldAccessExpr, first_token);
1372	node->data.field_access_expr.struct_expr = primary_expr;	888	node->data.field_access_expr.struct_expr = primary_expr;
1373	ast_buf_from_token(pc, name_token, &node->data.field_access_expr.field_name);	889	node->data.field_access_expr.field_name = token_buf(name_token);
1374		890
1375	normalize_parent_ptrs(node);	891	normalize_parent_ptrs(node);
1376	primary_expr = node;	892	primary_expr = node;
...	@@ -1819,10 +1335,10 @@ static AstNode ast_parse_if_expr(ParseContext pc, int *token_index, bool manda	...	@@ -1819,10 +1335,10 @@ static AstNode ast_parse_if_expr(ParseContext pc, int *token_index, bool manda
1819	*token_index += 1;	1335	*token_index += 1;
1820	node->data.if_var_expr.var_is_ptr = true;	1336	node->data.if_var_expr.var_is_ptr = true;
1821	Token *name_token = ast_eat_token(pc, token_index, TokenIdSymbol);	1337	Token *name_token = ast_eat_token(pc, token_index, TokenIdSymbol);
1822	ast_buf_from_token(pc, name_token, &node->data.if_var_expr.var_decl.symbol);	1338	node->data.if_var_expr.var_decl.symbol = token_buf(name_token);
1823	} else if (star_or_symbol->id == TokenIdSymbol) {	1339	} else if (star_or_symbol->id == TokenIdSymbol) {
1824	*token_index += 1;	1340	*token_index += 1;
1825	ast_buf_from_token(pc, star_or_symbol, &node->data.if_var_expr.var_decl.symbol);	1341	node->data.if_var_expr.var_decl.symbol = token_buf(star_or_symbol);
1826	} else {	1342	} else {
1827	ast_invalid_token_error(pc, star_or_symbol);	1343	ast_invalid_token_error(pc, star_or_symbol);
1828	}	1344	}
...	@@ -1974,7 +1490,7 @@ static AstNode ast_parse_variable_declaration_expr(ParseContext pc, int *token	...	@@ -1974,7 +1490,7 @@ static AstNode ast_parse_variable_declaration_expr(ParseContext pc, int *token
1974	node->data.variable_declaration.top_level_decl.directives = directives;	1490	node->data.variable_declaration.top_level_decl.directives = directives;
1975		1491
1976	Token *name_token = ast_eat_token(pc, token_index, TokenIdSymbol);	1492	Token *name_token = ast_eat_token(pc, token_index, TokenIdSymbol);
1977	ast_buf_from_token(pc, name_token, &node->data.variable_declaration.symbol);	1493	node->data.variable_declaration.symbol = token_buf(name_token);
1978		1494
1979	Token eq_or_colon = &pc->tokens->at(token_index);	1495	Token eq_or_colon = &pc->tokens->at(token_index);
1980	*token_index += 1;	1496	*token_index += 1;
...	@@ -2067,7 +1583,7 @@ static AstNode ast_parse_while_expr(ParseContext pc, int *token_index, bool ma	...	@@ -2067,7 +1583,7 @@ static AstNode ast_parse_while_expr(ParseContext pc, int *token_index, bool ma
2067	static AstNode ast_parse_symbol(ParseContext pc, int *token_index) {	1583	static AstNode ast_parse_symbol(ParseContext pc, int *token_index) {
2068	Token *token = ast_eat_token(pc, token_index, TokenIdSymbol);	1584	Token *token = ast_eat_token(pc, token_index, TokenIdSymbol);
2069	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);	1585	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);
2070	ast_buf_from_token(pc, token, &node->data.symbol_expr.symbol);	1586	node->data.symbol_expr.symbol = token_buf(token);
2071	return node;	1587	return node;
2072	}	1588	}
2073		1589
...	@@ -2405,7 +1921,7 @@ static AstNode ast_parse_label(ParseContext pc, int *token_index, bool mandato	...	@@ -2405,7 +1921,7 @@ static AstNode ast_parse_label(ParseContext pc, int *token_index, bool mandato
2405	*token_index += 2;	1921	*token_index += 2;
2406		1922
2407	AstNode *node = ast_create_node(pc, NodeTypeLabel, symbol_token);	1923	AstNode *node = ast_create_node(pc, NodeTypeLabel, symbol_token);
2408	ast_buf_from_token(pc, symbol_token, &node->data.label.name);	1924	node->data.label.name = token_buf(symbol_token);
2409	return node;	1925	return node;
2410	}	1926	}
2411		1927
...	@@ -2413,7 +1929,7 @@ static AstNode ast_create_void_expr(ParseContext pc, Token *token) {	...	@@ -2413,7 +1929,7 @@ static AstNode ast_create_void_expr(ParseContext pc, Token *token) {
2413	AstNode *node = ast_create_node(pc, NodeTypeContainerInitExpr, token);	1929	AstNode *node = ast_create_node(pc, NodeTypeContainerInitExpr, token);
2414	node->data.container_init_expr.type = ast_create_node(pc, NodeTypeSymbol, token);	1930	node->data.container_init_expr.type = ast_create_node(pc, NodeTypeSymbol, token);
2415	node->data.container_init_expr.kind = ContainerInitKindArray;	1931	node->data.container_init_expr.kind = ContainerInitKindArray;
2416	buf_init_from_str(&node->data.container_init_expr.type->data.symbol_expr.symbol, "void");	1932	node->data.container_init_expr.type->data.symbol_expr.symbol = pc->void_buf;
2417	normalize_parent_ptrs(node);	1933	normalize_parent_ptrs(node);
2418	return node;	1934	return node;
2419	}	1935	}
...	@@ -2508,9 +2024,9 @@ static AstNode ast_parse_fn_proto(ParseContext pc, int *token_index, bool mand	...	@@ -2508,9 +2024,9 @@ static AstNode ast_parse_fn_proto(ParseContext pc, int *token_index, bool mand
2508	Token fn_name = &pc->tokens->at(token_index);	2024	Token fn_name = &pc->tokens->at(token_index);
2509	if (fn_name->id == TokenIdSymbol) {	2025	if (fn_name->id == TokenIdSymbol) {
2510	*token_index += 1;	2026	*token_index += 1;
2511	ast_buf_from_token(pc, fn_name, &node->data.fn_proto.name);	2027	node->data.fn_proto.name = token_buf(fn_name);
2512	} else {	2028	} else {
2513	buf_resize(&node->data.fn_proto.name, 0);	2029	node->data.fn_proto.name = pc->empty_buf;
2514	}	2030	}
2515		2031
2516	ast_parse_param_decl_list(pc, token_index, &node->data.fn_proto.params, &node->data.fn_proto.is_var_args);	2032	ast_parse_param_decl_list(pc, token_index, &node->data.fn_proto.params, &node->data.fn_proto.is_var_args);
...	@@ -2663,7 +2179,7 @@ static AstNode ast_parse_container_decl(ParseContext pc, int *token_index,	...	@@ -2663,7 +2179,7 @@ static AstNode ast_parse_container_decl(ParseContext pc, int *token_index,
2663		2179
2664	AstNode *node = ast_create_node(pc, NodeTypeContainerDecl, first_token);	2180	AstNode *node = ast_create_node(pc, NodeTypeContainerDecl, first_token);
2665	node->data.struct_decl.kind = kind;	2181	node->data.struct_decl.kind = kind;
2666	ast_buf_from_token(pc, struct_name, &node->data.struct_decl.name);	2182	node->data.struct_decl.name = token_buf(struct_name);
2667	node->data.struct_decl.top_level_decl.visib_mod = visib_mod;	2183	node->data.struct_decl.top_level_decl.visib_mod = visib_mod;
2668	node->data.struct_decl.top_level_decl.directives = directives;	2184	node->data.struct_decl.top_level_decl.directives = directives;
2669		2185
...	@@ -2729,8 +2245,7 @@ static AstNode ast_parse_container_decl(ParseContext pc, int *token_index,	...	@@ -2729,8 +2245,7 @@ static AstNode ast_parse_container_decl(ParseContext pc, int *token_index,
2729		2245
2730	field_node->data.struct_field.top_level_decl.visib_mod = visib_mod;	2246	field_node->data.struct_field.top_level_decl.visib_mod = visib_mod;
2731	field_node->data.struct_field.top_level_decl.directives = directive_list;	2247	field_node->data.struct_field.top_level_decl.directives = directive_list;
2732		2248	field_node->data.struct_field.name = token_buf(token);
2733	ast_buf_from_token(pc, token, &field_node->data.struct_field.name);
2734		2249
2735	Token expr_or_comma = &pc->tokens->at(token_index);	2250	Token expr_or_comma = &pc->tokens->at(token_index);
2736	if (expr_or_comma->id == TokenIdComma) {	2251	if (expr_or_comma->id == TokenIdComma) {
...	@@ -2772,7 +2287,7 @@ static AstNode ast_parse_error_value_decl(ParseContext pc, int *token_index,	...	@@ -2772,7 +2287,7 @@ static AstNode ast_parse_error_value_decl(ParseContext pc, int *token_index,
2772	AstNode *node = ast_create_node(pc, NodeTypeErrorValueDecl, first_token);	2287	AstNode *node = ast_create_node(pc, NodeTypeErrorValueDecl, first_token);
2773	node->data.error_value_decl.top_level_decl.visib_mod = visib_mod;	2288	node->data.error_value_decl.top_level_decl.visib_mod = visib_mod;
2774	node->data.error_value_decl.top_level_decl.directives = directives;	2289	node->data.error_value_decl.top_level_decl.directives = directives;
2775	ast_buf_from_token(pc, name_tok, &node->data.error_value_decl.name);	2290	node->data.error_value_decl.name = token_buf(name_tok);
2776		2291
2777	normalize_parent_ptrs(node);	2292	normalize_parent_ptrs(node);
2778	return node;	2293	return node;
...	@@ -2795,7 +2310,7 @@ static AstNode ast_parse_type_decl(ParseContext pc, int *token_index,	...	@@ -2795,7 +2310,7 @@ static AstNode ast_parse_type_decl(ParseContext pc, int *token_index,
2795	ast_eat_token(pc, token_index, TokenIdEq);	2310	ast_eat_token(pc, token_index, TokenIdEq);
2796		2311
2797	AstNode *node = ast_create_node(pc, NodeTypeTypeDecl, first_token);	2312	AstNode *node = ast_create_node(pc, NodeTypeTypeDecl, first_token);
2798	ast_buf_from_token(pc, name_tok, &node->data.type_decl.symbol);	2313	node->data.type_decl.symbol = token_buf(name_tok);
2799	node->data.type_decl.child_type = ast_parse_prefix_op_expr(pc, token_index, true);	2314	node->data.type_decl.child_type = ast_parse_prefix_op_expr(pc, token_index, true);
2800		2315
2801	ast_eat_token(pc, token_index, TokenIdSemicolon);	2316	ast_eat_token(pc, token_index, TokenIdSemicolon);
...	@@ -2901,6 +2416,8 @@ AstNode ast_parse(Buf buf, ZigList<Token> tokens, ImportTableEntry owner,	...	@@ -2901,6 +2416,8 @@ AstNode ast_parse(Buf buf, ZigList<Token> tokens, ImportTableEntry owner,
2901	ErrColor err_color, uint32_t *next_node_index)	2416	ErrColor err_color, uint32_t *next_node_index)
2902	{	2417	{
2903	ParseContext pc = {0};	2418	ParseContext pc = {0};
		2419	pc.void_buf = buf_create_from_str("void");
		2420	pc.empty_buf = buf_create_from_str("");
2904	pc.err_color = err_color;	2421	pc.err_color = err_color;
2905	pc.owner = owner;	2422	pc.owner = owner;
2906	pc.buf = buf;	2423	pc.buf = buf;

src/tokenizer.cpp+418-315

...	@@ -11,6 +11,9 @@	...	@@ -11,6 +11,9 @@
11	#include <stdarg.h>	11	#include <stdarg.h>
12	#include <stdlib.h>	12	#include <stdlib.h>
13	#include <stdio.h>	13	#include <stdio.h>
		14	#include <inttypes.h>
		15	#include <limits.h>
		16	#include <errno.h>
14		17
15	#define WHITESPACE \	18	#define WHITESPACE \
16	' ': \	19	' ': \
...	@@ -30,7 +33,7 @@	...	@@ -30,7 +33,7 @@
30	'0': \	33	'0': \
31	case DIGIT_NON_ZERO	34	case DIGIT_NON_ZERO
32		35
33	#define ALPHA_EXCEPT_CR \	36	#define ALPHA_EXCEPT_C \
34	'a': \	37	'a': \
35	case 'b': \	38	case 'b': \
36	/case 'c':/ \	39	/case 'c':/ \
...	@@ -48,7 +51,7 @@	...	@@ -48,7 +51,7 @@
48	case 'o': \	51	case 'o': \
49	case 'p': \	52	case 'p': \
50	case 'q': \	53	case 'q': \
51	/case 'r':/ \	54	case 'r': \
52	case 's': \	55	case 's': \
53	case 't': \	56	case 't': \
54	case 'u': \	57	case 'u': \
...	@@ -85,77 +88,93 @@	...	@@ -85,77 +88,93 @@
85	case 'Z'	88	case 'Z'
86		89
87	#define ALPHA \	90	#define ALPHA \
88	ALPHA_EXCEPT_CR: \	91	ALPHA_EXCEPT_C: \
89	case 'c': \
90	case 'r'
91
92	#define SYMBOL_CHAR \
93	SYMBOL_CHAR_EXCEPT_C: \
94	case 'c'	92	case 'c'
95		93
96	#define SYMBOL_CHAR_EXCEPT_C \	94	#define SYMBOL_CHAR \
97	ALPHA_EXCEPT_CR: \	95	ALPHA_EXCEPT_C: \
98	case 'r': \
99	case DIGIT: \	96	case DIGIT: \
100	case '_'	97	case '_': \
		98	case 'c'
101		99
102	#define SYMBOL_START \	100	#define SYMBOL_START \
103	ALPHA: \	101	ALPHA: \
104	case '_'	102	case '_'
105		103
106	#define HEX_DIGIT \	104	struct ZigKeyword {
107	'a': \	105	const char *text;
108	case 'b': \	106	TokenId token_id;
109	case 'c': \	107	};
110	case 'd': \
111	case 'e': \
112	case 'f': \
113	case 'A': \
114	case 'B': \
115	case 'C': \
116	case 'D': \
117	case 'E': \
118	case 'F': \
119	case DIGIT
120		108
121	const char * zig_keywords[] = {	109	static const struct ZigKeyword zig_keywords[] = {
122	"true", "false", "null", "fn", "return", "var", "const", "extern",	110	{"asm", TokenIdKeywordAsm},
123	"pub", "export", "use", "if", "else", "goto", "asm",	111	{"break", TokenIdKeywordBreak},
124	"volatile", "struct", "enum", "while", "for", "continue", "break",	112	{"const", TokenIdKeywordConst},
125	"null", "noalias", "switch", "undefined", "error", "type", "inline",	113	{"continue", TokenIdKeywordContinue},
126	"defer", "union",	114	{"defer", TokenIdKeywordDefer},
		115	{"else", TokenIdKeywordElse},
		116	{"enum", TokenIdKeywordEnum},
		117	{"error", TokenIdKeywordError},
		118	{"export", TokenIdKeywordExport},
		119	{"extern", TokenIdKeywordExtern},
		120	{"false", TokenIdKeywordFalse},
		121	{"fn", TokenIdKeywordFn},
		122	{"for", TokenIdKeywordFor},
		123	{"goto", TokenIdKeywordGoto},
		124	{"if", TokenIdKeywordIf},
		125	{"inline", TokenIdKeywordInline},
		126	{"noalias", TokenIdKeywordNoAlias},
		127	{"null", TokenIdKeywordNull},
		128	{"pub", TokenIdKeywordPub},
		129	{"return", TokenIdKeywordReturn},
		130	{"struct", TokenIdKeywordStruct},
		131	{"switch", TokenIdKeywordSwitch},
		132	{"true", TokenIdKeywordTrue},
		133	{"type", TokenIdKeywordType},
		134	{"undefined", TokenIdKeywordUndefined},
		135	{"union", TokenIdKeywordUnion},
		136	{"use", TokenIdKeywordUse},
		137	{"var", TokenIdKeywordVar},
		138	{"volatile", TokenIdKeywordVolatile},
		139	{"while", TokenIdKeywordWhile},
127	};	140	};
128		141
129	bool is_zig_keyword(Buf *buf) {	142	bool is_zig_keyword(Buf *buf) {
130	for (int i = 0; i < array_length(zig_keywords); i += 1) {	143	for (int i = 0; i < array_length(zig_keywords); i += 1) {
131	if (buf_eql_str(buf, zig_keywords[i])) {	144	if (buf_eql_str(buf, zig_keywords[i].text)) {
132	return true;	145	return true;
133	}	146	}
134	}	147	}
135	return false;	148	return false;
136	}	149	}
137		150
		151	static bool is_symbol_char(uint8_t c) {
		152	switch (c) {
		153	case SYMBOL_CHAR:
		154	return true;
		155	default:
		156	return false;
		157	}
		158	}
		159
138	enum TokenizeState {	160	enum TokenizeState {
139	TokenizeStateStart,	161	TokenizeStateStart,
140	TokenizeStateSymbol,	162	TokenizeStateSymbol,
141	TokenizeStateSymbolFirst,	163	TokenizeStateSymbolFirstC,
142	TokenizeStateSymbolFirstRaw,
143	TokenizeStateFirstR,
144	TokenizeStateZero, // "0", which might lead to "0x"	164	TokenizeStateZero, // "0", which might lead to "0x"
145	TokenizeStateNumber, // "123", "0x123"	165	TokenizeStateNumber, // "123", "0x123"
		166	TokenizeStateNumberDot,
146	TokenizeStateFloatFraction, // "123.456", "0x123.456"	167	TokenizeStateFloatFraction, // "123.456", "0x123.456"
147	TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"	168	TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
148	TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"	169	TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
149	TokenizeStateString,	170	TokenizeStateString,
150	TokenizeStateStringEscape,	171	TokenizeStateStringEscape,
151	TokenizeStateRawString,
152	TokenizeStateRawStringContents,
153	TokenizeStateRawStringMaybeEnd,
154	TokenizeStateCharLiteral,	172	TokenizeStateCharLiteral,
155	TokenizeStateCharLiteralEnd,	173	TokenizeStateCharLiteralEnd,
156	TokenizeStateSawStar,	174	TokenizeStateSawStar,
157	TokenizeStateSawStarPercent,	175	TokenizeStateSawStarPercent,
158	TokenizeStateSawSlash,	176	TokenizeStateSawSlash,
		177	TokenizeStateSawBackslash,
159	TokenizeStateSawPercent,	178	TokenizeStateSawPercent,
160	TokenizeStateSawPlus,	179	TokenizeStateSawPlus,
161	TokenizeStateSawPlusPercent,	180	TokenizeStateSawPlusPercent,
...	@@ -167,6 +186,9 @@ enum TokenizeState {	...	@@ -167,6 +186,9 @@ enum TokenizeState {
167	TokenizeStateSawPipe,	186	TokenizeStateSawPipe,
168	TokenizeStateSawPipePipe,	187	TokenizeStateSawPipePipe,
169	TokenizeStateLineComment,	188	TokenizeStateLineComment,
		189	TokenizeStateLineString,
		190	TokenizeStateLineStringEnd,
		191	TokenizeStateLineStringContinue,
170	TokenizeStateSawEq,	192	TokenizeStateSawEq,
171	TokenizeStateSawBang,	193	TokenizeStateSawBang,
172	TokenizeStateSawLessThan,	194	TokenizeStateSawLessThan,
...	@@ -178,7 +200,7 @@ enum TokenizeState {	...	@@ -178,7 +200,7 @@ enum TokenizeState {
178	TokenizeStateSawDotDot,	200	TokenizeStateSawDotDot,
179	TokenizeStateSawQuestionMark,	201	TokenizeStateSawQuestionMark,
180	TokenizeStateSawAtSign,	202	TokenizeStateSawAtSign,
181	TokenizeStateHex,	203	TokenizeStateCharCode,
182	TokenizeStateError,	204	TokenizeStateError,
183	};	205	};
184		206
...	@@ -192,10 +214,16 @@ struct Tokenize {	...	@@ -192,10 +214,16 @@ struct Tokenize {
192	int column;	214	int column;
193	Token *cur_tok;	215	Token *cur_tok;
194	Tokenization *out;	216	Tokenization *out;
195	int raw_string_id_start;	217	uint32_t radix;
196	int raw_string_id_end;	218	int32_t exp_add_amt;
197	int raw_string_id_cmp_pos;	219	bool is_exp_negative;
198	int hex_chars_left;	220	bool is_num_lit_float;
		221	size_t char_code_index;
		222	size_t char_code_end;
		223	bool unicode;
		224	uint32_t char_code;
		225	int exponent_in_bin_or_dec;
		226	BigNum specified_exponent;
199	};	227	};
200		228
201	__attribute__ ((format (printf, 2, 3)))	229	__attribute__ ((format (printf, 2, 3)))
...	@@ -216,19 +244,28 @@ static void tokenize_error(Tokenize t, const char format, ...) {	...	@@ -216,19 +244,28 @@ static void tokenize_error(Tokenize t, const char format, ...) {
216	va_end(ap);	244	va_end(ap);
217	}	245	}
218		246
		247	static void set_token_id(Tokenize t, Token token, TokenId id) {
		248	token->id = id;
		249
		250	if (id == TokenIdNumberLiteral) {
		251	token->data.num_lit.overflow = false;
		252	} else if (id == TokenIdStringLiteral \|\| id == TokenIdSymbol) {
		253	memset(&token->data.str_lit.str, 0, sizeof(Buf));
		254	buf_resize(&token->data.str_lit.str, 0);
		255	token->data.str_lit.is_c_str = false;
		256	}
		257	}
		258
219	static void begin_token(Tokenize *t, TokenId id) {	259	static void begin_token(Tokenize *t, TokenId id) {
220	assert(!t->cur_tok);	260	assert(!t->cur_tok);
221	t->tokens->add_one();	261	t->tokens->add_one();
222	Token *token = &t->tokens->last();	262	Token *token = &t->tokens->last();
223	token->start_line = t->line;	263	token->start_line = t->line;
224	token->start_column = t->column;	264	token->start_column = t->column;
225	token->id = id;
226	token->start_pos = t->pos;	265	token->start_pos = t->pos;
227	token->radix = 0;	266
228	token->decimal_point_pos = 0;	267	set_token_id(t, token, id);
229	token->exponent_marker_pos = 0;	268
230	token->raw_string_start = 0;
231	token->raw_string_end = 0;
232	t->cur_tok = token;	269	t->cur_tok = token;
233	}	270	}
234		271
...	@@ -237,83 +274,82 @@ static void cancel_token(Tokenize *t) {	...	@@ -237,83 +274,82 @@ static void cancel_token(Tokenize *t) {
237	t->cur_tok = nullptr;	274	t->cur_tok = nullptr;
238	}	275	}
239		276
		277	static void end_float_token(Tokenize *t) {
		278	t->cur_tok->data.num_lit.bignum.kind = BigNumKindFloat;
		279
		280	if (t->radix == 10) {
		281	char *str_begin = buf_ptr(t->buf) + t->cur_tok->start_pos;
		282	char *str_end;
		283	errno = 0;
		284	t->cur_tok->data.num_lit.bignum.data.x_float = strtod(str_begin, &str_end);
		285	if (errno) {
		286	t->cur_tok->data.num_lit.overflow = true;
		287	return;
		288	}
		289	assert(str_end == buf_ptr(t->buf) + t->cur_tok->end_pos);
		290	return;
		291	}
		292
		293
		294	if (t->specified_exponent.data.x_uint >= INT_MAX) {
		295	t->cur_tok->data.num_lit.overflow = true;
		296	return;
		297	}
		298
		299	int64_t specified_exponent = t->specified_exponent.data.x_uint;
		300	if (t->is_exp_negative) {
		301	specified_exponent = -specified_exponent;
		302	}
		303	t->exponent_in_bin_or_dec += specified_exponent;
		304
		305	uint64_t significand = t->cur_tok->data.num_lit.bignum.data.x_uint;
		306	uint64_t significand_bits;
		307	uint64_t exponent_bits;
		308	if (significand == 0) {
		309	// 0 is all 0's
		310	significand_bits = 0;
		311	exponent_bits = 0;
		312	} else {
		313	// normalize the significand
		314	if (t->radix == 10) {
		315	zig_panic("TODO: decimal floats");
		316	} else {
		317	int significand_magnitude_in_bin = __builtin_clzll(1) - __builtin_clzll(significand);
		318	t->exponent_in_bin_or_dec += significand_magnitude_in_bin;
		319	if (!(-1023 <= t->exponent_in_bin_or_dec && t->exponent_in_bin_or_dec < 1023)) {
		320	t->cur_tok->data.num_lit.overflow = true;
		321	} else {
		322	// this should chop off exactly one 1 bit from the top.
		323	significand_bits = ((uint64_t)significand << (52 - significand_magnitude_in_bin)) & 0xfffffffffffffULL;
		324	exponent_bits = t->exponent_in_bin_or_dec + 1023;
		325	}
		326	}
		327	}
		328	uint64_t double_bits = (exponent_bits << 52) \| significand_bits;
		329	memcpy(&t->cur_tok->data.num_lit.bignum.data.x_float, &double_bits, sizeof(double));
		330	}
		331
240	static void end_token(Tokenize *t) {	332	static void end_token(Tokenize *t) {
241	assert(t->cur_tok);	333	assert(t->cur_tok);
242	t->cur_tok->end_pos = t->pos + 1;	334	t->cur_tok->end_pos = t->pos + 1;
243		335
244	// normalize number literal parsing stuff
245	if (t->cur_tok->id == TokenIdNumberLiteral) {	336	if (t->cur_tok->id == TokenIdNumberLiteral) {
246	if (t->cur_tok->exponent_marker_pos == 0) {	337	if (t->cur_tok->data.num_lit.overflow) {
247	t->cur_tok->exponent_marker_pos = t->cur_tok->end_pos;	338	return;
248	}	339	}
249	if (t->cur_tok->decimal_point_pos == 0) {	340	if (t->is_num_lit_float) {
250	t->cur_tok->decimal_point_pos = t->cur_tok->exponent_marker_pos;	341	end_float_token(t);
251	}	342	}
252	}	343	} else if (t->cur_tok->id == TokenIdSymbol) {
253		344	char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;
254	char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;	345	int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
255	int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
256		346
257	if (mem_eql_str(token_mem, token_len, "fn")) {	347	for (size_t i = 0; i < array_length(zig_keywords); i += 1) {
258	t->cur_tok->id = TokenIdKeywordFn;	348	if (mem_eql_str(token_mem, token_len, zig_keywords[i].text)) {
259	} else if (mem_eql_str(token_mem, token_len, "return")) {	349	t->cur_tok->id = zig_keywords[i].token_id;
260	t->cur_tok->id = TokenIdKeywordReturn;	350	break;
261	} else if (mem_eql_str(token_mem, token_len, "var")) {	351	}
262	t->cur_tok->id = TokenIdKeywordVar;	352	}
263	} else if (mem_eql_str(token_mem, token_len, "const")) {
264	t->cur_tok->id = TokenIdKeywordConst;
265	} else if (mem_eql_str(token_mem, token_len, "extern")) {
266	t->cur_tok->id = TokenIdKeywordExtern;
267	} else if (mem_eql_str(token_mem, token_len, "pub")) {
268	t->cur_tok->id = TokenIdKeywordPub;
269	} else if (mem_eql_str(token_mem, token_len, "export")) {
270	t->cur_tok->id = TokenIdKeywordExport;
271	} else if (mem_eql_str(token_mem, token_len, "use")) {
272	t->cur_tok->id = TokenIdKeywordUse;
273	} else if (mem_eql_str(token_mem, token_len, "true")) {
274	t->cur_tok->id = TokenIdKeywordTrue;
275	} else if (mem_eql_str(token_mem, token_len, "false")) {
276	t->cur_tok->id = TokenIdKeywordFalse;
277	} else if (mem_eql_str(token_mem, token_len, "if")) {
278	t->cur_tok->id = TokenIdKeywordIf;
279	} else if (mem_eql_str(token_mem, token_len, "else")) {
280	t->cur_tok->id = TokenIdKeywordElse;
281	} else if (mem_eql_str(token_mem, token_len, "goto")) {
282	t->cur_tok->id = TokenIdKeywordGoto;
283	} else if (mem_eql_str(token_mem, token_len, "volatile")) {
284	t->cur_tok->id = TokenIdKeywordVolatile;
285	} else if (mem_eql_str(token_mem, token_len, "asm")) {
286	t->cur_tok->id = TokenIdKeywordAsm;
287	} else if (mem_eql_str(token_mem, token_len, "struct")) {
288	t->cur_tok->id = TokenIdKeywordStruct;
289	} else if (mem_eql_str(token_mem, token_len, "enum")) {
290	t->cur_tok->id = TokenIdKeywordEnum;
291	} else if (mem_eql_str(token_mem, token_len, "union")) {
292	t->cur_tok->id = TokenIdKeywordUnion;
293	} else if (mem_eql_str(token_mem, token_len, "for")) {
294	t->cur_tok->id = TokenIdKeywordFor;
295	} else if (mem_eql_str(token_mem, token_len, "while")) {
296	t->cur_tok->id = TokenIdKeywordWhile;
297	} else if (mem_eql_str(token_mem, token_len, "continue")) {
298	t->cur_tok->id = TokenIdKeywordContinue;
299	} else if (mem_eql_str(token_mem, token_len, "break")) {
300	t->cur_tok->id = TokenIdKeywordBreak;
301	} else if (mem_eql_str(token_mem, token_len, "null")) {
302	t->cur_tok->id = TokenIdKeywordNull;
303	} else if (mem_eql_str(token_mem, token_len, "noalias")) {
304	t->cur_tok->id = TokenIdKeywordNoAlias;
305	} else if (mem_eql_str(token_mem, token_len, "switch")) {
306	t->cur_tok->id = TokenIdKeywordSwitch;
307	} else if (mem_eql_str(token_mem, token_len, "undefined")) {
308	t->cur_tok->id = TokenIdKeywordUndefined;
309	} else if (mem_eql_str(token_mem, token_len, "error")) {
310	t->cur_tok->id = TokenIdKeywordError;
311	} else if (mem_eql_str(token_mem, token_len, "type")) {
312	t->cur_tok->id = TokenIdKeywordType;
313	} else if (mem_eql_str(token_mem, token_len, "inline")) {
314	t->cur_tok->id = TokenIdKeywordInline;
315	} else if (mem_eql_str(token_mem, token_len, "defer")) {
316	t->cur_tok->id = TokenIdKeywordDefer;
317	}	353	}
318		354
319	t->cur_tok = nullptr;	355	t->cur_tok = nullptr;
...	@@ -327,7 +363,7 @@ static bool is_exponent_signifier(uint8_t c, int radix) {	...	@@ -327,7 +363,7 @@ static bool is_exponent_signifier(uint8_t c, int radix) {
327	}	363	}
328	}	364	}
329		365
330	int get_digit_value(uint8_t c) {	366	static uint32_t get_digit_value(uint8_t c) {
331	if ('0' <= c && c <= '9') {	367	if ('0' <= c && c <= '9') {
332	return c - '0';	368	return c - '0';
333	}	369	}
...	@@ -337,7 +373,19 @@ int get_digit_value(uint8_t c) {	...	@@ -337,7 +373,19 @@ int get_digit_value(uint8_t c) {
337	if ('a' <= c && c <= 'z') {	373	if ('a' <= c && c <= 'z') {
338	return c - 'a' + 10;	374	return c - 'a' + 10;
339	}	375	}
340	return -1;	376	return UINT32_MAX;
		377	}
		378
		379	void handle_string_escape(Tokenize *t, uint8_t c) {
		380	if (t->cur_tok->id == TokenIdCharLiteral) {
		381	t->cur_tok->data.char_lit.c = c;
		382	t->state = TokenizeStateCharLiteralEnd;
		383	} else if (t->cur_tok->id == TokenIdStringLiteral \|\| t->cur_tok->id == TokenIdSymbol) {
		384	buf_append_char(&t->cur_tok->data.str_lit.str, c);
		385	t->state = TokenizeStateString;
		386	} else {
		387	zig_unreachable();
		388	}
341	}	389	}
342		390
343	void tokenize(Buf buf, Tokenization out) {	391	void tokenize(Buf buf, Tokenization out) {
...	@@ -359,27 +407,35 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -359,27 +407,35 @@ void tokenize(Buf buf, Tokenization out) {
359	case WHITESPACE:	407	case WHITESPACE:
360	break;	408	break;
361	case 'c':	409	case 'c':
362	t.state = TokenizeStateSymbolFirst;	410	t.state = TokenizeStateSymbolFirstC;
363	begin_token(&t, TokenIdSymbol);	411	begin_token(&t, TokenIdSymbol);
		412	buf_append_char(&t.cur_tok->data.str_lit.str, c);
364	break;	413	break;
365	case 'r':	414	case ALPHA_EXCEPT_C:
366	t.state = TokenizeStateFirstR;
367	begin_token(&t, TokenIdSymbol);
368	break;
369	case ALPHA_EXCEPT_CR:
370	case '_':	415	case '_':
371	t.state = TokenizeStateSymbol;	416	t.state = TokenizeStateSymbol;
372	begin_token(&t, TokenIdSymbol);	417	begin_token(&t, TokenIdSymbol);
		418	buf_append_char(&t.cur_tok->data.str_lit.str, c);
373	break;	419	break;
374	case '0':	420	case '0':
375	t.state = TokenizeStateZero;	421	t.state = TokenizeStateZero;
376	begin_token(&t, TokenIdNumberLiteral);	422	begin_token(&t, TokenIdNumberLiteral);
377	t.cur_tok->radix = 10;	423	t.radix = 10;
		424	t.exp_add_amt = 1;
		425	t.exponent_in_bin_or_dec = 0;
		426	t.is_num_lit_float = false;
		427	bignum_init_unsigned(&t.cur_tok->data.num_lit.bignum, 0);
		428	bignum_init_unsigned(&t.specified_exponent, 0);
378	break;	429	break;
379	case DIGIT_NON_ZERO:	430	case DIGIT_NON_ZERO:
380	t.state = TokenizeStateNumber;	431	t.state = TokenizeStateNumber;
381	begin_token(&t, TokenIdNumberLiteral);	432	begin_token(&t, TokenIdNumberLiteral);
382	t.cur_tok->radix = 10;	433	t.radix = 10;
		434	t.exp_add_amt = 1;
		435	t.exponent_in_bin_or_dec = 0;
		436	t.is_num_lit_float = false;
		437	bignum_init_unsigned(&t.cur_tok->data.num_lit.bignum, get_digit_value(c));
		438	bignum_init_unsigned(&t.specified_exponent, 0);
383	break;	439	break;
384	case '"':	440	case '"':
385	begin_token(&t, TokenIdStringLiteral);	441	begin_token(&t, TokenIdStringLiteral);
...	@@ -437,6 +493,10 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -437,6 +493,10 @@ void tokenize(Buf buf, Tokenization out) {
437	begin_token(&t, TokenIdSlash);	493	begin_token(&t, TokenIdSlash);
438	t.state = TokenizeStateSawSlash;	494	t.state = TokenizeStateSawSlash;
439	break;	495	break;
		496	case '\\':
		497	begin_token(&t, TokenIdStringLiteral);
		498	t.state = TokenizeStateSawBackslash;
		499	break;
440	case '%':	500	case '%':
441	begin_token(&t, TokenIdPercent);	501	begin_token(&t, TokenIdPercent);
442	t.state = TokenizeStateSawPercent;	502	t.state = TokenizeStateSawPercent;
...	@@ -500,12 +560,12 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -500,12 +560,12 @@ void tokenize(Buf buf, Tokenization out) {
500	case TokenizeStateSawQuestionMark:	560	case TokenizeStateSawQuestionMark:
501	switch (c) {	561	switch (c) {
502	case '?':	562	case '?':
503	t.cur_tok->id = TokenIdDoubleQuestion;	563	set_token_id(&t, t.cur_tok, TokenIdDoubleQuestion);
504	end_token(&t);	564	end_token(&t);
505	t.state = TokenizeStateStart;	565	t.state = TokenizeStateStart;
506	break;	566	break;
507	case '=':	567	case '=':
508	t.cur_tok->id = TokenIdMaybeAssign;	568	set_token_id(&t, t.cur_tok, TokenIdMaybeAssign);
509	end_token(&t);	569	end_token(&t);
510	t.state = TokenizeStateStart;	570	t.state = TokenizeStateStart;
511	break;	571	break;
...	@@ -520,7 +580,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -520,7 +580,7 @@ void tokenize(Buf buf, Tokenization out) {
520	switch (c) {	580	switch (c) {
521	case '.':	581	case '.':
522	t.state = TokenizeStateSawDotDot;	582	t.state = TokenizeStateSawDotDot;
523	t.cur_tok->id = TokenIdEllipsis;	583	set_token_id(&t, t.cur_tok, TokenIdEllipsis);
524	break;	584	break;
525	default:	585	default:
526	t.pos -= 1;	586	t.pos -= 1;
...	@@ -542,12 +602,12 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -542,12 +602,12 @@ void tokenize(Buf buf, Tokenization out) {
542	case TokenizeStateSawGreaterThan:	602	case TokenizeStateSawGreaterThan:
543	switch (c) {	603	switch (c) {
544	case '=':	604	case '=':
545	t.cur_tok->id = TokenIdCmpGreaterOrEq;	605	set_token_id(&t, t.cur_tok, TokenIdCmpGreaterOrEq);
546	end_token(&t);	606	end_token(&t);
547	t.state = TokenizeStateStart;	607	t.state = TokenizeStateStart;
548	break;	608	break;
549	case '>':	609	case '>':
550	t.cur_tok->id = TokenIdBitShiftRight;	610	set_token_id(&t, t.cur_tok, TokenIdBitShiftRight);
551	t.state = TokenizeStateSawGreaterThanGreaterThan;	611	t.state = TokenizeStateSawGreaterThanGreaterThan;
552	break;	612	break;
553	default:	613	default:
...	@@ -560,7 +620,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -560,7 +620,7 @@ void tokenize(Buf buf, Tokenization out) {
560	case TokenizeStateSawGreaterThanGreaterThan:	620	case TokenizeStateSawGreaterThanGreaterThan:
561	switch (c) {	621	switch (c) {
562	case '=':	622	case '=':
563	t.cur_tok->id = TokenIdBitShiftRightEq;	623	set_token_id(&t, t.cur_tok, TokenIdBitShiftRightEq);
564	end_token(&t);	624	end_token(&t);
565	t.state = TokenizeStateStart;	625	t.state = TokenizeStateStart;
566	break;	626	break;
...	@@ -574,12 +634,12 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -574,12 +634,12 @@ void tokenize(Buf buf, Tokenization out) {
574	case TokenizeStateSawLessThan:	634	case TokenizeStateSawLessThan:
575	switch (c) {	635	switch (c) {
576	case '=':	636	case '=':
577	t.cur_tok->id = TokenIdCmpLessOrEq;	637	set_token_id(&t, t.cur_tok, TokenIdCmpLessOrEq);
578	end_token(&t);	638	end_token(&t);
579	t.state = TokenizeStateStart;	639	t.state = TokenizeStateStart;
580	break;	640	break;
581	case '<':	641	case '<':
582	t.cur_tok->id = TokenIdBitShiftLeft;	642	set_token_id(&t, t.cur_tok, TokenIdBitShiftLeft);
583	t.state = TokenizeStateSawLessThanLessThan;	643	t.state = TokenizeStateSawLessThanLessThan;
584	break;	644	break;
585	default:	645	default:
...	@@ -592,12 +652,12 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -592,12 +652,12 @@ void tokenize(Buf buf, Tokenization out) {
592	case TokenizeStateSawLessThanLessThan:	652	case TokenizeStateSawLessThanLessThan:
593	switch (c) {	653	switch (c) {
594	case '=':	654	case '=':
595	t.cur_tok->id = TokenIdBitShiftLeftEq;	655	set_token_id(&t, t.cur_tok, TokenIdBitShiftLeftEq);
596	end_token(&t);	656	end_token(&t);
597	t.state = TokenizeStateStart;	657	t.state = TokenizeStateStart;
598	break;	658	break;
599	case '%':	659	case '%':
600	t.cur_tok->id = TokenIdBitShiftLeftPercent;	660	set_token_id(&t, t.cur_tok, TokenIdBitShiftLeftPercent);
601	t.state = TokenizeStateSawShiftLeftPercent;	661	t.state = TokenizeStateSawShiftLeftPercent;
602	break;	662	break;
603	default:	663	default:
...	@@ -610,7 +670,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -610,7 +670,7 @@ void tokenize(Buf buf, Tokenization out) {
610	case TokenizeStateSawShiftLeftPercent:	670	case TokenizeStateSawShiftLeftPercent:
611	switch (c) {	671	switch (c) {
612	case '=':	672	case '=':
613	t.cur_tok->id = TokenIdBitShiftLeftPercentEq;	673	set_token_id(&t, t.cur_tok, TokenIdBitShiftLeftPercentEq);
614	end_token(&t);	674	end_token(&t);
615	t.state = TokenizeStateStart;	675	t.state = TokenizeStateStart;
616	break;	676	break;
...	@@ -624,7 +684,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -624,7 +684,7 @@ void tokenize(Buf buf, Tokenization out) {
624	case TokenizeStateSawBang:	684	case TokenizeStateSawBang:
625	switch (c) {	685	switch (c) {
626	case '=':	686	case '=':
627	t.cur_tok->id = TokenIdCmpNotEq;	687	set_token_id(&t, t.cur_tok, TokenIdCmpNotEq);
628	end_token(&t);	688	end_token(&t);
629	t.state = TokenizeStateStart;	689	t.state = TokenizeStateStart;
630	break;	690	break;
...	@@ -638,12 +698,12 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -638,12 +698,12 @@ void tokenize(Buf buf, Tokenization out) {
638	case TokenizeStateSawEq:	698	case TokenizeStateSawEq:
639	switch (c) {	699	switch (c) {
640	case '=':	700	case '=':
641	t.cur_tok->id = TokenIdCmpEq;	701	set_token_id(&t, t.cur_tok, TokenIdCmpEq);
642	end_token(&t);	702	end_token(&t);
643	t.state = TokenizeStateStart;	703	t.state = TokenizeStateStart;
644	break;	704	break;
645	case '>':	705	case '>':
646	t.cur_tok->id = TokenIdFatArrow;	706	set_token_id(&t, t.cur_tok, TokenIdFatArrow);
647	end_token(&t);	707	end_token(&t);
648	t.state = TokenizeStateStart;	708	t.state = TokenizeStateStart;
649	break;	709	break;
...	@@ -657,17 +717,17 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -657,17 +717,17 @@ void tokenize(Buf buf, Tokenization out) {
657	case TokenizeStateSawStar:	717	case TokenizeStateSawStar:
658	switch (c) {	718	switch (c) {
659	case '=':	719	case '=':
660	t.cur_tok->id = TokenIdTimesEq;	720	set_token_id(&t, t.cur_tok, TokenIdTimesEq);
661	end_token(&t);	721	end_token(&t);
662	t.state = TokenizeStateStart;	722	t.state = TokenizeStateStart;
663	break;	723	break;
664	case '*':	724	case '*':
665	t.cur_tok->id = TokenIdStarStar;	725	set_token_id(&t, t.cur_tok, TokenIdStarStar);
666	end_token(&t);	726	end_token(&t);
667	t.state = TokenizeStateStart;	727	t.state = TokenizeStateStart;
668	break;	728	break;
669	case '%':	729	case '%':
670	t.cur_tok->id = TokenIdTimesPercent;	730	set_token_id(&t, t.cur_tok, TokenIdTimesPercent);
671	t.state = TokenizeStateSawStarPercent;	731	t.state = TokenizeStateSawStarPercent;
672	break;	732	break;
673	default:	733	default:
...	@@ -680,7 +740,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -680,7 +740,7 @@ void tokenize(Buf buf, Tokenization out) {
680	case TokenizeStateSawStarPercent:	740	case TokenizeStateSawStarPercent:
681	switch (c) {	741	switch (c) {
682	case '=':	742	case '=':
683	t.cur_tok->id = TokenIdTimesPercentEq;	743	set_token_id(&t, t.cur_tok, TokenIdTimesPercentEq);
684	end_token(&t);	744	end_token(&t);
685	t.state = TokenizeStateStart;	745	t.state = TokenizeStateStart;
686	break;	746	break;
...	@@ -694,17 +754,17 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -694,17 +754,17 @@ void tokenize(Buf buf, Tokenization out) {
694	case TokenizeStateSawPercent:	754	case TokenizeStateSawPercent:
695	switch (c) {	755	switch (c) {
696	case '=':	756	case '=':
697	t.cur_tok->id = TokenIdModEq;	757	set_token_id(&t, t.cur_tok, TokenIdModEq);
698	end_token(&t);	758	end_token(&t);
699	t.state = TokenizeStateStart;	759	t.state = TokenizeStateStart;
700	break;	760	break;
701	case '.':	761	case '.':
702	t.cur_tok->id = TokenIdPercentDot;	762	set_token_id(&t, t.cur_tok, TokenIdPercentDot);
703	end_token(&t);	763	end_token(&t);
704	t.state = TokenizeStateStart;	764	t.state = TokenizeStateStart;
705	break;	765	break;
706	case '%':	766	case '%':
707	t.cur_tok->id = TokenIdPercentPercent;	767	set_token_id(&t, t.cur_tok, TokenIdPercentPercent);
708	end_token(&t);	768	end_token(&t);
709	t.state = TokenizeStateStart;	769	t.state = TokenizeStateStart;
710	break;	770	break;
...	@@ -718,17 +778,17 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -718,17 +778,17 @@ void tokenize(Buf buf, Tokenization out) {
718	case TokenizeStateSawPlus:	778	case TokenizeStateSawPlus:
719	switch (c) {	779	switch (c) {
720	case '=':	780	case '=':
721	t.cur_tok->id = TokenIdPlusEq;	781	set_token_id(&t, t.cur_tok, TokenIdPlusEq);
722	end_token(&t);	782	end_token(&t);
723	t.state = TokenizeStateStart;	783	t.state = TokenizeStateStart;
724	break;	784	break;
725	case '+':	785	case '+':
726	t.cur_tok->id = TokenIdPlusPlus;	786	set_token_id(&t, t.cur_tok, TokenIdPlusPlus);
727	end_token(&t);	787	end_token(&t);
728	t.state = TokenizeStateStart;	788	t.state = TokenizeStateStart;
729	break;	789	break;
730	case '%':	790	case '%':
731	t.cur_tok->id = TokenIdPlusPercent;	791	set_token_id(&t, t.cur_tok, TokenIdPlusPercent);
732	t.state = TokenizeStateSawPlusPercent;	792	t.state = TokenizeStateSawPlusPercent;
733	break;	793	break;
734	default:	794	default:
...	@@ -741,7 +801,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -741,7 +801,7 @@ void tokenize(Buf buf, Tokenization out) {
741	case TokenizeStateSawPlusPercent:	801	case TokenizeStateSawPlusPercent:
742	switch (c) {	802	switch (c) {
743	case '=':	803	case '=':
744	t.cur_tok->id = TokenIdPlusPercentEq;	804	set_token_id(&t, t.cur_tok, TokenIdPlusPercentEq);
745	end_token(&t);	805	end_token(&t);
746	t.state = TokenizeStateStart;	806	t.state = TokenizeStateStart;
747	break;	807	break;
...	@@ -755,11 +815,11 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -755,11 +815,11 @@ void tokenize(Buf buf, Tokenization out) {
755	case TokenizeStateSawAmpersand:	815	case TokenizeStateSawAmpersand:
756	switch (c) {	816	switch (c) {
757	case '&':	817	case '&':
758	t.cur_tok->id = TokenIdBoolAnd;	818	set_token_id(&t, t.cur_tok, TokenIdBoolAnd);
759	t.state = TokenizeStateSawAmpersandAmpersand;	819	t.state = TokenizeStateSawAmpersandAmpersand;
760	break;	820	break;
761	case '=':	821	case '=':
762	t.cur_tok->id = TokenIdBitAndEq;	822	set_token_id(&t, t.cur_tok, TokenIdBitAndEq);
763	end_token(&t);	823	end_token(&t);
764	t.state = TokenizeStateStart;	824	t.state = TokenizeStateStart;
765	break;	825	break;
...	@@ -773,7 +833,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -773,7 +833,7 @@ void tokenize(Buf buf, Tokenization out) {
773	case TokenizeStateSawAmpersandAmpersand:	833	case TokenizeStateSawAmpersandAmpersand:
774	switch (c) {	834	switch (c) {
775	case '=':	835	case '=':
776	t.cur_tok->id = TokenIdBoolAndEq;	836	set_token_id(&t, t.cur_tok, TokenIdBoolAndEq);
777	end_token(&t);	837	end_token(&t);
778	t.state = TokenizeStateStart;	838	t.state = TokenizeStateStart;
779	break;	839	break;
...	@@ -787,7 +847,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -787,7 +847,7 @@ void tokenize(Buf buf, Tokenization out) {
787	case TokenizeStateSawCaret:	847	case TokenizeStateSawCaret:
788	switch (c) {	848	switch (c) {
789	case '=':	849	case '=':
790	t.cur_tok->id = TokenIdBitXorEq;	850	set_token_id(&t, t.cur_tok, TokenIdBitXorEq);
791	end_token(&t);	851	end_token(&t);
792	t.state = TokenizeStateStart;	852	t.state = TokenizeStateStart;
793	break;	853	break;
...	@@ -801,11 +861,11 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -801,11 +861,11 @@ void tokenize(Buf buf, Tokenization out) {
801	case TokenizeStateSawPipe:	861	case TokenizeStateSawPipe:
802	switch (c) {	862	switch (c) {
803	case '\|':	863	case '\|':
804	t.cur_tok->id = TokenIdBoolOr;	864	set_token_id(&t, t.cur_tok, TokenIdBoolOr);
805	t.state = TokenizeStateSawPipePipe;	865	t.state = TokenizeStateSawPipePipe;
806	break;	866	break;
807	case '=':	867	case '=':
808	t.cur_tok->id = TokenIdBitOrEq;	868	set_token_id(&t, t.cur_tok, TokenIdBitOrEq);
809	end_token(&t);	869	end_token(&t);
810	t.state = TokenizeStateStart;	870	t.state = TokenizeStateStart;
811	break;	871	break;
...	@@ -819,7 +879,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -819,7 +879,7 @@ void tokenize(Buf buf, Tokenization out) {
819	case TokenizeStateSawPipePipe:	879	case TokenizeStateSawPipePipe:
820	switch (c) {	880	switch (c) {
821	case '=':	881	case '=':
822	t.cur_tok->id = TokenIdBoolOrEq;	882	set_token_id(&t, t.cur_tok, TokenIdBoolOrEq);
823	end_token(&t);	883	end_token(&t);
824	t.state = TokenizeStateStart;	884	t.state = TokenizeStateStart;
825	break;	885	break;
...	@@ -837,7 +897,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -837,7 +897,7 @@ void tokenize(Buf buf, Tokenization out) {
837	t.state = TokenizeStateLineComment;	897	t.state = TokenizeStateLineComment;
838	break;	898	break;
839	case '=':	899	case '=':
840	t.cur_tok->id = TokenIdDivEq;	900	set_token_id(&t, t.cur_tok, TokenIdDivEq);
841	end_token(&t);	901	end_token(&t);
842	t.state = TokenizeStateStart;	902	t.state = TokenizeStateStart;
843	break;	903	break;
...	@@ -848,24 +908,32 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -848,24 +908,32 @@ void tokenize(Buf buf, Tokenization out) {
848	continue;	908	continue;
849	}	909	}
850	break;	910	break;
851	case TokenizeStateLineComment:	911	case TokenizeStateSawBackslash:
		912	switch (c) {
		913	case '\\':
		914	t.state = TokenizeStateLineString;
		915	break;
		916	default:
		917	tokenize_error(&t, "invalid character: '%c'", c);
		918	break;
		919	}
		920	break;
		921	case TokenizeStateLineString:
852	switch (c) {	922	switch (c) {
853	case '\n':	923	case '\n':
854	t.state = TokenizeStateStart;	924	t.state = TokenizeStateLineStringEnd;
855	break;	925	break;
856	default:	926	default:
857	// do nothing	927	buf_append_char(&t.cur_tok->data.str_lit.str, c);
858	break;	928	break;
859	}	929	}
860	break;	930	break;
861	case TokenizeStateSymbolFirst:	931	case TokenizeStateLineStringEnd:
862	switch (c) {	932	switch (c) {
863	case '"':	933	case WHITESPACE:
864	t.cur_tok->id = TokenIdStringLiteral;
865	t.state = TokenizeStateString;
866	break;	934	break;
867	case SYMBOL_CHAR:	935	case '\\':
868	t.state = TokenizeStateSymbol;	936	t.state = TokenizeStateLineStringContinue;
869	break;	937	break;
870	default:	938	default:
871	t.pos -= 1;	939	t.pos -= 1;
...	@@ -874,29 +942,38 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -874,29 +942,38 @@ void tokenize(Buf buf, Tokenization out) {
874	continue;	942	continue;
875	}	943	}
876	break;	944	break;
877	case TokenizeStateSymbolFirstRaw:	945	case TokenizeStateLineStringContinue:
878	switch (c) {	946	switch (c) {
879	case '"':	947	case '\\':
880	t.cur_tok->id = TokenIdStringLiteral;	948	t.state = TokenizeStateLineString;
881	t.state = TokenizeStateRawString;	949	buf_append_char(&t.cur_tok->data.str_lit.str, '\n');
882	t.raw_string_id_start = t.pos + 1;
883	break;
884	case SYMBOL_CHAR:
885	t.state = TokenizeStateSymbol;
886	break;	950	break;
887	default:	951	default:
888	t.pos -= 1;	952	tokenize_error(&t, "invalid character: '%c'", c);
889	end_token(&t);	953	break;
		954	}
		955	break;
		956	case TokenizeStateLineComment:
		957	switch (c) {
		958	case '\n':
890	t.state = TokenizeStateStart;	959	t.state = TokenizeStateStart;
891	continue;	960	break;
		961	default:
		962	// do nothing
		963	break;
892	}	964	}
893	break;	965	break;
894	case TokenizeStateSawAtSign:	966	case TokenizeStateSymbolFirstC:
895	switch (c) {	967	switch (c) {
896	case '"':	968	case '"':
897	t.cur_tok->id = TokenIdSymbol;	969	set_token_id(&t, t.cur_tok, TokenIdStringLiteral);
		970	t.cur_tok->data.str_lit.is_c_str = true;
898	t.state = TokenizeStateString;	971	t.state = TokenizeStateString;
899	break;	972	break;
		973	case SYMBOL_CHAR:
		974	t.state = TokenizeStateSymbol;
		975	buf_append_char(&t.cur_tok->data.str_lit.str, c);
		976	break;
900	default:	977	default:
901	t.pos -= 1;	978	t.pos -= 1;
902	end_token(&t);	979	end_token(&t);
...	@@ -904,18 +981,11 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -904,18 +981,11 @@ void tokenize(Buf buf, Tokenization out) {
904	continue;	981	continue;
905	}	982	}
906	break;	983	break;
907	case TokenizeStateFirstR:	984	case TokenizeStateSawAtSign:
908	switch (c) {	985	switch (c) {
909	case '"':	986	case '"':
910	t.cur_tok->id = TokenIdStringLiteral;	987	set_token_id(&t, t.cur_tok, TokenIdSymbol);
911	t.state = TokenizeStateRawString;	988	t.state = TokenizeStateString;
912	t.raw_string_id_start = t.pos + 1;
913	break;
914	case 'c':
915	t.state = TokenizeStateSymbolFirstRaw;
916	break;
917	case SYMBOL_CHAR_EXCEPT_C:
918	t.state = TokenizeStateSymbol;
919	break;	989	break;
920	default:	990	default:
921	t.pos -= 1;	991	t.pos -= 1;
...	@@ -927,6 +997,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -927,6 +997,7 @@ void tokenize(Buf buf, Tokenization out) {
927	case TokenizeStateSymbol:	997	case TokenizeStateSymbol:
928	switch (c) {	998	switch (c) {
929	case SYMBOL_CHAR:	999	case SYMBOL_CHAR:
		1000	buf_append_char(&t.cur_tok->data.str_lit.str, c);
930	break;	1001	break;
931	default:	1002	default:
932	t.pos -= 1;	1003	t.pos -= 1;
...	@@ -942,108 +1013,124 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -942,108 +1013,124 @@ void tokenize(Buf buf, Tokenization out) {
942	t.state = TokenizeStateStart;	1013	t.state = TokenizeStateStart;
943	break;	1014	break;
944	case '\n':	1015	case '\n':
945	tokenize_error(&t, "use raw string for multiline string literal");	1016	tokenize_error(&t, "newline not allowed in string literal");
946	break;	1017	break;
947	case '\\':	1018	case '\\':
948	t.state = TokenizeStateStringEscape;	1019	t.state = TokenizeStateStringEscape;
949	break;	1020	break;
950	default:	1021	default:
		1022	buf_append_char(&t.cur_tok->data.str_lit.str, c);
951	break;	1023	break;
952	}	1024	}
953	break;	1025	break;
954	case TokenizeStateStringEscape:	1026	case TokenizeStateStringEscape:
955	switch (c) {	1027	switch (c) {
956	case 'x':	1028	case 'x':
957	t.state = TokenizeStateHex;	1029	t.state = TokenizeStateCharCode;
958	t.hex_chars_left = 2;	1030	t.radix = 16;
		1031	t.char_code = 0;
		1032	t.char_code_index = 0;
		1033	t.char_code_end = 2;
		1034	t.unicode = false;
959	break;	1035	break;
960	case 'u':	1036	case 'u':
961	t.state = TokenizeStateHex;	1037	t.state = TokenizeStateCharCode;
962	t.hex_chars_left = 4;	1038	t.radix = 16;
		1039	t.char_code = 0;
		1040	t.char_code_index = 0;
		1041	t.char_code_end = 4;
		1042	t.unicode = true;
963	break;	1043	break;
964	case 'U':	1044	case 'U':
965	t.state = TokenizeStateHex;	1045	t.state = TokenizeStateCharCode;
966	t.hex_chars_left = 6;	1046	t.radix = 16;
		1047	t.char_code = 0;
		1048	t.char_code_index = 0;
		1049	t.char_code_end = 6;
		1050	t.unicode = true;
967	break;	1051	break;
968	case 'n':	1052	case 'n':
		1053	handle_string_escape(&t, '\n');
		1054	break;
969	case 'r':	1055	case 'r':
		1056	handle_string_escape(&t, '\r');
		1057	break;
970	case '\\':	1058	case '\\':
		1059	handle_string_escape(&t, '\\');
		1060	break;
971	case 't':	1061	case 't':
		1062	handle_string_escape(&t, '\t');
		1063	break;
972	case '\'':	1064	case '\'':
		1065	handle_string_escape(&t, '\'');
		1066	break;
973	case '"':	1067	case '"':
974	if (t.cur_tok->id == TokenIdCharLiteral) {	1068	handle_string_escape(&t, '\"');
975	t.state = TokenizeStateCharLiteralEnd;
976	} else if (t.cur_tok->id == TokenIdStringLiteral) {
977	t.state = TokenizeStateString;
978	} else {
979	zig_unreachable();
980	}
981	break;	1069	break;
982	default:	1070	default:
983	tokenize_error(&t, "invalid character: '%c'", c);	1071	tokenize_error(&t, "invalid character: '%c'", c);
984	}	1072	}
985	break;	1073	break;
986	case TokenizeStateHex:	1074	case TokenizeStateCharCode:
987	switch (c) {	1075	{
988	case HEX_DIGIT:	1076	uint32_t digit_value = get_digit_value(c);
989	t.hex_chars_left -= 1;	1077	if (digit_value >= t.radix) {
990	if (t.hex_chars_left == 0) {	1078	tokenize_error(&t, "invalid digit: '%c'", c);
991	if (t.cur_tok->id == TokenIdCharLiteral) {	1079	}
992	t.state = TokenizeStateCharLiteralEnd;	1080	t.char_code *= t.radix;
993	} else if (t.cur_tok->id == TokenIdStringLiteral) {	1081	t.char_code += digit_value;
994	t.state = TokenizeStateString;	1082	t.char_code_index += 1;
995	} else if (t.cur_tok->id == TokenIdSymbol) {	1083
996	t.state = TokenizeStateString;	1084	if (t.char_code_index >= t.char_code_end) {
		1085	if (t.unicode) {
		1086	if (t.char_code <= 0x7f) {
		1087	// 00000000 00000000 00000000 0xxxxxxx
		1088	handle_string_escape(&t, t.char_code);
		1089	} else if (t.cur_tok->id == TokenIdCharLiteral) {
		1090	tokenize_error(&t, "unicode value too large for character literal: %x", t.char_code);
		1091	} else if (t.char_code <= 0x7ff) {
		1092	// 00000000 00000000 00000xxx xx000000
		1093	handle_string_escape(&t, 0xc0 \| (t.char_code >> 6));
		1094	// 00000000 00000000 00000000 00xxxxxx
		1095	handle_string_escape(&t, 0x80 \| (t.char_code & 0x3f));
		1096	} else if (t.char_code <= 0xffff) {
		1097	// 00000000 00000000 xxxx0000 00000000
		1098	handle_string_escape(&t, 0xe0 \| (t.char_code >> 12));
		1099	// 00000000 00000000 0000xxxx xx000000
		1100	handle_string_escape(&t, 0x80 \| ((t.char_code >> 6) & 0x3f));
		1101	// 00000000 00000000 00000000 00xxxxxx
		1102	handle_string_escape(&t, 0x80 \| (t.char_code & 0x3f));
		1103	} else if (t.char_code <= 0x10ffff) {
		1104	// 00000000 000xxx00 00000000 00000000
		1105	handle_string_escape(&t, 0xf0 \| (t.char_code >> 18));
		1106	// 00000000 000000xx xxxx0000 00000000
		1107	handle_string_escape(&t, 0x80 \| ((t.char_code >> 12) & 0x3f));
		1108	// 00000000 00000000 0000xxxx xx000000
		1109	handle_string_escape(&t, 0x80 \| ((t.char_code >> 6) & 0x3f));
		1110	// 00000000 00000000 00000000 00xxxxxx
		1111	handle_string_escape(&t, 0x80 \| (t.char_code & 0x3f));
997	} else {	1112	} else {
998	zig_unreachable();	1113	tokenize_error(&t, "unicode value out of range: %x", t.char_code);
999	}	1114	}
		1115	} else {
		1116	if (t.cur_tok->id == TokenIdCharLiteral && t.char_code >= sizeof(uint8_t)) {
		1117	tokenize_error(&t, "value too large for character literal: '%x'",
		1118	t.char_code);
		1119	}
		1120	handle_string_escape(&t, t.char_code);
1000	}	1121	}
1001	break;
1002	default:
1003	tokenize_error(&t, "invalid character: '%c'", c);
1004	}
1005	break;
1006	case TokenizeStateRawString:
1007	if (c == '(') {
1008	t.raw_string_id_end = t.pos;
1009	t.cur_tok->raw_string_start = t.pos + 1;
1010	t.state = TokenizeStateRawStringContents;
1011	}
1012	break;
1013	case TokenizeStateRawStringContents:
1014	if (c == ')') {
1015	t.state = TokenizeStateRawStringMaybeEnd;
1016	t.raw_string_id_cmp_pos = t.raw_string_id_start;
1017	t.cur_tok->raw_string_end = t.pos;
1018	}
1019	break;
1020	case TokenizeStateRawStringMaybeEnd:
1021	if (t.raw_string_id_cmp_pos >= t.raw_string_id_end &&
1022	c == '"')
1023	{
1024	end_token(&t);
1025	t.state = TokenizeStateStart;
1026	} else if (c != buf_ptr(t.buf)[t.raw_string_id_cmp_pos]) {
1027	if (c == ')') {
1028	t.raw_string_id_cmp_pos = t.raw_string_id_start;
1029	t.cur_tok->raw_string_end = t.pos;
1030	} else {
1031	t.state = TokenizeStateRawStringContents;
1032	}	1122	}
1033	} else {
1034	t.raw_string_id_cmp_pos += 1;
1035	}	1123	}
1036	break;	1124	break;
1037	case TokenizeStateCharLiteral:	1125	case TokenizeStateCharLiteral:
1038	switch (c) {	1126	switch (c) {
1039	case '\'':	1127	case '\'':
1040	end_token(&t);	1128	tokenize_error(&t, "expected character");
1041	t.state = TokenizeStateStart;
1042	break;
1043	case '\\':	1129	case '\\':
1044	t.state = TokenizeStateStringEscape;	1130	t.state = TokenizeStateStringEscape;
1045	break;	1131	break;
1046	default:	1132	default:
		1133	t.cur_tok->data.char_lit.c = c;
1047	t.state = TokenizeStateCharLiteralEnd;	1134	t.state = TokenizeStateCharLiteralEnd;
1048	break;	1135	break;
1049	}	1136	}
...	@@ -1061,15 +1148,17 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -1061,15 +1148,17 @@ void tokenize(Buf buf, Tokenization out) {
1061	case TokenizeStateZero:	1148	case TokenizeStateZero:
1062	switch (c) {	1149	switch (c) {
1063	case 'b':	1150	case 'b':
1064	t.cur_tok->radix = 2;	1151	t.radix = 2;
1065	t.state = TokenizeStateNumber;	1152	t.state = TokenizeStateNumber;
1066	break;	1153	break;
1067	case 'o':	1154	case 'o':
1068	t.cur_tok->radix = 8;	1155	t.radix = 8;
		1156	t.exp_add_amt = 3;
1069	t.state = TokenizeStateNumber;	1157	t.state = TokenizeStateNumber;
1070	break;	1158	break;
1071	case 'x':	1159	case 'x':
1072	t.cur_tok->radix = 16;	1160	t.radix = 16;
		1161	t.exp_add_amt = 4;
1073	t.state = TokenizeStateNumber;	1162	t.state = TokenizeStateNumber;
1074	break;	1163	break;
1075	default:	1164	default:
...	@@ -1082,113 +1171,127 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -1082,113 +1171,127 @@ void tokenize(Buf buf, Tokenization out) {
1082	case TokenizeStateNumber:	1171	case TokenizeStateNumber:
1083	{	1172	{
1084	if (c == '.') {	1173	if (c == '.') {
1085	if (t.pos + 1 < buf_len(t.buf)) {	1174	t.state = TokenizeStateNumberDot;
1086	uint8_t next_c = buf_ptr(t.buf)[t.pos + 1];
1087	if (next_c == '.') {
1088	t.pos -= 1;
1089	end_token(&t);
1090	t.state = TokenizeStateStart;
1091	continue;
1092	}
1093	}
1094	t.cur_tok->decimal_point_pos = t.pos;
1095	t.state = TokenizeStateFloatFraction;
1096	break;	1175	break;
1097	}	1176	}
1098	if (is_exponent_signifier(c, t.cur_tok->radix)) {	1177	if (is_exponent_signifier(c, t.radix)) {
1099	t.cur_tok->exponent_marker_pos = t.pos;
1100	t.state = TokenizeStateFloatExponentUnsigned;	1178	t.state = TokenizeStateFloatExponentUnsigned;
		1179	t.is_num_lit_float = true;
1101	break;	1180	break;
1102	}	1181	}
1103	if (c == '_') {	1182	uint32_t digit_value = get_digit_value(c);
1104	tokenize_error(&t, "invalid character: '%c'", c);	1183	if (digit_value >= t.radix) {
1105	break;	1184	if (is_symbol_char(c)) {
1106	}
1107	int digit_value = get_digit_value(c);
1108	if (digit_value >= 0) {
1109	if (digit_value >= t.cur_tok->radix) {
1110	tokenize_error(&t, "invalid character: '%c'", c);	1185	tokenize_error(&t, "invalid character: '%c'", c);
1111	break;
1112	}	1186	}
1113	// normal digit
1114	} else {
1115	// not my char	1187	// not my char
1116	t.pos -= 1;	1188	t.pos -= 1;
1117	end_token(&t);	1189	end_token(&t);
1118	t.state = TokenizeStateStart;	1190	t.state = TokenizeStateStart;
1119	continue;	1191	continue;
1120	}	1192	}
		1193	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
		1194	bignum_multiply_by_scalar(&t.cur_tok->data.num_lit.bignum, t.radix);
		1195	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
		1196	bignum_increment_by_scalar(&t.cur_tok->data.num_lit.bignum, digit_value);
1121	break;	1197	break;
1122	}	1198	}
		1199	case TokenizeStateNumberDot:
		1200	if (c == '.') {
		1201	t.pos -= 2;
		1202	end_token(&t);
		1203	t.state = TokenizeStateStart;
		1204	continue;
		1205	}
		1206	t.pos -= 1;
		1207	t.state = TokenizeStateFloatFraction;
		1208	t.is_num_lit_float = true;
		1209	continue;
1123	case TokenizeStateFloatFraction:	1210	case TokenizeStateFloatFraction:
1124	{	1211	{
1125	if (is_exponent_signifier(c, t.cur_tok->radix)) {	1212	if (is_exponent_signifier(c, t.radix)) {
1126	t.cur_tok->exponent_marker_pos = t.pos;
1127	t.state = TokenizeStateFloatExponentUnsigned;	1213	t.state = TokenizeStateFloatExponentUnsigned;
1128	break;	1214	break;
1129	}	1215	}
1130	if (c == '_') {	1216	uint32_t digit_value = get_digit_value(c);
1131	tokenize_error(&t, "invalid character: '%c'", c);	1217	if (digit_value >= t.radix) {
1132	break;	1218	if (is_symbol_char(c)) {
1133	}
1134	int digit_value = get_digit_value(c);
1135	if (digit_value >= 0) {
1136	if (digit_value >= t.cur_tok->radix) {
1137	tokenize_error(&t, "invalid character: '%c'", c);	1219	tokenize_error(&t, "invalid character: '%c'", c);
1138	break;
1139	}	1220	}
1140	// normal digit
1141	} else {
1142	// not my char	1221	// not my char
1143	t.pos -= 1;	1222	t.pos -= 1;
1144	end_token(&t);	1223	end_token(&t);
1145	t.state = TokenizeStateStart;	1224	t.state = TokenizeStateStart;
1146	continue;	1225	continue;
1147	}	1226	}
		1227	t.exponent_in_bin_or_dec -= t.exp_add_amt;
		1228	if (t.radix == 10) {
		1229	// For now we use strtod to parse decimal floats, so we just have to get to the
		1230	// end of the token.
		1231	break;
		1232	}
		1233	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
		1234	bignum_multiply_by_scalar(&t.cur_tok->data.num_lit.bignum, t.radix);
		1235	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
		1236	bignum_increment_by_scalar(&t.cur_tok->data.num_lit.bignum, digit_value);
1148	break;	1237	break;
1149	}	1238	}
1150	case TokenizeStateFloatExponentUnsigned:	1239	case TokenizeStateFloatExponentUnsigned:
1151	switch (c) {	1240	switch (c) {
1152	case '+':	1241	case '+':
		1242	t.is_exp_negative = false;
		1243	t.state = TokenizeStateFloatExponentNumber;
		1244	break;
1153	case '-':	1245	case '-':
		1246	t.is_exp_negative = true;
1154	t.state = TokenizeStateFloatExponentNumber;	1247	t.state = TokenizeStateFloatExponentNumber;
1155	break;	1248	break;
1156	default:	1249	default:
1157	// reinterpret as normal exponent number	1250	// reinterpret as normal exponent number
1158	t.pos -= 1;	1251	t.pos -= 1;
		1252	t.is_exp_negative = false;
1159	t.state = TokenizeStateFloatExponentNumber;	1253	t.state = TokenizeStateFloatExponentNumber;
1160	continue;	1254	continue;
1161	}	1255	}
1162	break;	1256	break;
1163	case TokenizeStateFloatExponentNumber:	1257	case TokenizeStateFloatExponentNumber:
1164	switch (c) {	1258	{
1165	case DIGIT:	1259	uint32_t digit_value = get_digit_value(c);
1166	break;	1260	if (digit_value >= t.radix) {
1167	case ALPHA:	1261	if (is_symbol_char(c)) {
1168	case '_':	1262	tokenize_error(&t, "invalid character: '%c'", c);
1169	tokenize_error(&t, "invalid character: '%c'", c);	1263	}
1170	break;	1264	// not my char
1171	default:
1172	t.pos -= 1;	1265	t.pos -= 1;
1173	end_token(&t);	1266	end_token(&t);
1174	t.state = TokenizeStateStart;	1267	t.state = TokenizeStateStart;
1175	continue;	1268	continue;
		1269	}
		1270	if (t.radix == 10) {
		1271	// For now we use strtod to parse decimal floats, so we just have to get to the
		1272	// end of the token.
		1273	break;
		1274	}
		1275	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
		1276	bignum_multiply_by_scalar(&t.specified_exponent, 10);
		1277	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
		1278	bignum_increment_by_scalar(&t.specified_exponent, digit_value);
1176	}	1279	}
1177	break;	1280	break;
1178	case TokenizeStateSawDash:	1281	case TokenizeStateSawDash:
1179	switch (c) {	1282	switch (c) {
1180	case '>':	1283	case '>':
1181	t.cur_tok->id = TokenIdArrow;	1284	set_token_id(&t, t.cur_tok, TokenIdArrow);
1182	end_token(&t);	1285	end_token(&t);
1183	t.state = TokenizeStateStart;	1286	t.state = TokenizeStateStart;
1184	break;	1287	break;
1185	case '=':	1288	case '=':
1186	t.cur_tok->id = TokenIdMinusEq;	1289	set_token_id(&t, t.cur_tok, TokenIdMinusEq);
1187	end_token(&t);	1290	end_token(&t);
1188	t.state = TokenizeStateStart;	1291	t.state = TokenizeStateStart;
1189	break;	1292	break;
1190	case '%':	1293	case '%':
1191	t.cur_tok->id = TokenIdMinusPercent;	1294	set_token_id(&t, t.cur_tok, TokenIdMinusPercent);
1192	t.state = TokenizeStateSawMinusPercent;	1295	t.state = TokenizeStateSawMinusPercent;
1193	break;	1296	break;
1194	default:	1297	default:
...	@@ -1201,7 +1304,7 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -1201,7 +1304,7 @@ void tokenize(Buf buf, Tokenization out) {
1201	case TokenizeStateSawMinusPercent:	1304	case TokenizeStateSawMinusPercent:
1202	switch (c) {	1305	switch (c) {
1203	case '=':	1306	case '=':
1204	t.cur_tok->id = TokenIdMinusPercentEq;	1307	set_token_id(&t, t.cur_tok, TokenIdMinusPercentEq);
1205	end_token(&t);	1308	end_token(&t);
1206	t.state = TokenizeStateStart;	1309	t.state = TokenizeStateStart;
1207	break;	1310	break;
...	@@ -1226,11 +1329,14 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -1226,11 +1329,14 @@ void tokenize(Buf buf, Tokenization out) {
1226	case TokenizeStateStart:	1329	case TokenizeStateStart:
1227	case TokenizeStateError:	1330	case TokenizeStateError:
1228	break;	1331	break;
		1332	case TokenizeStateNumberDot:
		1333	tokenize_error(&t, "unterminated number literal");
		1334	break;
1229	case TokenizeStateString:	1335	case TokenizeStateString:
1230	tokenize_error(&t, "unterminated string");	1336	tokenize_error(&t, "unterminated string");
1231	break;	1337	break;
1232	case TokenizeStateStringEscape:	1338	case TokenizeStateStringEscape:
1233	case TokenizeStateHex:	1339	case TokenizeStateCharCode:
1234	if (t.cur_tok->id == TokenIdStringLiteral) {	1340	if (t.cur_tok->id == TokenIdStringLiteral) {
1235	tokenize_error(&t, "unterminated string");	1341	tokenize_error(&t, "unterminated string");
1236	} else if (t.cur_tok->id == TokenIdCharLiteral) {	1342	} else if (t.cur_tok->id == TokenIdCharLiteral) {
...	@@ -1239,19 +1345,12 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -1239,19 +1345,12 @@ void tokenize(Buf buf, Tokenization out) {
1239	zig_unreachable();	1345	zig_unreachable();
1240	}	1346	}
1241	break;	1347	break;
1242	case TokenizeStateRawString:
1243	case TokenizeStateRawStringContents:
1244	case TokenizeStateRawStringMaybeEnd:
1245	tokenize_error(&t, "unterminated raw string");
1246	break;
1247	case TokenizeStateCharLiteral:	1348	case TokenizeStateCharLiteral:
1248	case TokenizeStateCharLiteralEnd:	1349	case TokenizeStateCharLiteralEnd:
1249	tokenize_error(&t, "unterminated character literal");	1350	tokenize_error(&t, "unterminated character literal");
1250	break;	1351	break;
1251	case TokenizeStateSymbol:	1352	case TokenizeStateSymbol:
1252	case TokenizeStateSymbolFirst:	1353	case TokenizeStateSymbolFirstC:
1253	case TokenizeStateSymbolFirstRaw:
1254	case TokenizeStateFirstR:
1255	case TokenizeStateZero:	1354	case TokenizeStateZero:
1256	case TokenizeStateNumber:	1355	case TokenizeStateNumber:
1257	case TokenizeStateFloatFraction:	1356	case TokenizeStateFloatFraction:
...	@@ -1280,9 +1379,13 @@ void tokenize(Buf buf, Tokenization out) {	...	@@ -1280,9 +1379,13 @@ void tokenize(Buf buf, Tokenization out) {
1280	case TokenizeStateSawPlusPercent:	1379	case TokenizeStateSawPlusPercent:
1281	case TokenizeStateSawMinusPercent:	1380	case TokenizeStateSawMinusPercent:
1282	case TokenizeStateSawShiftLeftPercent:	1381	case TokenizeStateSawShiftLeftPercent:
		1382	case TokenizeStateLineString:
		1383	case TokenizeStateLineStringEnd:
1283	end_token(&t);	1384	end_token(&t);
1284	break;	1385	break;
1285	case TokenizeStateSawDotDot:	1386	case TokenizeStateSawDotDot:
		1387	case TokenizeStateSawBackslash:
		1388	case TokenizeStateLineStringContinue:
1286	tokenize_error(&t, "unexpected EOF");	1389	tokenize_error(&t, "unexpected EOF");
1287	break;	1390	break;
1288	case TokenizeStateLineComment:	1391	case TokenizeStateLineComment:

src/tokenizer.hpp+26-9

...	@@ -9,6 +9,7 @@	...	@@ -9,6 +9,7 @@
9	#define ZIG_TOKENIZER_HPP	9	#define ZIG_TOKENIZER_HPP
10		10
11	#include "buffer.hpp"	11	#include "buffer.hpp"
		12	#include "bignum.hpp"
12		13
13	enum TokenId {	14	enum TokenId {
14	TokenIdEof,	15	TokenIdEof,
...	@@ -111,6 +112,22 @@ enum TokenId {	...	@@ -111,6 +112,22 @@ enum TokenId {
111	TokenIdPercentDot,	112	TokenIdPercentDot,
112	};	113	};
113		114
		115	struct TokenNumLit {
		116	BigNum bignum;
		117	// overflow is true if when parsing the number, we discovered it would not
		118	// fit without losing data in a uint64_t or double
		119	bool overflow;
		120	};
		121
		122	struct TokenStrLit {
		123	Buf str;
		124	bool is_c_str;
		125	};
		126
		127	struct TokenCharLit {
		128	uint8_t c;
		129	};
		130
114	struct Token {	131	struct Token {
115	TokenId id;	132	TokenId id;
116	int start_pos;	133	int start_pos;
...	@@ -118,14 +135,16 @@ struct Token {	...	@@ -118,14 +135,16 @@ struct Token {
118	int start_line;	135	int start_line;
119	int start_column;	136	int start_column;
120		137
121	// for id == TokenIdNumberLiteral	138	union {
122	int radix; // if != 10, then skip the first 2 characters	139	// TokenIdNumberLiteral
123	int decimal_point_pos; // either exponent_marker_pos or the position of the '.'	140	TokenNumLit num_lit;
124	int exponent_marker_pos; // either end_pos or the position of the 'e'/'p'
125		141
126	// for id == TokenIdStringLiteral	142	// TokenIdStringLiteral or TokenIdSymbol
127	int raw_string_start;	143	TokenStrLit str_lit;
128	int raw_string_end;	144
		145	// TokenIdCharLiteral
		146	TokenCharLit char_lit;
		147	} data;
129	};	148	};
130		149
131	struct Tokenization {	150	struct Tokenization {
...	@@ -142,8 +161,6 @@ void tokenize(Buf buf, Tokenization out_tokenization);	...	@@ -142,8 +161,6 @@ void tokenize(Buf buf, Tokenization out_tokenization);
142		161
143	void print_tokens(Buf buf, ZigList<Token> tokens);	162	void print_tokens(Buf buf, ZigList<Token> tokens);
144		163
145	int get_digit_value(uint8_t c);
146
147	const char * token_name(TokenId id);	164	const char * token_name(TokenId id);
148		165
149	bool valid_symbol_starter(uint8_t c);	166	bool valid_symbol_starter(uint8_t c);

test/run_tests.cpp+2-2

...	@@ -1173,7 +1173,7 @@ fn f() {	...	@@ -1173,7 +1173,7 @@ fn f() {
1173	add_compile_fail_case("normal string with newline", R"SOURCE(	1173	add_compile_fail_case("normal string with newline", R"SOURCE(
1174	const foo = "a	1174	const foo = "a
1175	b";	1175	b";
1176	)SOURCE", 1, ".tmp_source.zig:2:13: error: use raw string for multiline string literal");	1176	)SOURCE", 1, ".tmp_source.zig:2:13: error: newline not allowed in string literal");
1177		1177
1178	add_compile_fail_case("invalid comparison for function pointers", R"SOURCE(	1178	add_compile_fail_case("invalid comparison for function pointers", R"SOURCE(
1179	fn foo() {}	1179	fn foo() {}
...	@@ -1760,7 +1760,7 @@ struct type {	...	@@ -1760,7 +1760,7 @@ struct type {
1760	)SOURCE", 3,	1760	)SOURCE", 3,
1761	R"(pub const FOO = c"aoeu\x13 derp")",	1761	R"(pub const FOO = c"aoeu\x13 derp")",
1762	R"(pub const FOO2 = c"aoeu\x134 derp")",	1762	R"(pub const FOO2 = c"aoeu\x134 derp")",
1763	R"(pub const FOO_CHAR = '\x3f')");	1763	R"(pub const FOO_CHAR = '?')");
1764	}	1764	}
1765		1765
1766	static void run_self_hosted_test(bool is_release_mode) {	1766	static void run_self_hosted_test(bool is_release_mode) {

test/self_hosted.zig+6-10

 @@ -684,17 +684,13 @@ fn count_trailing_zeroes() {
 #attribute("test")
 fn multiline_string() {
-    const s1 = r"AOEU(
+    const s1 =
-one
+        \\one
-two)
+        \\two)
-three)AOEU";
+        \\three
-    const s2 = "\none\ntwo)\nthree";
+    ;
-    const s3 = r"(
+    const s2 = "one\ntwo)\nthree";
-one
-two)
-three)";
     assert(str.eql(s1, s2));
-    assert(str.eql(s3, s2));
 }

...	@@ -1431,7 +1431,7 @@ static LLVMValueRef gen_field_access_expr(CodeGen g, AstNode node, bool is_lva	...	@@ -1431,7 +1431,7 @@ static LLVMValueRef gen_field_access_expr(CodeGen g, AstNode node, bool is_lva
1431	TypeTableEntry *struct_type = get_expr_type(struct_expr);	1431	TypeTableEntry *struct_type = get_expr_type(struct_expr);
1432		1432
1433	if (struct_type->id == TypeTableEntryIdArray) {	1433	if (struct_type->id == TypeTableEntryIdArray) {
1434	Buf *name = &node->data.field_access_expr.field_name;	1434	Buf *name = node->data.field_access_expr.field_name;
1435	assert(buf_eql_str(name, "len"));	1435	assert(buf_eql_str(name, "len"));
1436	return LLVMConstInt(g->builtin_types.entry_usize->type_ref,	1436	return LLVMConstInt(g->builtin_types.entry_usize->type_ref,
1437	struct_type->data.array.len, false);	1437	struct_type->data.array.len, false);
...	@@ -2726,18 +2726,18 @@ static LLVMValueRef gen_block(CodeGen g, AstNode block_node, TypeTableEntry *i	...	@@ -2726,18 +2726,18 @@ static LLVMValueRef gen_block(CodeGen g, AstNode block_node, TypeTableEntry *i
2726	}	2726	}
2727		2727
2728	static int find_asm_index(CodeGen g, AstNode node, AsmToken *tok) {	2728	static int find_asm_index(CodeGen g, AstNode node, AsmToken *tok) {
2729	const char *ptr = buf_ptr(&node->data.asm_expr.asm_template) + tok->start + 2;	2729	const char *ptr = buf_ptr(node->data.asm_expr.asm_template) + tok->start + 2;
2730	int len = tok->end - tok->start - 2;	2730	int len = tok->end - tok->start - 2;
2731	int result = 0;	2731	int result = 0;
2732	for (int i = 0; i < node->data.asm_expr.output_list.length; i += 1, result += 1) {	2732	for (int i = 0; i < node->data.asm_expr.output_list.length; i += 1, result += 1) {
2733	AsmOutput *asm_output = node->data.asm_expr.output_list.at(i);	2733	AsmOutput *asm_output = node->data.asm_expr.output_list.at(i);
2734	if (buf_eql_mem(&asm_output->asm_symbolic_name, ptr, len)) {	2734	if (buf_eql_mem(asm_output->asm_symbolic_name, ptr, len)) {
2735	return result;	2735	return result;
2736	}	2736	}
2737	}	2737	}
2738	for (int i = 0; i < node->data.asm_expr.input_list.length; i += 1, result += 1) {	2738	for (int i = 0; i < node->data.asm_expr.input_list.length; i += 1, result += 1) {
2739	AsmInput *asm_input = node->data.asm_expr.input_list.at(i);	2739	AsmInput *asm_input = node->data.asm_expr.input_list.at(i);
2740	if (buf_eql_mem(&asm_input->asm_symbolic_name, ptr, len)) {	2740	if (buf_eql_mem(asm_input->asm_symbolic_name, ptr, len)) {
2741	return result;	2741	return result;
2742	}	2742	}
2743	}	2743	}
...	@@ -2749,7 +2749,7 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {	...	@@ -2749,7 +2749,7 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {
2749		2749
2750	AstNodeAsmExpr *asm_expr = &node->data.asm_expr;	2750	AstNodeAsmExpr *asm_expr = &node->data.asm_expr;
2751		2751
2752	Buf *src_template = &asm_expr->asm_template;	2752	Buf *src_template = asm_expr->asm_template;
2753		2753
2754	Buf llvm_template = BUF_INIT;	2754	Buf llvm_template = BUF_INIT;
2755	buf_resize(&llvm_template, 0);	2755	buf_resize(&llvm_template, 0);
...	@@ -2796,11 +2796,11 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {	...	@@ -2796,11 +2796,11 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {
2796	for (int i = 0; i < asm_expr->output_list.length; i += 1, total_index += 1) {	2796	for (int i = 0; i < asm_expr->output_list.length; i += 1, total_index += 1) {
2797	AsmOutput *asm_output = asm_expr->output_list.at(i);	2797	AsmOutput *asm_output = asm_expr->output_list.at(i);
2798	bool is_return = (asm_output->return_type != nullptr);	2798	bool is_return = (asm_output->return_type != nullptr);
2799	assert(*buf_ptr(&asm_output->constraint) == '=');	2799	assert(*buf_ptr(asm_output->constraint) == '=');
2800	if (is_return) {	2800	if (is_return) {
2801	buf_appendf(&constraint_buf, "=%s", buf_ptr(&asm_output->constraint) + 1);	2801	buf_appendf(&constraint_buf, "=%s", buf_ptr(asm_output->constraint) + 1);
2802	} else {	2802	} else {
2803	buf_appendf(&constraint_buf, "=*%s", buf_ptr(&asm_output->constraint) + 1);	2803	buf_appendf(&constraint_buf, "=*%s", buf_ptr(asm_output->constraint) + 1);
2804	}	2804	}
2805	if (total_index + 1 < total_constraint_count) {	2805	if (total_index + 1 < total_constraint_count) {
2806	buf_append_char(&constraint_buf, ',');	2806	buf_append_char(&constraint_buf, ',');
...	@@ -2816,7 +2816,7 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {	...	@@ -2816,7 +2816,7 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {
2816	}	2816	}
2817	for (int i = 0; i < asm_expr->input_list.length; i += 1, total_index += 1, param_index += 1) {	2817	for (int i = 0; i < asm_expr->input_list.length; i += 1, total_index += 1, param_index += 1) {
2818	AsmInput *asm_input = asm_expr->input_list.at(i);	2818	AsmInput *asm_input = asm_expr->input_list.at(i);
2819	buf_append_buf(&constraint_buf, &asm_input->constraint);	2819	buf_append_buf(&constraint_buf, asm_input->constraint);
2820	if (total_index + 1 < total_constraint_count) {	2820	if (total_index + 1 < total_constraint_count) {
2821	buf_append_char(&constraint_buf, ',');	2821	buf_append_char(&constraint_buf, ',');
2822	}	2822	}
...	@@ -2885,7 +2885,7 @@ static LLVMValueRef gen_container_init_expr(CodeGen g, AstNode node) {	...	@@ -2885,7 +2885,7 @@ static LLVMValueRef gen_container_init_expr(CodeGen g, AstNode node) {
2885	if (type_struct_field->type_entry->id == TypeTableEntryIdVoid) {	2885	if (type_struct_field->type_entry->id == TypeTableEntryIdVoid) {
2886	continue;	2886	continue;
2887	}	2887	}
2888	assert(buf_eql_buf(type_struct_field->name, &field_node->data.struct_val_field.name));	2888	assert(buf_eql_buf(type_struct_field->name, field_node->data.struct_val_field.name));
2889		2889
2890	set_debug_source_node(g, field_node);	2890	set_debug_source_node(g, field_node);
2891	LLVMValueRef field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, type_struct_field->gen_index, "");	2891	LLVMValueRef field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, type_struct_field->gen_index, "");
...	@@ -3853,7 +3853,7 @@ static void generate_error_name_table(CodeGen *g) {	...	@@ -3853,7 +3853,7 @@ static void generate_error_name_table(CodeGen *g) {
3853	for (int i = 1; i < g->error_decls.length; i += 1) {	3853	for (int i = 1; i < g->error_decls.length; i += 1) {
3854	AstNode *error_decl_node = g->error_decls.at(i);	3854	AstNode *error_decl_node = g->error_decls.at(i);
3855	assert(error_decl_node->type == NodeTypeErrorValueDecl);	3855	assert(error_decl_node->type == NodeTypeErrorValueDecl);
3856	Buf *name = &error_decl_node->data.error_value_decl.name;	3856	Buf *name = error_decl_node->data.error_value_decl.name;
3857		3857
3858	LLVMValueRef str_init = LLVMConstString(buf_ptr(name), buf_len(name), true);	3858	LLVMValueRef str_init = LLVMConstString(buf_ptr(name), buf_len(name), true);
3859	LLVMValueRef str_global = LLVMAddGlobal(g->module, LLVMTypeOf(str_init), "");	3859	LLVMValueRef str_global = LLVMAddGlobal(g->module, LLVMTypeOf(str_init), "");
...	@@ -3882,7 +3882,7 @@ static void build_label_blocks(CodeGen g, FnTableEntry fn) {	...	@@ -3882,7 +3882,7 @@ static void build_label_blocks(CodeGen g, FnTableEntry fn) {
3882	LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn->fn_value, "entry");	3882	LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn->fn_value, "entry");
3883	for (int i = 0; i < fn->all_labels.length; i += 1) {	3883	for (int i = 0; i < fn->all_labels.length; i += 1) {
3884	LabelTableEntry *label = fn->all_labels.at(i);	3884	LabelTableEntry *label = fn->all_labels.at(i);
3885	Buf *name = &label->decl_node->data.label.name;	3885	Buf *name = label->decl_node->data.label.name;
3886	label->basic_block = LLVMAppendBasicBlock(fn->fn_value, buf_ptr(name));	3886	label->basic_block = LLVMAppendBasicBlock(fn->fn_value, buf_ptr(name));
3887	}	3887	}
3888	LLVMPositionBuilderAtEnd(g->builder, entry_block);	3888	LLVMPositionBuilderAtEnd(g->builder, entry_block);
...	@@ -4951,7 +4951,7 @@ void codegen_generate_h_file(CodeGen *g) {	...	@@ -4951,7 +4951,7 @@ void codegen_generate_h_file(CodeGen *g) {
4951	buf_appendf(&h_buf, "%s %s %s(",	4951	buf_appendf(&h_buf, "%s %s %s(",
4952	buf_ptr(export_macro),	4952	buf_ptr(export_macro),
4953	buf_ptr(&return_type_c),	4953	buf_ptr(&return_type_c),
4954	buf_ptr(&fn_proto->name));	4954	buf_ptr(fn_proto->name));
4955		4955
4956	Buf param_type_c = BUF_INIT;	4956	Buf param_type_c = BUF_INIT;
4957	if (fn_proto->params.length) {	4957	if (fn_proto->params.length) {
...	@@ -4961,7 +4961,7 @@ void codegen_generate_h_file(CodeGen *g) {	...	@@ -4961,7 +4961,7 @@ void codegen_generate_h_file(CodeGen *g) {
4961	to_c_type(g, param_type, &param_type_c);	4961	to_c_type(g, param_type, &param_type_c);
4962	buf_appendf(&h_buf, "%s %s",	4962	buf_appendf(&h_buf, "%s %s",
4963	buf_ptr(&param_type_c),	4963	buf_ptr(&param_type_c),
4964	buf_ptr(&param_decl_node->data.param_decl.name));	4964	buf_ptr(param_decl_node->data.param_decl.name));
4965	if (param_i < fn_proto->params.length - 1)	4965	if (param_i < fn_proto->params.length - 1)
4966	buf_appendf(&h_buf, ", ");	4966	buf_appendf(&h_buf, ", ");
4967	}	4967	}

...	@@ -104,14 +104,14 @@ static AstNode create_node(Context c, NodeType type) {	...	@@ -104,14 +104,14 @@ static AstNode create_node(Context c, NodeType type) {
104		104
105	static AstNode create_symbol_node(Context c, const char *type_name) {	105	static AstNode create_symbol_node(Context c, const char *type_name) {
106	AstNode *node = create_node(c, NodeTypeSymbol);	106	AstNode *node = create_node(c, NodeTypeSymbol);
107	buf_init_from_str(&node->data.symbol_expr.symbol, type_name);	107	node->data.symbol_expr.symbol = buf_create_from_str(type_name);
108	return node;	108	return node;
109	}	109	}
110		110
111	static AstNode create_field_access_node(Context c, const char lhs, const char rhs) {	111	static AstNode create_field_access_node(Context c, const char lhs, const char rhs) {
112	AstNode *node = create_node(c, NodeTypeFieldAccessExpr);	112	AstNode *node = create_node(c, NodeTypeFieldAccessExpr);
113	node->data.field_access_expr.struct_expr = create_symbol_node(c, lhs);	113	node->data.field_access_expr.struct_expr = create_symbol_node(c, lhs);
114	buf_init_from_str(&node->data.field_access_expr.field_name, rhs);	114	node->data.field_access_expr.field_name = buf_create_from_str(rhs);
115	normalize_parent_ptrs(node);	115	normalize_parent_ptrs(node);
116	return node;	116	return node;
117	}	117	}
...	@@ -120,7 +120,7 @@ static AstNode create_typed_var_decl_node(Context c, bool is_const, const char	...	@@ -120,7 +120,7 @@ static AstNode create_typed_var_decl_node(Context c, bool is_const, const char
120	AstNode type_node, AstNode init_node)	120	AstNode type_node, AstNode init_node)
121	{	121	{
122	AstNode *node = create_node(c, NodeTypeVariableDeclaration);	122	AstNode *node = create_node(c, NodeTypeVariableDeclaration);
123	buf_init_from_str(&node->data.variable_declaration.symbol, var_name);	123	node->data.variable_declaration.symbol = buf_create_from_str(var_name);
124	node->data.variable_declaration.is_const = is_const;	124	node->data.variable_declaration.is_const = is_const;
125	node->data.variable_declaration.top_level_decl.visib_mod = c->visib_mod;	125	node->data.variable_declaration.top_level_decl.visib_mod = c->visib_mod;
126	node->data.variable_declaration.expr = init_node;	126	node->data.variable_declaration.expr = init_node;
...	@@ -146,7 +146,7 @@ static AstNode create_prefix_node(Context c, PrefixOp op, AstNode *child_node)	...	@@ -146,7 +146,7 @@ static AstNode create_prefix_node(Context c, PrefixOp op, AstNode *child_node)
146	static AstNode create_struct_field_node(Context c, const char name, AstNode type_node) {	146	static AstNode create_struct_field_node(Context c, const char name, AstNode type_node) {
147	assert(type_node);	147	assert(type_node);
148	AstNode *node = create_node(c, NodeTypeStructField);	148	AstNode *node = create_node(c, NodeTypeStructField);
149	buf_init_from_str(&node->data.struct_field.name, name);	149	node->data.struct_field.name = buf_create_from_str(name);
150	node->data.struct_field.top_level_decl.visib_mod = VisibModPub;	150	node->data.struct_field.top_level_decl.visib_mod = VisibModPub;
151	node->data.struct_field.type = type_node;	151	node->data.struct_field.type = type_node;
152		152
...	@@ -157,7 +157,7 @@ static AstNode create_struct_field_node(Context c, const char name, AstNode	...	@@ -157,7 +157,7 @@ static AstNode create_struct_field_node(Context c, const char name, AstNode
157	static AstNode create_param_decl_node(Context c, const char name, AstNode type_node, bool is_noalias) {	157	static AstNode create_param_decl_node(Context c, const char name, AstNode type_node, bool is_noalias) {
158	assert(type_node);	158	assert(type_node);
159	AstNode *node = create_node(c, NodeTypeParamDecl);	159	AstNode *node = create_node(c, NodeTypeParamDecl);
160	buf_init_from_str(&node->data.param_decl.name, name);	160	node->data.param_decl.name = buf_create_from_str(name);
161	node->data.param_decl.type = type_node;	161	node->data.param_decl.type = type_node;
162	node->data.param_decl.is_noalias = is_noalias;	162	node->data.param_decl.is_noalias = is_noalias;
163		163
...	@@ -171,17 +171,18 @@ static AstNode create_char_lit_node(Context c, uint8_t value) {	...	@@ -171,17 +171,18 @@ static AstNode create_char_lit_node(Context c, uint8_t value) {
171	return node;	171	return node;
172	}	172	}
173		173
		174	// accepts ownership of buf
174	static AstNode create_str_lit_node(Context c, Buf *buf) {	175	static AstNode create_str_lit_node(Context c, Buf *buf) {
175	AstNode *node = create_node(c, NodeTypeStringLiteral);	176	AstNode *node = create_node(c, NodeTypeStringLiteral);
176	buf_init_from_buf(&node->data.string_literal.buf, buf);	177	node->data.string_literal.buf = buf;
177	node->data.string_literal.c = true;	178	node->data.string_literal.c = true;
178	return node;	179	return node;
179	}	180	}
180		181
181	static AstNode create_num_lit_float(Context c, double x) {	182	static AstNode create_num_lit_float(Context c, double x) {
182	AstNode *node = create_node(c, NodeTypeNumberLiteral);	183	AstNode *node = create_node(c, NodeTypeNumberLiteral);
183	node->data.number_literal.kind = NumLitFloat;	184	node->data.number_literal.bignum = allocate_nonzero<BigNum>(1);
184	node->data.number_literal.data.x_float = x;	185	bignum_init_float(node->data.number_literal.bignum, x);
185	return node;	186	return node;
186	}	187	}
187		188
...	@@ -193,8 +194,8 @@ static AstNode create_num_lit_float_negative(Context c, double x, bool negativ	...	@@ -193,8 +194,8 @@ static AstNode create_num_lit_float_negative(Context c, double x, bool negativ
193		194
194	static AstNode create_num_lit_unsigned(Context c, uint64_t x) {	195	static AstNode create_num_lit_unsigned(Context c, uint64_t x) {
195	AstNode *node = create_node(c, NodeTypeNumberLiteral);	196	AstNode *node = create_node(c, NodeTypeNumberLiteral);
196	node->data.number_literal.kind = NumLitUInt;	197	node->data.number_literal.bignum = allocate_nonzero<BigNum>(1);
197	node->data.number_literal.data.x_uint = x;	198	bignum_init_unsigned(node->data.number_literal.bignum, x);
198	return node;	199	return node;
199	}	200	}
200		201
...	@@ -221,7 +222,7 @@ static AstNode create_num_lit_signed(Context c, int64_t x) {	...	@@ -221,7 +222,7 @@ static AstNode create_num_lit_signed(Context c, int64_t x) {
221		222
222	static AstNode create_type_decl_node(Context c, const char name, AstNode child_type_node) {	223	static AstNode create_type_decl_node(Context c, const char name, AstNode child_type_node) {
223	AstNode *node = create_node(c, NodeTypeTypeDecl);	224	AstNode *node = create_node(c, NodeTypeTypeDecl);
224	buf_init_from_str(&node->data.type_decl.symbol, name);	225	node->data.type_decl.symbol = buf_create_from_str(name);
225	node->data.type_decl.top_level_decl.visib_mod = c->visib_mod;	226	node->data.type_decl.top_level_decl.visib_mod = c->visib_mod;
226	node->data.type_decl.child_type = child_type_node;	227	node->data.type_decl.child_type = child_type_node;
227		228
...	@@ -240,7 +241,7 @@ static AstNode create_fn_proto_node(Context c, Buf name, TypeTableEntry fn_t	...	@@ -240,7 +241,7 @@ static AstNode create_fn_proto_node(Context c, Buf name, TypeTableEntry fn_t
240	AstNode *node = create_node(c, NodeTypeFnProto);	241	AstNode *node = create_node(c, NodeTypeFnProto);
241	node->data.fn_proto.is_inline = true;	242	node->data.fn_proto.is_inline = true;
242	node->data.fn_proto.top_level_decl.visib_mod = c->visib_mod;	243	node->data.fn_proto.top_level_decl.visib_mod = c->visib_mod;
243	buf_init_from_buf(&node->data.fn_proto.name, name);	244	node->data.fn_proto.name = name;
244	node->data.fn_proto.return_type = make_type_node(c, fn_type->data.fn.fn_type_id.return_type);	245	node->data.fn_proto.return_type = make_type_node(c, fn_type->data.fn.fn_type_id.return_type);
245		246
246	for (int i = 0; i < fn_type->data.fn.fn_type_id.param_count; i += 1) {	247	for (int i = 0; i < fn_type->data.fn.fn_type_id.param_count; i += 1) {
...	@@ -273,7 +274,7 @@ static AstNode create_inline_fn_node(Context c, Buf fn_name, Buf var_name, T	...	@@ -273,7 +274,7 @@ static AstNode create_inline_fn_node(Context c, Buf fn_name, Buf var_name, T
273	fn_call_node->data.fn_call_expr.fn_ref_expr = unwrap_node;	274	fn_call_node->data.fn_call_expr.fn_ref_expr = unwrap_node;
274	for (int i = 0; i < fn_type->data.fn.fn_type_id.param_count; i += 1) {	275	for (int i = 0; i < fn_type->data.fn.fn_type_id.param_count; i += 1) {
275	AstNode *decl_node = node->data.fn_def.fn_proto->data.fn_proto.params.at(i);	276	AstNode *decl_node = node->data.fn_def.fn_proto->data.fn_proto.params.at(i);
276	Buf *param_name = &decl_node->data.param_decl.name;	277	Buf *param_name = decl_node->data.param_decl.name;
277	fn_call_node->data.fn_call_expr.params.append(create_symbol_node(c, buf_ptr(param_name)));	278	fn_call_node->data.fn_call_expr.params.append(create_symbol_node(c, buf_ptr(param_name)));
278	}	279	}
279		280
...	@@ -686,10 +687,9 @@ static TypeTableEntry resolve_qual_type(Context c, QualType qt, const Decl *de	...	@@ -686,10 +687,9 @@ static TypeTableEntry resolve_qual_type(Context c, QualType qt, const Decl *de
686	}	687	}
687		688
688	static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {	689	static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {
689	Buf fn_name = BUF_INIT;	690	Buf *fn_name = buf_create_from_str(decl_name(fn_decl));
690	buf_init_from_str(&fn_name, decl_name(fn_decl));
691		691
692	if (c->fn_table.maybe_get(&fn_name)) {	692	if (c->fn_table.maybe_get(fn_name)) {
693	// we already saw this function	693	// we already saw this function
694	return;	694	return;
695	}	695	}
...	@@ -697,14 +697,14 @@ static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {	...	@@ -697,14 +697,14 @@ static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {
697	TypeTableEntry *fn_type = resolve_qual_type(c, fn_decl->getType(), fn_decl);	697	TypeTableEntry *fn_type = resolve_qual_type(c, fn_decl->getType(), fn_decl);
698		698
699	if (fn_type->id == TypeTableEntryIdInvalid) {	699	if (fn_type->id == TypeTableEntryIdInvalid) {
700	emit_warning(c, fn_decl, "ignoring function '%s' - unable to resolve type", buf_ptr(&fn_name));	700	emit_warning(c, fn_decl, "ignoring function '%s' - unable to resolve type", buf_ptr(fn_name));
701	return;	701	return;
702	}	702	}
703	assert(fn_type->id == TypeTableEntryIdFn);	703	assert(fn_type->id == TypeTableEntryIdFn);
704		704
705		705
706	AstNode *node = create_node(c, NodeTypeFnProto);	706	AstNode *node = create_node(c, NodeTypeFnProto);
707	buf_init_from_buf(&node->data.fn_proto.name, &fn_name);	707	node->data.fn_proto.name = fn_name;
708		708
709	node->data.fn_proto.is_extern = fn_type->data.fn.fn_type_id.is_extern;	709	node->data.fn_proto.is_extern = fn_type->data.fn.fn_type_id.is_extern;
710	node->data.fn_proto.top_level_decl.visib_mod = c->visib_mod;	710	node->data.fn_proto.top_level_decl.visib_mod = c->visib_mod;
...	@@ -731,7 +731,7 @@ static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {	...	@@ -731,7 +731,7 @@ static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {
731		731
732	normalize_parent_ptrs(node);	732	normalize_parent_ptrs(node);
733		733
734	c->fn_table.put(buf_create_from_buf(&fn_name), true);	734	c->fn_table.put(buf_create_from_buf(fn_name), true);
735	c->root->data.root.top_level_decls.append(node);	735	c->root->data.root.top_level_decls.append(node);
736	}	736	}
737		737
...	@@ -937,7 +937,7 @@ static void visit_enum_decl(Context c, const EnumDecl enum_decl) {	...	@@ -937,7 +937,7 @@ static void visit_enum_decl(Context c, const EnumDecl enum_decl) {
937	if (enum_type->data.enumeration.complete) {	937	if (enum_type->data.enumeration.complete) {
938	// now create top level decl for the type	938	// now create top level decl for the type
939	AstNode *enum_node = create_node(c, NodeTypeContainerDecl);	939	AstNode *enum_node = create_node(c, NodeTypeContainerDecl);
940	buf_init_from_buf(&enum_node->data.struct_decl.name, &enum_type->name);	940	enum_node->data.struct_decl.name = &enum_type->name;
941	enum_node->data.struct_decl.kind = ContainerKindEnum;	941	enum_node->data.struct_decl.kind = ContainerKindEnum;
942	enum_node->data.struct_decl.top_level_decl.visib_mod = VisibModExport;	942	enum_node->data.struct_decl.top_level_decl.visib_mod = VisibModExport;
943	enum_node->data.struct_decl.type_entry = enum_type;	943	enum_node->data.struct_decl.type_entry = enum_type;
...	@@ -1114,7 +1114,7 @@ static void visit_record_decl(Context c, const RecordDecl record_decl) {	...	@@ -1114,7 +1114,7 @@ static void visit_record_decl(Context c, const RecordDecl record_decl) {
1114	if (struct_type->data.structure.complete) {	1114	if (struct_type->data.structure.complete) {
1115	// now create a top level decl node for the type	1115	// now create a top level decl node for the type
1116	AstNode *struct_node = create_node(c, NodeTypeContainerDecl);	1116	AstNode *struct_node = create_node(c, NodeTypeContainerDecl);
1117	buf_init_from_buf(&struct_node->data.struct_decl.name, &struct_type->name);	1117	struct_node->data.struct_decl.name = &struct_type->name;
1118	struct_node->data.struct_decl.kind = ContainerKindStruct;	1118	struct_node->data.struct_decl.kind = ContainerKindStruct;
1119	struct_node->data.struct_decl.top_level_decl.visib_mod = VisibModExport;	1119	struct_node->data.struct_decl.top_level_decl.visib_mod = VisibModExport;
1120	struct_node->data.struct_decl.type_entry = struct_type;	1120	struct_node->data.struct_decl.type_entry = struct_type;
...	@@ -1284,7 +1284,7 @@ static void render_aliases(Context *c) {	...	@@ -1284,7 +1284,7 @@ static void render_aliases(Context *c) {
1284	for (int i = 0; i < c->aliases.length; i += 1) {	1284	for (int i = 0; i < c->aliases.length; i += 1) {
1285	AstNode *alias_node = c->aliases.at(i);	1285	AstNode *alias_node = c->aliases.at(i);
1286	assert(alias_node->type == NodeTypeVariableDeclaration);	1286	assert(alias_node->type == NodeTypeVariableDeclaration);
1287	Buf *name = &alias_node->data.variable_declaration.symbol;	1287	Buf *name = alias_node->data.variable_declaration.symbol;
1288	if (name_exists(c, name)) {	1288	if (name_exists(c, name)) {
1289	continue;	1289	continue;
1290	}	1290	}
...	@@ -1327,7 +1327,7 @@ static void process_macro(Context c, CTokenize ctok, Buf name, const char ch	...	@@ -1327,7 +1327,7 @@ static void process_macro(Context c, CTokenize ctok, Buf name, const char ch
1327	case CTokIdStrLit:	1327	case CTokIdStrLit:
1328	if (is_last && is_first) {	1328	if (is_last && is_first) {
1329	AstNode *var_node = create_var_decl_node(c, buf_ptr(name),	1329	AstNode *var_node = create_var_decl_node(c, buf_ptr(name),
1330	create_str_lit_node(c, &tok->data.str_lit));	1330	create_str_lit_node(c, buf_create_from_buf(&tok->data.str_lit)));
1331	c->macro_table.put(name, var_node);	1331	c->macro_table.put(name, var_node);
1332	}	1332	}
1333	return;	1333	return;

...	@@ -1053,7 +1053,7 @@ static void resolve_function_proto(CodeGen g, AstNode node, FnTableEntry *fn_t	...	@@ -1053,7 +1053,7 @@ static void resolve_function_proto(CodeGen g, AstNode node, FnTableEntry *fn_t
1053	if (fn_proto->top_level_decl.directives) {	1053	if (fn_proto->top_level_decl.directives) {
1054	for (int i = 0; i < fn_proto->top_level_decl.directives->length; i += 1) {	1054	for (int i = 0; i < fn_proto->top_level_decl.directives->length; i += 1) {
1055	AstNode *directive_node = fn_proto->top_level_decl.directives->at(i);	1055	AstNode *directive_node = fn_proto->top_level_decl.directives->at(i);
1056	Buf *name = &directive_node->data.directive.name;	1056	Buf *name = directive_node->data.directive.name;
1057		1057
1058	if (buf_eql_str(name, "attribute")) {	1058	if (buf_eql_str(name, "attribute")) {
1059	if (fn_table_entry->fn_def_node) {	1059	if (fn_table_entry->fn_def_node) {
...	@@ -1251,7 +1251,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt	...	@@ -1251,7 +1251,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt
1251	for (uint32_t i = 0; i < field_count; i += 1) {	1251	for (uint32_t i = 0; i < field_count; i += 1) {
1252	AstNode *field_node = decl_node->data.struct_decl.fields.at(i);	1252	AstNode *field_node = decl_node->data.struct_decl.fields.at(i);
1253	TypeEnumField *type_enum_field = &enum_type->data.enumeration.fields[i];	1253	TypeEnumField *type_enum_field = &enum_type->data.enumeration.fields[i];
1254	type_enum_field->name = &field_node->data.struct_field.name;	1254	type_enum_field->name = field_node->data.struct_field.name;
1255	TypeTableEntry *field_type = analyze_type_expr(g, import, context,	1255	TypeTableEntry *field_type = analyze_type_expr(g, import, context,
1256	field_node->data.struct_field.type);	1256	field_node->data.struct_field.type);
1257	type_enum_field->type_entry = field_type;	1257	type_enum_field->type_entry = field_type;
...	@@ -1365,7 +1365,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt	...	@@ -1365,7 +1365,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt
1365	uint64_t debug_align_in_bits = 8*LLVMABISizeOfType(g->target_data_ref, enum_type->type_ref);	1365	uint64_t debug_align_in_bits = 8*LLVMABISizeOfType(g->target_data_ref, enum_type->type_ref);
1366	LLVMZigDIType *replacement_di_type = LLVMZigCreateDebugStructType(g->dbuilder,	1366	LLVMZigDIType *replacement_di_type = LLVMZigCreateDebugStructType(g->dbuilder,
1367	LLVMZigFileToScope(import->di_file),	1367	LLVMZigFileToScope(import->di_file),
1368	buf_ptr(&decl_node->data.struct_decl.name),	1368	buf_ptr(decl_node->data.struct_decl.name),
1369	import->di_file, decl_node->line + 1,	1369	import->di_file, decl_node->line + 1,
1370	debug_size_in_bits,	1370	debug_size_in_bits,
1371	debug_align_in_bits,	1371	debug_align_in_bits,
...	@@ -1381,7 +1381,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt	...	@@ -1381,7 +1381,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt
1381	uint64_t tag_debug_size_in_bits = 8*LLVMStoreSizeOfType(g->target_data_ref, tag_type_entry->type_ref);	1381	uint64_t tag_debug_size_in_bits = 8*LLVMStoreSizeOfType(g->target_data_ref, tag_type_entry->type_ref);
1382	uint64_t tag_debug_align_in_bits = 8*LLVMABISizeOfType(g->target_data_ref, tag_type_entry->type_ref);	1382	uint64_t tag_debug_align_in_bits = 8*LLVMABISizeOfType(g->target_data_ref, tag_type_entry->type_ref);
1383	LLVMZigDIType *tag_di_type = LLVMZigCreateDebugEnumerationType(g->dbuilder,	1383	LLVMZigDIType *tag_di_type = LLVMZigCreateDebugEnumerationType(g->dbuilder,
1384	LLVMZigFileToScope(import->di_file), buf_ptr(&decl_node->data.struct_decl.name),	1384	LLVMZigFileToScope(import->di_file), buf_ptr(decl_node->data.struct_decl.name),
1385	import->di_file, decl_node->line + 1,	1385	import->di_file, decl_node->line + 1,
1386	tag_debug_size_in_bits,	1386	tag_debug_size_in_bits,
1387	tag_debug_align_in_bits,	1387	tag_debug_align_in_bits,
...	@@ -1441,7 +1441,7 @@ static void resolve_struct_type(CodeGen g, ImportTableEntry import, TypeTableE	...	@@ -1441,7 +1441,7 @@ static void resolve_struct_type(CodeGen g, ImportTableEntry import, TypeTableE
1441	for (int i = 0; i < field_count; i += 1) {	1441	for (int i = 0; i < field_count; i += 1) {
1442	AstNode *field_node = decl_node->data.struct_decl.fields.at(i);	1442	AstNode *field_node = decl_node->data.struct_decl.fields.at(i);
1443	TypeStructField *type_struct_field = &struct_type->data.structure.fields[i];	1443	TypeStructField *type_struct_field = &struct_type->data.structure.fields[i];
1444	type_struct_field->name = &field_node->data.struct_field.name;	1444	type_struct_field->name = field_node->data.struct_field.name;
1445	TypeTableEntry *field_type = analyze_type_expr(g, import, context,	1445	TypeTableEntry *field_type = analyze_type_expr(g, import, context,
1446	field_node->data.struct_field.type);	1446	field_node->data.struct_field.type);
1447	type_struct_field->type_entry = field_type;	1447	type_struct_field->type_entry = field_type;