Commit d0b11af2bd445d10383049d2d7574c19a95c9006 | zig

author	Andrew Kelley <andrew@ziglang.org> 2016-08-01 23:11:31-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2016-08-04 22:52:38-07:00
log	d0b11af2bd445d10383049d2d7574c19a95c9006
tree	c841cdb1773668133c16d3a8be389630608ad523
parent	0450b73e3e4a488a2a20f2b8933443dcbe5fd5e3

new multiline string syntax

This patch also moves a bunch of the parser code into the tokenizer.

Closes #162.

15 files changed, 802 insertions(+), 1093 deletions(-)

doc
- langref.md
- vim
  - syntax
    zig.vim
src
test
- run_tests.cpp
- self_hosted.zig

doc/langref.md+65-36

...	...	@@ -7,27 +7,27 @@ Root = many(TopLevelDecl) "EOF"
7	7
8	8	TopLevelDecl = many(Directive) option(VisibleMod) (FnDef \| ExternDecl \| ContainerDecl \| GlobalVarDecl \| ErrorValueDecl \| TypeDecl \| UseDecl)
9	9
10		TypeDecl = "type" "Symbol" "=" TypeExpr ";"
	10	TypeDecl = "type" Symbol "=" TypeExpr ";"
11	11
12		ErrorValueDecl = "error" "Symbol" ";"
	12	ErrorValueDecl = "error" Symbol ";"
13	13
14	14	GlobalVarDecl = VariableDeclaration ";"
15	15
16		VariableDeclaration = ("var" \| "const") "Symbol" option(":" TypeExpr) "=" Expression
	16	VariableDeclaration = ("var" \| "const") Symbol option(":" TypeExpr) "=" Expression
17	17
18		ContainerDecl = ("struct" \| "enum" \| "union") "Symbol" option(ParamDeclList) "{" many(StructMember) "}"
	18	ContainerDecl = ("struct" \| "enum" \| "union") Symbol option(ParamDeclList) "{" many(StructMember) "}"
19	19
20	20	StructMember = many(Directive) option(VisibleMod) (StructField \| FnDef \| GlobalVarDecl \| ContainerDecl)
21	21
22		StructField = "Symbol" option(":" Expression) ",")
	22	StructField = Symbol option(":" Expression) ",")
23	23
24	24	UseDecl = "use" Expression ";"
25	25
26	26	ExternDecl = "extern" (FnProto \| VariableDeclaration) ";"
27	27
28		FnProto = "fn" option("Symbol") ParamDeclList option("->" TypeExpr)
	28	FnProto = "fn" option(Symbol) ParamDeclList option("->" TypeExpr)
29	29
30		Directive = "#" "Symbol" "(" Expression ")"
	30	Directive = "#" Symbol "(" Expression ")"
31	31
32	32	VisibleMod = "pub" \| "export"
33	33
...	...	@@ -35,13 +35,13 @@ FnDef = option("inline" \| "extern") FnProto Block
35	35
36	36	ParamDeclList = "(" list(ParamDecl, ",") ")"
37	37
38		ParamDecl = option("noalias" \| "inline") option("Symbol" ":") TypeExpr \| "..."
	38	ParamDecl = option("noalias" \| "inline") option(Symbol ":") TypeExpr \| "..."
39	39
40	40	Block = "{" list(option(Statement), ";") "}"
41	41
42	42	Statement = Label \| VariableDeclaration ";" \| Defer ";" \| NonBlockExpression ";" \| BlockExpression
43	43
44		Label = "Symbol" ":"
	44	Label = Symbol ":"
45	45
46	46	Expression = BlockExpression \| NonBlockExpression
47	47
...	...	@@ -49,23 +49,23 @@ TypeExpr = PrefixOpExpression
49	49
50	50	NonBlockExpression = ReturnExpression \| AssignmentExpression
51	51
52		AsmExpression = "asm" option("volatile") "(" "String" option(AsmOutput) ")"
	52	AsmExpression = "asm" option("volatile") "(" String option(AsmOutput) ")"
53	53
54	54	AsmOutput = ":" list(AsmOutputItem, ",") option(AsmInput)
55	55
56	56	AsmInput = ":" list(AsmInputItem, ",") option(AsmClobbers)
57	57
58		AsmOutputItem = "[" "Symbol" "]" "String" "(" ("Symbol" \| "->" TypeExpr) ")"
	58	AsmOutputItem = "[" Symbol "]" String "(" (Symbol \| "->" TypeExpr) ")"
59	59
60		AsmInputItem = "[" "Symbol" "]" "String" "(" Expression ")"
	60	AsmInputItem = "[" Symbol "]" String "(" Expression ")"
61	61
62		AsmClobbers= ":" list("String", ",")
	62	AsmClobbers= ":" list(String, ",")
63	63
64	64	UnwrapExpression = BoolOrExpression (UnwrapMaybe \| UnwrapError) \| BoolOrExpression
65	65
66	66	UnwrapMaybe = "??" Expression
67	67
68		UnwrapError = "%%" option("\|" "Symbol" "\|") Expression
	68	UnwrapError = "%%" option("\|" Symbol "\|") Expression
69	69
70	70	AssignmentExpression = UnwrapExpression AssignmentOperator UnwrapExpression \| UnwrapExpression
71	71
...	...	@@ -75,13 +75,13 @@ BlockExpression = IfExpression \| Block \| WhileExpression \| ForExpression \| Switc
75	75
76	76	SwitchExpression = "switch" "(" Expression ")" "{" many(SwitchProng) "}"
77	77
78		SwitchProng = (list(SwitchItem, ",") \| "else") "=>" option("\|" "Symbol" "\|") Expression ","
	78	SwitchProng = (list(SwitchItem, ",") \| "else") "=>" option("\|" Symbol "\|") Expression ","
79	79
80	80	SwitchItem = Expression \| (Expression "..." Expression)
81	81
82	82	WhileExpression = "while" "(" Expression option(";" Expression) ")" Expression
83	83
84		ForExpression = "for" "(" Expression ")" option("\|" option("*") "Symbol" option("," "Symbol") "\|") Expression
	84	ForExpression = "for" "(" Expression ")" option("\|" option("*") Symbol option("," Symbol) "\|") Expression
85	85
86	86	BoolOrExpression = BoolAndExpression "\|\|" BoolOrExpression \| BoolAndExpression
87	87
...	...	@@ -93,7 +93,7 @@ IfExpression = IfVarExpression \| IfBoolExpression
93	93
94	94	IfBoolExpression = "if" "(" Expression ")" Expression option(Else)
95	95
96		IfVarExpression = "if" "(" ("const" \| "var") option("*") "Symbol" option(":" TypeExpr) "?=" Expression ")" Expression Option(Else)
	96	IfVarExpression = "if" "(" ("const" \| "var") option("*") Symbol option(":" TypeExpr) "?=" Expression ")" Expression Option(Else)
97	97
98	98	Else = "else" Expression
99	99
...	...	@@ -127,7 +127,7 @@ PrefixOpExpression = PrefixOp PrefixOpExpression \| SuffixOpExpression
127	127
128	128	SuffixOpExpression = PrimaryExpression option(FnCallExpression \| ArrayAccessExpression \| FieldAccessExpression \| SliceExpression)
129	129
130		FieldAccessExpression = "." "Symbol"
	130	FieldAccessExpression = "." Symbol
131	131
132	132	FnCallExpression = "(" list(Expression, ",") ")"
133	133
...	...	@@ -139,15 +139,15 @@ ContainerInitExpression = "{" ContainerInitBody "}"
139	139
140	140	ContainerInitBody = list(StructLiteralField, ",") \| list(Expression, ",")
141	141
142		StructLiteralField = "." "Symbol" "=" Expression
	142	StructLiteralField = "." Symbol "=" Expression
143	143
144	144	PrefixOp = "!" \| "-" \| "~" \| "*" \| ("&" option("const")) \| "?" \| "%" \| "%%" \| "??" \| "-%"
145	145
146		PrimaryExpression = "Number" \| "String" \| "CharLiteral" \| KeywordLiteral \| GroupedExpression \| GotoExpression \| BlockExpression \| "Symbol" \| ("@" "Symbol" FnCallExpression) \| ArrayType \| (option("extern") FnProto) \| AsmExpression \| ("error" "." "Symbol")
	146	PrimaryExpression = Number \| String \| CharLiteral \| KeywordLiteral \| GroupedExpression \| GotoExpression \| BlockExpression \| Symbol \| ("@" Symbol FnCallExpression) \| ArrayType \| (option("extern") FnProto) \| AsmExpression \| ("error" "." Symbol)
147	147
148	148	ArrayType = "[" option(Expression) "]" option("const") TypeExpr
149	149
150		GotoExpression = "goto" "Symbol"
	150	GotoExpression = "goto" Symbol
151	151
152	152	GroupedExpression = "(" Expression ")"
153	153
...	...	@@ -265,14 +265,13 @@ from codegen.
265	265	### Literals
266	266
267	267	#### Character and String Literals
	268
268	269	```
269	270	Literal Example Characters Escapes Null Term Type
270	271
271	272	Byte 'H' All ASCII Byte No u8
272	273	UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8
273	274	UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8
274		UTF-8 Raw String r"X(hello)X" All Unicode None No [5]u8
275		UTF-8 Raw C String rc"X(hello)X" All Unicode None Yes &const u8
276	275	```
277	276
278	277	### Escapes
...	...	@@ -291,26 +290,56 @@ UTF-8 Raw C String rc"X(hello)X" All Unicode None Yes &const
291	290
292	291	Note that the maximum valid Unicode point is 0x10ffff.
293	292
294		##### Raw Strings
	293	##### Multiline String Literals
295	294
296		Raw string literals have no escapes and can span across multiple lines. To
297		start a raw string, use 'r"' or 'rc"' followed by unique bytes followed by '('.
298		To end a raw string, use ')' followed by the same unique bytes, followed by '"'.
	295	Multiline string literals have no escapes and can span across multiple lines.
	296	To start a multiline string literal, use the `\\` token. Just like a comment,
	297	the string literal goes until the end of the line. The end of the line is not
	298	included in the string literal.
299	299
	300	However, if the next line begins with `\\` then a newline is appended and
	301	the string literal continues.
300	302
301		#### Numeric Literals
	303	Example:
302	304
	305	```zig
	306	const hello_world_in_c =
	307	\\#include <stdio.h>
	308	\\
	309	\\int main(int argc, char **argv) {
	310	\\ printf("hello world\n");
	311	\\ return 0;
	312	\\}
	313	;
303	314	```
304		Number literals Example Exponentiation
305
306		Decimal integer 98222 N/A
307		Hex integer 0xff N/A
308		Octal integer 0o77 N/A
309		Binary integer 0b11110000 N/A
310		Floating-point 123.0E+77 Optional
311		Hex floating point TODO TODO
	315
	316	For a multiline C string literal, prepend `c` to each `\\`. Example:
	317
	318	```zig
	319	const c_string_literal =
	320	c\\#include <stdio.h>
	321	c\\
	322	c\\int main(int argc, char **argv) {
	323	c\\ printf("hello world\n");
	324	c\\ return 0;
	325	c\\}
	326	;
312	327	```
313	328
	329	In this example the variable `c_string_literal` has type `&const char` and
	330	has a terminating null byte.
	331
	332	#### Number Literals
	333
	334	Number literals \| Example \| Exponentiation
	335	--------------------\|-------------\|--------------
	336	Decimal integer \| 98222 \| N/A
	337	Hex integer \| 0xff \| N/A
	338	Octal integer \| 0o77 \| N/A
	339	Binary integer \| 0b11110000 \| N/A
	340	Floating point \| 123.0E+77 \| Optional
	341	Hex floating point \| 0x103.70p-5 \| Optional
	342
314	343	### Identifiers
315	344
316	345	TODO

doc/vim/syntax/zig.vim+16-18

...	...	@@ -1,11 +1,12 @@
1	1	" Vim syntax file
2	2	" Language: Zig
3	3	" Maintainer: Andrew Kelley
4		" Latest Revision: 28 July 2016
	4	" Latest Revision: 03 August 2016
5	5
6	6	if exists("b:current_syntax")
7	7	finish
8	8	endif
	9	let b:current_syntax = "zig"
9	10
10	11	syn keyword zigStorage const var extern export pub noalias inline noinline
11	12	syn keyword zigStructure struct enum union
...	...	@@ -24,33 +25,30 @@ syn keyword zigBoolean true false
24	25	syn match zigOperator display "\%(+%\?\\|-%\?\\|/\\|*%\?\\|=\\|\^\\|&\\|?\\|\|\\|!\\|>\\|<\\|%\\|<<%\?\\|>>\\|&&\\|\|\|\)=\?"
25	26	syn match zigArrowCharacter display "->"
26	27
27		syn match zigDecNumber display "\<[0-9][0-9_]*\%([iu]\%(size\\|8\\|16\\|32\\|64\)\)\="
28		syn match zigHexNumber display "\<0x[a-fA-F0-9_]\+\%([iu]\%(size\\|8\\|16\\|32\\|64\)\)\="
29		syn match zigOctNumber display "\<0o[0-7_]\+\%([iu]\%(size\\|8\\|16\\|32\\|64\)\)\="
30		syn match zigBinNumber display "\<0b[01_]\+\%([iu]\%(size\\|8\\|16\\|32\\|64\)\)\="
	28	syn match zigDecNumber display "\<[0-9]*\%(.[0-9]\+\)\=\%([eE][+-]\?[0-9]\+\)\="
	29	syn match zigHexNumber display "\<0x[a-fA-F0-9]\+\%(.[a-fA-F0-9]\+\%([pP][+-]\?[0-9]\+\)\?\)\="
	30	syn match zigOctNumber display "\<0o[0-7]\+"
	31	syn match zigBinNumber display "\<0b[01]\+\%(.[01]\+\%([eE][+-]\?[0-9]\+\)\?\)\="
31	32
32	33
33	34	syn match zigCharacterInvalid display contained /b\?'\zs[\n\r\t']\ze'/
34	35	syn match zigCharacterInvalidUnicode display contained /b'\zs[^[:cntrl:][:graph:][:alnum:][:space:]]\ze'/
35	36	syn match zigCharacter /b'$[^\\]\\|\\\(.\\|x\x\{2}$\)'/ contains=zigEscape,zigEscapeError,zigCharacterInvalid,zigCharacterInvalidUnicode
36		syn match zigCharacter /'$[^\\]\\|\\\(.\\|x\x\{2}\\|u\x\{4}\\|U\x\{8}\\|u{\x\{1,6}}$\)'/ contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigCharacterInvalid
37
38		syn match zigShebang /\%^#![^[].*/
	37	syn match zigCharacter /'$[^\\]\\|\\\(.\\|x\x\{2}\\|u\x\{4}\\|U\x\{6}$\)'/ contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigCharacterInvalid
39	38
40	39	syn region zigCommentLine start="//" end="$" contains=zigTodo,@Spell
41	40	syn region zigCommentLineDoc start="//\%(//\@!\\|!\)" end="$" contains=zigTodo,@Spell
42	41
	42	" TODO match only the first '\\' within the zigMultilineString as zigMultilineStringPrefix
	43	syn match zigMultilineStringPrefix display contained /c\?\\\\/
	44	syn region zigMultilineString start="c\?\\\\" end="$" contains=zigMultilineStringPrefix
	45
43	46	syn keyword zigTodo contained TODO XXX
44	47
45	48	syn match zigEscapeError display contained /\\./
46		syn match zigEscape display contained /\\$[nrt0\\'"]\\|x\x\{2}$/
47		syn match zigEscapeUnicode display contained /\\$u\x\{4}\\|U\x\{8}$/
48		syn match zigEscapeUnicode display contained /\\u{\x\{1,6}}/
49		syn match zigStringContinuation display contained /\\\n\s*/
50		syn region zigString start=+c\?"+ skip=+\\\\\\|\\"+ end=+"+ oneline contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigStringContinuation,@Spell
51		syn region zigString start='r"\z([^)]*\)(' end=')\z1"' contains=@Spell
52
53		let b:current_syntax = "zig"
	49	syn match zigEscape display contained /\\$[nrt\\'"]\\|x\x\{2}$/
	50	syn match zigEscapeUnicode display contained /\\$u\x\{4}\\|U\x\{6}$/
	51	syn region zigString start=+c\?"+ skip=+\\\\\\|\\"+ end=+"+ oneline contains=zigEscape,zigEscapeUnicode,zigEscapeError,@Spell
54	52
55	53	hi def link zigDecNumber zigNumber
56	54	hi def link zigHexNumber zigNumber
...	...	@@ -59,12 +57,12 @@ hi def link zigBinNumber zigNumber
59	57
60	58	hi def link zigKeyword Keyword
61	59	hi def link zigType Type
62		hi def link zigShebang Comment
63	60	hi def link zigCommentLine Comment
64	61	hi def link zigCommentLineDoc SpecialComment
65	62	hi def link zigTodo Todo
66		hi def link zigStringContinuation Special
67	63	hi def link zigString String
	64	hi def link zigMultilineString String
	65	hi def link zigMultilineStringPrefix Comment
68	66	hi def link zigCharacterInvalid Error
69	67	hi def link zigCharacterInvalidUnicode zigCharacterInvalid
70	68	hi def link zigCharacter Character

src/all_types.hpp+21-32

...	...	@@ -194,7 +194,7 @@ struct AstNodeRoot {
194	194
195	195	struct AstNodeFnProto {
196	196	TopLevelDecl top_level_decl;
197		Buf name;
	197	Buf *name;
198	198	ZigList<AstNode *> params;
199	199	AstNode *return_type;
200	200	bool is_var_args;
...	...	@@ -229,7 +229,7 @@ struct AstNodeFnDecl {
229	229	};
230	230
231	231	struct AstNodeParamDecl {
232		Buf name;
	232	Buf *name;
233	233	AstNode *type;
234	234	bool is_noalias;
235	235	bool is_inline;
...	...	@@ -279,7 +279,7 @@ struct AstNodeDefer {
279	279
280	280	struct AstNodeVariableDeclaration {
281	281	TopLevelDecl top_level_decl;
282		Buf symbol;
	282	Buf *symbol;
283	283	bool is_const;
284	284	bool is_extern;
285	285	// one or both of type and expr will be non null
...	...	@@ -293,7 +293,7 @@ struct AstNodeVariableDeclaration {
293	293
294	294	struct AstNodeTypeDecl {
295	295	TopLevelDecl top_level_decl;
296		Buf symbol;
	296	Buf *symbol;
297	297	AstNode *child_type;
298	298
299	299	// populated by semantic analyzer
...	...	@@ -305,7 +305,7 @@ struct AstNodeTypeDecl {
305	305
306	306	struct AstNodeErrorValueDecl {
307	307	TopLevelDecl top_level_decl;
308		Buf name;
	308	Buf *name;
309	309
310	310	// populated by semantic analyzer
311	311	ErrorTableEntry *err;
...	...	@@ -434,7 +434,7 @@ struct AstNodeSliceExpr {
434	434
435	435	struct AstNodeFieldAccessExpr {
436	436	AstNode *struct_expr;
437		Buf field_name;
	437	Buf *field_name;
438	438
439	439	// populated by semantic analyzer
440	440	TypeStructField *type_struct_field;
...	...	@@ -448,7 +448,7 @@ struct AstNodeFieldAccessExpr {
448	448	};
449	449
450	450	struct AstNodeDirective {
451		Buf name;
	451	Buf *name;
452	452	AstNode *expr;
453	453	};
454	454
...	...	@@ -555,7 +555,7 @@ struct AstNodeSwitchRange {
555	555	};
556	556
557	557	struct AstNodeLabel {
558		Buf name;
	558	Buf *name;
559	559
560	560	// populated by semantic analyzer
561	561	Expr resolved_expr;
...	...	@@ -563,7 +563,7 @@ struct AstNodeLabel {
563	563	};
564	564
565	565	struct AstNodeGoto {
566		Buf name;
	566	Buf *name;
567	567
568	568	// populated by semantic analyzer
569	569	Expr resolved_expr;
...	...	@@ -571,9 +571,9 @@ struct AstNodeGoto {
571	571	};
572	572
573	573	struct AsmOutput {
574		Buf asm_symbolic_name;
575		Buf constraint;
576		Buf variable_name;
	574	Buf *asm_symbolic_name;
	575	Buf *constraint;
	576	Buf *variable_name;
577	577	AstNode *return_type; // null unless "=r" and return
578	578
579	579	// populated by semantic analyzer
...	...	@@ -581,8 +581,8 @@ struct AsmOutput {
581	581	};
582	582
583	583	struct AsmInput {
584		Buf asm_symbolic_name;
585		Buf constraint;
	584	Buf *asm_symbolic_name;
	585	Buf *constraint;
586	586	AstNode *expr;
587	587	};
588	588
...	...	@@ -593,8 +593,7 @@ struct SrcPos {
593	593
594	594	struct AstNodeAsmExpr {
595	595	bool is_volatile;
596		Buf asm_template;
597		ZigList<SrcPos> offset_map;
	596	Buf *asm_template;
598	597	ZigList<AsmToken> token_list;
599	598	ZigList<AsmOutput*> output_list;
600	599	ZigList<AsmInput*> input_list;
...	...	@@ -613,7 +612,7 @@ enum ContainerKind {
613	612
614	613	struct AstNodeStructDecl {
615	614	TopLevelDecl top_level_decl;
616		Buf name;
	615	Buf *name;
617	616	ContainerKind kind;
618	617	ZigList<AstNode *> generic_params;
619	618	bool generic_params_is_var_args; // always an error but it can happen from parsing
...	...	@@ -629,12 +628,12 @@ struct AstNodeStructDecl {
629	628
630	629	struct AstNodeStructField {
631	630	TopLevelDecl top_level_decl;
632		Buf name;
	631	Buf *name;
633	632	AstNode *type;
634	633	};
635	634
636	635	struct AstNodeStringLiteral {
637		Buf buf;
	636	Buf *buf;
638	637	bool c;
639	638
640	639	// populated by semantic analyzer:
...	...	@@ -648,29 +647,19 @@ struct AstNodeCharLiteral {
648	647	Expr resolved_expr;
649	648	};
650	649
651		enum NumLit {
652		NumLitFloat,
653		NumLitUInt,
654		};
655
656	650	struct AstNodeNumberLiteral {
657		NumLit kind;
	651	BigNum *bignum;
658	652
659	653	// overflow is true if when parsing the number, we discovered it would not
660	654	// fit without losing data in a uint64_t or double
661	655	bool overflow;
662	656
663		union {
664		uint64_t x_uint;
665		double x_float;
666		} data;
667
668	657	// populated by semantic analyzer
669	658	Expr resolved_expr;
670	659	};
671	660
672	661	struct AstNodeStructValueField {
673		Buf name;
	662	Buf *name;
674	663	AstNode *expr;
675	664
676	665	// populated by semantic analyzer
...	...	@@ -706,7 +695,7 @@ struct AstNodeUndefinedLiteral {
706	695	};
707	696
708	697	struct AstNodeSymbolExpr {
709		Buf symbol;
	698	Buf *symbol;
710	699
711	700	// populated by semantic analyzer
712	701	Expr resolved_expr;

src/analyze.cpp+58-58

...	...	@@ -1053,7 +1053,7 @@ static void resolve_function_proto(CodeGen g, AstNode node, FnTableEntry *fn_t
1053	1053	if (fn_proto->top_level_decl.directives) {
1054	1054	for (int i = 0; i < fn_proto->top_level_decl.directives->length; i += 1) {
1055	1055	AstNode *directive_node = fn_proto->top_level_decl.directives->at(i);
1056		Buf *name = &directive_node->data.directive.name;
	1056	Buf *name = directive_node->data.directive.name;
1057	1057
1058	1058	if (buf_eql_str(name, "attribute")) {
1059	1059	if (fn_table_entry->fn_def_node) {
...	...	@@ -1251,7 +1251,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt
1251	1251	for (uint32_t i = 0; i < field_count; i += 1) {
1252	1252	AstNode *field_node = decl_node->data.struct_decl.fields.at(i);
1253	1253	TypeEnumField *type_enum_field = &enum_type->data.enumeration.fields[i];
1254		type_enum_field->name = &field_node->data.struct_field.name;
	1254	type_enum_field->name = field_node->data.struct_field.name;
1255	1255	TypeTableEntry *field_type = analyze_type_expr(g, import, context,
1256	1256	field_node->data.struct_field.type);
1257	1257	type_enum_field->type_entry = field_type;
...	...	@@ -1365,7 +1365,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt
1365	1365	uint64_t debug_align_in_bits = 8*LLVMABISizeOfType(g->target_data_ref, enum_type->type_ref);
1366	1366	LLVMZigDIType *replacement_di_type = LLVMZigCreateDebugStructType(g->dbuilder,
1367	1367	LLVMZigFileToScope(import->di_file),
1368		buf_ptr(&decl_node->data.struct_decl.name),
	1368	buf_ptr(decl_node->data.struct_decl.name),
1369	1369	import->di_file, decl_node->line + 1,
1370	1370	debug_size_in_bits,
1371	1371	debug_align_in_bits,
...	...	@@ -1381,7 +1381,7 @@ static void resolve_enum_type(CodeGen g, ImportTableEntry import, TypeTableEnt
1381	1381	uint64_t tag_debug_size_in_bits = 8*LLVMStoreSizeOfType(g->target_data_ref, tag_type_entry->type_ref);
1382	1382	uint64_t tag_debug_align_in_bits = 8*LLVMABISizeOfType(g->target_data_ref, tag_type_entry->type_ref);
1383	1383	LLVMZigDIType *tag_di_type = LLVMZigCreateDebugEnumerationType(g->dbuilder,
1384		LLVMZigFileToScope(import->di_file), buf_ptr(&decl_node->data.struct_decl.name),
	1384	LLVMZigFileToScope(import->di_file), buf_ptr(decl_node->data.struct_decl.name),
1385	1385	import->di_file, decl_node->line + 1,
1386	1386	tag_debug_size_in_bits,
1387	1387	tag_debug_align_in_bits,
...	...	@@ -1441,7 +1441,7 @@ static void resolve_struct_type(CodeGen g, ImportTableEntry import, TypeTableE
1441	1441	for (int i = 0; i < field_count; i += 1) {
1442	1442	AstNode *field_node = decl_node->data.struct_decl.fields.at(i);
1443	1443	TypeStructField *type_struct_field = &struct_type->data.structure.fields[i];
1444		type_struct_field->name = &field_node->data.struct_field.name;
	1444	type_struct_field->name = field_node->data.struct_field.name;
1445	1445	TypeTableEntry *field_type = analyze_type_expr(g, import, context,
1446	1446	field_node->data.struct_field.type);
1447	1447	type_struct_field->type_entry = field_type;
...	...	@@ -1514,7 +1514,7 @@ static void resolve_struct_type(CodeGen g, ImportTableEntry import, TypeTableE
1514	1514	uint64_t debug_align_in_bits = 8*LLVMABISizeOfType(g->target_data_ref, struct_type->type_ref);
1515	1515	LLVMZigDIType *replacement_di_type = LLVMZigCreateDebugStructType(g->dbuilder,
1516	1516	LLVMZigFileToScope(import->di_file),
1517		buf_ptr(&decl_node->data.struct_decl.name),
	1517	buf_ptr(decl_node->data.struct_decl.name),
1518	1518	import->di_file, decl_node->line + 1,
1519	1519	debug_size_in_bits,
1520	1520	debug_align_in_bits,
...	...	@@ -1570,7 +1570,7 @@ static void preview_fn_proto_instance(CodeGen g, ImportTableEntry import, AstN
1570	1570	assert(!is_generic_instance \|\| !is_generic_fn);
1571	1571
1572	1572	AstNode *parent_decl = proto_node->data.fn_proto.top_level_decl.parent_decl;
1573		Buf *proto_name = &proto_node->data.fn_proto.name;
	1573	Buf *proto_name = proto_node->data.fn_proto.name;
1574	1574
1575	1575	AstNode *fn_def_node = proto_node->data.fn_proto.fn_def_node;
1576	1576	bool is_extern = proto_node->data.fn_proto.is_extern;
...	...	@@ -1645,7 +1645,7 @@ static void scan_struct_decl(CodeGen g, ImportTableEntry import, BlockContext
1645	1645	return;
1646	1646	}
1647	1647
1648		Buf *name = &node->data.struct_decl.name;
	1648	Buf *name = node->data.struct_decl.name;
1649	1649	TypeTableEntry *container_type = get_partial_container_type(g, import, context,
1650	1650	node->data.struct_decl.kind, node, buf_ptr(name));
1651	1651	node->data.struct_decl.type_entry = container_type;
...	...	@@ -1692,7 +1692,7 @@ static void preview_error_value_decl(CodeGen g, AstNode node) {
1692	1692	ErrorTableEntry *err = allocate<ErrorTableEntry>(1);
1693	1693
1694	1694	err->decl_node = node;
1695		buf_init_from_buf(&err->name, &node->data.error_value_decl.name);
	1695	buf_init_from_buf(&err->name, node->data.error_value_decl.name);
1696	1696
1697	1697	auto existing_entry = g->error_table.maybe_get(&err->name);
1698	1698	if (existing_entry) {
...	...	@@ -1749,7 +1749,7 @@ static void resolve_top_level_decl(CodeGen g, AstNode node, bool pointer_only)
1749	1749	case NodeTypeTypeDecl:
1750	1750	{
1751	1751	AstNode *type_node = node->data.type_decl.child_type;
1752		Buf *decl_name = &node->data.type_decl.symbol;
	1752	Buf *decl_name = node->data.type_decl.symbol;
1753	1753
1754	1754	TypeTableEntry *entry;
1755	1755	if (node->data.type_decl.override_type) {
...	...	@@ -2479,12 +2479,12 @@ static TypeTableEntry analyze_container_init_expr(CodeGen g, ImportTableEntry
2479	2479	val_field_node->block_context = context;
2480	2480
2481	2481	TypeStructField *type_field = find_struct_type_field(container_type,
2482		&val_field_node->data.struct_val_field.name);
	2482	val_field_node->data.struct_val_field.name);
2483	2483
2484	2484	if (!type_field) {
2485	2485	add_node_error(g, val_field_node,
2486	2486	buf_sprintf("no member named '%s' in '%s'",
2487		buf_ptr(&val_field_node->data.struct_val_field.name), buf_ptr(&container_type->name)));
	2487	buf_ptr(val_field_node->data.struct_val_field.name), buf_ptr(&container_type->name)));
2488	2488	continue;
2489	2489	}
2490	2490
...	...	@@ -2604,7 +2604,7 @@ static TypeTableEntry analyze_field_access_expr(CodeGen g, ImportTableEntry *i
2604	2604
2605	2605	AstNode **struct_expr_node = &node->data.field_access_expr.struct_expr;
2606	2606	TypeTableEntry struct_type = analyze_expression(g, import, context, nullptr, struct_expr_node);
2607		Buf *field_name = &node->data.field_access_expr.field_name;
	2607	Buf *field_name = node->data.field_access_expr.field_name;
2608	2608
2609	2609	bool wrapped_in_fn_call = node->data.field_access_expr.is_fn_call;
2610	2610
...	...	@@ -2965,28 +2965,33 @@ static TypeTableEntry resolve_expr_const_val_as_string_lit(CodeGen g, AstNode
2965	2965	return get_array_type(g, g->builtin_types.entry_u8, buf_len(str));
2966	2966	}
2967	2967
2968
2969		static TypeTableEntry resolve_expr_const_val_as_unsigned_num_lit(CodeGen g, AstNode *node,
2970		TypeTableEntry *expected_type, uint64_t x, bool depends_on_compile_var)
	2968	static TypeTableEntry resolve_expr_const_val_as_bignum(CodeGen g, AstNode *node,
	2969	TypeTableEntry expected_type, BigNum bignum, bool depends_on_compile_var)
2971	2970	{
2972	2971	Expr *expr = get_resolved_expr(node);
2973	2972	expr->const_val.ok = true;
2974	2973	expr->const_val.depends_on_compile_var = depends_on_compile_var;
2975	2974
2976		bignum_init_unsigned(&expr->const_val.data.x_bignum, x);
2977
2978		return g->builtin_types.entry_num_lit_int;
	2975	bignum_init_bignum(&expr->const_val.data.x_bignum, bignum);
	2976	if (bignum->kind == BigNumKindInt) {
	2977	return g->builtin_types.entry_num_lit_int;
	2978	} else if (bignum->kind == BigNumKindFloat) {
	2979	return g->builtin_types.entry_num_lit_float;
	2980	} else {
	2981	zig_unreachable();
	2982	}
2979	2983	}
2980	2984
2981		static TypeTableEntry resolve_expr_const_val_as_float_num_lit(CodeGen g, AstNode *node,
2982		TypeTableEntry *expected_type, double x)
	2985	static TypeTableEntry resolve_expr_const_val_as_unsigned_num_lit(CodeGen g, AstNode *node,
	2986	TypeTableEntry *expected_type, uint64_t x, bool depends_on_compile_var)
2983	2987	{
2984	2988	Expr *expr = get_resolved_expr(node);
2985	2989	expr->const_val.ok = true;
	2990	expr->const_val.depends_on_compile_var = depends_on_compile_var;
2986	2991
2987		bignum_init_float(&expr->const_val.data.x_bignum, x);
	2992	bignum_init_unsigned(&expr->const_val.data.x_bignum, x);
2988	2993
2989		return g->builtin_types.entry_num_lit_float;
	2994	return g->builtin_types.entry_num_lit_int;
2990	2995	}
2991	2996
2992	2997	static TypeTableEntry analyze_error_literal_expr(CodeGen g, ImportTableEntry *import,
...	...	@@ -3073,7 +3078,7 @@ static TypeTableEntry analyze_symbol_expr(CodeGen g, ImportTableEntry *import,
3073	3078	return resolve_expr_const_val_as_type(g, node, node->data.symbol_expr.override_type_entry, false);
3074	3079	}
3075	3080
3076		Buf *variable_name = &node->data.symbol_expr.symbol;
	3081	Buf *variable_name = node->data.symbol_expr.symbol;
3077	3082
3078	3083	auto primitive_table_entry = g->primitive_type_table.maybe_get(variable_name);
3079	3084	if (primitive_table_entry) {
...	...	@@ -3177,7 +3182,7 @@ static TypeTableEntry analyze_lvalue(CodeGen g, ImportTableEntry *import, Bloc
3177	3182	return g->builtin_types.entry_invalid;
3178	3183	}
3179	3184	if (purpose != LValPurposeAddressOf) {
3180		Buf *name = &lhs_node->data.symbol_expr.symbol;
	3185	Buf *name = lhs_node->data.symbol_expr.symbol;
3181	3186	VariableTableEntry *var = find_variable(g, block_context, name);
3182	3187	if (var) {
3183	3188	if (var->is_const) {
...	...	@@ -3742,7 +3747,7 @@ static TypeTableEntry analyze_unwrap_error_expr(CodeGen g, ImportTableEntry *i
3742	3747	if (var_node) {
3743	3748	child_context = new_block_context(node, parent_context);
3744	3749	var_node->block_context = child_context;
3745		Buf *var_name = &var_node->data.symbol_expr.symbol;
	3750	Buf *var_name = var_node->data.symbol_expr.symbol;
3746	3751	node->data.unwrap_err_expr.var = add_local_var(g, var_node, import, child_context, var_name,
3747	3752	g->builtin_types.entry_pure_error, true, nullptr);
3748	3753	} else {
...	...	@@ -3827,7 +3832,7 @@ static VariableTableEntry analyze_variable_declaration_raw(CodeGen g, ImportTa
3827	3832	assert(type != nullptr); // should have been caught by the parser
3828	3833
3829	3834	VariableTableEntry *var = add_local_var(g, source_node, import, context,
3830		&variable_declaration->symbol, type, is_const,
	3835	variable_declaration->symbol, type, is_const,
3831	3836	expr_is_maybe ? nullptr : variable_declaration->expr);
3832	3837
3833	3838	variable_declaration->variable = var;
...	...	@@ -3886,15 +3891,7 @@ static TypeTableEntry analyze_number_literal_expr(CodeGen g, ImportTableEntry
3886	3891	return g->builtin_types.entry_invalid;
3887	3892	}
3888	3893
3889		if (node->data.number_literal.kind == NumLitUInt) {
3890		return resolve_expr_const_val_as_unsigned_num_lit(g, node,
3891		expected_type, node->data.number_literal.data.x_uint, false);
3892		} else if (node->data.number_literal.kind == NumLitFloat) {
3893		return resolve_expr_const_val_as_float_num_lit(g, node,
3894		expected_type, node->data.number_literal.data.x_float);
3895		} else {
3896		zig_unreachable();
3897		}
	3894	return resolve_expr_const_val_as_bignum(g, node, expected_type, node->data.number_literal.bignum, false);
3898	3895	}
3899	3896
3900	3897	static TypeTableEntry analyze_array_type(CodeGen g, ImportTableEntry import, BlockContext context,
...	...	@@ -4034,13 +4031,13 @@ static TypeTableEntry analyze_for_expr(CodeGen g, ImportTableEntry *import, Bl
4034	4031
4035	4032	AstNode *elem_var_node = node->data.for_expr.elem_node;
4036	4033	elem_var_node->block_context = child_context;
4037		Buf *elem_var_name = &elem_var_node->data.symbol_expr.symbol;
	4034	Buf *elem_var_name = elem_var_node->data.symbol_expr.symbol;
4038	4035	node->data.for_expr.elem_var = add_local_var(g, elem_var_node, import, child_context, elem_var_name,
4039	4036	var_type, true, nullptr);
4040	4037
4041	4038	AstNode *index_var_node = node->data.for_expr.index_node;
4042	4039	if (index_var_node) {
4043		Buf *index_var_name = &index_var_node->data.symbol_expr.symbol;
	4040	Buf *index_var_name = index_var_node->data.symbol_expr.symbol;
4044	4041	index_var_node->block_context = child_context;
4045	4042	node->data.for_expr.index_var = add_local_var(g, index_var_node, import, child_context, index_var_name,
4046	4043	g->builtin_types.entry_usize, true, nullptr);
...	...	@@ -4952,7 +4949,7 @@ static TypeTableEntry analyze_builtin_fn_call_expr(CodeGen g, ImportTableEntry
4952	4949	assert(node->type == NodeTypeFnCallExpr);
4953	4950
4954	4951	AstNode *fn_ref_expr = node->data.fn_call_expr.fn_ref_expr;
4955		Buf *name = &fn_ref_expr->data.symbol_expr.symbol;
	4952	Buf *name = fn_ref_expr->data.symbol_expr.symbol;
4956	4953
4957	4954	auto entry = g->builtin_fn_table.maybe_get(name);
4958	4955
...	...	@@ -5476,7 +5473,7 @@ static TypeTableEntry analyze_fn_call_with_inline_args(CodeGen g, ImportTableE
5476	5473	ConstExprValue const_val = &get_resolved_expr(param_node)->const_val;
5477	5474	if (const_val->ok) {
5478	5475	VariableTableEntry *var = add_local_var(g, generic_param_decl_node, decl_node->owner, child_context,
5479		&generic_param_decl_node->data.param_decl.name, param_type, true, *param_node);
	5476	generic_param_decl_node->data.param_decl.name, param_type, true, *param_node);
5480	5477	// This generic function instance could be called with anything, so when this variable is read it
5481	5478	// needs to know that it depends on compile time variable data.
5482	5479	var->force_depends_on_compile_var = true;
...	...	@@ -5570,7 +5567,7 @@ static TypeTableEntry analyze_generic_fn_call(CodeGen g, ImportTableEntry *imp
5570	5567	ConstExprValue const_val = &get_resolved_expr(param_node)->const_val;
5571	5568	if (const_val->ok) {
5572	5569	VariableTableEntry *var = add_local_var(g, generic_param_decl_node, decl_node->owner, child_context,
5573		&generic_param_decl_node->data.param_decl.name, param_type, true, *param_node);
	5570	generic_param_decl_node->data.param_decl.name, param_type, true, *param_node);
5574	5571	var->force_depends_on_compile_var = true;
5575	5572	} else {
5576	5573	add_node_error(g, *param_node, buf_sprintf("unable to evaluate constant expression"));
...	...	@@ -5964,7 +5961,7 @@ static TypeTableEntry analyze_switch_expr(CodeGen g, ImportTableEntry *import,
5964	5961
5965	5962	if (expr_type->id == TypeTableEntryIdEnum) {
5966	5963	if (item_node->type == NodeTypeSymbol) {
5967		Buf *field_name = &item_node->data.symbol_expr.symbol;
	5964	Buf *field_name = item_node->data.symbol_expr.symbol;
5968	5965	TypeEnumField *type_enum_field = get_enum_field(expr_type, field_name);
5969	5966	if (type_enum_field) {
5970	5967	item_node->data.symbol_expr.enum_field = type_enum_field;
...	...	@@ -6000,7 +5997,7 @@ static TypeTableEntry analyze_switch_expr(CodeGen g, ImportTableEntry *import,
6000	5997	}
6001	5998	} else if (expr_type->id == TypeTableEntryIdErrorUnion) {
6002	5999	if (item_node->type == NodeTypeSymbol) {
6003		Buf *err_name = &item_node->data.symbol_expr.symbol;
	6000	Buf *err_name = item_node->data.symbol_expr.symbol;
6004	6001	bool is_ok_case = buf_eql_str(err_name, "Ok");
6005	6002	auto err_table_entry = is_ok_case ? nullptr: g->error_table.maybe_get(err_name);
6006	6003	if (is_ok_case \|\| err_table_entry) {
...	...	@@ -6072,7 +6069,7 @@ static TypeTableEntry analyze_switch_expr(CodeGen g, ImportTableEntry *import,
6072	6069	AstNode *var_node = prong_node->data.switch_prong.var_symbol;
6073	6070	if (var_node) {
6074	6071	assert(var_node->type == NodeTypeSymbol);
6075		Buf *var_name = &var_node->data.symbol_expr.symbol;
	6072	Buf *var_name = var_node->data.symbol_expr.symbol;
6076	6073	var_node->block_context = child_context;
6077	6074	prong_node->data.switch_prong.var = add_local_var(g, var_node, import,
6078	6075	child_context, var_name, var_type, true, nullptr);
...	...	@@ -6228,9 +6225,9 @@ static TypeTableEntry analyze_string_literal_expr(CodeGen g, ImportTableEntry
6228	6225	TypeTableEntry expected_type, AstNode node)
6229	6226	{
6230	6227	if (node->data.string_literal.c) {
6231		return resolve_expr_const_val_as_c_string_lit(g, node, &node->data.string_literal.buf);
	6228	return resolve_expr_const_val_as_c_string_lit(g, node, node->data.string_literal.buf);
6232	6229	} else {
6233		return resolve_expr_const_val_as_string_lit(g, node, &node->data.string_literal.buf);
	6230	return resolve_expr_const_val_as_string_lit(g, node, node->data.string_literal.buf);
6234	6231	}
6235	6232	}
6236	6233
...	...	@@ -6255,7 +6252,7 @@ static TypeTableEntry analyze_block_expr(CodeGen g, ImportTableEntry *import,
6255	6252	child->data.label.label_entry = label;
6256	6253	fn_table_entry->all_labels.append(label);
6257	6254
6258		child_context->label_table.put(&child->data.label.name, label);
	6255	child_context->label_table.put(child->data.label.name, label);
6259	6256
6260	6257	return_type = g->builtin_types.entry_void;
6261	6258	continue;
...	...	@@ -6316,7 +6313,7 @@ static TypeTableEntry analyze_asm_expr(CodeGen g, ImportTableEntry *import, Bl
6316	6313	break;
6317	6314	}
6318	6315	} else {
6319		Buf *variable_name = &asm_output->variable_name;
	6316	Buf *variable_name = asm_output->variable_name;
6320	6317	VariableTableEntry *var = find_variable(g, context, variable_name);
6321	6318	if (var) {
6322	6319	asm_output->variable = var;
...	...	@@ -6351,7 +6348,7 @@ static TypeTableEntry analyze_goto_pass1(CodeGen g, ImportTableEntry *import,
6351	6348
6352	6349	static void analyze_goto_pass2(CodeGen g, ImportTableEntry import, AstNode *node) {
6353	6350	assert(node->type == NodeTypeGoto);
6354		Buf *label_name = &node->data.goto_expr.name;
	6351	Buf *label_name = node->data.goto_expr.name;
6355	6352	BlockContext *context = node->block_context;
6356	6353	assert(context);
6357	6354	LabelTableEntry *label = find_label(g, context, label_name);
...	...	@@ -6549,11 +6546,11 @@ static void analyze_fn_body(CodeGen g, FnTableEntry fn_table_entry) {
6549	6546	buf_sprintf("byvalue struct parameters not yet supported on extern functions"));
6550	6547	}
6551	6548
6552		if (buf_len(&param_decl->name) == 0) {
	6549	if (buf_len(param_decl->name) == 0) {
6553	6550	add_node_error(g, param_decl_node, buf_sprintf("missing parameter name"));
6554	6551	}
6555	6552
6556		VariableTableEntry *var = add_local_var(g, param_decl_node, import, context, &param_decl->name,
	6553	VariableTableEntry *var = add_local_var(g, param_decl_node, import, context, param_decl->name,
6557	6554	type, true, nullptr);
6558	6555	var->src_arg_index = i;
6559	6556	param_decl_node->data.param_decl.variable = var;
...	...	@@ -6583,7 +6580,7 @@ static void analyze_fn_body(CodeGen g, FnTableEntry fn_table_entry) {
6583	6580	if (!label->used) {
6584	6581	add_node_error(g, label->decl_node,
6585	6582	buf_sprintf("label '%s' defined but not used",
6586		buf_ptr(&label->decl_node->data.label.name)));
	6583	buf_ptr(label->decl_node->data.label.name)));
6587	6584	}
6588	6585	}
6589	6586
...	...	@@ -6640,7 +6637,7 @@ static void scan_decls(CodeGen g, ImportTableEntry import, BlockContext *conte
6640	6637	break;
6641	6638	case NodeTypeContainerDecl:
6642	6639	{
6643		Buf *name = &node->data.struct_decl.name;
	6640	Buf *name = node->data.struct_decl.name;
6644	6641	add_top_level_decl(g, import, context, node, name);
6645	6642	if (node->data.struct_decl.generic_params.length == 0) {
6646	6643	scan_struct_decl(g, import, context, node);
...	...	@@ -6653,20 +6650,20 @@ static void scan_decls(CodeGen g, ImportTableEntry import, BlockContext *conte
6653	6650	break;
6654	6651	case NodeTypeVariableDeclaration:
6655	6652	{
6656		Buf *name = &node->data.variable_declaration.symbol;
	6653	Buf *name = node->data.variable_declaration.symbol;
6657	6654	add_top_level_decl(g, import, context, node, name);
6658	6655	break;
6659	6656	}
6660	6657	case NodeTypeTypeDecl:
6661	6658	{
6662		Buf *name = &node->data.type_decl.symbol;
	6659	Buf *name = node->data.type_decl.symbol;
6663	6660	add_top_level_decl(g, import, context, node, name);
6664	6661	break;
6665	6662	}
6666	6663	case NodeTypeFnProto:
6667	6664	{
6668	6665	// if the name is missing, we immediately announce an error
6669		Buf *fn_name = &node->data.fn_proto.name;
	6666	Buf *fn_name = node->data.fn_proto.name;
6670	6667	if (buf_len(fn_name) == 0) {
6671	6668	node->data.fn_proto.skip = true;
6672	6669	add_node_error(g, node, buf_sprintf("missing function name"));
...	...	@@ -6851,6 +6848,9 @@ ImportTableEntry add_source_file(CodeGen g, PackageTableEntry *package,
6851	6848	assert(import_entry->root);
6852	6849	if (g->verbose) {
6853	6850	ast_print(stderr, import_entry->root, 0);
	6851	//fprintf(stderr, "\nReformatted Source:\n");
	6852	//fprintf(stderr, "---------------------\n");
	6853	//ast_render(stderr, import_entry->root, 4);
6854	6854	}
6855	6855
6856	6856	import_entry->di_file = LLVMZigCreateFile(g->dbuilder, buf_ptr(src_basename), buf_ptr(src_dirname));
...	...	@@ -6868,7 +6868,7 @@ ImportTableEntry add_source_file(CodeGen g, PackageTableEntry *package,
6868	6868	if (top_level_decl->type == NodeTypeFnDef) {
6869	6869	AstNode *proto_node = top_level_decl->data.fn_def.fn_proto;
6870	6870	assert(proto_node->type == NodeTypeFnProto);
6871		Buf *proto_name = &proto_node->data.fn_proto.name;
	6871	Buf *proto_name = proto_node->data.fn_proto.name;
6872	6872
6873	6873	bool is_private = (proto_node->data.fn_proto.top_level_decl.visib_mod == VisibModPrivate);
6874	6874
...	...	@@ -7064,7 +7064,7 @@ bool is_node_void_expr(AstNode *node) {
7064	7064	{
7065	7065	AstNode *type_node = node->data.container_init_expr.type;
7066	7066	if (type_node->type == NodeTypeSymbol &&
7067		buf_eql_str(&type_node->data.symbol_expr.symbol, "void"))
	7067	buf_eql_str(type_node->data.symbol_expr.symbol, "void"))
7068	7068	{
7069	7069	return true;
7070	7070	}

src/ast_render.cpp+62-22

...	...	@@ -78,6 +78,24 @@ static const char *visib_mod_string(VisibMod mod) {
78	78	zig_unreachable();
79	79	}
80	80
	81	static const char *return_string(ReturnKind kind) {
	82	switch (kind) {
	83	case ReturnKindUnconditional: return "return";
	84	case ReturnKindError: return "%return";
	85	case ReturnKindMaybe: return "?return";
	86	}
	87	zig_unreachable();
	88	}
	89
	90	static const char *defer_string(ReturnKind kind) {
	91	switch (kind) {
	92	case ReturnKindUnconditional: return "defer";
	93	case ReturnKindError: return "%defer";
	94	case ReturnKindMaybe: return "?defer";
	95	}
	96	zig_unreachable();
	97	}
	98
81	99	static const char *extern_string(bool is_extern) {
82	100	return is_extern ? "extern " : "";
83	101	}
...	...	@@ -243,7 +261,7 @@ static bool is_node_void(AstNode *node) {
243	261	if (node->type == NodeTypeSymbol) {
244	262	if (node->data.symbol_expr.override_type_entry) {
245	263	return node->data.symbol_expr.override_type_entry->id == TypeTableEntryIdVoid;
246		} else if (buf_eql_str(&node->data.symbol_expr.symbol, "void")) {
	264	} else if (buf_eql_str(node->data.symbol_expr.symbol, "void")) {
247	265	return true;
248	266	}
249	267	}
...	...	@@ -260,7 +278,12 @@ static bool is_digit(uint8_t c) {
260	278	}
261	279
262	280	static bool is_printable(uint8_t c) {
263		return is_alpha_under(c) \|\| is_digit(c) \|\| c == ' ';
	281	static const uint8_t printables[] =
	282	" abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.~`!@#$%^&*()_-+=\\{}[];'\"?/<>,";
	283	for (size_t i = 0; i < array_length(printables); i += 1) {
	284	if (c == printables[i]) return true;
	285	}
	286	return false;
264	287	}
265	288
266	289	static void string_literal_escape(Buf source, Buf dest) {
...	...	@@ -353,18 +376,18 @@ static void render_node(AstRender ar, AstNode node) {
353	376	const char *extern_str = extern_string(node->data.fn_proto.is_extern);
354	377	const char *inline_str = inline_string(node->data.fn_proto.is_inline);
355	378	fprintf(ar->f, "%s%s%sfn ", pub_str, inline_str, extern_str);
356		print_symbol(ar, &node->data.fn_proto.name);
	379	print_symbol(ar, node->data.fn_proto.name);
357	380	fprintf(ar->f, "(");
358	381	int arg_count = node->data.fn_proto.params.length;
359	382	bool is_var_args = node->data.fn_proto.is_var_args;
360	383	for (int arg_i = 0; arg_i < arg_count; arg_i += 1) {
361	384	AstNode *param_decl = node->data.fn_proto.params.at(arg_i);
362	385	assert(param_decl->type == NodeTypeParamDecl);
363		if (buf_len(&param_decl->data.param_decl.name) > 0) {
	386	if (buf_len(param_decl->data.param_decl.name) > 0) {
364	387	const char *noalias_str = param_decl->data.param_decl.is_noalias ? "noalias " : "";
365	388	const char *inline_str = param_decl->data.param_decl.is_inline ? "inline " : "";
366	389	fprintf(ar->f, "%s%s", noalias_str, inline_str);
367		print_symbol(ar, &param_decl->data.param_decl.name);
	390	print_symbol(ar, param_decl->data.param_decl.name);
368	391	fprintf(ar->f, ": ");
369	392	}
370	393	render_node(ar, param_decl->data.param_decl.type);
...	...	@@ -417,21 +440,31 @@ static void render_node(AstRender ar, AstNode node) {
417	440	fprintf(ar->f, "}");
418	441	break;
419	442	case NodeTypeDirective:
420		fprintf(ar->f, "#%s(", buf_ptr(&node->data.directive.name));
	443	fprintf(ar->f, "#%s(", buf_ptr(node->data.directive.name));
421	444	render_node(ar, node->data.directive.expr);
422	445	fprintf(ar->f, ")\n");
423	446	break;
424	447	case NodeTypeReturnExpr:
425		zig_panic("TODO");
	448	{
	449	const char *return_str = return_string(node->data.return_expr.kind);
	450	fprintf(ar->f, "%s ", return_str);
	451	render_node(ar, node->data.return_expr.expr);
	452	break;
	453	}
426	454	case NodeTypeDefer:
427		zig_panic("TODO");
	455	{
	456	const char *defer_str = defer_string(node->data.defer.kind);
	457	fprintf(ar->f, "%s ", defer_str);
	458	render_node(ar, node->data.return_expr.expr);
	459	break;
	460	}
428	461	case NodeTypeVariableDeclaration:
429	462	{
430	463	const char *pub_str = visib_mod_string(node->data.variable_declaration.top_level_decl.visib_mod);
431	464	const char *extern_str = extern_string(node->data.variable_declaration.is_extern);
432	465	const char *const_or_var = const_or_var_string(node->data.variable_declaration.is_const);
433	466	fprintf(ar->f, "%s%s%s ", pub_str, extern_str, const_or_var);
434		print_symbol(ar, &node->data.variable_declaration.symbol);
	467	print_symbol(ar, node->data.variable_declaration.symbol);
435	468
436	469	if (node->data.variable_declaration.type) {
437	470	fprintf(ar->f, ": ");
...	...	@@ -446,7 +479,7 @@ static void render_node(AstRender ar, AstNode node) {
446	479	case NodeTypeTypeDecl:
447	480	{
448	481	const char *pub_str = visib_mod_string(node->data.type_decl.top_level_decl.visib_mod);
449		const char *var_name = buf_ptr(&node->data.type_decl.symbol);
	482	const char *var_name = buf_ptr(node->data.type_decl.symbol);
450	483	fprintf(ar->f, "%stype %s = ", pub_str, var_name);
451	484	render_node(ar, node->data.type_decl.child_type);
452	485	break;
...	...	@@ -463,12 +496,15 @@ static void render_node(AstRender ar, AstNode node) {
463	496	case NodeTypeUnwrapErrorExpr:
464	497	zig_panic("TODO");
465	498	case NodeTypeNumberLiteral:
466		switch (node->data.number_literal.kind) {
467		case NumLitUInt:
468		fprintf(ar->f, "%" PRIu64, node->data.number_literal.data.x_uint);
	499	switch (node->data.number_literal.bignum->kind) {
	500	case BigNumKindInt:
	501	{
	502	const char *negative_str = node->data.number_literal.bignum->is_negative ? "-" : "";
	503	fprintf(ar->f, "%s%llu", negative_str, node->data.number_literal.bignum->data.x_uint);
	504	}
469	505	break;
470		case NumLitFloat:
471		fprintf(ar->f, "%f", node->data.number_literal.data.x_float);
	506	case BigNumKindFloat:
	507	fprintf(ar->f, "%f", node->data.number_literal.bignum->data.x_float);
472	508	break;
473	509	}
474	510	break;
...	...	@@ -478,7 +514,7 @@ static void render_node(AstRender ar, AstNode node) {
478	514	fprintf(ar->f, "c");
479	515	}
480	516	Buf tmp_buf = BUF_INIT;
481		string_literal_escape(&node->data.string_literal.buf, &tmp_buf);
	517	string_literal_escape(node->data.string_literal.buf, &tmp_buf);
482	518	fprintf(ar->f, "\"%s\"", buf_ptr(&tmp_buf));
483	519	}
484	520	break;
...	...	@@ -498,7 +534,7 @@ static void render_node(AstRender ar, AstNode node) {
498	534	if (override_type) {
499	535	fprintf(ar->f, "%s", buf_ptr(&override_type->name));
500	536	} else {
501		fprintf(ar->f, "%s", buf_ptr(&node->data.symbol_expr.symbol));
	537	print_symbol(ar, node->data.symbol_expr.symbol);
502	538	}
503	539	}
504	540	break;
...	...	@@ -513,10 +549,14 @@ static void render_node(AstRender ar, AstNode node) {
513	549	case NodeTypeFnCallExpr:
514	550	if (node->data.fn_call_expr.is_builtin) {
515	551	fprintf(ar->f, "@");
	552	} else {
	553	fprintf(ar->f, "(");
516	554	}
517		fprintf(ar->f, "(");
518	555	render_node(ar, node->data.fn_call_expr.fn_ref_expr);
519		fprintf(ar->f, ")(");
	556	if (!node->data.fn_call_expr.is_builtin) {
	557	fprintf(ar->f, ")");
	558	}
	559	fprintf(ar->f, "(");
520	560	for (int i = 0; i < node->data.fn_call_expr.params.length; i += 1) {
521	561	AstNode *param = node->data.fn_call_expr.params.at(i);
522	562	if (i != 0) {
...	...	@@ -537,7 +577,7 @@ static void render_node(AstRender ar, AstNode node) {
537	577	case NodeTypeFieldAccessExpr:
538	578	{
539	579	AstNode *lhs = node->data.field_access_expr.struct_expr;
540		Buf *rhs = &node->data.field_access_expr.field_name;
	580	Buf *rhs = node->data.field_access_expr.field_name;
541	581	render_node(ar, lhs);
542	582	fprintf(ar->f, ".");
543	583	print_symbol(ar, rhs);
...	...	@@ -577,7 +617,7 @@ static void render_node(AstRender ar, AstNode node) {
577	617	zig_panic("TODO");
578	618	case NodeTypeContainerDecl:
579	619	{
580		const char *struct_name = buf_ptr(&node->data.struct_decl.name);
	620	const char *struct_name = buf_ptr(node->data.struct_decl.name);
581	621	const char *pub_str = visib_mod_string(node->data.struct_decl.top_level_decl.visib_mod);
582	622	const char *container_str = container_string(node->data.struct_decl.kind);
583	623	fprintf(ar->f, "%s%s %s {\n", pub_str, container_str, struct_name);
...	...	@@ -586,7 +626,7 @@ static void render_node(AstRender ar, AstNode node) {
586	626	AstNode *field_node = node->data.struct_decl.fields.at(field_i);
587	627	assert(field_node->type == NodeTypeStructField);
588	628	print_indent(ar);
589		print_symbol(ar, &field_node->data.struct_field.name);
	629	print_symbol(ar, field_node->data.struct_field.name);
590	630	if (!is_node_void(field_node->data.struct_field.type)) {
591	631	fprintf(ar->f, ": ");
592	632	render_node(ar, field_node->data.struct_field.type);

src/bignum.cpp+17

...	...	@@ -6,6 +6,7 @@
6	6	*/
7	7
8	8	#include "bignum.hpp"
	9	#include "buffer.hpp"
9	10
10	11	#include <assert.h>
11	12	#include <math.h>
...	...	@@ -41,6 +42,10 @@ void bignum_init_signed(BigNum *dest, int64_t x) {
41	42	}
42	43	}
43	44
	45	void bignum_init_bignum(BigNum dest, BigNum src) {
	46	memcpy(dest, src, sizeof(BigNum));
	47	}
	48
44	49	bool bignum_fits_in_bits(BigNum *bn, int bit_count, bool is_signed) {
45	50	assert(bn->kind == BigNumKindInt);
46	51
...	...	@@ -343,3 +348,15 @@ bool bignum_cmp_gte(BigNum op1, BigNum op2) {
343	348	return true;
344	349	}
345	350	}
	351
	352	bool bignum_increment_by_scalar(BigNum *bignum, uint64_t scalar) {
	353	assert(bignum->kind == BigNumKindInt);
	354	assert(!bignum->is_negative);
	355	return __builtin_uaddll_overflow(bignum->data.x_uint, scalar, &bignum->data.x_uint);
	356	}
	357
	358	bool bignum_multiply_by_scalar(BigNum *bignum, uint64_t scalar) {
	359	assert(bignum->kind == BigNumKindInt);
	360	assert(!bignum->is_negative);
	361	return __builtin_umulll_overflow(bignum->data.x_uint, scalar, &bignum->data.x_uint);
	362	}

src/bignum.hpp+10-1

...	...	@@ -5,7 +5,8 @@
5	5	* See http://opensource.org/licenses/MIT
6	6	*/
7	7
8		#include "buffer.hpp"
	8	#ifndef ZIG_BIGNUM_HPP
	9	#define ZIG_BIGNUM_HPP
9	10
10	11	#include <stdint.h>
11	12
...	...	@@ -26,6 +27,7 @@ struct BigNum {
26	27	void bignum_init_float(BigNum *dest, double x);
27	28	void bignum_init_unsigned(BigNum *dest, uint64_t x);
28	29	void bignum_init_signed(BigNum *dest, int64_t x);
	30	void bignum_init_bignum(BigNum dest, BigNum src);
29	31
30	32	bool bignum_fits_in_bits(BigNum *bn, int bit_count, bool is_signed);
31	33	uint64_t bignum_to_twos_complement(BigNum *bn);
...	...	@@ -57,4 +59,11 @@ bool bignum_cmp_gt(BigNum op1, BigNum op2);
57	59	bool bignum_cmp_lte(BigNum op1, BigNum op2);
58	60	bool bignum_cmp_gte(BigNum op1, BigNum op2);
59	61
	62	// helper functions
	63	bool bignum_increment_by_scalar(BigNum *bignum, uint64_t scalar);
	64	bool bignum_multiply_by_scalar(BigNum *bignum, uint64_t scalar);
	65
	66	struct Buf;
60	67	Buf bignum_to_buf(BigNum bn);
	68
	69	#endif

src/codegen.cpp+14-14

...	...	@@ -1431,7 +1431,7 @@ static LLVMValueRef gen_field_access_expr(CodeGen g, AstNode node, bool is_lva
1431	1431	TypeTableEntry *struct_type = get_expr_type(struct_expr);
1432	1432
1433	1433	if (struct_type->id == TypeTableEntryIdArray) {
1434		Buf *name = &node->data.field_access_expr.field_name;
	1434	Buf *name = node->data.field_access_expr.field_name;
1435	1435	assert(buf_eql_str(name, "len"));
1436	1436	return LLVMConstInt(g->builtin_types.entry_usize->type_ref,
1437	1437	struct_type->data.array.len, false);
...	...	@@ -2726,18 +2726,18 @@ static LLVMValueRef gen_block(CodeGen g, AstNode block_node, TypeTableEntry *i
2726	2726	}
2727	2727
2728	2728	static int find_asm_index(CodeGen g, AstNode node, AsmToken *tok) {
2729		const char *ptr = buf_ptr(&node->data.asm_expr.asm_template) + tok->start + 2;
	2729	const char *ptr = buf_ptr(node->data.asm_expr.asm_template) + tok->start + 2;
2730	2730	int len = tok->end - tok->start - 2;
2731	2731	int result = 0;
2732	2732	for (int i = 0; i < node->data.asm_expr.output_list.length; i += 1, result += 1) {
2733	2733	AsmOutput *asm_output = node->data.asm_expr.output_list.at(i);
2734		if (buf_eql_mem(&asm_output->asm_symbolic_name, ptr, len)) {
	2734	if (buf_eql_mem(asm_output->asm_symbolic_name, ptr, len)) {
2735	2735	return result;
2736	2736	}
2737	2737	}
2738	2738	for (int i = 0; i < node->data.asm_expr.input_list.length; i += 1, result += 1) {
2739	2739	AsmInput *asm_input = node->data.asm_expr.input_list.at(i);
2740		if (buf_eql_mem(&asm_input->asm_symbolic_name, ptr, len)) {
	2740	if (buf_eql_mem(asm_input->asm_symbolic_name, ptr, len)) {
2741	2741	return result;
2742	2742	}
2743	2743	}
...	...	@@ -2749,7 +2749,7 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {
2749	2749
2750	2750	AstNodeAsmExpr *asm_expr = &node->data.asm_expr;
2751	2751
2752		Buf *src_template = &asm_expr->asm_template;
	2752	Buf *src_template = asm_expr->asm_template;
2753	2753
2754	2754	Buf llvm_template = BUF_INIT;
2755	2755	buf_resize(&llvm_template, 0);
...	...	@@ -2796,11 +2796,11 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {
2796	2796	for (int i = 0; i < asm_expr->output_list.length; i += 1, total_index += 1) {
2797	2797	AsmOutput *asm_output = asm_expr->output_list.at(i);
2798	2798	bool is_return = (asm_output->return_type != nullptr);
2799		assert(*buf_ptr(&asm_output->constraint) == '=');
	2799	assert(*buf_ptr(asm_output->constraint) == '=');
2800	2800	if (is_return) {
2801		buf_appendf(&constraint_buf, "=%s", buf_ptr(&asm_output->constraint) + 1);
	2801	buf_appendf(&constraint_buf, "=%s", buf_ptr(asm_output->constraint) + 1);
2802	2802	} else {
2803		buf_appendf(&constraint_buf, "=*%s", buf_ptr(&asm_output->constraint) + 1);
	2803	buf_appendf(&constraint_buf, "=*%s", buf_ptr(asm_output->constraint) + 1);
2804	2804	}
2805	2805	if (total_index + 1 < total_constraint_count) {
2806	2806	buf_append_char(&constraint_buf, ',');
...	...	@@ -2816,7 +2816,7 @@ static LLVMValueRef gen_asm_expr(CodeGen g, AstNode node) {
2816	2816	}
2817	2817	for (int i = 0; i < asm_expr->input_list.length; i += 1, total_index += 1, param_index += 1) {
2818	2818	AsmInput *asm_input = asm_expr->input_list.at(i);
2819		buf_append_buf(&constraint_buf, &asm_input->constraint);
	2819	buf_append_buf(&constraint_buf, asm_input->constraint);
2820	2820	if (total_index + 1 < total_constraint_count) {
2821	2821	buf_append_char(&constraint_buf, ',');
2822	2822	}
...	...	@@ -2885,7 +2885,7 @@ static LLVMValueRef gen_container_init_expr(CodeGen g, AstNode node) {
2885	2885	if (type_struct_field->type_entry->id == TypeTableEntryIdVoid) {
2886	2886	continue;
2887	2887	}
2888		assert(buf_eql_buf(type_struct_field->name, &field_node->data.struct_val_field.name));
	2888	assert(buf_eql_buf(type_struct_field->name, field_node->data.struct_val_field.name));
2889	2889
2890	2890	set_debug_source_node(g, field_node);
2891	2891	LLVMValueRef field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, type_struct_field->gen_index, "");
...	...	@@ -3853,7 +3853,7 @@ static void generate_error_name_table(CodeGen *g) {
3853	3853	for (int i = 1; i < g->error_decls.length; i += 1) {
3854	3854	AstNode *error_decl_node = g->error_decls.at(i);
3855	3855	assert(error_decl_node->type == NodeTypeErrorValueDecl);
3856		Buf *name = &error_decl_node->data.error_value_decl.name;
	3856	Buf *name = error_decl_node->data.error_value_decl.name;
3857	3857
3858	3858	LLVMValueRef str_init = LLVMConstString(buf_ptr(name), buf_len(name), true);
3859	3859	LLVMValueRef str_global = LLVMAddGlobal(g->module, LLVMTypeOf(str_init), "");
...	...	@@ -3882,7 +3882,7 @@ static void build_label_blocks(CodeGen g, FnTableEntry fn) {
3882	3882	LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn->fn_value, "entry");
3883	3883	for (int i = 0; i < fn->all_labels.length; i += 1) {
3884	3884	LabelTableEntry *label = fn->all_labels.at(i);
3885		Buf *name = &label->decl_node->data.label.name;
	3885	Buf *name = label->decl_node->data.label.name;
3886	3886	label->basic_block = LLVMAppendBasicBlock(fn->fn_value, buf_ptr(name));
3887	3887	}
3888	3888	LLVMPositionBuilderAtEnd(g->builder, entry_block);
...	...	@@ -4951,7 +4951,7 @@ void codegen_generate_h_file(CodeGen *g) {
4951	4951	buf_appendf(&h_buf, "%s %s %s(",
4952	4952	buf_ptr(export_macro),
4953	4953	buf_ptr(&return_type_c),
4954		buf_ptr(&fn_proto->name));
	4954	buf_ptr(fn_proto->name));
4955	4955
4956	4956	Buf param_type_c = BUF_INIT;
4957	4957	if (fn_proto->params.length) {
...	...	@@ -4961,7 +4961,7 @@ void codegen_generate_h_file(CodeGen *g) {
4961	4961	to_c_type(g, param_type, &param_type_c);
4962	4962	buf_appendf(&h_buf, "%s %s",
4963	4963	buf_ptr(&param_type_c),
4964		buf_ptr(&param_decl_node->data.param_decl.name));
	4964	buf_ptr(param_decl_node->data.param_decl.name));
4965	4965	if (param_i < fn_proto->params.length - 1)
4966	4966	buf_appendf(&h_buf, ", ");
4967	4967	}

src/eval.cpp+7-13

...	...	@@ -427,7 +427,7 @@ static EvalVar find_var(EvalFn ef, Buf *name) {
427	427	static bool eval_symbol_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {
428	428	assert(node->type == NodeTypeSymbol);
429	429
430		Buf *name = &node->data.symbol_expr.symbol;
	430	Buf *name = node->data.symbol_expr.symbol;
431	431	EvalVar *var = find_var(ef, name);
432	432	assert(var);
433	433
...	...	@@ -924,7 +924,7 @@ static bool eval_field_access_expr(EvalFn ef, AstNode node, ConstExprValue *ou
924	924	TypeTableEntry *struct_type = get_resolved_expr(struct_expr)->type_entry;
925	925
926	926	if (struct_type->id == TypeTableEntryIdArray) {
927		Buf *name = &node->data.field_access_expr.field_name;
	927	Buf *name = node->data.field_access_expr.field_name;
928	928	assert(buf_eql_str(name, "len"));
929	929	zig_panic("TODO");
930	930	} else if (struct_type->id == TypeTableEntryIdStruct \|\| (struct_type->id == TypeTableEntryIdPointer &&
...	...	@@ -971,7 +971,7 @@ static bool eval_for_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {
971	971	if (eval_expr(ef, array_node, &array_val)) return true;
972	972
973	973	assert(elem_node->type == NodeTypeSymbol);
974		Buf *elem_var_name = &elem_node->data.symbol_expr.symbol;
	974	Buf *elem_var_name = elem_node->data.symbol_expr.symbol;
975	975
976	976	if (node->data.for_expr.elem_is_ptr) {
977	977	zig_panic("TODO");
...	...	@@ -980,7 +980,7 @@ static bool eval_for_expr(EvalFn ef, AstNode node, ConstExprValue *out_val) {
980	980	Buf *index_var_name = nullptr;
981	981	if (index_node) {
982	982	assert(index_node->type == NodeTypeSymbol);
983		index_var_name = &index_node->data.symbol_expr.symbol;
	983	index_var_name = index_node->data.symbol_expr.symbol;
984	984	}
985	985
986	986	uint64_t it_index = 0;
...	...	@@ -1164,7 +1164,7 @@ static bool eval_var_decl_expr(EvalFn ef, AstNode node, ConstExprValue *out_va
1164	1164
1165	1165	my_scope->vars.add_one();
1166	1166	EvalVar *var = &my_scope->vars.last();
1167		var->name = &node->data.variable_declaration.symbol;
	1167	var->name = node->data.variable_declaration.symbol;
1168	1168
1169	1169	if (eval_expr(ef, node->data.variable_declaration.expr, &var->value)) return true;
1170	1170
...	...	@@ -1178,13 +1178,7 @@ static bool eval_number_literal_expr(EvalFn ef, AstNode node, ConstExprValue *
1178	1178	assert(!node->data.number_literal.overflow);
1179	1179
1180	1180	out_val->ok = true;
1181		if (node->data.number_literal.kind == NumLitUInt) {
1182		bignum_init_unsigned(&out_val->data.x_bignum, node->data.number_literal.data.x_uint);
1183		} else if (node->data.number_literal.kind == NumLitFloat) {
1184		bignum_init_float(&out_val->data.x_bignum, node->data.number_literal.data.x_float);
1185		} else {
1186		zig_unreachable();
1187		}
	1181	bignum_init_bignum(&out_val->data.x_bignum, node->data.number_literal.bignum);
1188	1182
1189	1183	return false;
1190	1184	}
...	...	@@ -1339,7 +1333,7 @@ static bool eval_fn_args(EvalFnRoot efr, FnTableEntry fn, ConstExprValue *args
1339	1333
1340	1334	root_scope->vars.add_one();
1341	1335	EvalVar *eval_var = &root_scope->vars.last();
1342		eval_var->name = &decl_param_node->data.param_decl.name;
	1336	eval_var->name = decl_param_node->data.param_decl.name;
1343	1337	eval_var->value = *src_const_val;
1344	1338	}
1345	1339

src/parseh.cpp+23-23

...	...	@@ -104,14 +104,14 @@ static AstNode create_node(Context c, NodeType type) {
104	104
105	105	static AstNode create_symbol_node(Context c, const char *type_name) {
106	106	AstNode *node = create_node(c, NodeTypeSymbol);
107		buf_init_from_str(&node->data.symbol_expr.symbol, type_name);
	107	node->data.symbol_expr.symbol = buf_create_from_str(type_name);
108	108	return node;
109	109	}
110	110
111	111	static AstNode create_field_access_node(Context c, const char lhs, const char rhs) {
112	112	AstNode *node = create_node(c, NodeTypeFieldAccessExpr);
113	113	node->data.field_access_expr.struct_expr = create_symbol_node(c, lhs);
114		buf_init_from_str(&node->data.field_access_expr.field_name, rhs);
	114	node->data.field_access_expr.field_name = buf_create_from_str(rhs);
115	115	normalize_parent_ptrs(node);
116	116	return node;
117	117	}
...	...	@@ -120,7 +120,7 @@ static AstNode create_typed_var_decl_node(Context c, bool is_const, const char
120	120	AstNode type_node, AstNode init_node)
121	121	{
122	122	AstNode *node = create_node(c, NodeTypeVariableDeclaration);
123		buf_init_from_str(&node->data.variable_declaration.symbol, var_name);
	123	node->data.variable_declaration.symbol = buf_create_from_str(var_name);
124	124	node->data.variable_declaration.is_const = is_const;
125	125	node->data.variable_declaration.top_level_decl.visib_mod = c->visib_mod;
126	126	node->data.variable_declaration.expr = init_node;
...	...	@@ -146,7 +146,7 @@ static AstNode create_prefix_node(Context c, PrefixOp op, AstNode *child_node)
146	146	static AstNode create_struct_field_node(Context c, const char name, AstNode type_node) {
147	147	assert(type_node);
148	148	AstNode *node = create_node(c, NodeTypeStructField);
149		buf_init_from_str(&node->data.struct_field.name, name);
	149	node->data.struct_field.name = buf_create_from_str(name);
150	150	node->data.struct_field.top_level_decl.visib_mod = VisibModPub;
151	151	node->data.struct_field.type = type_node;
152	152
...	...	@@ -157,7 +157,7 @@ static AstNode create_struct_field_node(Context c, const char name, AstNode
157	157	static AstNode create_param_decl_node(Context c, const char name, AstNode type_node, bool is_noalias) {
158	158	assert(type_node);
159	159	AstNode *node = create_node(c, NodeTypeParamDecl);
160		buf_init_from_str(&node->data.param_decl.name, name);
	160	node->data.param_decl.name = buf_create_from_str(name);
161	161	node->data.param_decl.type = type_node;
162	162	node->data.param_decl.is_noalias = is_noalias;
163	163
...	...	@@ -171,17 +171,18 @@ static AstNode create_char_lit_node(Context c, uint8_t value) {
171	171	return node;
172	172	}
173	173
	174	// accepts ownership of buf
174	175	static AstNode create_str_lit_node(Context c, Buf *buf) {
175	176	AstNode *node = create_node(c, NodeTypeStringLiteral);
176		buf_init_from_buf(&node->data.string_literal.buf, buf);
	177	node->data.string_literal.buf = buf;
177	178	node->data.string_literal.c = true;
178	179	return node;
179	180	}
180	181
181	182	static AstNode create_num_lit_float(Context c, double x) {
182	183	AstNode *node = create_node(c, NodeTypeNumberLiteral);
183		node->data.number_literal.kind = NumLitFloat;
184		node->data.number_literal.data.x_float = x;
	184	node->data.number_literal.bignum = allocate_nonzero<BigNum>(1);
	185	bignum_init_float(node->data.number_literal.bignum, x);
185	186	return node;
186	187	}
187	188
...	...	@@ -193,8 +194,8 @@ static AstNode create_num_lit_float_negative(Context c, double x, bool negativ
193	194
194	195	static AstNode create_num_lit_unsigned(Context c, uint64_t x) {
195	196	AstNode *node = create_node(c, NodeTypeNumberLiteral);
196		node->data.number_literal.kind = NumLitUInt;
197		node->data.number_literal.data.x_uint = x;
	197	node->data.number_literal.bignum = allocate_nonzero<BigNum>(1);
	198	bignum_init_unsigned(node->data.number_literal.bignum, x);
198	199	return node;
199	200	}
200	201
...	...	@@ -221,7 +222,7 @@ static AstNode create_num_lit_signed(Context c, int64_t x) {
221	222
222	223	static AstNode create_type_decl_node(Context c, const char name, AstNode child_type_node) {
223	224	AstNode *node = create_node(c, NodeTypeTypeDecl);
224		buf_init_from_str(&node->data.type_decl.symbol, name);
	225	node->data.type_decl.symbol = buf_create_from_str(name);
225	226	node->data.type_decl.top_level_decl.visib_mod = c->visib_mod;
226	227	node->data.type_decl.child_type = child_type_node;
227	228
...	...	@@ -240,7 +241,7 @@ static AstNode create_fn_proto_node(Context c, Buf name, TypeTableEntry fn_t
240	241	AstNode *node = create_node(c, NodeTypeFnProto);
241	242	node->data.fn_proto.is_inline = true;
242	243	node->data.fn_proto.top_level_decl.visib_mod = c->visib_mod;
243		buf_init_from_buf(&node->data.fn_proto.name, name);
	244	node->data.fn_proto.name = name;
244	245	node->data.fn_proto.return_type = make_type_node(c, fn_type->data.fn.fn_type_id.return_type);
245	246
246	247	for (int i = 0; i < fn_type->data.fn.fn_type_id.param_count; i += 1) {
...	...	@@ -273,7 +274,7 @@ static AstNode create_inline_fn_node(Context c, Buf fn_name, Buf var_name, T
273	274	fn_call_node->data.fn_call_expr.fn_ref_expr = unwrap_node;
274	275	for (int i = 0; i < fn_type->data.fn.fn_type_id.param_count; i += 1) {
275	276	AstNode *decl_node = node->data.fn_def.fn_proto->data.fn_proto.params.at(i);
276		Buf *param_name = &decl_node->data.param_decl.name;
	277	Buf *param_name = decl_node->data.param_decl.name;
277	278	fn_call_node->data.fn_call_expr.params.append(create_symbol_node(c, buf_ptr(param_name)));
278	279	}
279	280
...	...	@@ -686,10 +687,9 @@ static TypeTableEntry resolve_qual_type(Context c, QualType qt, const Decl *de
686	687	}
687	688
688	689	static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {
689		Buf fn_name = BUF_INIT;
690		buf_init_from_str(&fn_name, decl_name(fn_decl));
	690	Buf *fn_name = buf_create_from_str(decl_name(fn_decl));
691	691
692		if (c->fn_table.maybe_get(&fn_name)) {
	692	if (c->fn_table.maybe_get(fn_name)) {
693	693	// we already saw this function
694	694	return;
695	695	}
...	...	@@ -697,14 +697,14 @@ static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {
697	697	TypeTableEntry *fn_type = resolve_qual_type(c, fn_decl->getType(), fn_decl);
698	698
699	699	if (fn_type->id == TypeTableEntryIdInvalid) {
700		emit_warning(c, fn_decl, "ignoring function '%s' - unable to resolve type", buf_ptr(&fn_name));
	700	emit_warning(c, fn_decl, "ignoring function '%s' - unable to resolve type", buf_ptr(fn_name));
701	701	return;
702	702	}
703	703	assert(fn_type->id == TypeTableEntryIdFn);
704	704
705	705
706	706	AstNode *node = create_node(c, NodeTypeFnProto);
707		buf_init_from_buf(&node->data.fn_proto.name, &fn_name);
	707	node->data.fn_proto.name = fn_name;
708	708
709	709	node->data.fn_proto.is_extern = fn_type->data.fn.fn_type_id.is_extern;
710	710	node->data.fn_proto.top_level_decl.visib_mod = c->visib_mod;
...	...	@@ -731,7 +731,7 @@ static void visit_fn_decl(Context c, const FunctionDecl fn_decl) {
731	731
732	732	normalize_parent_ptrs(node);
733	733
734		c->fn_table.put(buf_create_from_buf(&fn_name), true);
	734	c->fn_table.put(buf_create_from_buf(fn_name), true);
735	735	c->root->data.root.top_level_decls.append(node);
736	736	}
737	737
...	...	@@ -937,7 +937,7 @@ static void visit_enum_decl(Context c, const EnumDecl enum_decl) {
937	937	if (enum_type->data.enumeration.complete) {
938	938	// now create top level decl for the type
939	939	AstNode *enum_node = create_node(c, NodeTypeContainerDecl);
940		buf_init_from_buf(&enum_node->data.struct_decl.name, &enum_type->name);
	940	enum_node->data.struct_decl.name = &enum_type->name;
941	941	enum_node->data.struct_decl.kind = ContainerKindEnum;
942	942	enum_node->data.struct_decl.top_level_decl.visib_mod = VisibModExport;
943	943	enum_node->data.struct_decl.type_entry = enum_type;
...	...	@@ -1114,7 +1114,7 @@ static void visit_record_decl(Context c, const RecordDecl record_decl) {
1114	1114	if (struct_type->data.structure.complete) {
1115	1115	// now create a top level decl node for the type
1116	1116	AstNode *struct_node = create_node(c, NodeTypeContainerDecl);
1117		buf_init_from_buf(&struct_node->data.struct_decl.name, &struct_type->name);
	1117	struct_node->data.struct_decl.name = &struct_type->name;
1118	1118	struct_node->data.struct_decl.kind = ContainerKindStruct;
1119	1119	struct_node->data.struct_decl.top_level_decl.visib_mod = VisibModExport;
1120	1120	struct_node->data.struct_decl.type_entry = struct_type;
...	...	@@ -1284,7 +1284,7 @@ static void render_aliases(Context *c) {
1284	1284	for (int i = 0; i < c->aliases.length; i += 1) {
1285	1285	AstNode *alias_node = c->aliases.at(i);
1286	1286	assert(alias_node->type == NodeTypeVariableDeclaration);
1287		Buf *name = &alias_node->data.variable_declaration.symbol;
	1287	Buf *name = alias_node->data.variable_declaration.symbol;
1288	1288	if (name_exists(c, name)) {
1289	1289	continue;
1290	1290	}
...	...	@@ -1327,7 +1327,7 @@ static void process_macro(Context c, CTokenize ctok, Buf name, const char ch
1327	1327	case CTokIdStrLit:
1328	1328	if (is_last && is_first) {
1329	1329	AstNode *var_node = create_var_decl_node(c, buf_ptr(name),
1330		create_str_lit_node(c, &tok->data.str_lit));
	1330	create_str_lit_node(c, buf_create_from_buf(&tok->data.str_lit)));
1331	1331	c->macro_table.put(name, var_node);
1332	1332	}
1333	1333	return;

src/parser.cpp+57-540

...	...	@@ -21,6 +21,9 @@ struct ParseContext {
21	21	ImportTableEntry *owner;
22	22	ErrColor err_color;
23	23	uint32_t *next_node_index;
	24	// These buffers are used freqently so we preallocate them once here.
	25	Buf *void_buf;
	26	Buf *empty_buf;
24	27	};
25	28
26	29	__attribute__ ((format (printf, 4, 5)))
...	...	@@ -29,7 +32,9 @@ static void ast_asm_error(ParseContext pc, AstNode node, int offset, const cha
29	32	assert(node->type == NodeTypeAsmExpr);
30	33
31	34
32		SrcPos pos = node->data.asm_expr.offset_map.at(offset);
	35	// TODO calculate or otherwise keep track of originating line/column number for strings
	36	//SrcPos pos = node->data.asm_expr.offset_map.at(offset);
	37	SrcPos pos = { node->line, node->column };
33	38
34	39	va_list ap;
35	40	va_start(ap, format);
...	...	@@ -83,12 +88,12 @@ static AstNode ast_create_node(ParseContext pc, NodeType type, Token *first_to
83	88
84	89	static AstNode ast_create_void_type_node(ParseContext pc, Token *token) {
85	90	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);
86		buf_init_from_str(&node->data.symbol_expr.symbol, "void");
	91	node->data.symbol_expr.symbol = pc->void_buf;
87	92	return node;
88	93	}
89	94
90	95	static void parse_asm_template(ParseContext pc, AstNode node) {
91		Buf *asm_template = &node->data.asm_expr.asm_template;
	96	Buf *asm_template = node->data.asm_expr.asm_template;
92	97
93	98	enum State {
94	99	StateStart,
...	...	@@ -170,514 +175,29 @@ static void parse_asm_template(ParseContext pc, AstNode node) {
170	175	}
171	176	}
172	177
173		static uint8_t parse_char_literal(ParseContext pc, Token token) {
174		// skip the single quotes at beginning and end
175		// convert escape sequences
176		bool escape = false;
177		int return_count = 0;
178		uint8_t return_value;
179		for (int i = token->start_pos + 1; i < token->end_pos - 1; i += 1) {
180		uint8_t c = ((uint8_t)buf_ptr(pc->buf) + i);
181		if (escape) {
182		switch (c) {
183		case '\\':
184		return_value = '\\';
185		return_count += 1;
186		break;
187		case 'r':
188		return_value = '\r';
189		return_count += 1;
190		break;
191		case 'n':
192		return_value = '\n';
193		return_count += 1;
194		break;
195		case 't':
196		return_value = '\t';
197		return_count += 1;
198		break;
199		case '\'':
200		return_value = '\'';
201		return_count += 1;
202		break;
203		default:
204		ast_error(pc, token, "invalid escape character");
205		}
206		escape = false;
207		} else if (c == '\\') {
208		escape = true;
209		} else {
210		return_value = c;
211		return_count += 1;
212		}
213		}
214		if (return_count == 0) {
215		ast_error(pc, token, "character literal too short");
216		} else if (return_count > 1) {
217		ast_error(pc, token, "character literal too long");
218		}
219		return return_value;
220		}
221
222		static uint32_t get_hex_digit(uint8_t c) {
223		switch (c) {
224		case '0': return 0;
225		case '1': return 1;
226		case '2': return 2;
227		case '3': return 3;
228		case '4': return 4;
229		case '5': return 5;
230		case '6': return 6;
231		case '7': return 7;
232		case '8': return 8;
233		case '9': return 9;
234
235		case 'a':
236		case 'A':
237		return 10;
238		case 'b':
239		case 'B':
240		return 11;
241		case 'c':
242		case 'C':
243		return 12;
244		case 'd':
245		case 'D':
246		return 13;
247		case 'e':
248		case 'E':
249		return 14;
250		case 'f':
251		case 'F':
252		return 15;
253		default:
254		return UINT32_MAX;
255		}
256		}
257
258		static void parse_string_literal(ParseContext pc, Token token, Buf buf, bool out_c_str,
259		ZigList<SrcPos> *offset_map)
260		{
261		if (token->raw_string_start > 0) {
262		uint8_t c1 = ((uint8_t)buf_ptr(pc->buf) + token->start_pos);
263		uint8_t c2 = ((uint8_t)buf_ptr(pc->buf) + token->start_pos + 1);
264		assert(c1 == 'r');
265		if (out_c_str) {
266		*out_c_str = (c2 == 'c');
267		}
268		const char *str = buf_ptr(pc->buf) + token->raw_string_start;
269		buf_init_from_mem(buf, str, token->raw_string_end - token->raw_string_start);
270		if (offset_map) {
271		SrcPos pos = {token->start_line, token->start_column};
272		for (int i = token->start_pos; i < token->raw_string_start; i += 1) {
273		uint8_t c = buf_ptr(pc->buf)[i];
274		if (c == '\n') {
275		pos.line += 1;
276		pos.column = 0;
277		} else {
278		pos.column += 1;
279		}
280		}
281		for (int i = token->raw_string_start; i < token->raw_string_end; i += 1) {
282		offset_map->append(pos);
283
284		uint8_t c = buf_ptr(pc->buf)[i];
285		if (c == '\n') {
286		pos.line += 1;
287		pos.column = 0;
288		} else {
289		pos.column += 1;
290		}
291		}
292		}
293		return;
294		}
295
296		// skip the double quotes at beginning and end
297		// convert escape sequences
298		// detect c string literal
299
300		enum State {
301		StatePre,
302		StateSkipQuot,
303		StateStart,
304		StateEscape,
305		StateHex1,
306		StateHex2,
307		StateUnicode,
308		};
309
310		buf_resize(buf, 0);
311
312		int unicode_index;
313		int unicode_end;
314
315		State state = StatePre;
316		SrcPos pos = {token->start_line, token->start_column};
317		uint32_t hex_value = 0;
318		for (int i = token->start_pos; i < token->end_pos - 1; i += 1) {
319		uint8_t c = ((uint8_t)buf_ptr(pc->buf) + i);
320
321		switch (state) {
322		case StatePre:
323		switch (c) {
324		case '@':
325		state = StateSkipQuot;
326		break;
327		case 'c':
328		if (out_c_str) {
329		*out_c_str = true;
330		} else {
331		ast_error(pc, token, "C string literal not allowed here");
332		}
333		state = StateSkipQuot;
334		break;
335		case '"':
336		state = StateStart;
337		break;
338		default:
339		ast_error(pc, token, "invalid string character");
340		}
341		break;
342		case StateSkipQuot:
343		state = StateStart;
344		break;
345		case StateStart:
346		if (c == '\\') {
347		state = StateEscape;
348		} else {
349		buf_append_char(buf, c);
350		if (offset_map) offset_map->append(pos);
351		}
352		break;
353		case StateEscape:
354		switch (c) {
355		case '\\':
356		buf_append_char(buf, '\\');
357		if (offset_map) offset_map->append(pos);
358		state = StateStart;
359		break;
360		case 'r':
361		buf_append_char(buf, '\r');
362		if (offset_map) offset_map->append(pos);
363		state = StateStart;
364		break;
365		case 'n':
366		buf_append_char(buf, '\n');
367		if (offset_map) offset_map->append(pos);
368		state = StateStart;
369		break;
370		case 't':
371		buf_append_char(buf, '\t');
372		if (offset_map) offset_map->append(pos);
373		state = StateStart;
374		break;
375		case '"':
376		buf_append_char(buf, '"');
377		if (offset_map) offset_map->append(pos);
378		state = StateStart;
379		break;
380		case '\'':
381		buf_append_char(buf, '\'');
382		if (offset_map) offset_map->append(pos);
383		state = StateStart;
384		break;
385		case 'x':
386		state = StateHex1;
387		break;
388		case 'u':
389		state = StateUnicode;
390		unicode_index = 0;
391		unicode_end = 4;
392		hex_value = 0;
393		break;
394		case 'U':
395		state = StateUnicode;
396		unicode_index = 0;
397		unicode_end = 6;
398		hex_value = 0;
399		break;
400		default:
401		ast_error(pc, token, "invalid escape character");
402		}
403		break;
404		case StateHex1:
405		{
406		uint32_t hex_digit = get_hex_digit(c);
407		if (hex_digit == UINT32_MAX) {
408		ast_error(pc, token, "invalid hex digit: '%c'", c);
409		}
410		hex_value = hex_digit * 16;
411		state = StateHex2;
412		break;
413		}
414		case StateHex2:
415		{
416		uint32_t hex_digit = get_hex_digit(c);
417		if (hex_digit == UINT32_MAX) {
418		ast_error(pc, token, "invalid hex digit: '%c'", c);
419		}
420		hex_value += hex_digit;
421		assert(hex_value >= 0 && hex_value <= 255);
422		buf_append_char(buf, hex_value);
423		state = StateStart;
424		break;
425		}
426		case StateUnicode:
427		{
428		uint32_t hex_digit = get_hex_digit(c);
429		if (hex_digit == UINT32_MAX) {
430		ast_error(pc, token, "invalid hex digit: '%c'", c);
431		}
432		hex_value *= 16;
433		hex_value += hex_digit;
434		unicode_index += 1;
435		if (unicode_index >= unicode_end) {
436		if (hex_value <= 0x7f) {
437		// 00000000 00000000 00000000 0xxxxxxx
438		buf_append_char(buf, hex_value);
439		} else if (hex_value <= 0x7ff) {
440		// 00000000 00000000 00000xxx xx000000
441		buf_append_char(buf, (unsigned char)(0xc0 \| (hex_value >> 6)));
442		// 00000000 00000000 00000000 00xxxxxx
443		buf_append_char(buf, (unsigned char)(0x80 \| (hex_value & 0x3f)));
444		} else if (hex_value <= 0xffff) {
445		// 00000000 00000000 xxxx0000 00000000
446		buf_append_char(buf, (unsigned char)(0xe0 \| (hex_value >> 12)));
447		// 00000000 00000000 0000xxxx xx000000
448		buf_append_char(buf, (unsigned char)(0x80 \| ((hex_value >> 6) & 0x3f)));
449		// 00000000 00000000 00000000 00xxxxxx
450		buf_append_char(buf, (unsigned char)(0x80 \| (hex_value & 0x3f)));
451		} else if (hex_value <= 0x10ffff) {
452		// 00000000 000xxx00 00000000 00000000
453		buf_append_char(buf, (unsigned char)(0xf0 \| (hex_value >> 18)));
454		// 00000000 000000xx xxxx0000 00000000
455		buf_append_char(buf, (unsigned char)(0x80 \| ((hex_value >> 12) & 0x3f)));
456		// 00000000 00000000 0000xxxx xx000000
457		buf_append_char(buf, (unsigned char)(0x80 \| ((hex_value >> 6) & 0x3f)));
458		// 00000000 00000000 00000000 00xxxxxx
459		buf_append_char(buf, (unsigned char)(0x80 \| (hex_value & 0x3f)));
460		} else {
461		ast_error(pc, token, "unicode value out of range: %x", hex_value);
462		}
463		state = StateStart;
464		}
465		break;
466		}
467		}
468		if (c == '\n') {
469		pos.line += 1;
470		pos.column = 0;
471		} else {
472		pos.column += 1;
473		}
474		}
475		assert(state == StateStart);
476		if (offset_map) offset_map->append(pos);
	178	static Buf token_buf(Token token) {
	179	assert(token->id == TokenIdStringLiteral \|\| token->id == TokenIdSymbol);
	180	return &token->data.str_lit.str;
477	181	}
478	182
479		static void ast_buf_from_token(ParseContext pc, Token token, Buf *buf) {
480		uint8_t first_char = (uint8_t )buf_ptr(pc->buf) + token->start_pos;
481		bool at_sign = *first_char == '@';
482		if (at_sign) {
483		parse_string_literal(pc, token, buf, nullptr, nullptr);
484		} else {
485		buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
486		}
	183	static BigNum token_bignum(Token token) {
	184	assert(token->id == TokenIdNumberLiteral);
	185	return &token->data.num_lit.bignum;
487	186	}
488	187
489
490		static unsigned long long parse_int_digits(ParseContext *pc, int digits_start, int digits_end, int radix,
491		int skip_index, bool *overflow)
492		{
493		unsigned long long x = 0;
494
495		for (int i = digits_start; i < digits_end; i++) {
496		if (i == skip_index)
497		continue;
498		uint8_t c = ((uint8_t)buf_ptr(pc->buf) + i);
499		unsigned long long digit = get_digit_value(c);
500
501		// x *= radix;
502		if (__builtin_umulll_overflow(x, radix, &x)) {
503		*overflow = true;
504		return 0;
505		}
506
507		// x += digit
508		if (__builtin_uaddll_overflow(x, digit, &x)) {
509		*overflow = true;
510		return 0;
511		}
512		}
513		return x;
	188	static uint8_t token_char_lit(Token *token) {
	189	assert(token->id == TokenIdCharLiteral);
	190	return token->data.char_lit.c;
514	191	}
515	192
516		static void parse_number_literal(ParseContext pc, Token token, AstNodeNumberLiteral *num_lit) {
517		assert(token->id == TokenIdNumberLiteral);
518
519		int whole_number_start = token->start_pos;
520		if (token->radix != 10) {
521		// skip the "0x"
522		whole_number_start += 2;
523		}
524
525		int whole_number_end = token->decimal_point_pos;
526		if (whole_number_end <= whole_number_start) {
527		// TODO: error for empty whole number part
528		num_lit->overflow = true;
529		return;
530		}
531
532		if (token->decimal_point_pos == token->end_pos) {
533		// integer
534		unsigned long long whole_number = parse_int_digits(pc, whole_number_start, whole_number_end,
535		token->radix, -1, &num_lit->overflow);
536		if (num_lit->overflow) return;
537
538		num_lit->data.x_uint = whole_number;
539		num_lit->kind = NumLitUInt;
	193	static void ast_buf_from_token(ParseContext pc, Token token, Buf *buf) {
	194	if (token->id == TokenIdSymbol) {
	195	buf_init_from_buf(buf, token_buf(token));
540	196	} else {
541		// float
542
543		if (token->radix == 10) {
544		// use a third-party base-10 float parser
545		char *str_begin = buf_ptr(pc->buf) + whole_number_start;
546		char *str_end;
547		errno = 0;
548		double x = strtod(str_begin, &str_end);
549		if (errno) {
550		// TODO: forward error to user
551		num_lit->overflow = true;
552		return;
553		}
554		assert(str_end == buf_ptr(pc->buf) + token->end_pos);
555		num_lit->data.x_float = x;
556		num_lit->kind = NumLitFloat;
557		return;
558		}
559
560		if (token->decimal_point_pos < token->exponent_marker_pos) {
561		// fraction
562		int fraction_start = token->decimal_point_pos + 1;
563		int fraction_end = token->exponent_marker_pos;
564		if (fraction_end <= fraction_start) {
565		// TODO: error for empty fraction part
566		num_lit->overflow = true;
567		return;
568		}
569		}
570
571		// trim leading and trailing zeros in the significand digit sequence
572		int significand_start = whole_number_start;
573		for (; significand_start < token->exponent_marker_pos; significand_start++) {
574		if (significand_start == token->decimal_point_pos)
575		continue;
576		uint8_t c = ((uint8_t)buf_ptr(pc->buf) + significand_start);
577		if (c != '0')
578		break;
579		}
580		int significand_end = token->exponent_marker_pos;
581		for (; significand_end - 1 > significand_start; significand_end--) {
582		if (significand_end - 1 <= token->decimal_point_pos) {
583		significand_end = token->decimal_point_pos;
584		break;
585		}
586		uint8_t c = ((uint8_t)buf_ptr(pc->buf) + significand_end - 1);
587		if (c != '0')
588		break;
589		}
590
591		unsigned long long significand_as_int = parse_int_digits(pc, significand_start, significand_end,
592		token->radix, token->decimal_point_pos, &num_lit->overflow);
593		if (num_lit->overflow) return;
594
595		int exponent_in_bin_or_dec = 0;
596		if (significand_end > token->decimal_point_pos) {
597		exponent_in_bin_or_dec = token->decimal_point_pos + 1 - significand_end;
598		if (token->radix == 2) {
599		// already good
600		} else if (token->radix == 8) {
601		exponent_in_bin_or_dec *= 3;
602		} else if (token->radix == 10) {
603		// already good
604		} else if (token->radix == 16) {
605		exponent_in_bin_or_dec *= 4;
606		} else zig_unreachable();
607		}
608
609		if (token->exponent_marker_pos < token->end_pos) {
610		// exponent
611		int exponent_start = token->exponent_marker_pos + 1;
612		int exponent_end = token->end_pos;
613		if (exponent_end <= exponent_start) {
614		// TODO: error for empty exponent part
615		num_lit->overflow = true;
616		return;
617		}
618		bool is_exponent_negative = false;
619		uint8_t c = ((uint8_t)buf_ptr(pc->buf) + exponent_start);
620		if (c == '+') {
621		exponent_start += 1;
622		} else if (c == '-') {
623		exponent_start += 1;
624		is_exponent_negative = true;
625		}
626
627		if (exponent_end <= exponent_start) {
628		// TODO: error for empty exponent part
629		num_lit->overflow = true;
630		return;
631		}
632
633		unsigned long long specified_exponent = parse_int_digits(pc, exponent_start, exponent_end,
634		10, -1, &num_lit->overflow);
635		// TODO: this check is a little silly
636		if (specified_exponent >= LLONG_MAX) {
637		num_lit->overflow = true;
638		return;
639		}
640
641		if (is_exponent_negative) {
642		exponent_in_bin_or_dec -= specified_exponent;
643		} else {
644		exponent_in_bin_or_dec += specified_exponent;
645		}
646		}
647
648		uint64_t significand_bits;
649		uint64_t exponent_bits;
650		if (significand_as_int != 0) {
651		// normalize the significand
652		if (token->radix == 10) {
653		zig_panic("TODO: decimal floats");
654		} else {
655		int significand_magnitude_in_bin = __builtin_clzll(1) - __builtin_clzll(significand_as_int);
656		exponent_in_bin_or_dec += significand_magnitude_in_bin;
657		if (!(-1023 <= exponent_in_bin_or_dec && exponent_in_bin_or_dec < 1023)) {
658		num_lit->overflow = true;
659		return;
660		}
661
662		// this should chop off exactly one 1 bit from the top.
663		significand_bits = ((uint64_t)significand_as_int << (52 - significand_magnitude_in_bin)) & 0xfffffffffffffULL;
664		exponent_bits = exponent_in_bin_or_dec + 1023;
665		}
666		} else {
667		// 0 is all 0's
668		significand_bits = 0;
669		exponent_bits = 0;
670		}
671
672		uint64_t double_bits = (exponent_bits << 52) \| significand_bits;
673		double x = (double )&double_bits;
674
675		num_lit->data.x_float = x;
676		num_lit->kind = NumLitFloat;
	197	buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
677	198	}
678	199	}
679	200
680
681	201	__attribute__ ((noreturn))
682	202	static void ast_invalid_token_error(ParseContext pc, Token token) {
683	203	Buf token_value = BUF_INIT;
...	...	@@ -723,7 +243,7 @@ static AstNode ast_parse_directive(ParseContext pc, int *token_index) {
723	243
724	244	Token *name_symbol = ast_eat_token(pc, token_index, TokenIdSymbol);
725	245
726		ast_buf_from_token(pc, name_symbol, &node->data.directive.name);
	246	node->data.directive.name = token_buf(name_symbol);
727	247
728	248	node->data.directive.expr = ast_parse_grouped_expr(pc, token_index, true);
729	249
...	...	@@ -769,12 +289,12 @@ static AstNode ast_parse_param_decl(ParseContext pc, int *token_index) {
769	289	token = &pc->tokens->at(*token_index);
770	290	}
771	291
772		buf_resize(&node->data.param_decl.name, 0);
	292	node->data.param_decl.name = pc->empty_buf;
773	293
774	294	if (token->id == TokenIdSymbol) {
775	295	Token next_token = &pc->tokens->at(token_index + 1);
776	296	if (next_token->id == TokenIdColon) {
777		ast_buf_from_token(pc, token, &node->data.param_decl.name);
	297	node->data.param_decl.name = token_buf(token);
778	298	*token_index += 2;
779	299	}
780	300	}
...	...	@@ -915,8 +435,8 @@ static void ast_parse_asm_input_item(ParseContext pc, int token_index, AstNode
915	435	ast_eat_token(pc, token_index, TokenIdRParen);
916	436
917	437	AsmInput *asm_input = allocate<AsmInput>(1);
918		ast_buf_from_token(pc, alias, &asm_input->asm_symbolic_name);
919		parse_string_literal(pc, constraint, &asm_input->constraint, nullptr, nullptr);
	438	asm_input->asm_symbolic_name = token_buf(alias);
	439	asm_input->constraint = token_buf(constraint);
920	440	asm_input->expr = expr_node;
921	441	node->data.asm_expr.input_list.append(asm_input);
922	442	}
...	...	@@ -938,7 +458,7 @@ static void ast_parse_asm_output_item(ParseContext pc, int token_index, AstNod
938	458	Token token = &pc->tokens->at(token_index);
939	459	*token_index += 1;
940	460	if (token->id == TokenIdSymbol) {
941		ast_buf_from_token(pc, token, &asm_output->variable_name);
	461	asm_output->variable_name = token_buf(token);
942	462	} else if (token->id == TokenIdArrow) {
943	463	asm_output->return_type = ast_parse_prefix_op_expr(pc, token_index, true);
944	464	} else {
...	...	@@ -947,8 +467,8 @@ static void ast_parse_asm_output_item(ParseContext pc, int token_index, AstNod
947	467
948	468	ast_eat_token(pc, token_index, TokenIdRParen);
949	469
950		ast_buf_from_token(pc, alias, &asm_output->asm_symbolic_name);
951		parse_string_literal(pc, constraint, &asm_output->constraint, nullptr, nullptr);
	470	asm_output->asm_symbolic_name = token_buf(alias);
	471	asm_output->constraint = token_buf(constraint);
952	472	node->data.asm_expr.output_list.append(asm_output);
953	473	}
954	474
...	...	@@ -968,8 +488,7 @@ static void ast_parse_asm_clobbers(ParseContext pc, int token_index, AstNode *
968	488	ast_expect_token(pc, string_tok, TokenIdStringLiteral);
969	489	*token_index += 1;
970	490
971		Buf *clobber_buf = buf_alloc();
972		parse_string_literal(pc, string_tok, clobber_buf, nullptr, nullptr);
	491	Buf *clobber_buf = token_buf(string_tok);
973	492	node->data.asm_expr.clobber_list.append(clobber_buf);
974	493
975	494	Token comma = &pc->tokens->at(token_index);
...	...	@@ -1072,19 +591,14 @@ static AstNode ast_parse_asm_expr(ParseContext pc, int *token_index, bool mand
1072	591	ast_expect_token(pc, lparen_tok, TokenIdLParen);
1073	592	*token_index += 1;
1074	593
1075		Token template_tok = &pc->tokens->at(token_index);
1076		ast_expect_token(pc, template_tok, TokenIdStringLiteral);
1077		*token_index += 1;
	594	Token *template_tok = ast_eat_token(pc, token_index, TokenIdStringLiteral);
1078	595
1079		parse_string_literal(pc, template_tok, &node->data.asm_expr.asm_template, nullptr,
1080		&node->data.asm_expr.offset_map);
	596	node->data.asm_expr.asm_template = token_buf(template_tok);
1081	597	parse_asm_template(pc, node);
1082	598
1083	599	ast_parse_asm_output(pc, token_index, node);
1084	600
1085		Token rparen_tok = &pc->tokens->at(token_index);
1086		ast_expect_token(pc, rparen_tok, TokenIdRParen);
1087		*token_index += 1;
	601	ast_eat_token(pc, token_index, TokenIdRParen);
1088	602
1089	603	normalize_parent_ptrs(node);
1090	604	return node;
...	...	@@ -1099,17 +613,19 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool
1099	613
1100	614	if (token->id == TokenIdNumberLiteral) {
1101	615	AstNode *node = ast_create_node(pc, NodeTypeNumberLiteral, token);
1102		parse_number_literal(pc, token, &node->data.number_literal);
	616	node->data.number_literal.bignum = token_bignum(token);
	617	node->data.number_literal.overflow = token->data.num_lit.overflow;
1103	618	*token_index += 1;
1104	619	return node;
1105	620	} else if (token->id == TokenIdStringLiteral) {
1106	621	AstNode *node = ast_create_node(pc, NodeTypeStringLiteral, token);
1107		parse_string_literal(pc, token, &node->data.string_literal.buf, &node->data.string_literal.c, nullptr);
	622	node->data.string_literal.buf = token_buf(token);
	623	node->data.string_literal.c = token->data.str_lit.is_c_str;
1108	624	*token_index += 1;
1109	625	return node;
1110	626	} else if (token->id == TokenIdCharLiteral) {
1111	627	AstNode *node = ast_create_node(pc, NodeTypeCharLiteral, token);
1112		node->data.char_literal.value = parse_char_literal(pc, token);
	628	node->data.char_literal.value = token_char_lit(token);
1113	629	*token_index += 1;
1114	630	return node;
1115	631	} else if (token->id == TokenIdKeywordTrue) {
...	...	@@ -1155,7 +671,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool
1155	671	*token_index += 1;
1156	672	Token *name_tok = ast_eat_token(pc, token_index, TokenIdSymbol);
1157	673	AstNode *name_node = ast_create_node(pc, NodeTypeSymbol, name_tok);
1158		ast_buf_from_token(pc, name_tok, &name_node->data.symbol_expr.symbol);
	674	name_node->data.symbol_expr.symbol = token_buf(name_tok);
1159	675
1160	676	AstNode *node = ast_create_node(pc, NodeTypeFnCallExpr, token);
1161	677	node->data.fn_call_expr.fn_ref_expr = name_node;
...	...	@@ -1168,7 +684,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool
1168	684	} else if (token->id == TokenIdSymbol) {
1169	685	*token_index += 1;
1170	686	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);
1171		ast_buf_from_token(pc, token, &node->data.symbol_expr.symbol);
	687	node->data.symbol_expr.symbol = token_buf(token);
1172	688	return node;
1173	689	} else if (token->id == TokenIdKeywordGoto) {
1174	690	AstNode *node = ast_create_node(pc, NodeTypeGoto, token);
...	...	@@ -1178,7 +694,7 @@ static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool
1178	694	*token_index += 1;
1179	695	ast_expect_token(pc, dest_symbol, TokenIdSymbol);
1180	696
1181		ast_buf_from_token(pc, dest_symbol, &node->data.goto_expr.name);
	697	node->data.goto_expr.name = token_buf(dest_symbol);
1182	698	return node;
1183	699	}
1184	700
...	...	@@ -1243,7 +759,7 @@ static AstNode ast_parse_curly_suffix_expr(ParseContext pc, int *token_index,
1243	759
1244	760	AstNode *field_node = ast_create_node(pc, NodeTypeStructValueField, token);
1245	761
1246		ast_buf_from_token(pc, field_name_tok, &field_node->data.struct_val_field.name);
	762	field_node->data.struct_val_field.name = token_buf(field_name_tok);
1247	763	field_node->data.struct_val_field.expr = ast_parse_expression(pc, token_index, true);
1248	764
1249	765	normalize_parent_ptrs(field_node);
...	...	@@ -1370,7 +886,7 @@ static AstNode ast_parse_suffix_op_expr(ParseContext pc, int *token_index, boo
1370	886
1371	887	AstNode *node = ast_create_node(pc, NodeTypeFieldAccessExpr, first_token);
1372	888	node->data.field_access_expr.struct_expr = primary_expr;
1373		ast_buf_from_token(pc, name_token, &node->data.field_access_expr.field_name);
	889	node->data.field_access_expr.field_name = token_buf(name_token);
1374	890
1375	891	normalize_parent_ptrs(node);
1376	892	primary_expr = node;
...	...	@@ -1819,10 +1335,10 @@ static AstNode ast_parse_if_expr(ParseContext pc, int *token_index, bool manda
1819	1335	*token_index += 1;
1820	1336	node->data.if_var_expr.var_is_ptr = true;
1821	1337	Token *name_token = ast_eat_token(pc, token_index, TokenIdSymbol);
1822		ast_buf_from_token(pc, name_token, &node->data.if_var_expr.var_decl.symbol);
	1338	node->data.if_var_expr.var_decl.symbol = token_buf(name_token);
1823	1339	} else if (star_or_symbol->id == TokenIdSymbol) {
1824	1340	*token_index += 1;
1825		ast_buf_from_token(pc, star_or_symbol, &node->data.if_var_expr.var_decl.symbol);
	1341	node->data.if_var_expr.var_decl.symbol = token_buf(star_or_symbol);
1826	1342	} else {
1827	1343	ast_invalid_token_error(pc, star_or_symbol);
1828	1344	}
...	...	@@ -1974,7 +1490,7 @@ static AstNode ast_parse_variable_declaration_expr(ParseContext pc, int *token
1974	1490	node->data.variable_declaration.top_level_decl.directives = directives;
1975	1491
1976	1492	Token *name_token = ast_eat_token(pc, token_index, TokenIdSymbol);
1977		ast_buf_from_token(pc, name_token, &node->data.variable_declaration.symbol);
	1493	node->data.variable_declaration.symbol = token_buf(name_token);
1978	1494
1979	1495	Token eq_or_colon = &pc->tokens->at(token_index);
1980	1496	*token_index += 1;
...	...	@@ -2067,7 +1583,7 @@ static AstNode ast_parse_while_expr(ParseContext pc, int *token_index, bool ma
2067	1583	static AstNode ast_parse_symbol(ParseContext pc, int *token_index) {
2068	1584	Token *token = ast_eat_token(pc, token_index, TokenIdSymbol);
2069	1585	AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);
2070		ast_buf_from_token(pc, token, &node->data.symbol_expr.symbol);
	1586	node->data.symbol_expr.symbol = token_buf(token);
2071	1587	return node;
2072	1588	}
2073	1589
...	...	@@ -2405,7 +1921,7 @@ static AstNode ast_parse_label(ParseContext pc, int *token_index, bool mandato
2405	1921	*token_index += 2;
2406	1922
2407	1923	AstNode *node = ast_create_node(pc, NodeTypeLabel, symbol_token);
2408		ast_buf_from_token(pc, symbol_token, &node->data.label.name);
	1924	node->data.label.name = token_buf(symbol_token);
2409	1925	return node;
2410	1926	}
2411	1927
...	...	@@ -2413,7 +1929,7 @@ static AstNode ast_create_void_expr(ParseContext pc, Token *token) {
2413	1929	AstNode *node = ast_create_node(pc, NodeTypeContainerInitExpr, token);
2414	1930	node->data.container_init_expr.type = ast_create_node(pc, NodeTypeSymbol, token);
2415	1931	node->data.container_init_expr.kind = ContainerInitKindArray;
2416		buf_init_from_str(&node->data.container_init_expr.type->data.symbol_expr.symbol, "void");
	1932	node->data.container_init_expr.type->data.symbol_expr.symbol = pc->void_buf;
2417	1933	normalize_parent_ptrs(node);
2418	1934	return node;
2419	1935	}
...	...	@@ -2508,9 +2024,9 @@ static AstNode ast_parse_fn_proto(ParseContext pc, int *token_index, bool mand
2508	2024	Token fn_name = &pc->tokens->at(token_index);
2509	2025	if (fn_name->id == TokenIdSymbol) {
2510	2026	*token_index += 1;
2511		ast_buf_from_token(pc, fn_name, &node->data.fn_proto.name);
	2027	node->data.fn_proto.name = token_buf(fn_name);
2512	2028	} else {
2513		buf_resize(&node->data.fn_proto.name, 0);
	2029	node->data.fn_proto.name = pc->empty_buf;
2514	2030	}
2515	2031
2516	2032	ast_parse_param_decl_list(pc, token_index, &node->data.fn_proto.params, &node->data.fn_proto.is_var_args);
...	...	@@ -2663,7 +2179,7 @@ static AstNode ast_parse_container_decl(ParseContext pc, int *token_index,
2663	2179
2664	2180	AstNode *node = ast_create_node(pc, NodeTypeContainerDecl, first_token);
2665	2181	node->data.struct_decl.kind = kind;
2666		ast_buf_from_token(pc, struct_name, &node->data.struct_decl.name);
	2182	node->data.struct_decl.name = token_buf(struct_name);
2667	2183	node->data.struct_decl.top_level_decl.visib_mod = visib_mod;
2668	2184	node->data.struct_decl.top_level_decl.directives = directives;
2669	2185
...	...	@@ -2729,8 +2245,7 @@ static AstNode ast_parse_container_decl(ParseContext pc, int *token_index,
2729	2245
2730	2246	field_node->data.struct_field.top_level_decl.visib_mod = visib_mod;
2731	2247	field_node->data.struct_field.top_level_decl.directives = directive_list;
2732
2733		ast_buf_from_token(pc, token, &field_node->data.struct_field.name);
	2248	field_node->data.struct_field.name = token_buf(token);
2734	2249
2735	2250	Token expr_or_comma = &pc->tokens->at(token_index);
2736	2251	if (expr_or_comma->id == TokenIdComma) {
...	...	@@ -2772,7 +2287,7 @@ static AstNode ast_parse_error_value_decl(ParseContext pc, int *token_index,
2772	2287	AstNode *node = ast_create_node(pc, NodeTypeErrorValueDecl, first_token);
2773	2288	node->data.error_value_decl.top_level_decl.visib_mod = visib_mod;
2774	2289	node->data.error_value_decl.top_level_decl.directives = directives;
2775		ast_buf_from_token(pc, name_tok, &node->data.error_value_decl.name);
	2290	node->data.error_value_decl.name = token_buf(name_tok);
2776	2291
2777	2292	normalize_parent_ptrs(node);
2778	2293	return node;
...	...	@@ -2795,7 +2310,7 @@ static AstNode ast_parse_type_decl(ParseContext pc, int *token_index,
2795	2310	ast_eat_token(pc, token_index, TokenIdEq);
2796	2311
2797	2312	AstNode *node = ast_create_node(pc, NodeTypeTypeDecl, first_token);
2798		ast_buf_from_token(pc, name_tok, &node->data.type_decl.symbol);
	2313	node->data.type_decl.symbol = token_buf(name_tok);
2799	2314	node->data.type_decl.child_type = ast_parse_prefix_op_expr(pc, token_index, true);
2800	2315
2801	2316	ast_eat_token(pc, token_index, TokenIdSemicolon);
...	...	@@ -2901,6 +2416,8 @@ AstNode ast_parse(Buf buf, ZigList<Token> tokens, ImportTableEntry owner,
2901	2416	ErrColor err_color, uint32_t *next_node_index)
2902	2417	{
2903	2418	ParseContext pc = {0};
	2419	pc.void_buf = buf_create_from_str("void");
	2420	pc.empty_buf = buf_create_from_str("");
2904	2421	pc.err_color = err_color;
2905	2422	pc.owner = owner;
2906	2423	pc.buf = buf;

src/tokenizer.cpp+418-315

...	...	@@ -11,6 +11,9 @@
11	11	#include <stdarg.h>
12	12	#include <stdlib.h>
13	13	#include <stdio.h>
	14	#include <inttypes.h>
	15	#include <limits.h>
	16	#include <errno.h>
14	17
15	18	#define WHITESPACE \
16	19	' ': \
...	...	@@ -30,7 +33,7 @@
30	33	'0': \
31	34	case DIGIT_NON_ZERO
32	35
33		#define ALPHA_EXCEPT_CR \
	36	#define ALPHA_EXCEPT_C \
34	37	'a': \
35	38	case 'b': \
36	39	/case 'c':/ \
...	...	@@ -48,7 +51,7 @@
48	51	case 'o': \
49	52	case 'p': \
50	53	case 'q': \
51		/case 'r':/ \
	54	case 'r': \
52	55	case 's': \
53	56	case 't': \
54	57	case 'u': \
...	...	@@ -85,77 +88,93 @@
85	88	case 'Z'
86	89
87	90	#define ALPHA \
88		ALPHA_EXCEPT_CR: \
89		case 'c': \
90		case 'r'
91
92		#define SYMBOL_CHAR \
93		SYMBOL_CHAR_EXCEPT_C: \
	91	ALPHA_EXCEPT_C: \
94	92	case 'c'
95	93
96		#define SYMBOL_CHAR_EXCEPT_C \
97		ALPHA_EXCEPT_CR: \
98		case 'r': \
	94	#define SYMBOL_CHAR \
	95	ALPHA_EXCEPT_C: \
99	96	case DIGIT: \
100		case '_'
	97	case '_': \
	98	case 'c'
101	99
102	100	#define SYMBOL_START \
103	101	ALPHA: \
104	102	case '_'
105	103
106		#define HEX_DIGIT \
107		'a': \
108		case 'b': \
109		case 'c': \
110		case 'd': \
111		case 'e': \
112		case 'f': \
113		case 'A': \
114		case 'B': \
115		case 'C': \
116		case 'D': \
117		case 'E': \
118		case 'F': \
119		case DIGIT
	104	struct ZigKeyword {
	105	const char *text;
	106	TokenId token_id;
	107	};
120	108
121		const char * zig_keywords[] = {
122		"true", "false", "null", "fn", "return", "var", "const", "extern",
123		"pub", "export", "use", "if", "else", "goto", "asm",
124		"volatile", "struct", "enum", "while", "for", "continue", "break",
125		"null", "noalias", "switch", "undefined", "error", "type", "inline",
126		"defer", "union",
	109	static const struct ZigKeyword zig_keywords[] = {
	110	{"asm", TokenIdKeywordAsm},
	111	{"break", TokenIdKeywordBreak},
	112	{"const", TokenIdKeywordConst},
	113	{"continue", TokenIdKeywordContinue},
	114	{"defer", TokenIdKeywordDefer},
	115	{"else", TokenIdKeywordElse},
	116	{"enum", TokenIdKeywordEnum},
	117	{"error", TokenIdKeywordError},
	118	{"export", TokenIdKeywordExport},
	119	{"extern", TokenIdKeywordExtern},
	120	{"false", TokenIdKeywordFalse},
	121	{"fn", TokenIdKeywordFn},
	122	{"for", TokenIdKeywordFor},
	123	{"goto", TokenIdKeywordGoto},
	124	{"if", TokenIdKeywordIf},
	125	{"inline", TokenIdKeywordInline},
	126	{"noalias", TokenIdKeywordNoAlias},
	127	{"null", TokenIdKeywordNull},
	128	{"pub", TokenIdKeywordPub},
	129	{"return", TokenIdKeywordReturn},
	130	{"struct", TokenIdKeywordStruct},
	131	{"switch", TokenIdKeywordSwitch},
	132	{"true", TokenIdKeywordTrue},
	133	{"type", TokenIdKeywordType},
	134	{"undefined", TokenIdKeywordUndefined},
	135	{"union", TokenIdKeywordUnion},
	136	{"use", TokenIdKeywordUse},
	137	{"var", TokenIdKeywordVar},
	138	{"volatile", TokenIdKeywordVolatile},
	139	{"while", TokenIdKeywordWhile},
127	140	};
128	141
129	142	bool is_zig_keyword(Buf *buf) {
130	143	for (int i = 0; i < array_length(zig_keywords); i += 1) {
131		if (buf_eql_str(buf, zig_keywords[i])) {
	144	if (buf_eql_str(buf, zig_keywords[i].text)) {
132	145	return true;
133	146	}
134	147	}
135	148	return false;
136	149	}
137	150
	151	static bool is_symbol_char(uint8_t c) {
	152	switch (c) {
	153	case SYMBOL_CHAR:
	154	return true;
	155	default:
	156	return false;
	157	}
	158	}
	159
138	160	enum TokenizeState {
139	161	TokenizeStateStart,
140	162	TokenizeStateSymbol,
141		TokenizeStateSymbolFirst,
142		TokenizeStateSymbolFirstRaw,
143		TokenizeStateFirstR,
	163	TokenizeStateSymbolFirstC,
144	164	TokenizeStateZero, // "0", which might lead to "0x"
145	165	TokenizeStateNumber, // "123", "0x123"
	166	TokenizeStateNumberDot,
146	167	TokenizeStateFloatFraction, // "123.456", "0x123.456"
147	168	TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
148	169	TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
149	170	TokenizeStateString,
150	171	TokenizeStateStringEscape,
151		TokenizeStateRawString,
152		TokenizeStateRawStringContents,
153		TokenizeStateRawStringMaybeEnd,
154	172	TokenizeStateCharLiteral,
155	173	TokenizeStateCharLiteralEnd,
156	174	TokenizeStateSawStar,
157	175	TokenizeStateSawStarPercent,
158	176	TokenizeStateSawSlash,
	177	TokenizeStateSawBackslash,
159	178	TokenizeStateSawPercent,
160	179	TokenizeStateSawPlus,
161	180	TokenizeStateSawPlusPercent,
...	...	@@ -167,6 +186,9 @@ enum TokenizeState {
167	186	TokenizeStateSawPipe,
168	187	TokenizeStateSawPipePipe,
169	188	TokenizeStateLineComment,
	189	TokenizeStateLineString,
	190	TokenizeStateLineStringEnd,
	191	TokenizeStateLineStringContinue,
170	192	TokenizeStateSawEq,
171	193	TokenizeStateSawBang,
172	194	TokenizeStateSawLessThan,
...	...	@@ -178,7 +200,7 @@ enum TokenizeState {
178	200	TokenizeStateSawDotDot,
179	201	TokenizeStateSawQuestionMark,
180	202	TokenizeStateSawAtSign,
181		TokenizeStateHex,
	203	TokenizeStateCharCode,
182	204	TokenizeStateError,
183	205	};
184	206
...	...	@@ -192,10 +214,16 @@ struct Tokenize {
192	214	int column;
193	215	Token *cur_tok;
194	216	Tokenization *out;
195		int raw_string_id_start;
196		int raw_string_id_end;
197		int raw_string_id_cmp_pos;
198		int hex_chars_left;
	217	uint32_t radix;
	218	int32_t exp_add_amt;
	219	bool is_exp_negative;
	220	bool is_num_lit_float;
	221	size_t char_code_index;
	222	size_t char_code_end;
	223	bool unicode;
	224	uint32_t char_code;
	225	int exponent_in_bin_or_dec;
	226	BigNum specified_exponent;
199	227	};
200	228
201	229	__attribute__ ((format (printf, 2, 3)))
...	...	@@ -216,19 +244,28 @@ static void tokenize_error(Tokenize t, const char format, ...) {
216	244	va_end(ap);
217	245	}
218	246
	247	static void set_token_id(Tokenize t, Token token, TokenId id) {
	248	token->id = id;
	249
	250	if (id == TokenIdNumberLiteral) {
	251	token->data.num_lit.overflow = false;
	252	} else if (id == TokenIdStringLiteral \|\| id == TokenIdSymbol) {
	253	memset(&token->data.str_lit.str, 0, sizeof(Buf));
	254	buf_resize(&token->data.str_lit.str, 0);
	255	token->data.str_lit.is_c_str = false;
	256	}
	257	}
	258
219	259	static void begin_token(Tokenize *t, TokenId id) {
220	260	assert(!t->cur_tok);
221	261	t->tokens->add_one();
222	262	Token *token = &t->tokens->last();
223	263	token->start_line = t->line;
224	264	token->start_column = t->column;
225		token->id = id;
226	265	token->start_pos = t->pos;
227		token->radix = 0;
228		token->decimal_point_pos = 0;
229		token->exponent_marker_pos = 0;
230		token->raw_string_start = 0;
231		token->raw_string_end = 0;
	266
	267	set_token_id(t, token, id);
	268
232	269	t->cur_tok = token;
233	270	}
234	271
...	...	@@ -237,83 +274,82 @@ static void cancel_token(Tokenize *t) {
237	274	t->cur_tok = nullptr;
238	275	}
239	276
	277	static void end_float_token(Tokenize *t) {
	278	t->cur_tok->data.num_lit.bignum.kind = BigNumKindFloat;
	279
	280	if (t->radix == 10) {
	281	char *str_begin = buf_ptr(t->buf) + t->cur_tok->start_pos;
	282	char *str_end;
	283	errno = 0;
	284	t->cur_tok->data.num_lit.bignum.data.x_float = strtod(str_begin, &str_end);
	285	if (errno) {
	286	t->cur_tok->data.num_lit.overflow = true;
	287	return;
	288	}
	289	assert(str_end == buf_ptr(t->buf) + t->cur_tok->end_pos);
	290	return;
	291	}
	292
	293
	294	if (t->specified_exponent.data.x_uint >= INT_MAX) {
	295	t->cur_tok->data.num_lit.overflow = true;
	296	return;
	297	}
	298
	299	int64_t specified_exponent = t->specified_exponent.data.x_uint;
	300	if (t->is_exp_negative) {
	301	specified_exponent = -specified_exponent;
	302	}
	303	t->exponent_in_bin_or_dec += specified_exponent;
	304
	305	uint64_t significand = t->cur_tok->data.num_lit.bignum.data.x_uint;
	306	uint64_t significand_bits;
	307	uint64_t exponent_bits;
	308	if (significand == 0) {
	309	// 0 is all 0's
	310	significand_bits = 0;
	311	exponent_bits = 0;
	312	} else {
	313	// normalize the significand
	314	if (t->radix == 10) {
	315	zig_panic("TODO: decimal floats");
	316	} else {
	317	int significand_magnitude_in_bin = __builtin_clzll(1) - __builtin_clzll(significand);
	318	t->exponent_in_bin_or_dec += significand_magnitude_in_bin;
	319	if (!(-1023 <= t->exponent_in_bin_or_dec && t->exponent_in_bin_or_dec < 1023)) {
	320	t->cur_tok->data.num_lit.overflow = true;
	321	} else {
	322	// this should chop off exactly one 1 bit from the top.
	323	significand_bits = ((uint64_t)significand << (52 - significand_magnitude_in_bin)) & 0xfffffffffffffULL;
	324	exponent_bits = t->exponent_in_bin_or_dec + 1023;
	325	}
	326	}
	327	}
	328	uint64_t double_bits = (exponent_bits << 52) \| significand_bits;
	329	memcpy(&t->cur_tok->data.num_lit.bignum.data.x_float, &double_bits, sizeof(double));
	330	}
	331
240	332	static void end_token(Tokenize *t) {
241	333	assert(t->cur_tok);
242	334	t->cur_tok->end_pos = t->pos + 1;
243	335
244		// normalize number literal parsing stuff
245	336	if (t->cur_tok->id == TokenIdNumberLiteral) {
246		if (t->cur_tok->exponent_marker_pos == 0) {
247		t->cur_tok->exponent_marker_pos = t->cur_tok->end_pos;
	337	if (t->cur_tok->data.num_lit.overflow) {
	338	return;
248	339	}
249		if (t->cur_tok->decimal_point_pos == 0) {
250		t->cur_tok->decimal_point_pos = t->cur_tok->exponent_marker_pos;
	340	if (t->is_num_lit_float) {
	341	end_float_token(t);
251	342	}
252		}
253
254		char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;
255		int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
	343	} else if (t->cur_tok->id == TokenIdSymbol) {
	344	char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;
	345	int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
256	346
257		if (mem_eql_str(token_mem, token_len, "fn")) {
258		t->cur_tok->id = TokenIdKeywordFn;
259		} else if (mem_eql_str(token_mem, token_len, "return")) {
260		t->cur_tok->id = TokenIdKeywordReturn;
261		} else if (mem_eql_str(token_mem, token_len, "var")) {
262		t->cur_tok->id = TokenIdKeywordVar;
263		} else if (mem_eql_str(token_mem, token_len, "const")) {
264		t->cur_tok->id = TokenIdKeywordConst;
265		} else if (mem_eql_str(token_mem, token_len, "extern")) {
266		t->cur_tok->id = TokenIdKeywordExtern;
267		} else if (mem_eql_str(token_mem, token_len, "pub")) {
268		t->cur_tok->id = TokenIdKeywordPub;
269		} else if (mem_eql_str(token_mem, token_len, "export")) {
270		t->cur_tok->id = TokenIdKeywordExport;
271		} else if (mem_eql_str(token_mem, token_len, "use")) {
272		t->cur_tok->id = TokenIdKeywordUse;
273		} else if (mem_eql_str(token_mem, token_len, "true")) {
274		t->cur_tok->id = TokenIdKeywordTrue;
275		} else if (mem_eql_str(token_mem, token_len, "false")) {
276		t->cur_tok->id = TokenIdKeywordFalse;
277		} else if (mem_eql_str(token_mem, token_len, "if")) {
278		t->cur_tok->id = TokenIdKeywordIf;
279		} else if (mem_eql_str(token_mem, token_len, "else")) {
280		t->cur_tok->id = TokenIdKeywordElse;
281		} else if (mem_eql_str(token_mem, token_len, "goto")) {
282		t->cur_tok->id = TokenIdKeywordGoto;
283		} else if (mem_eql_str(token_mem, token_len, "volatile")) {
284		t->cur_tok->id = TokenIdKeywordVolatile;
285		} else if (mem_eql_str(token_mem, token_len, "asm")) {
286		t->cur_tok->id = TokenIdKeywordAsm;
287		} else if (mem_eql_str(token_mem, token_len, "struct")) {
288		t->cur_tok->id = TokenIdKeywordStruct;
289		} else if (mem_eql_str(token_mem, token_len, "enum")) {
290		t->cur_tok->id = TokenIdKeywordEnum;
291		} else if (mem_eql_str(token_mem, token_len, "union")) {
292		t->cur_tok->id = TokenIdKeywordUnion;
293		} else if (mem_eql_str(token_mem, token_len, "for")) {
294		t->cur_tok->id = TokenIdKeywordFor;
295		} else if (mem_eql_str(token_mem, token_len, "while")) {
296		t->cur_tok->id = TokenIdKeywordWhile;
297		} else if (mem_eql_str(token_mem, token_len, "continue")) {
298		t->cur_tok->id = TokenIdKeywordContinue;
299		} else if (mem_eql_str(token_mem, token_len, "break")) {
300		t->cur_tok->id = TokenIdKeywordBreak;
301		} else if (mem_eql_str(token_mem, token_len, "null")) {
302		t->cur_tok->id = TokenIdKeywordNull;
303		} else if (mem_eql_str(token_mem, token_len, "noalias")) {
304		t->cur_tok->id = TokenIdKeywordNoAlias;
305		} else if (mem_eql_str(token_mem, token_len, "switch")) {
306		t->cur_tok->id = TokenIdKeywordSwitch;
307		} else if (mem_eql_str(token_mem, token_len, "undefined")) {
308		t->cur_tok->id = TokenIdKeywordUndefined;
309		} else if (mem_eql_str(token_mem, token_len, "error")) {
310		t->cur_tok->id = TokenIdKeywordError;
311		} else if (mem_eql_str(token_mem, token_len, "type")) {
312		t->cur_tok->id = TokenIdKeywordType;
313		} else if (mem_eql_str(token_mem, token_len, "inline")) {
314		t->cur_tok->id = TokenIdKeywordInline;
315		} else if (mem_eql_str(token_mem, token_len, "defer")) {
316		t->cur_tok->id = TokenIdKeywordDefer;
	347	for (size_t i = 0; i < array_length(zig_keywords); i += 1) {
	348	if (mem_eql_str(token_mem, token_len, zig_keywords[i].text)) {
	349	t->cur_tok->id = zig_keywords[i].token_id;
	350	break;
	351	}
	352	}
317	353	}
318	354
319	355	t->cur_tok = nullptr;
...	...	@@ -327,7 +363,7 @@ static bool is_exponent_signifier(uint8_t c, int radix) {
327	363	}
328	364	}
329	365
330		int get_digit_value(uint8_t c) {
	366	static uint32_t get_digit_value(uint8_t c) {
331	367	if ('0' <= c && c <= '9') {
332	368	return c - '0';
333	369	}
...	...	@@ -337,7 +373,19 @@ int get_digit_value(uint8_t c) {
337	373	if ('a' <= c && c <= 'z') {
338	374	return c - 'a' + 10;
339	375	}
340		return -1;
	376	return UINT32_MAX;
	377	}
	378
	379	void handle_string_escape(Tokenize *t, uint8_t c) {
	380	if (t->cur_tok->id == TokenIdCharLiteral) {
	381	t->cur_tok->data.char_lit.c = c;
	382	t->state = TokenizeStateCharLiteralEnd;
	383	} else if (t->cur_tok->id == TokenIdStringLiteral \|\| t->cur_tok->id == TokenIdSymbol) {
	384	buf_append_char(&t->cur_tok->data.str_lit.str, c);
	385	t->state = TokenizeStateString;
	386	} else {
	387	zig_unreachable();
	388	}
341	389	}
342	390
343	391	void tokenize(Buf buf, Tokenization out) {
...	...	@@ -359,27 +407,35 @@ void tokenize(Buf buf, Tokenization out) {
359	407	case WHITESPACE:
360	408	break;
361	409	case 'c':
362		t.state = TokenizeStateSymbolFirst;
	410	t.state = TokenizeStateSymbolFirstC;
363	411	begin_token(&t, TokenIdSymbol);
	412	buf_append_char(&t.cur_tok->data.str_lit.str, c);
364	413	break;
365		case 'r':
366		t.state = TokenizeStateFirstR;
367		begin_token(&t, TokenIdSymbol);
368		break;
369		case ALPHA_EXCEPT_CR:
	414	case ALPHA_EXCEPT_C:
370	415	case '_':
371	416	t.state = TokenizeStateSymbol;
372	417	begin_token(&t, TokenIdSymbol);
	418	buf_append_char(&t.cur_tok->data.str_lit.str, c);
373	419	break;
374	420	case '0':
375	421	t.state = TokenizeStateZero;
376	422	begin_token(&t, TokenIdNumberLiteral);
377		t.cur_tok->radix = 10;
	423	t.radix = 10;
	424	t.exp_add_amt = 1;
	425	t.exponent_in_bin_or_dec = 0;
	426	t.is_num_lit_float = false;
	427	bignum_init_unsigned(&t.cur_tok->data.num_lit.bignum, 0);
	428	bignum_init_unsigned(&t.specified_exponent, 0);
378	429	break;
379	430	case DIGIT_NON_ZERO:
380	431	t.state = TokenizeStateNumber;
381	432	begin_token(&t, TokenIdNumberLiteral);
382		t.cur_tok->radix = 10;
	433	t.radix = 10;
	434	t.exp_add_amt = 1;
	435	t.exponent_in_bin_or_dec = 0;
	436	t.is_num_lit_float = false;
	437	bignum_init_unsigned(&t.cur_tok->data.num_lit.bignum, get_digit_value(c));
	438	bignum_init_unsigned(&t.specified_exponent, 0);
383	439	break;
384	440	case '"':
385	441	begin_token(&t, TokenIdStringLiteral);
...	...	@@ -437,6 +493,10 @@ void tokenize(Buf buf, Tokenization out) {
437	493	begin_token(&t, TokenIdSlash);
438	494	t.state = TokenizeStateSawSlash;
439	495	break;
	496	case '\\':
	497	begin_token(&t, TokenIdStringLiteral);
	498	t.state = TokenizeStateSawBackslash;
	499	break;
440	500	case '%':
441	501	begin_token(&t, TokenIdPercent);
442	502	t.state = TokenizeStateSawPercent;
...	...	@@ -500,12 +560,12 @@ void tokenize(Buf buf, Tokenization out) {
500	560	case TokenizeStateSawQuestionMark:
501	561	switch (c) {
502	562	case '?':
503		t.cur_tok->id = TokenIdDoubleQuestion;
	563	set_token_id(&t, t.cur_tok, TokenIdDoubleQuestion);
504	564	end_token(&t);
505	565	t.state = TokenizeStateStart;
506	566	break;
507	567	case '=':
508		t.cur_tok->id = TokenIdMaybeAssign;
	568	set_token_id(&t, t.cur_tok, TokenIdMaybeAssign);
509	569	end_token(&t);
510	570	t.state = TokenizeStateStart;
511	571	break;
...	...	@@ -520,7 +580,7 @@ void tokenize(Buf buf, Tokenization out) {
520	580	switch (c) {
521	581	case '.':
522	582	t.state = TokenizeStateSawDotDot;
523		t.cur_tok->id = TokenIdEllipsis;
	583	set_token_id(&t, t.cur_tok, TokenIdEllipsis);
524	584	break;
525	585	default:
526	586	t.pos -= 1;
...	...	@@ -542,12 +602,12 @@ void tokenize(Buf buf, Tokenization out) {
542	602	case TokenizeStateSawGreaterThan:
543	603	switch (c) {
544	604	case '=':
545		t.cur_tok->id = TokenIdCmpGreaterOrEq;
	605	set_token_id(&t, t.cur_tok, TokenIdCmpGreaterOrEq);
546	606	end_token(&t);
547	607	t.state = TokenizeStateStart;
548	608	break;
549	609	case '>':
550		t.cur_tok->id = TokenIdBitShiftRight;
	610	set_token_id(&t, t.cur_tok, TokenIdBitShiftRight);
551	611	t.state = TokenizeStateSawGreaterThanGreaterThan;
552	612	break;
553	613	default:
...	...	@@ -560,7 +620,7 @@ void tokenize(Buf buf, Tokenization out) {
560	620	case TokenizeStateSawGreaterThanGreaterThan:
561	621	switch (c) {
562	622	case '=':
563		t.cur_tok->id = TokenIdBitShiftRightEq;
	623	set_token_id(&t, t.cur_tok, TokenIdBitShiftRightEq);
564	624	end_token(&t);
565	625	t.state = TokenizeStateStart;
566	626	break;
...	...	@@ -574,12 +634,12 @@ void tokenize(Buf buf, Tokenization out) {
574	634	case TokenizeStateSawLessThan:
575	635	switch (c) {
576	636	case '=':
577		t.cur_tok->id = TokenIdCmpLessOrEq;
	637	set_token_id(&t, t.cur_tok, TokenIdCmpLessOrEq);
578	638	end_token(&t);
579	639	t.state = TokenizeStateStart;
580	640	break;
581	641	case '<':
582		t.cur_tok->id = TokenIdBitShiftLeft;
	642	set_token_id(&t, t.cur_tok, TokenIdBitShiftLeft);
583	643	t.state = TokenizeStateSawLessThanLessThan;
584	644	break;
585	645	default:
...	...	@@ -592,12 +652,12 @@ void tokenize(Buf buf, Tokenization out) {
592	652	case TokenizeStateSawLessThanLessThan:
593	653	switch (c) {
594	654	case '=':
595		t.cur_tok->id = TokenIdBitShiftLeftEq;
	655	set_token_id(&t, t.cur_tok, TokenIdBitShiftLeftEq);
596	656	end_token(&t);
597	657	t.state = TokenizeStateStart;
598	658	break;
599	659	case '%':
600		t.cur_tok->id = TokenIdBitShiftLeftPercent;
	660	set_token_id(&t, t.cur_tok, TokenIdBitShiftLeftPercent);
601	661	t.state = TokenizeStateSawShiftLeftPercent;
602	662	break;
603	663	default:
...	...	@@ -610,7 +670,7 @@ void tokenize(Buf buf, Tokenization out) {
610	670	case TokenizeStateSawShiftLeftPercent:
611	671	switch (c) {
612	672	case '=':
613		t.cur_tok->id = TokenIdBitShiftLeftPercentEq;
	673	set_token_id(&t, t.cur_tok, TokenIdBitShiftLeftPercentEq);
614	674	end_token(&t);
615	675	t.state = TokenizeStateStart;
616	676	break;
...	...	@@ -624,7 +684,7 @@ void tokenize(Buf buf, Tokenization out) {
624	684	case TokenizeStateSawBang:
625	685	switch (c) {
626	686	case '=':
627		t.cur_tok->id = TokenIdCmpNotEq;
	687	set_token_id(&t, t.cur_tok, TokenIdCmpNotEq);
628	688	end_token(&t);
629	689	t.state = TokenizeStateStart;
630	690	break;
...	...	@@ -638,12 +698,12 @@ void tokenize(Buf buf, Tokenization out) {
638	698	case TokenizeStateSawEq:
639	699	switch (c) {
640	700	case '=':
641		t.cur_tok->id = TokenIdCmpEq;
	701	set_token_id(&t, t.cur_tok, TokenIdCmpEq);
642	702	end_token(&t);
643	703	t.state = TokenizeStateStart;
644	704	break;
645	705	case '>':
646		t.cur_tok->id = TokenIdFatArrow;
	706	set_token_id(&t, t.cur_tok, TokenIdFatArrow);
647	707	end_token(&t);
648	708	t.state = TokenizeStateStart;
649	709	break;
...	...	@@ -657,17 +717,17 @@ void tokenize(Buf buf, Tokenization out) {
657	717	case TokenizeStateSawStar:
658	718	switch (c) {
659	719	case '=':
660		t.cur_tok->id = TokenIdTimesEq;
	720	set_token_id(&t, t.cur_tok, TokenIdTimesEq);
661	721	end_token(&t);
662	722	t.state = TokenizeStateStart;
663	723	break;
664	724	case '*':
665		t.cur_tok->id = TokenIdStarStar;
	725	set_token_id(&t, t.cur_tok, TokenIdStarStar);
666	726	end_token(&t);
667	727	t.state = TokenizeStateStart;
668	728	break;
669	729	case '%':
670		t.cur_tok->id = TokenIdTimesPercent;
	730	set_token_id(&t, t.cur_tok, TokenIdTimesPercent);
671	731	t.state = TokenizeStateSawStarPercent;
672	732	break;
673	733	default:
...	...	@@ -680,7 +740,7 @@ void tokenize(Buf buf, Tokenization out) {
680	740	case TokenizeStateSawStarPercent:
681	741	switch (c) {
682	742	case '=':
683		t.cur_tok->id = TokenIdTimesPercentEq;
	743	set_token_id(&t, t.cur_tok, TokenIdTimesPercentEq);
684	744	end_token(&t);
685	745	t.state = TokenizeStateStart;
686	746	break;
...	...	@@ -694,17 +754,17 @@ void tokenize(Buf buf, Tokenization out) {
694	754	case TokenizeStateSawPercent:
695	755	switch (c) {
696	756	case '=':
697		t.cur_tok->id = TokenIdModEq;
	757	set_token_id(&t, t.cur_tok, TokenIdModEq);
698	758	end_token(&t);
699	759	t.state = TokenizeStateStart;
700	760	break;
701	761	case '.':
702		t.cur_tok->id = TokenIdPercentDot;
	762	set_token_id(&t, t.cur_tok, TokenIdPercentDot);
703	763	end_token(&t);
704	764	t.state = TokenizeStateStart;
705	765	break;
706	766	case '%':
707		t.cur_tok->id = TokenIdPercentPercent;
	767	set_token_id(&t, t.cur_tok, TokenIdPercentPercent);
708	768	end_token(&t);
709	769	t.state = TokenizeStateStart;
710	770	break;
...	...	@@ -718,17 +778,17 @@ void tokenize(Buf buf, Tokenization out) {
718	778	case TokenizeStateSawPlus:
719	779	switch (c) {
720	780	case '=':
721		t.cur_tok->id = TokenIdPlusEq;
	781	set_token_id(&t, t.cur_tok, TokenIdPlusEq);
722	782	end_token(&t);
723	783	t.state = TokenizeStateStart;
724	784	break;
725	785	case '+':
726		t.cur_tok->id = TokenIdPlusPlus;
	786	set_token_id(&t, t.cur_tok, TokenIdPlusPlus);
727	787	end_token(&t);
728	788	t.state = TokenizeStateStart;
729	789	break;
730	790	case '%':
731		t.cur_tok->id = TokenIdPlusPercent;
	791	set_token_id(&t, t.cur_tok, TokenIdPlusPercent);
732	792	t.state = TokenizeStateSawPlusPercent;
733	793	break;
734	794	default:
...	...	@@ -741,7 +801,7 @@ void tokenize(Buf buf, Tokenization out) {
741	801	case TokenizeStateSawPlusPercent:
742	802	switch (c) {
743	803	case '=':
744		t.cur_tok->id = TokenIdPlusPercentEq;
	804	set_token_id(&t, t.cur_tok, TokenIdPlusPercentEq);
745	805	end_token(&t);
746	806	t.state = TokenizeStateStart;
747	807	break;
...	...	@@ -755,11 +815,11 @@ void tokenize(Buf buf, Tokenization out) {
755	815	case TokenizeStateSawAmpersand:
756	816	switch (c) {
757	817	case '&':
758		t.cur_tok->id = TokenIdBoolAnd;
	818	set_token_id(&t, t.cur_tok, TokenIdBoolAnd);
759	819	t.state = TokenizeStateSawAmpersandAmpersand;
760	820	break;
761	821	case '=':
762		t.cur_tok->id = TokenIdBitAndEq;
	822	set_token_id(&t, t.cur_tok, TokenIdBitAndEq);
763	823	end_token(&t);
764	824	t.state = TokenizeStateStart;
765	825	break;
...	...	@@ -773,7 +833,7 @@ void tokenize(Buf buf, Tokenization out) {
773	833	case TokenizeStateSawAmpersandAmpersand:
774	834	switch (c) {
775	835	case '=':
776		t.cur_tok->id = TokenIdBoolAndEq;
	836	set_token_id(&t, t.cur_tok, TokenIdBoolAndEq);
777	837	end_token(&t);
778	838	t.state = TokenizeStateStart;
779	839	break;
...	...	@@ -787,7 +847,7 @@ void tokenize(Buf buf, Tokenization out) {
787	847	case TokenizeStateSawCaret:
788	848	switch (c) {
789	849	case '=':
790		t.cur_tok->id = TokenIdBitXorEq;
	850	set_token_id(&t, t.cur_tok, TokenIdBitXorEq);
791	851	end_token(&t);
792	852	t.state = TokenizeStateStart;
793	853	break;
...	...	@@ -801,11 +861,11 @@ void tokenize(Buf buf, Tokenization out) {
801	861	case TokenizeStateSawPipe:
802	862	switch (c) {
803	863	case '\|':
804		t.cur_tok->id = TokenIdBoolOr;
	864	set_token_id(&t, t.cur_tok, TokenIdBoolOr);
805	865	t.state = TokenizeStateSawPipePipe;
806	866	break;
807	867	case '=':
808		t.cur_tok->id = TokenIdBitOrEq;
	868	set_token_id(&t, t.cur_tok, TokenIdBitOrEq);
809	869	end_token(&t);
810	870	t.state = TokenizeStateStart;
811	871	break;
...	...	@@ -819,7 +879,7 @@ void tokenize(Buf buf, Tokenization out) {
819	879	case TokenizeStateSawPipePipe:
820	880	switch (c) {
821	881	case '=':
822		t.cur_tok->id = TokenIdBoolOrEq;
	882	set_token_id(&t, t.cur_tok, TokenIdBoolOrEq);
823	883	end_token(&t);
824	884	t.state = TokenizeStateStart;
825	885	break;
...	...	@@ -837,7 +897,7 @@ void tokenize(Buf buf, Tokenization out) {
837	897	t.state = TokenizeStateLineComment;
838	898	break;
839	899	case '=':
840		t.cur_tok->id = TokenIdDivEq;
	900	set_token_id(&t, t.cur_tok, TokenIdDivEq);
841	901	end_token(&t);
842	902	t.state = TokenizeStateStart;
843	903	break;
...	...	@@ -848,24 +908,32 @@ void tokenize(Buf buf, Tokenization out) {
848	908	continue;
849	909	}
850	910	break;
851		case TokenizeStateLineComment:
	911	case TokenizeStateSawBackslash:
	912	switch (c) {
	913	case '\\':
	914	t.state = TokenizeStateLineString;
	915	break;
	916	default:
	917	tokenize_error(&t, "invalid character: '%c'", c);
	918	break;
	919	}
	920	break;
	921	case TokenizeStateLineString:
852	922	switch (c) {
853	923	case '\n':
854		t.state = TokenizeStateStart;
	924	t.state = TokenizeStateLineStringEnd;
855	925	break;
856	926	default:
857		// do nothing
	927	buf_append_char(&t.cur_tok->data.str_lit.str, c);
858	928	break;
859	929	}
860	930	break;
861		case TokenizeStateSymbolFirst:
	931	case TokenizeStateLineStringEnd:
862	932	switch (c) {
863		case '"':
864		t.cur_tok->id = TokenIdStringLiteral;
865		t.state = TokenizeStateString;
	933	case WHITESPACE:
866	934	break;
867		case SYMBOL_CHAR:
868		t.state = TokenizeStateSymbol;
	935	case '\\':
	936	t.state = TokenizeStateLineStringContinue;
869	937	break;
870	938	default:
871	939	t.pos -= 1;
...	...	@@ -874,29 +942,38 @@ void tokenize(Buf buf, Tokenization out) {
874	942	continue;
875	943	}
876	944	break;
877		case TokenizeStateSymbolFirstRaw:
	945	case TokenizeStateLineStringContinue:
878	946	switch (c) {
879		case '"':
880		t.cur_tok->id = TokenIdStringLiteral;
881		t.state = TokenizeStateRawString;
882		t.raw_string_id_start = t.pos + 1;
883		break;
884		case SYMBOL_CHAR:
885		t.state = TokenizeStateSymbol;
	947	case '\\':
	948	t.state = TokenizeStateLineString;
	949	buf_append_char(&t.cur_tok->data.str_lit.str, '\n');
886	950	break;
887	951	default:
888		t.pos -= 1;
889		end_token(&t);
	952	tokenize_error(&t, "invalid character: '%c'", c);
	953	break;
	954	}
	955	break;
	956	case TokenizeStateLineComment:
	957	switch (c) {
	958	case '\n':
890	959	t.state = TokenizeStateStart;
891		continue;
	960	break;
	961	default:
	962	// do nothing
	963	break;
892	964	}
893	965	break;
894		case TokenizeStateSawAtSign:
	966	case TokenizeStateSymbolFirstC:
895	967	switch (c) {
896	968	case '"':
897		t.cur_tok->id = TokenIdSymbol;
	969	set_token_id(&t, t.cur_tok, TokenIdStringLiteral);
	970	t.cur_tok->data.str_lit.is_c_str = true;
898	971	t.state = TokenizeStateString;
899	972	break;
	973	case SYMBOL_CHAR:
	974	t.state = TokenizeStateSymbol;
	975	buf_append_char(&t.cur_tok->data.str_lit.str, c);
	976	break;
900	977	default:
901	978	t.pos -= 1;
902	979	end_token(&t);
...	...	@@ -904,18 +981,11 @@ void tokenize(Buf buf, Tokenization out) {
904	981	continue;
905	982	}
906	983	break;
907		case TokenizeStateFirstR:
	984	case TokenizeStateSawAtSign:
908	985	switch (c) {
909	986	case '"':
910		t.cur_tok->id = TokenIdStringLiteral;
911		t.state = TokenizeStateRawString;
912		t.raw_string_id_start = t.pos + 1;
913		break;
914		case 'c':
915		t.state = TokenizeStateSymbolFirstRaw;
916		break;
917		case SYMBOL_CHAR_EXCEPT_C:
918		t.state = TokenizeStateSymbol;
	987	set_token_id(&t, t.cur_tok, TokenIdSymbol);
	988	t.state = TokenizeStateString;
919	989	break;
920	990	default:
921	991	t.pos -= 1;
...	...	@@ -927,6 +997,7 @@ void tokenize(Buf buf, Tokenization out) {
927	997	case TokenizeStateSymbol:
928	998	switch (c) {
929	999	case SYMBOL_CHAR:
	1000	buf_append_char(&t.cur_tok->data.str_lit.str, c);
930	1001	break;
931	1002	default:
932	1003	t.pos -= 1;
...	...	@@ -942,108 +1013,124 @@ void tokenize(Buf buf, Tokenization out) {
942	1013	t.state = TokenizeStateStart;
943	1014	break;
944	1015	case '\n':
945		tokenize_error(&t, "use raw string for multiline string literal");
	1016	tokenize_error(&t, "newline not allowed in string literal");
946	1017	break;
947	1018	case '\\':
948	1019	t.state = TokenizeStateStringEscape;
949	1020	break;
950	1021	default:
	1022	buf_append_char(&t.cur_tok->data.str_lit.str, c);
951	1023	break;
952	1024	}
953	1025	break;
954	1026	case TokenizeStateStringEscape:
955	1027	switch (c) {
956	1028	case 'x':
957		t.state = TokenizeStateHex;
958		t.hex_chars_left = 2;
	1029	t.state = TokenizeStateCharCode;
	1030	t.radix = 16;
	1031	t.char_code = 0;
	1032	t.char_code_index = 0;
	1033	t.char_code_end = 2;
	1034	t.unicode = false;
959	1035	break;
960	1036	case 'u':
961		t.state = TokenizeStateHex;
962		t.hex_chars_left = 4;
	1037	t.state = TokenizeStateCharCode;
	1038	t.radix = 16;
	1039	t.char_code = 0;
	1040	t.char_code_index = 0;
	1041	t.char_code_end = 4;
	1042	t.unicode = true;
963	1043	break;
964	1044	case 'U':
965		t.state = TokenizeStateHex;
966		t.hex_chars_left = 6;
	1045	t.state = TokenizeStateCharCode;
	1046	t.radix = 16;
	1047	t.char_code = 0;
	1048	t.char_code_index = 0;
	1049	t.char_code_end = 6;
	1050	t.unicode = true;
967	1051	break;
968	1052	case 'n':
	1053	handle_string_escape(&t, '\n');
	1054	break;
969	1055	case 'r':
	1056	handle_string_escape(&t, '\r');
	1057	break;
970	1058	case '\\':
	1059	handle_string_escape(&t, '\\');
	1060	break;
971	1061	case 't':
	1062	handle_string_escape(&t, '\t');
	1063	break;
972	1064	case '\'':
	1065	handle_string_escape(&t, '\'');
	1066	break;
973	1067	case '"':
974		if (t.cur_tok->id == TokenIdCharLiteral) {
975		t.state = TokenizeStateCharLiteralEnd;
976		} else if (t.cur_tok->id == TokenIdStringLiteral) {
977		t.state = TokenizeStateString;
978		} else {
979		zig_unreachable();
980		}
	1068	handle_string_escape(&t, '\"');
981	1069	break;
982	1070	default:
983	1071	tokenize_error(&t, "invalid character: '%c'", c);
984	1072	}
985	1073	break;
986		case TokenizeStateHex:
987		switch (c) {
988		case HEX_DIGIT:
989		t.hex_chars_left -= 1;
990		if (t.hex_chars_left == 0) {
991		if (t.cur_tok->id == TokenIdCharLiteral) {
992		t.state = TokenizeStateCharLiteralEnd;
993		} else if (t.cur_tok->id == TokenIdStringLiteral) {
994		t.state = TokenizeStateString;
995		} else if (t.cur_tok->id == TokenIdSymbol) {
996		t.state = TokenizeStateString;
	1074	case TokenizeStateCharCode:
	1075	{
	1076	uint32_t digit_value = get_digit_value(c);
	1077	if (digit_value >= t.radix) {
	1078	tokenize_error(&t, "invalid digit: '%c'", c);
	1079	}
	1080	t.char_code *= t.radix;
	1081	t.char_code += digit_value;
	1082	t.char_code_index += 1;
	1083
	1084	if (t.char_code_index >= t.char_code_end) {
	1085	if (t.unicode) {
	1086	if (t.char_code <= 0x7f) {
	1087	// 00000000 00000000 00000000 0xxxxxxx
	1088	handle_string_escape(&t, t.char_code);
	1089	} else if (t.cur_tok->id == TokenIdCharLiteral) {
	1090	tokenize_error(&t, "unicode value too large for character literal: %x", t.char_code);
	1091	} else if (t.char_code <= 0x7ff) {
	1092	// 00000000 00000000 00000xxx xx000000
	1093	handle_string_escape(&t, 0xc0 \| (t.char_code >> 6));
	1094	// 00000000 00000000 00000000 00xxxxxx
	1095	handle_string_escape(&t, 0x80 \| (t.char_code & 0x3f));
	1096	} else if (t.char_code <= 0xffff) {
	1097	// 00000000 00000000 xxxx0000 00000000
	1098	handle_string_escape(&t, 0xe0 \| (t.char_code >> 12));
	1099	// 00000000 00000000 0000xxxx xx000000
	1100	handle_string_escape(&t, 0x80 \| ((t.char_code >> 6) & 0x3f));
	1101	// 00000000 00000000 00000000 00xxxxxx
	1102	handle_string_escape(&t, 0x80 \| (t.char_code & 0x3f));
	1103	} else if (t.char_code <= 0x10ffff) {
	1104	// 00000000 000xxx00 00000000 00000000
	1105	handle_string_escape(&t, 0xf0 \| (t.char_code >> 18));
	1106	// 00000000 000000xx xxxx0000 00000000
	1107	handle_string_escape(&t, 0x80 \| ((t.char_code >> 12) & 0x3f));
	1108	// 00000000 00000000 0000xxxx xx000000
	1109	handle_string_escape(&t, 0x80 \| ((t.char_code >> 6) & 0x3f));
	1110	// 00000000 00000000 00000000 00xxxxxx
	1111	handle_string_escape(&t, 0x80 \| (t.char_code & 0x3f));
997	1112	} else {
998		zig_unreachable();
	1113	tokenize_error(&t, "unicode value out of range: %x", t.char_code);
999	1114	}
	1115	} else {
	1116	if (t.cur_tok->id == TokenIdCharLiteral && t.char_code >= sizeof(uint8_t)) {
	1117	tokenize_error(&t, "value too large for character literal: '%x'",
	1118	t.char_code);
	1119	}
	1120	handle_string_escape(&t, t.char_code);
1000	1121	}
1001		break;
1002		default:
1003		tokenize_error(&t, "invalid character: '%c'", c);
1004		}
1005		break;
1006		case TokenizeStateRawString:
1007		if (c == '(') {
1008		t.raw_string_id_end = t.pos;
1009		t.cur_tok->raw_string_start = t.pos + 1;
1010		t.state = TokenizeStateRawStringContents;
1011		}
1012		break;
1013		case TokenizeStateRawStringContents:
1014		if (c == ')') {
1015		t.state = TokenizeStateRawStringMaybeEnd;
1016		t.raw_string_id_cmp_pos = t.raw_string_id_start;
1017		t.cur_tok->raw_string_end = t.pos;
1018		}
1019		break;
1020		case TokenizeStateRawStringMaybeEnd:
1021		if (t.raw_string_id_cmp_pos >= t.raw_string_id_end &&
1022		c == '"')
1023		{
1024		end_token(&t);
1025		t.state = TokenizeStateStart;
1026		} else if (c != buf_ptr(t.buf)[t.raw_string_id_cmp_pos]) {
1027		if (c == ')') {
1028		t.raw_string_id_cmp_pos = t.raw_string_id_start;
1029		t.cur_tok->raw_string_end = t.pos;
1030		} else {
1031		t.state = TokenizeStateRawStringContents;
1032	1122	}
1033		} else {
1034		t.raw_string_id_cmp_pos += 1;
1035	1123	}
1036	1124	break;
1037	1125	case TokenizeStateCharLiteral:
1038	1126	switch (c) {
1039	1127	case '\'':
1040		end_token(&t);
1041		t.state = TokenizeStateStart;
1042		break;
	1128	tokenize_error(&t, "expected character");
1043	1129	case '\\':
1044	1130	t.state = TokenizeStateStringEscape;
1045	1131	break;
1046	1132	default:
	1133	t.cur_tok->data.char_lit.c = c;
1047	1134	t.state = TokenizeStateCharLiteralEnd;
1048	1135	break;
1049	1136	}
...	...	@@ -1061,15 +1148,17 @@ void tokenize(Buf buf, Tokenization out) {
1061	1148	case TokenizeStateZero:
1062	1149	switch (c) {
1063	1150	case 'b':
1064		t.cur_tok->radix = 2;
	1151	t.radix = 2;
1065	1152	t.state = TokenizeStateNumber;
1066	1153	break;
1067	1154	case 'o':
1068		t.cur_tok->radix = 8;
	1155	t.radix = 8;
	1156	t.exp_add_amt = 3;
1069	1157	t.state = TokenizeStateNumber;
1070	1158	break;
1071	1159	case 'x':
1072		t.cur_tok->radix = 16;
	1160	t.radix = 16;
	1161	t.exp_add_amt = 4;
1073	1162	t.state = TokenizeStateNumber;
1074	1163	break;
1075	1164	default:
...	...	@@ -1082,113 +1171,127 @@ void tokenize(Buf buf, Tokenization out) {
1082	1171	case TokenizeStateNumber:
1083	1172	{
1084	1173	if (c == '.') {
1085		if (t.pos + 1 < buf_len(t.buf)) {
1086		uint8_t next_c = buf_ptr(t.buf)[t.pos + 1];
1087		if (next_c == '.') {
1088		t.pos -= 1;
1089		end_token(&t);
1090		t.state = TokenizeStateStart;
1091		continue;
1092		}
1093		}
1094		t.cur_tok->decimal_point_pos = t.pos;
1095		t.state = TokenizeStateFloatFraction;
	1174	t.state = TokenizeStateNumberDot;
1096	1175	break;
1097	1176	}
1098		if (is_exponent_signifier(c, t.cur_tok->radix)) {
1099		t.cur_tok->exponent_marker_pos = t.pos;
	1177	if (is_exponent_signifier(c, t.radix)) {
1100	1178	t.state = TokenizeStateFloatExponentUnsigned;
	1179	t.is_num_lit_float = true;
1101	1180	break;
1102	1181	}
1103		if (c == '_') {
1104		tokenize_error(&t, "invalid character: '%c'", c);
1105		break;
1106		}
1107		int digit_value = get_digit_value(c);
1108		if (digit_value >= 0) {
1109		if (digit_value >= t.cur_tok->radix) {
	1182	uint32_t digit_value = get_digit_value(c);
	1183	if (digit_value >= t.radix) {
	1184	if (is_symbol_char(c)) {
1110	1185	tokenize_error(&t, "invalid character: '%c'", c);
1111		break;
1112	1186	}
1113		// normal digit
1114		} else {
1115	1187	// not my char
1116	1188	t.pos -= 1;
1117	1189	end_token(&t);
1118	1190	t.state = TokenizeStateStart;
1119	1191	continue;
1120	1192	}
	1193	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
	1194	bignum_multiply_by_scalar(&t.cur_tok->data.num_lit.bignum, t.radix);
	1195	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
	1196	bignum_increment_by_scalar(&t.cur_tok->data.num_lit.bignum, digit_value);
1121	1197	break;
1122	1198	}
	1199	case TokenizeStateNumberDot:
	1200	if (c == '.') {
	1201	t.pos -= 2;
	1202	end_token(&t);
	1203	t.state = TokenizeStateStart;
	1204	continue;
	1205	}
	1206	t.pos -= 1;
	1207	t.state = TokenizeStateFloatFraction;
	1208	t.is_num_lit_float = true;
	1209	continue;
1123	1210	case TokenizeStateFloatFraction:
1124	1211	{
1125		if (is_exponent_signifier(c, t.cur_tok->radix)) {
1126		t.cur_tok->exponent_marker_pos = t.pos;
	1212	if (is_exponent_signifier(c, t.radix)) {
1127	1213	t.state = TokenizeStateFloatExponentUnsigned;
1128	1214	break;
1129	1215	}
1130		if (c == '_') {
1131		tokenize_error(&t, "invalid character: '%c'", c);
1132		break;
1133		}
1134		int digit_value = get_digit_value(c);
1135		if (digit_value >= 0) {
1136		if (digit_value >= t.cur_tok->radix) {
	1216	uint32_t digit_value = get_digit_value(c);
	1217	if (digit_value >= t.radix) {
	1218	if (is_symbol_char(c)) {
1137	1219	tokenize_error(&t, "invalid character: '%c'", c);
1138		break;
1139	1220	}
1140		// normal digit
1141		} else {
1142	1221	// not my char
1143	1222	t.pos -= 1;
1144	1223	end_token(&t);
1145	1224	t.state = TokenizeStateStart;
1146	1225	continue;
1147	1226	}
	1227	t.exponent_in_bin_or_dec -= t.exp_add_amt;
	1228	if (t.radix == 10) {
	1229	// For now we use strtod to parse decimal floats, so we just have to get to the
	1230	// end of the token.
	1231	break;
	1232	}
	1233	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
	1234	bignum_multiply_by_scalar(&t.cur_tok->data.num_lit.bignum, t.radix);
	1235	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
	1236	bignum_increment_by_scalar(&t.cur_tok->data.num_lit.bignum, digit_value);
1148	1237	break;
1149	1238	}
1150	1239	case TokenizeStateFloatExponentUnsigned:
1151	1240	switch (c) {
1152	1241	case '+':
	1242	t.is_exp_negative = false;
	1243	t.state = TokenizeStateFloatExponentNumber;
	1244	break;
1153	1245	case '-':
	1246	t.is_exp_negative = true;
1154	1247	t.state = TokenizeStateFloatExponentNumber;
1155	1248	break;
1156	1249	default:
1157	1250	// reinterpret as normal exponent number
1158	1251	t.pos -= 1;
	1252	t.is_exp_negative = false;
1159	1253	t.state = TokenizeStateFloatExponentNumber;
1160	1254	continue;
1161	1255	}
1162	1256	break;
1163	1257	case TokenizeStateFloatExponentNumber:
1164		switch (c) {
1165		case DIGIT:
1166		break;
1167		case ALPHA:
1168		case '_':
1169		tokenize_error(&t, "invalid character: '%c'", c);
1170		break;
1171		default:
	1258	{
	1259	uint32_t digit_value = get_digit_value(c);
	1260	if (digit_value >= t.radix) {
	1261	if (is_symbol_char(c)) {
	1262	tokenize_error(&t, "invalid character: '%c'", c);
	1263	}
	1264	// not my char
1172	1265	t.pos -= 1;
1173	1266	end_token(&t);
1174	1267	t.state = TokenizeStateStart;
1175	1268	continue;
	1269	}
	1270	if (t.radix == 10) {
	1271	// For now we use strtod to parse decimal floats, so we just have to get to the
	1272	// end of the token.
	1273	break;
	1274	}
	1275	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
	1276	bignum_multiply_by_scalar(&t.specified_exponent, 10);
	1277	t.cur_tok->data.num_lit.overflow = t.cur_tok->data.num_lit.overflow \|\|
	1278	bignum_increment_by_scalar(&t.specified_exponent, digit_value);
1176	1279	}
1177	1280	break;
1178	1281	case TokenizeStateSawDash:
1179	1282	switch (c) {
1180	1283	case '>':
1181		t.cur_tok->id = TokenIdArrow;
	1284	set_token_id(&t, t.cur_tok, TokenIdArrow);
1182	1285	end_token(&t);
1183	1286	t.state = TokenizeStateStart;
1184	1287	break;
1185	1288	case '=':
1186		t.cur_tok->id = TokenIdMinusEq;
	1289	set_token_id(&t, t.cur_tok, TokenIdMinusEq);
1187	1290	end_token(&t);
1188	1291	t.state = TokenizeStateStart;
1189	1292	break;
1190	1293	case '%':
1191		t.cur_tok->id = TokenIdMinusPercent;
	1294	set_token_id(&t, t.cur_tok, TokenIdMinusPercent);
1192	1295	t.state = TokenizeStateSawMinusPercent;
1193	1296	break;
1194	1297	default:
...	...	@@ -1201,7 +1304,7 @@ void tokenize(Buf buf, Tokenization out) {
1201	1304	case TokenizeStateSawMinusPercent:
1202	1305	switch (c) {
1203	1306	case '=':
1204		t.cur_tok->id = TokenIdMinusPercentEq;
	1307	set_token_id(&t, t.cur_tok, TokenIdMinusPercentEq);
1205	1308	end_token(&t);
1206	1309	t.state = TokenizeStateStart;
1207	1310	break;
...	...	@@ -1226,11 +1329,14 @@ void tokenize(Buf buf, Tokenization out) {
1226	1329	case TokenizeStateStart:
1227	1330	case TokenizeStateError:
1228	1331	break;
	1332	case TokenizeStateNumberDot:
	1333	tokenize_error(&t, "unterminated number literal");
	1334	break;
1229	1335	case TokenizeStateString:
1230	1336	tokenize_error(&t, "unterminated string");
1231	1337	break;
1232	1338	case TokenizeStateStringEscape:
1233		case TokenizeStateHex:
	1339	case TokenizeStateCharCode:
1234	1340	if (t.cur_tok->id == TokenIdStringLiteral) {
1235	1341	tokenize_error(&t, "unterminated string");
1236	1342	} else if (t.cur_tok->id == TokenIdCharLiteral) {
...	...	@@ -1239,19 +1345,12 @@ void tokenize(Buf buf, Tokenization out) {
1239	1345	zig_unreachable();
1240	1346	}
1241	1347	break;
1242		case TokenizeStateRawString:
1243		case TokenizeStateRawStringContents:
1244		case TokenizeStateRawStringMaybeEnd:
1245		tokenize_error(&t, "unterminated raw string");
1246		break;
1247	1348	case TokenizeStateCharLiteral:
1248	1349	case TokenizeStateCharLiteralEnd:
1249	1350	tokenize_error(&t, "unterminated character literal");
1250	1351	break;
1251	1352	case TokenizeStateSymbol:
1252		case TokenizeStateSymbolFirst:
1253		case TokenizeStateSymbolFirstRaw:
1254		case TokenizeStateFirstR:
	1353	case TokenizeStateSymbolFirstC:
1255	1354	case TokenizeStateZero:
1256	1355	case TokenizeStateNumber:
1257	1356	case TokenizeStateFloatFraction:
...	...	@@ -1280,9 +1379,13 @@ void tokenize(Buf buf, Tokenization out) {
1280	1379	case TokenizeStateSawPlusPercent:
1281	1380	case TokenizeStateSawMinusPercent:
1282	1381	case TokenizeStateSawShiftLeftPercent:
	1382	case TokenizeStateLineString:
	1383	case TokenizeStateLineStringEnd:
1283	1384	end_token(&t);
1284	1385	break;
1285	1386	case TokenizeStateSawDotDot:
	1387	case TokenizeStateSawBackslash:
	1388	case TokenizeStateLineStringContinue:
1286	1389	tokenize_error(&t, "unexpected EOF");
1287	1390	break;
1288	1391	case TokenizeStateLineComment:

src/tokenizer.hpp+26-9

...	...	@@ -9,6 +9,7 @@
9	9	#define ZIG_TOKENIZER_HPP
10	10
11	11	#include "buffer.hpp"
	12	#include "bignum.hpp"
12	13
13	14	enum TokenId {
14	15	TokenIdEof,
...	...	@@ -111,6 +112,22 @@ enum TokenId {
111	112	TokenIdPercentDot,
112	113	};
113	114
	115	struct TokenNumLit {
	116	BigNum bignum;
	117	// overflow is true if when parsing the number, we discovered it would not
	118	// fit without losing data in a uint64_t or double
	119	bool overflow;
	120	};
	121
	122	struct TokenStrLit {
	123	Buf str;
	124	bool is_c_str;
	125	};
	126
	127	struct TokenCharLit {
	128	uint8_t c;
	129	};
	130
114	131	struct Token {
115	132	TokenId id;
116	133	int start_pos;
...	...	@@ -118,14 +135,16 @@ struct Token {
118	135	int start_line;
119	136	int start_column;
120	137
121		// for id == TokenIdNumberLiteral
122		int radix; // if != 10, then skip the first 2 characters
123		int decimal_point_pos; // either exponent_marker_pos or the position of the '.'
124		int exponent_marker_pos; // either end_pos or the position of the 'e'/'p'
	138	union {
	139	// TokenIdNumberLiteral
	140	TokenNumLit num_lit;
125	141
126		// for id == TokenIdStringLiteral
127		int raw_string_start;
128		int raw_string_end;
	142	// TokenIdStringLiteral or TokenIdSymbol
	143	TokenStrLit str_lit;
	144
	145	// TokenIdCharLiteral
	146	TokenCharLit char_lit;
	147	} data;
129	148	};
130	149
131	150	struct Tokenization {
...	...	@@ -142,8 +161,6 @@ void tokenize(Buf buf, Tokenization out_tokenization);
142	161
143	162	void print_tokens(Buf buf, ZigList<Token> tokens);
144	163
145		int get_digit_value(uint8_t c);
146
147	164	const char * token_name(TokenId id);
148	165
149	166	bool valid_symbol_starter(uint8_t c);

test/run_tests.cpp+2-2

...	...	@@ -1173,7 +1173,7 @@ fn f() {
1173	1173	add_compile_fail_case("normal string with newline", R"SOURCE(
1174	1174	const foo = "a
1175	1175	b";
1176		)SOURCE", 1, ".tmp_source.zig:2:13: error: use raw string for multiline string literal");
	1176	)SOURCE", 1, ".tmp_source.zig:2:13: error: newline not allowed in string literal");
1177	1177
1178	1178	add_compile_fail_case("invalid comparison for function pointers", R"SOURCE(
1179	1179	fn foo() {}
...	...	@@ -1760,7 +1760,7 @@ struct type {
1760	1760	)SOURCE", 3,
1761	1761	R"(pub const FOO = c"aoeu\x13 derp")",
1762	1762	R"(pub const FOO2 = c"aoeu\x134 derp")",
1763		R"(pub const FOO_CHAR = '\x3f')");
	1763	R"(pub const FOO_CHAR = '?')");
1764	1764	}
1765	1765
1766	1766	static void run_self_hosted_test(bool is_release_mode) {

test/self_hosted.zig+6-10

...	...	@@ -684,17 +684,13 @@ fn count_trailing_zeroes() {
684	684
685	685	#attribute("test")
686	686	fn multiline_string() {
687		const s1 = r"AOEU(
688		one
689		two)
690		three)AOEU";
691		const s2 = "\none\ntwo)\nthree";
692		const s3 = r"(
693		one
694		two)
695		three)";
	687	const s1 =
	688	\\one
	689	\\two)
	690	\\three
	691	;
	692	const s2 = "one\ntwo)\nthree";
696	693	assert(str.eql(s1, s2));
697		assert(str.eql(s3, s2));
698	694	}
699	695
700	696