Commit e5d1f0eea592e6efc4c4665eb473de95ef106e8a | zig

author	Andrew Kelley <andrew@ziglang.org> 2015-11-28 00:40:54-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2015-11-29 02:43:05-07:00
log	e5d1f0eea592e6efc4c4665eb473de95ef106e8a
tree	e37bba95e3e2d2941ec4d9ffb9f09358a0bc0fed
parent	f6529341a2413a3eb72920418d19ea7d2aca089b

parse and codegen for math expressions

7 files changed, 1396 insertions(+), 183 deletions(-)

README.md
doc
- vim
  - syntax
    zig.vim
src

README.md+57-9

...	...	@@ -30,6 +30,16 @@ readable, safe, optimal, and concise code to solve any computing problem.
30	30	This mode should automatically provide test coverage.
31	31	* Memory zeroed by default, unless you initialize with "uninitialized".
32	32
	33	### Building
	34
	35	```
	36	mkdir build
	37	cd build
	38	cmake ..
	39	make
	40	./run_tests
	41	```
	42
33	43	## Roadmap
34	44
35	45	* Math expression
...	...	@@ -93,25 +103,63 @@ PointerType : token(Star) token(Const) Type \| token(Star) token(Mut) Type
93	103
94	104	Block : token(LBrace) many(Statement) token(RBrace)
95	105
96		Statement : ExpressionStatement \| ReturnStatement
	106	Statement : ExpressionStatement
97	107
98	108	ExpressionStatement : Expression token(Semicolon)
99	109
100		ReturnStatement : token(Return) option(Expression) token(Semicolon)
	110	Expression : BoolOrExpression \| ReturnExpression
	111
	112	BoolOrExpression : BoolAndExpression token(BoolOr) BoolAndExpression \| BoolAndExpression
	113
	114	ReturnExpression : token(Return) option(Expression)
	115
	116	BoolAndExpression : ComparisonExpression token(BoolAnd) ComparisonExpression \| ComparisonExpression
	117
	118	ComparisonExpression : BinaryOrExpression ComparisonOperator BinaryOrExpression \| BinaryOrExpression
	119
	120	ComparisonOperator : token(BoolEq) \| token(BoolNotEq) \| token(BoolLessThan) \| token(BoolGreaterThan) \| token(BoolLessEqual) \| token(BoolGreaterEqual)
	121
	122	BinaryOrExpression : BinaryXorExpression token(BinOr) BinaryXorExpression \| BinaryXorExpression
101	123
102		Expression : token(Number) \| token(String) \| token(Unreachable) \| FnCall
	124	BinaryXorExpression : BinaryAndExpression token(BinXor) BinaryAndExpression \| BinaryAndExpression
	125
	126	BinaryAndExpression : BitShiftExpression token(BinAnd) BitShiftExpression \| BitShiftExpression
	127
	128	BitShiftExpression : AdditionExpression BitShiftOperator AdditionExpression \| AdditionExpression
	129
	130	BitShiftOperator : token(BitShiftLeft \| token(BitShiftRight)
	131
	132	AdditionExpression : MultiplyExpression AdditionOperator MultiplyExpression \| MultiplyExpression
	133
	134	AdditionOperator : token(Plus) \| token(Minus)
	135
	136	MultiplyExpression : CastExpression MultiplyOperator CastExpression \| CastExpression
	137
	138	MultiplyOperator : token(Star) \| token(Slash) \| token(Percent)
	139
	140	CastExpression : PrimaryExpression token(as) Type \| PrimaryExpression
	141
	142	PrimaryExpression : token(Number) \| token(String) \| token(Unreachable) \| FnCall \| GroupedExpression \| Block
	143
	144	GroupedExpression : token(LParen) Expression token(RParen)
103	145
104	146	FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen)
105	147
106	148	Directive : token(NumberSign) token(Symbol) token(LParen) token(String) token(RParen)
107	149	```
108	150
109		### Building
	151	### Binary Operator Precedence
110	152
111	153	```
112		mkdir build
113		cd build
114		cmake ..
115		make
116		./run_tests
	154	as
	155	* / %
	156	+ -
	157	<< >>
	158	&
	159	^
	160	\|
	161	== != < > <= >=
	162	&&
	163	\|\|
	164	=
117	165	```

doc/vim/syntax/zig.vim+1-1

...	...	@@ -7,7 +7,7 @@ if exists("b:current_syntax")
7	7	finish
8	8	endif
9	9
10		syn keyword zigKeyword fn return mut const extern unreachable export pub
	10	syn keyword zigKeyword fn return mut const extern unreachable export pub as
11	11	syn keyword zigType bool i8 u8 i16 u16 i32 u32 i64 u64 isize usize f32 f64 f128 void
12	12
13	13	syn region zigCommentLine start="//" end="$" contains=zigTodo,@Spell

src/codegen.cpp+377-64

...	...	@@ -77,6 +77,7 @@ struct CodeGen {
77	77	ZigList<FnTableEntry *> fn_defs;
78	78	Buf *out_name;
79	79	OutType out_type;
	80	LLVMValueRef cur_fn;
80	81	};
81	82
82	83	struct TypeNode {
...	...	@@ -301,12 +302,23 @@ static void find_declarations(CodeGen g, AstNode node) {
301	302	// we handled directives in the parent function
302	303	break;
303	304	case NodeTypeFnDecl:
304		case NodeTypeStatementReturn:
	305	case NodeTypeReturnExpr:
305	306	case NodeTypeRoot:
306	307	case NodeTypeBlock:
307		case NodeTypeExpression:
	308	case NodeTypeBoolOrExpr:
308	309	case NodeTypeFnCall:
309	310	case NodeTypeRootExportDecl:
	311	case NodeTypeBoolAndExpr:
	312	case NodeTypeComparisonExpr:
	313	case NodeTypeBinOrExpr:
	314	case NodeTypeBinXorExpr:
	315	case NodeTypeBinAndExpr:
	316	case NodeTypeBitShiftExpr:
	317	case NodeTypeAddExpr:
	318	case NodeTypeMultExpr:
	319	case NodeTypeCastExpr:
	320	case NodeTypePrimaryExpr:
	321	case NodeTypeGroupedExpr:
310	322	zig_unreachable();
311	323	}
312	324	}
...	...	@@ -341,7 +353,7 @@ static void check_fn_def_control_flow(CodeGen g, AstNode node) {
341	353	bool prev_statement_return = false;
342	354	for (int i = 0; i < body_node->data.block.statements.length; i += 1) {
343	355	AstNode *statement_node = body_node->data.block.statements.at(i);
344		if (statement_node->type == NodeTypeStatementReturn) {
	356	if (statement_node->type == NodeTypeReturnExpr) {
345	357	if (type_id == TypeIdUnreachable) {
346	358	add_node_error(g, statement_node,
347	359	buf_sprintf("return statement in function with unreachable return type"));
...	...	@@ -464,23 +476,15 @@ static void analyze_node(CodeGen g, AstNode node) {
464	476	analyze_node(g, child);
465	477	}
466	478	break;
467		case NodeTypeStatementReturn:
468		if (node->data.statement_return.expression) {
469		analyze_node(g, node->data.statement_return.expression);
	479	case NodeTypeReturnExpr:
	480	if (node->data.return_expr.expr) {
	481	analyze_node(g, node->data.return_expr.expr);
470	482	}
471	483	break;
472		case NodeTypeExpression:
473		switch (node->data.expression.type) {
474		case AstNodeExpressionTypeNumber:
475		break;
476		case AstNodeExpressionTypeString:
477		break;
478		case AstNodeExpressionTypeFnCall:
479		analyze_node(g, node->data.expression.data.fn_call);
480		break;
481		case AstNodeExpressionTypeUnreachable:
482		break;
483		}
	484	case NodeTypeBoolOrExpr:
	485	analyze_node(g, node->data.bool_or_expr.op1);
	486	if (node->data.bool_or_expr.op2)
	487	analyze_node(g, node->data.bool_or_expr.op2);
484	488	break;
485	489	case NodeTypeFnCall:
486	490	{
...	...	@@ -511,6 +515,54 @@ static void analyze_node(CodeGen g, AstNode node) {
511	515	case NodeTypeDirective:
512	516	// we looked at directives in the parent node
513	517	break;
	518	case NodeTypeBoolAndExpr:
	519	zig_panic("TODO");
	520	break;
	521	case NodeTypeComparisonExpr:
	522	zig_panic("TODO");
	523	break;
	524	case NodeTypeBinOrExpr:
	525	zig_panic("TODO");
	526	break;
	527	case NodeTypeBinXorExpr:
	528	zig_panic("TODO");
	529	break;
	530	case NodeTypeBinAndExpr:
	531	zig_panic("TODO");
	532	break;
	533	case NodeTypeBitShiftExpr:
	534	zig_panic("TODO");
	535	break;
	536	case NodeTypeAddExpr:
	537	zig_panic("TODO");
	538	break;
	539	case NodeTypeMultExpr:
	540	zig_panic("TODO");
	541	break;
	542	case NodeTypeCastExpr:
	543	zig_panic("TODO");
	544	break;
	545	case NodeTypePrimaryExpr:
	546	switch (node->data.primary_expr.type) {
	547	case PrimaryExprTypeNumber:
	548	case PrimaryExprTypeString:
	549	case PrimaryExprTypeUnreachable:
	550	// nothing to do
	551	break;
	552	case PrimaryExprTypeFnCall:
	553	analyze_node(g, node->data.primary_expr.data.fn_call);
	554	break;
	555	case PrimaryExprTypeGroupedExpr:
	556	analyze_node(g, node->data.primary_expr.data.grouped_expr);
	557	break;
	558	case PrimaryExprTypeBlock:
	559	analyze_node(g, node->data.primary_expr.data.block);
	560	break;
	561	}
	562	break;
	563	case NodeTypeGroupedExpr:
	564	zig_panic("TODO");
	565	break;
514	566	}
515	567	}
516	568
...	...	@@ -649,34 +701,326 @@ static LLVMValueRef find_or_create_string(CodeGen g, Buf str) {
649	701	return global_value;
650	702	}
651	703
652		static LLVMValueRef gen_expr(CodeGen g, AstNode expr_node) {
653		assert(expr_node->type == NodeTypeExpression);
654		switch (expr_node->data.expression.type) {
655		case AstNodeExpressionTypeNumber:
	704	static LLVMValueRef gen_primary_expr(CodeGen g, AstNode node) {
	705	assert(node->type == NodeTypePrimaryExpr);
	706
	707	AstNodePrimaryExpr *prim_expr = &node->data.primary_expr;
	708
	709	switch (node->data.primary_expr.type) {
	710	case PrimaryExprTypeNumber:
656	711	{
657		Buf *number_str = &expr_node->data.expression.data.number;
	712	Buf *number_str = &prim_expr->data.number;
658	713	LLVMTypeRef number_type = LLVMInt32Type();
659	714	LLVMValueRef number_val = LLVMConstIntOfStringAndSize(number_type,
660	715	buf_ptr(number_str), buf_len(number_str), 10);
661	716	return number_val;
662	717	}
663		case AstNodeExpressionTypeString:
	718	case PrimaryExprTypeString:
664	719	{
665		Buf *str = &expr_node->data.expression.data.string;
	720	Buf *str = &prim_expr->data.string;
666	721	LLVMValueRef str_val = find_or_create_string(g, str);
667	722	LLVMValueRef indices[] = {
668	723	LLVMConstInt(LLVMInt32Type(), 0, false),
669	724	LLVMConstInt(LLVMInt32Type(), 0, false)
670	725	};
671		LLVMValueRef ptr_val = LLVMBuildInBoundsGEP(g->builder, str_val,
672		indices, 2, "");
673
	726	LLVMValueRef ptr_val = LLVMBuildInBoundsGEP(g->builder, str_val, indices, 2, "");
674	727	return ptr_val;
675	728	}
676		case AstNodeExpressionTypeFnCall:
677		return gen_fn_call(g, expr_node->data.expression.data.fn_call);
678		case AstNodeExpressionTypeUnreachable:
	729	case PrimaryExprTypeUnreachable:
	730	add_debug_source_node(g, node);
679	731	return LLVMBuildUnreachable(g->builder);
	732	case PrimaryExprTypeFnCall:
	733	return gen_fn_call(g, prim_expr->data.fn_call);
	734	case PrimaryExprTypeGroupedExpr:
	735	return gen_expr(g, prim_expr->data.grouped_expr);
	736	case PrimaryExprTypeBlock:
	737	break;
	738	}
	739
	740	zig_unreachable();
	741	}
	742
	743	static LLVMValueRef gen_cast_expr(CodeGen g, AstNode node) {
	744	assert(node->type == NodeTypeCastExpr);
	745
	746	LLVMValueRef expr = gen_primary_expr(g, node->data.cast_expr.primary_expr);
	747
	748	if (!node->data.cast_expr.type)
	749	return expr;
	750
	751	zig_panic("TODO cast expression");
	752	}
	753
	754	static LLVMValueRef gen_mult_expr(CodeGen g, AstNode node) {
	755	assert(node->type == NodeTypeMultExpr);
	756
	757	LLVMValueRef val1 = gen_cast_expr(g, node->data.mult_expr.op1);
	758
	759	if (!node->data.mult_expr.op2)
	760	return val1;
	761
	762	LLVMValueRef val2 = gen_cast_expr(g, node->data.mult_expr.op2);
	763
	764	switch (node->data.mult_expr.mult_op) {
	765	case MultOpMult:
	766	// TODO types so we know float vs int
	767	add_debug_source_node(g, node);
	768	return LLVMBuildMul(g->builder, val1, val2, "");
	769	case MultOpDiv:
	770	// TODO types so we know float vs int and signed vs unsigned
	771	add_debug_source_node(g, node);
	772	return LLVMBuildSDiv(g->builder, val1, val2, "");
	773	case MultOpMod:
	774	// TODO types so we know float vs int and signed vs unsigned
	775	add_debug_source_node(g, node);
	776	return LLVMBuildSRem(g->builder, val1, val2, "");
	777	case MultOpInvalid:
	778	zig_unreachable();
	779	}
	780	zig_unreachable();
	781	}
	782
	783	static LLVMValueRef gen_add_expr(CodeGen g, AstNode node) {
	784	assert(node->type == NodeTypeAddExpr);
	785
	786	LLVMValueRef val1 = gen_mult_expr(g, node->data.add_expr.op1);
	787
	788	if (!node->data.add_expr.op2)
	789	return val1;
	790
	791	LLVMValueRef val2 = gen_mult_expr(g, node->data.add_expr.op2);
	792
	793	switch (node->data.add_expr.add_op) {
	794	case AddOpAdd:
	795	add_debug_source_node(g, node);
	796	return LLVMBuildAdd(g->builder, val1, val2, "");
	797	case AddOpSub:
	798	add_debug_source_node(g, node);
	799	return LLVMBuildSub(g->builder, val1, val2, "");
	800	case AddOpInvalid:
	801	zig_unreachable();
	802	}
	803	zig_unreachable();
	804	}
	805
	806	static LLVMValueRef gen_bit_shift_expr(CodeGen g, AstNode node) {
	807	assert(node->type == NodeTypeBitShiftExpr);
	808
	809	LLVMValueRef val1 = gen_add_expr(g, node->data.bit_shift_expr.op1);
	810
	811	if (!node->data.bit_shift_expr.op2)
	812	return val1;
	813
	814	LLVMValueRef val2 = gen_add_expr(g, node->data.bit_shift_expr.op2);
	815
	816	switch (node->data.bit_shift_expr.bit_shift_op) {
	817	case BitShiftOpLeft:
	818	add_debug_source_node(g, node);
	819	return LLVMBuildShl(g->builder, val1, val2, "");
	820	case BitShiftOpRight:
	821	// TODO implement type system so that we know whether to do
	822	// logical or arithmetic shifting here.
	823	// signed -> arithmetic, unsigned -> logical
	824	add_debug_source_node(g, node);
	825	return LLVMBuildLShr(g->builder, val1, val2, "");
	826	case BitShiftOpInvalid:
	827	zig_unreachable();
	828	}
	829	zig_unreachable();
	830	}
	831
	832	static LLVMValueRef gen_bin_and_expr(CodeGen g, AstNode node) {
	833	assert(node->type == NodeTypeBinAndExpr);
	834
	835	LLVMValueRef val1 = gen_bit_shift_expr(g, node->data.bin_and_expr.op1);
	836
	837	if (!node->data.bin_and_expr.op2)
	838	return val1;
	839
	840	LLVMValueRef val2 = gen_bit_shift_expr(g, node->data.bin_and_expr.op2);
	841
	842	add_debug_source_node(g, node);
	843	return LLVMBuildAnd(g->builder, val1, val2, "");
	844	}
	845
	846	static LLVMValueRef gen_bin_xor_expr(CodeGen g, AstNode node) {
	847	assert(node->type == NodeTypeBinXorExpr);
	848
	849	LLVMValueRef val1 = gen_bin_and_expr(g, node->data.bin_xor_expr.op1);
	850
	851	if (!node->data.bin_xor_expr.op2)
	852	return val1;
	853
	854	LLVMValueRef val2 = gen_bin_and_expr(g, node->data.bin_xor_expr.op2);
	855
	856	add_debug_source_node(g, node);
	857	return LLVMBuildXor(g->builder, val1, val2, "");
	858	}
	859
	860	static LLVMValueRef gen_bin_or_expr(CodeGen g, AstNode node) {
	861	assert(node->type == NodeTypeBinOrExpr);
	862
	863	LLVMValueRef val1 = gen_bin_xor_expr(g, node->data.bin_or_expr.op1);
	864
	865	if (!node->data.bin_or_expr.op2)
	866	return val1;
	867
	868	LLVMValueRef val2 = gen_bin_xor_expr(g, node->data.bin_or_expr.op2);
	869
	870	add_debug_source_node(g, node);
	871	return LLVMBuildOr(g->builder, val1, val2, "");
	872	}
	873
	874	static LLVMIntPredicate cmp_op_to_int_predicate(CmpOp cmp_op, bool is_signed) {
	875	switch (cmp_op) {
	876	case CmpOpInvalid:
	877	zig_unreachable();
	878	case CmpOpEq:
	879	return LLVMIntEQ;
	880	case CmpOpNotEq:
	881	return LLVMIntNE;
	882	case CmpOpLessThan:
	883	return is_signed ? LLVMIntSLT : LLVMIntULT;
	884	case CmpOpGreaterThan:
	885	return is_signed ? LLVMIntSGT : LLVMIntUGT;
	886	case CmpOpLessOrEq:
	887	return is_signed ? LLVMIntSLE : LLVMIntULE;
	888	case CmpOpGreaterOrEq:
	889	return is_signed ? LLVMIntSGE : LLVMIntUGE;
	890	}
	891	zig_unreachable();
	892	}
	893
	894	static LLVMValueRef gen_cmp_expr(CodeGen g, AstNode node) {
	895	assert(node->type == NodeTypeComparisonExpr);
	896
	897	LLVMValueRef val1 = gen_bin_or_expr(g, node->data.comparison_expr.op1);
	898
	899	if (!node->data.comparison_expr.op2)
	900	return val1;
	901
	902	LLVMValueRef val2 = gen_bin_or_expr(g, node->data.comparison_expr.op2);
	903
	904	// TODO implement type system so that we know whether to do signed or unsigned comparison here
	905	LLVMIntPredicate pred = cmp_op_to_int_predicate(node->data.comparison_expr.cmp_op, true);
	906	add_debug_source_node(g, node);
	907	return LLVMBuildICmp(g->builder, pred, val1, val2, "");
	908	}
	909
	910	static LLVMValueRef gen_bool_and_expr(CodeGen g, AstNode node) {
	911	assert(node->type == NodeTypeBoolAndExpr);
	912
	913	LLVMValueRef val1 = gen_cmp_expr(g, node->data.bool_and_expr.op1);
	914
	915	if (!node->data.bool_and_expr.op2)
	916	return val1;
	917
	918	// block for when val1 == true
	919	LLVMBasicBlockRef true_block = LLVMAppendBasicBlock(g->cur_fn, "BoolAndTrue");
	920	// block for when val1 == false (don't even evaluate the second part)
	921	LLVMBasicBlockRef false_block = LLVMAppendBasicBlock(g->cur_fn, "BoolAndFalse");
	922
	923	LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(val1));
	924	add_debug_source_node(g, node);
	925	LLVMValueRef val1_i1 = LLVMBuildICmp(g->builder, LLVMIntEQ, val1, zero, "");
	926	LLVMBuildCondBr(g->builder, val1_i1, false_block, true_block);
	927
	928	LLVMPositionBuilderAtEnd(g->builder, true_block);
	929	LLVMValueRef val2 = gen_cmp_expr(g, node->data.bool_and_expr.op2);
	930	add_debug_source_node(g, node);
	931	LLVMValueRef val2_i1 = LLVMBuildICmp(g->builder, LLVMIntEQ, val2, zero, "");
	932
	933	LLVMPositionBuilderAtEnd(g->builder, false_block);
	934	add_debug_source_node(g, node);
	935	LLVMValueRef phi = LLVMBuildPhi(g->builder, LLVMInt1Type(), "");
	936	LLVMValueRef one_i1 = LLVMConstAllOnes(LLVMInt1Type());
	937	LLVMValueRef incoming_values[2] = {one_i1, val2_i1};
	938	LLVMBasicBlockRef incoming_blocks[2] = {LLVMGetInsertBlock(g->builder), true_block};
	939	LLVMAddIncoming(phi, incoming_values, incoming_blocks, 2);
	940
	941	return phi;
	942	}
	943
	944	static LLVMValueRef gen_bool_or_expr(CodeGen g, AstNode expr_node) {
	945	assert(expr_node->type == NodeTypeBoolOrExpr);
	946
	947	LLVMValueRef val1 = gen_bool_and_expr(g, expr_node->data.bool_or_expr.op1);
	948
	949	if (!expr_node->data.bool_or_expr.op2)
	950	return val1;
	951
	952	// block for when val1 == false
	953	LLVMBasicBlockRef false_block = LLVMAppendBasicBlock(g->cur_fn, "BoolOrFalse");
	954	// block for when val1 == true (don't even evaluate the second part)
	955	LLVMBasicBlockRef true_block = LLVMAppendBasicBlock(g->cur_fn, "BoolOrTrue");
	956
	957	LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(val1));
	958	add_debug_source_node(g, expr_node);
	959	LLVMValueRef val1_i1 = LLVMBuildICmp(g->builder, LLVMIntEQ, val1, zero, "");
	960	LLVMBuildCondBr(g->builder, val1_i1, false_block, true_block);
	961
	962	LLVMPositionBuilderAtEnd(g->builder, false_block);
	963	LLVMValueRef val2 = gen_bool_and_expr(g, expr_node->data.bool_or_expr.op2);
	964	add_debug_source_node(g, expr_node);
	965	LLVMValueRef val2_i1 = LLVMBuildICmp(g->builder, LLVMIntEQ, val2, zero, "");
	966
	967	LLVMPositionBuilderAtEnd(g->builder, true_block);
	968	add_debug_source_node(g, expr_node);
	969	LLVMValueRef phi = LLVMBuildPhi(g->builder, LLVMInt1Type(), "");
	970	LLVMValueRef one_i1 = LLVMConstAllOnes(LLVMInt1Type());
	971	LLVMValueRef incoming_values[2] = {one_i1, val2_i1};
	972	LLVMBasicBlockRef incoming_blocks[2] = {LLVMGetInsertBlock(g->builder), false_block};
	973	LLVMAddIncoming(phi, incoming_values, incoming_blocks, 2);
	974
	975	return phi;
	976	}
	977
	978	static LLVMValueRef gen_return_expr(CodeGen g, AstNode node) {
	979	assert(node->type == NodeTypeReturnExpr);
	980	AstNode *param_node = node->data.return_expr.expr;
	981	if (param_node) {
	982	LLVMValueRef value = gen_expr(g, param_node);
	983
	984	add_debug_source_node(g, node);
	985	return LLVMBuildRet(g->builder, value);
	986	} else {
	987	add_debug_source_node(g, node);
	988	return LLVMBuildRetVoid(g->builder);
	989	}
	990	}
	991	/*
	992	Expression : BoolOrExpression \| ReturnExpression
	993	*/
	994	static LLVMValueRef gen_expr(CodeGen g, AstNode node) {
	995	switch (node->type) {
	996	case NodeTypeBoolOrExpr:
	997	return gen_bool_or_expr(g, node);
	998	case NodeTypeReturnExpr:
	999	return gen_return_expr(g, node);
	1000	case NodeTypeRoot:
	1001	case NodeTypeRootExportDecl:
	1002	case NodeTypeFnProto:
	1003	case NodeTypeFnDef:
	1004	case NodeTypeFnDecl:
	1005	case NodeTypeParamDecl:
	1006	case NodeTypeType:
	1007	case NodeTypeBlock:
	1008	case NodeTypeFnCall:
	1009	case NodeTypeExternBlock:
	1010	case NodeTypeDirective:
	1011	case NodeTypeBoolAndExpr:
	1012	case NodeTypeComparisonExpr:
	1013	case NodeTypeBinOrExpr:
	1014	case NodeTypeBinXorExpr:
	1015	case NodeTypeBinAndExpr:
	1016	case NodeTypeBitShiftExpr:
	1017	case NodeTypeAddExpr:
	1018	case NodeTypeMultExpr:
	1019	case NodeTypeCastExpr:
	1020	case NodeTypePrimaryExpr:
	1021	return gen_primary_expr(g, node);
	1022	case NodeTypeGroupedExpr:
	1023	zig_unreachable();
680	1024	}
681	1025	zig_unreachable();
682	1026	}
...	...	@@ -692,39 +1036,7 @@ static void gen_block(CodeGen g, AstNode block_node, bool add_implicit_return)
692	1036
693	1037	for (int i = 0; i < block_node->data.block.statements.length; i += 1) {
694	1038	AstNode *statement_node = block_node->data.block.statements.at(i);
695		switch (statement_node->type) {
696		case NodeTypeStatementReturn:
697		{
698		AstNode *expr_node = statement_node->data.statement_return.expression;
699		if (expr_node) {
700		LLVMValueRef value = gen_expr(g, expr_node);
701
702		add_debug_source_node(g, statement_node);
703		LLVMBuildRet(g->builder, value);
704		} else {
705		add_debug_source_node(g, statement_node);
706		LLVMBuildRetVoid(g->builder);
707		}
708		break;
709		}
710		case NodeTypeExpression:
711		{
712		gen_expr(g, statement_node);
713		break;
714		}
715		case NodeTypeRoot:
716		case NodeTypeFnProto:
717		case NodeTypeFnDef:
718		case NodeTypeFnDecl:
719		case NodeTypeParamDecl:
720		case NodeTypeType:
721		case NodeTypeBlock:
722		case NodeTypeFnCall:
723		case NodeTypeExternBlock:
724		case NodeTypeDirective:
725		case NodeTypeRootExportDecl:
726		zig_unreachable();
727		}
	1039	gen_expr(g, statement_node);
728	1040	}
729	1041
730	1042	if (add_implicit_return) {
...	...	@@ -810,6 +1122,7 @@ void code_gen(CodeGen *g) {
810	1122	FnTableEntry *fn_table_entry = g->fn_defs.at(i);
811	1123	AstNode *fn_def_node = fn_table_entry->fn_def_node;
812	1124	LLVMValueRef fn = fn_table_entry->fn_value;
	1125	g->cur_fn = fn;
813	1126
814	1127	AstNode *proto_node = fn_table_entry->proto_node;
815	1128	assert(proto_node->type == NodeTypeFnProto);

src/parser.cpp+631-83

...	...	@@ -10,6 +10,47 @@
10	10	#include <stdarg.h>
11	11	#include <stdio.h>
12	12
	13	static const char *mult_op_str(MultOp mult_op) {
	14	switch (mult_op) {
	15	case MultOpInvalid: return "(invalid)";
	16	case MultOpMult: return "*";
	17	case MultOpDiv: return "/";
	18	case MultOpMod: return "%";
	19	}
	20	zig_unreachable();
	21	}
	22
	23	static const char *add_op_str(AddOp add_op) {
	24	switch (add_op) {
	25	case AddOpInvalid: return "(invalid)";
	26	case AddOpAdd: return "+";
	27	case AddOpSub: return "-";
	28	}
	29	zig_unreachable();
	30	}
	31
	32	static const char *bit_shift_op_str(BitShiftOp bit_shift_op) {
	33	switch (bit_shift_op) {
	34	case BitShiftOpInvalid: return "(invalid)";
	35	case BitShiftOpLeft: return "<<";
	36	case BitShiftOpRight: return ">>";
	37	}
	38	zig_unreachable();
	39	}
	40
	41	static const char *cmp_op_str(CmpOp cmp_op) {
	42	switch (cmp_op) {
	43	case CmpOpInvalid: return "(invalid)";
	44	case CmpOpEq: return "=";
	45	case CmpOpNotEq: return "!=";
	46	case CmpOpLessThan: return "<";
	47	case CmpOpGreaterThan: return ">";
	48	case CmpOpLessOrEq: return "<=";
	49	case CmpOpGreaterOrEq: return ">=";
	50	}
	51	zig_unreachable();
	52	}
	53
13	54	__attribute__ ((format (printf, 2, 3)))
14	55	__attribute__ ((noreturn))
15	56	static void ast_error(Token token, const char format, ...) {
...	...	@@ -43,16 +84,38 @@ const char *node_type_str(NodeType node_type) {
43	84	return "Type";
44	85	case NodeTypeBlock:
45	86	return "Block";
46		case NodeTypeStatementReturn:
47		return "StatementReturn";
48		case NodeTypeExpression:
49		return "Expression";
	87	case NodeTypeBoolOrExpr:
	88	return "BoolOrExpr";
50	89	case NodeTypeFnCall:
51	90	return "FnCall";
52	91	case NodeTypeExternBlock:
53	92	return "ExternBlock";
54	93	case NodeTypeDirective:
55	94	return "Directive";
	95	case NodeTypeReturnExpr:
	96	return "ReturnExpr";
	97	case NodeTypeBoolAndExpr:
	98	return "BoolAndExpr";
	99	case NodeTypeComparisonExpr:
	100	return "ComparisonExpr";
	101	case NodeTypeBinOrExpr:
	102	return "BinOrExpr";
	103	case NodeTypeBinXorExpr:
	104	return "BinXorExpr";
	105	case NodeTypeBinAndExpr:
	106	return "BinAndExpr";
	107	case NodeTypeBitShiftExpr:
	108	return "BitShiftExpr";
	109	case NodeTypeAddExpr:
	110	return "AddExpr";
	111	case NodeTypeMultExpr:
	112	return "MultExpr";
	113	case NodeTypeCastExpr:
	114	return "CastExpr";
	115	case NodeTypePrimaryExpr:
	116	return "PrimaryExpr";
	117	case NodeTypeGroupedExpr:
	118	return "GroupedExpr";
56	119	}
57	120	zig_unreachable();
58	121	}
...	...	@@ -133,10 +196,10 @@ void ast_print(AstNode *node, int indent) {
133	196	}
134	197	}
135	198	break;
136		case NodeTypeStatementReturn:
137		fprintf(stderr, "ReturnStatement\n");
138		if (node->data.statement_return.expression)
139		ast_print(node->data.statement_return.expression, indent + 2);
	199	case NodeTypeReturnExpr:
	200	fprintf(stderr, "%s\n", node_type_str(node->type));
	201	if (node->data.return_expr.expr)
	202	ast_print(node->data.return_expr.expr, indent + 2);
140	203	break;
141	204	case NodeTypeExternBlock:
142	205	{
...	...	@@ -151,22 +214,11 @@ void ast_print(AstNode *node, int indent) {
151	214	fprintf(stderr, "%s\n", node_type_str(node->type));
152	215	ast_print(node->data.fn_decl.fn_proto, indent + 2);
153	216	break;
154		case NodeTypeExpression:
155		switch (node->data.expression.type) {
156		case AstNodeExpressionTypeNumber:
157		fprintf(stderr, "NumberLiteralExpression %s\n", buf_ptr(&node->data.expression.data.number));
158		break;
159		case AstNodeExpressionTypeString:
160		fprintf(stderr, "StringLiteralExpression '%s'\n", buf_ptr(&node->data.expression.data.string));
161		break;
162		case AstNodeExpressionTypeFnCall:
163		fprintf(stderr, "FnCallExpression\n");
164		ast_print(node->data.expression.data.fn_call, indent + 2);
165		break;
166		case AstNodeExpressionTypeUnreachable:
167		fprintf(stderr, "UnreachableExpression\n");
168		break;
169		}
	217	case NodeTypeBoolOrExpr:
	218	fprintf(stderr, "%s\n", node_type_str(node->type));
	219	ast_print(node->data.bool_or_expr.op1, indent + 2);
	220	if (node->data.bool_or_expr.op2)
	221	ast_print(node->data.bool_or_expr.op2, indent + 2);
170	222	break;
171	223	case NodeTypeFnCall:
172	224	fprintf(stderr, "%s '%s'\n", node_type_str(node->type), buf_ptr(&node->data.fn_call.name));
...	...	@@ -178,6 +230,95 @@ void ast_print(AstNode *node, int indent) {
178	230	case NodeTypeDirective:
179	231	fprintf(stderr, "%s\n", node_type_str(node->type));
180	232	break;
	233	case NodeTypeBoolAndExpr:
	234	fprintf(stderr, "%s\n", node_type_str(node->type));
	235	ast_print(node->data.bool_and_expr.op1, indent + 2);
	236	if (node->data.bool_and_expr.op2)
	237	ast_print(node->data.bool_and_expr.op2, indent + 2);
	238	break;
	239	case NodeTypeComparisonExpr:
	240	fprintf(stderr, "%s %s\n", node_type_str(node->type),
	241	cmp_op_str(node->data.comparison_expr.cmp_op));
	242	ast_print(node->data.comparison_expr.op1, indent + 2);
	243	if (node->data.comparison_expr.op2)
	244	ast_print(node->data.comparison_expr.op2, indent + 2);
	245	break;
	246	case NodeTypeBinOrExpr:
	247	fprintf(stderr, "%s\n", node_type_str(node->type));
	248	ast_print(node->data.bin_or_expr.op1, indent + 2);
	249	if (node->data.bin_or_expr.op2)
	250	ast_print(node->data.bin_or_expr.op2, indent + 2);
	251	break;
	252	case NodeTypeBinXorExpr:
	253	fprintf(stderr, "%s\n", node_type_str(node->type));
	254	ast_print(node->data.bin_xor_expr.op1, indent + 2);
	255	if (node->data.bin_xor_expr.op2)
	256	ast_print(node->data.bin_xor_expr.op2, indent + 2);
	257	break;
	258	case NodeTypeBinAndExpr:
	259	fprintf(stderr, "%s\n", node_type_str(node->type));
	260	ast_print(node->data.bin_and_expr.op1, indent + 2);
	261	if (node->data.bin_and_expr.op2)
	262	ast_print(node->data.bin_and_expr.op2, indent + 2);
	263	break;
	264	case NodeTypeBitShiftExpr:
	265	fprintf(stderr, "%s %s\n", node_type_str(node->type),
	266	bit_shift_op_str(node->data.bit_shift_expr.bit_shift_op));
	267	ast_print(node->data.bit_shift_expr.op1, indent + 2);
	268	if (node->data.bit_shift_expr.op2)
	269	ast_print(node->data.bit_shift_expr.op2, indent + 2);
	270	break;
	271	case NodeTypeAddExpr:
	272	fprintf(stderr, "%s %s\n", node_type_str(node->type),
	273	add_op_str(node->data.add_expr.add_op));
	274	ast_print(node->data.add_expr.op1, indent + 2);
	275	if (node->data.add_expr.op2)
	276	ast_print(node->data.add_expr.op2, indent + 2);
	277	break;
	278	case NodeTypeMultExpr:
	279	fprintf(stderr, "%s %s\n", node_type_str(node->type),
	280	mult_op_str(node->data.mult_expr.mult_op));
	281	ast_print(node->data.mult_expr.op1, indent + 2);
	282	if (node->data.mult_expr.op2)
	283	ast_print(node->data.mult_expr.op2, indent + 2);
	284	break;
	285	case NodeTypeCastExpr:
	286	fprintf(stderr, "%s\n", node_type_str(node->type));
	287	ast_print(node->data.cast_expr.primary_expr, indent + 2);
	288	if (node->data.cast_expr.type)
	289	ast_print(node->data.cast_expr.type, indent + 2);
	290	break;
	291	case NodeTypePrimaryExpr:
	292	switch (node->data.primary_expr.type) {
	293	case PrimaryExprTypeNumber:
	294	fprintf(stderr, "PrimaryExpr Number %s\n",
	295	buf_ptr(&node->data.primary_expr.data.number));
	296	break;
	297	case PrimaryExprTypeString:
	298	fprintf(stderr, "PrimaryExpr String '%s'\n",
	299	buf_ptr(&node->data.primary_expr.data.string));
	300	break;
	301	case PrimaryExprTypeUnreachable:
	302	fprintf(stderr, "PrimaryExpr Unreachable\n");
	303	break;
	304	case PrimaryExprTypeFnCall:
	305	fprintf(stderr, "PrimaryExpr FnCall\n");
	306	ast_print(node->data.primary_expr.data.fn_call, indent + 2);
	307	break;
	308	case PrimaryExprTypeGroupedExpr:
	309	fprintf(stderr, "PrimaryExpr GroupedExpr\n");
	310	ast_print(node->data.primary_expr.data.grouped_expr, indent + 2);
	311	break;
	312	case PrimaryExprTypeBlock:
	313	fprintf(stderr, "PrimaryExpr Block\n");
	314	ast_print(node->data.primary_expr.data.block, indent + 2);
	315	break;
	316	}
	317	break;
	318	case NodeTypeGroupedExpr:
	319	fprintf(stderr, "%s\n", node_type_str(node->type));
	320	ast_print(node->data.grouped_expr.expr, indent + 2);
	321	break;
181	322	}
182	323	}
183	324
...	...	@@ -267,6 +408,7 @@ void ast_invalid_token_error(ParseContext pc, Token token) {
267	408	}
268	409
269	410	static AstNode ast_parse_expression(ParseContext pc, int *token_index, bool mandatory);
	411	static AstNode ast_parse_block(ParseContext pc, int *token_index, bool mandatory);
270	412
271	413
272	414	static void ast_expect_token(ParseContext pc, Token token, TokenId token_id) {
...	...	@@ -444,109 +586,515 @@ static void ast_parse_fn_call_param_list(ParseContext *pc, int token_index, int
444	586	zig_unreachable();
445	587	}
446	588
	589	/*
	590	GroupedExpression : token(LParen) Expression token(RParen)
	591	*/
	592	static AstNode ast_parse_grouped_expr(ParseContext pc, int *token_index, bool mandatory) {
	593	Token l_paren = &pc->tokens->at(token_index);
	594	if (l_paren->id != TokenIdLParen) {
	595	if (mandatory) {
	596	ast_invalid_token_error(pc, l_paren);
	597	} else {
	598	return nullptr;
	599	}
	600	}
	601
	602	*token_index += 1;
	603
	604	AstNode *node = ast_create_node(NodeTypeGroupedExpr, l_paren);
	605
	606	node->data.grouped_expr.expr = ast_parse_expression(pc, token_index, true);
	607
	608	return node;
	609	}
	610
447	611	/*
448	612	FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) ;
449	613	*/
450		static AstNode ast_parse_fn_call(ParseContext pc, int token_index, int *new_token_index) {
451		Token *fn_name = &pc->tokens->at(token_index);
452		token_index += 1;
453		ast_expect_token(pc, fn_name, TokenIdSymbol);
	614	static AstNode ast_parse_fn_call(ParseContext pc, int *token_index, bool mandatory) {
	615	Token fn_name = &pc->tokens->at(token_index);
	616	if (fn_name->id != TokenIdSymbol) {
	617	if (mandatory) {
	618	ast_invalid_token_error(pc, fn_name);
	619	} else {
	620	return nullptr;
	621	}
	622	}
	623
	624	*token_index += 1;
454	625
455	626	AstNode *node = ast_create_node(NodeTypeFnCall, fn_name);
456	627
457	628
458	629	ast_buf_from_token(pc, fn_name, &node->data.fn_call.name);
459	630
460		ast_parse_fn_call_param_list(pc, token_index, &token_index, &node->data.fn_call.params);
	631	ast_parse_fn_call_param_list(pc, *token_index, token_index, &node->data.fn_call.params);
461	632
462		*new_token_index = token_index;
463	633	return node;
464	634	}
465	635
466	636	/*
467		Expression : token(Number) \| token(String) \| token(Unreachable) \| FnCall
	637	PrimaryExpression : token(Number) \| token(String) \| token(Unreachable) \| FnCall \| GroupedExpression \| Block
468	638	*/
469		static AstNode ast_parse_expression(ParseContext pc, int *token_index, bool mandatory) {
	639	static AstNode ast_parse_primary_expr(ParseContext pc, int *token_index, bool mandatory) {
470	640	Token token = &pc->tokens->at(token_index);
471		AstNode *node = ast_create_node(NodeTypeExpression, token);
472		if (token->id == TokenIdKeywordUnreachable) {
473		node->data.expression.type = AstNodeExpressionTypeUnreachable;
474		*token_index += 1;
475		} else if (token->id == TokenIdSymbol) {
476		node->data.expression.type = AstNodeExpressionTypeFnCall;
477		node->data.expression.data.fn_call = ast_parse_fn_call(pc, *token_index, token_index);
478		} else if (token->id == TokenIdNumberLiteral) {
479		node->data.expression.type = AstNodeExpressionTypeNumber;
480		ast_buf_from_token(pc, token, &node->data.expression.data.number);
	641
	642	if (token->id == TokenIdNumberLiteral) {
	643	AstNode *node = ast_create_node(NodeTypePrimaryExpr, token);
	644	node->data.primary_expr.type = PrimaryExprTypeNumber;
	645	ast_buf_from_token(pc, token, &node->data.primary_expr.data.number);
481	646	*token_index += 1;
	647	return node;
482	648	} else if (token->id == TokenIdStringLiteral) {
483		node->data.expression.type = AstNodeExpressionTypeString;
484		parse_string_literal(pc, token, &node->data.expression.data.string);
	649	AstNode *node = ast_create_node(NodeTypePrimaryExpr, token);
	650	node->data.primary_expr.type = PrimaryExprTypeString;
	651	parse_string_literal(pc, token, &node->data.primary_expr.data.string);
485	652	*token_index += 1;
486		} else if (mandatory) {
487		ast_invalid_token_error(pc, token);
488		} else {
	653	return node;
	654	} else if (token->id == TokenIdKeywordUnreachable) {
	655	AstNode *node = ast_create_node(NodeTypePrimaryExpr, token);
	656	node->data.primary_expr.type = PrimaryExprTypeUnreachable;
	657	*token_index += 1;
	658	return node;
	659	}
	660
	661	AstNode *block_node = ast_parse_block(pc, token_index, false);
	662	if (block_node) {
	663	AstNode *node = ast_create_node(NodeTypePrimaryExpr, token);
	664	node->data.primary_expr.type = PrimaryExprTypeBlock;
	665	node->data.primary_expr.data.block = block_node;
	666	return node;
	667	}
	668
	669	AstNode *grouped_expr_node = ast_parse_grouped_expr(pc, token_index, false);
	670	if (grouped_expr_node) {
	671	AstNode *node = ast_create_node(NodeTypePrimaryExpr, token);
	672	node->data.primary_expr.type = PrimaryExprTypeGroupedExpr;
	673	node->data.primary_expr.data.grouped_expr = grouped_expr_node;
	674	return node;
	675	}
	676
	677	AstNode *fn_call_node = ast_parse_fn_call(pc, token_index, false);
	678	if (fn_call_node) {
	679	AstNode *node = ast_create_node(NodeTypePrimaryExpr, token);
	680	node->data.primary_expr.type = PrimaryExprTypeFnCall;
	681	node->data.primary_expr.data.fn_call = fn_call_node;
	682	return node;
	683	}
	684
	685	if (!mandatory)
489	686	return nullptr;
	687
	688	ast_invalid_token_error(pc, token);
	689	}
	690
	691	/*
	692	CastExpression : PrimaryExpression token(As) Type \| PrimaryExpression
	693	*/
	694	static AstNode ast_parse_cast_expression(ParseContext pc, int *token_index, bool mandatory) {
	695	AstNode *primary_expr = ast_parse_primary_expr(pc, token_index, mandatory);
	696	if (!primary_expr)
	697	return nullptr;
	698
	699	Token as_kw = &pc->tokens->at(token_index);
	700	if (as_kw->id != TokenIdKeywordAs)
	701	return primary_expr;
	702	*token_index += 1;
	703
	704	AstNode *node = ast_create_node(NodeTypeCastExpr, as_kw);
	705	node->data.cast_expr.primary_expr = primary_expr;
	706
	707	node->data.cast_expr.type = ast_parse_type(pc, *token_index, token_index);
	708
	709	return node;
	710	}
	711
	712	static MultOp tok_to_mult_op(Token *token) {
	713	switch (token->id) {
	714	case TokenIdStar: return MultOpMult;
	715	case TokenIdSlash: return MultOpDiv;
	716	case TokenIdPercent: return MultOpMod;
	717	default: return MultOpInvalid;
	718	}
	719	}
	720
	721	/*
	722	MultiplyOperator : token(Star) \| token(Slash) \| token(Percent)
	723	*/
	724	static MultOp ast_parse_mult_op(ParseContext pc, int token_index, bool mandatory) {
	725	Token token = &pc->tokens->at(token_index);
	726	MultOp result = tok_to_mult_op(token);
	727	if (result == MultOpInvalid) {
	728	if (mandatory) {
	729	ast_invalid_token_error(pc, token);
	730	} else {
	731	return MultOpInvalid;
	732	}
	733	}
	734	*token_index += 1;
	735	return result;
	736	}
	737
	738	/*
	739	MultiplyExpression : CastExpression MultiplyOperator CastExpression \| CastExpression
	740	*/
	741	static AstNode ast_parse_mult_expr(ParseContext pc, int *token_index, bool mandatory) {
	742	AstNode *operand_1 = ast_parse_cast_expression(pc, token_index, mandatory);
	743	if (!operand_1)
	744	return nullptr;
	745
	746	Token token = &pc->tokens->at(token_index);
	747	MultOp mult_op = ast_parse_mult_op(pc, token_index, false);
	748	if (mult_op == MultOpInvalid)
	749	return operand_1;
	750
	751	AstNode *operand_2 = ast_parse_cast_expression(pc, token_index, true);
	752
	753	AstNode *node = ast_create_node(NodeTypeMultExpr, token);
	754	node->data.mult_expr.op1 = operand_1;
	755	node->data.mult_expr.mult_op = mult_op;
	756	node->data.mult_expr.op2 = operand_2;
	757
	758	return node;
	759	}
	760
	761	static AddOp tok_to_add_op(Token *token) {
	762	switch (token->id) {
	763	case TokenIdPlus: return AddOpAdd;
	764	case TokenIdDash: return AddOpSub;
	765	default: return AddOpInvalid;
	766	}
	767	}
	768
	769	/*
	770	AdditionOperator : token(Plus) \| token(Minus)
	771	*/
	772	static AddOp ast_parse_add_op(ParseContext pc, int token_index, bool mandatory) {
	773	Token token = &pc->tokens->at(token_index);
	774	AddOp result = tok_to_add_op(token);
	775	if (result == AddOpInvalid) {
	776	if (mandatory) {
	777	ast_invalid_token_error(pc, token);
	778	} else {
	779	return AddOpInvalid;
	780	}
	781	}
	782	*token_index += 1;
	783	return result;
	784	}
	785
	786	/*
	787	AdditionExpression : MultiplyExpression AdditionOperator MultiplyExpression \| MultiplyExpression
	788	*/
	789	static AstNode ast_parse_add_expr(ParseContext pc, int *token_index, bool mandatory) {
	790	AstNode *operand_1 = ast_parse_mult_expr(pc, token_index, mandatory);
	791	if (!operand_1)
	792	return nullptr;
	793
	794	Token token = &pc->tokens->at(token_index);
	795	AddOp add_op = ast_parse_add_op(pc, token_index, false);
	796	if (add_op == AddOpInvalid)
	797	return operand_1;
	798
	799	AstNode *operand_2 = ast_parse_mult_expr(pc, token_index, true);
	800
	801	AstNode *node = ast_create_node(NodeTypeAddExpr, token);
	802	node->data.add_expr.op1 = operand_1;
	803	node->data.add_expr.add_op = add_op;
	804	node->data.add_expr.op2 = operand_2;
	805
	806	return node;
	807	}
	808
	809	static BitShiftOp tok_to_bit_shift_op(Token *token) {
	810	switch (token->id) {
	811	case TokenIdBitShiftLeft: return BitShiftOpLeft;
	812	case TokenIdBitShiftRight: return BitShiftOpRight;
	813	default: return BitShiftOpInvalid;
490	814	}
	815	}
	816
	817	/*
	818	BitShiftOperator : token(BitShiftLeft \| token(BitShiftRight)
	819	*/
	820	static BitShiftOp ast_parse_bit_shift_op(ParseContext pc, int token_index, bool mandatory) {
	821	Token token = &pc->tokens->at(token_index);
	822	BitShiftOp result = tok_to_bit_shift_op(token);
	823	if (result == BitShiftOpInvalid) {
	824	if (mandatory) {
	825	ast_invalid_token_error(pc, token);
	826	} else {
	827	return BitShiftOpInvalid;
	828	}
	829	}
	830	*token_index += 1;
	831	return result;
	832	}
	833
	834	/*
	835	BitShiftExpression : AdditionExpression BitShiftOperator AdditionExpression \| AdditionExpression
	836	*/
	837	static AstNode ast_parse_bit_shift_expr(ParseContext pc, int *token_index, bool mandatory) {
	838	AstNode *operand_1 = ast_parse_add_expr(pc, token_index, mandatory);
	839	if (!operand_1)
	840	return nullptr;
	841
	842	Token token = &pc->tokens->at(token_index);
	843	BitShiftOp bit_shift_op = ast_parse_bit_shift_op(pc, token_index, false);
	844	if (bit_shift_op == BitShiftOpInvalid)
	845	return operand_1;
	846
	847	AstNode *operand_2 = ast_parse_add_expr(pc, token_index, true);
	848
	849	AstNode *node = ast_create_node(NodeTypeBitShiftExpr, token);
	850	node->data.bit_shift_expr.op1 = operand_1;
	851	node->data.bit_shift_expr.bit_shift_op = bit_shift_op;
	852	node->data.bit_shift_expr.op2 = operand_2;
491	853
492	854	return node;
493	855	}
494	856
	857
495	858	/*
496		Statement : ExpressionStatement \| ReturnStatement ;
	859	BinaryAndExpression : BitShiftExpression token(BinAnd) BitShiftExpression \| BitShiftExpression
	860	*/
	861	static AstNode ast_parse_bin_and_expr(ParseContext pc, int *token_index, bool mandatory) {
	862	AstNode *operand_1 = ast_parse_bit_shift_expr(pc, token_index, mandatory);
	863	if (!operand_1)
	864	return nullptr;
	865
	866	Token token = &pc->tokens->at(token_index);
	867	if (token->id != TokenIdBinAnd)
	868	return operand_1;
	869	*token_index += 1;
	870
	871	AstNode *operand_2 = ast_parse_bit_shift_expr(pc, token_index, true);
497	872
498		ExpressionStatement : Expression token(Semicolon) ;
	873	AstNode *node = ast_create_node(NodeTypeBinAndExpr, token);
	874	node->data.bin_and_expr.op1 = operand_1;
	875	node->data.bin_and_expr.op2 = operand_2;
499	876
500		ReturnStatement : token(Return) option(Expression) token(Semicolon) ;
	877	return node;
	878	}
	879
	880	/*
	881	BinaryXorExpression : BinaryAndExpression token(BinXor) BinaryAndExpression \| BinaryAndExpression
501	882	*/
502		static AstNode ast_parse_statement(ParseContext pc, int token_index, int *new_token_index) {
503		Token *token = &pc->tokens->at(token_index);
504		if (token->id == TokenIdKeywordReturn) {
505		AstNode *node = ast_create_node(NodeTypeStatementReturn, token);
506		token_index += 1;
507		node->data.statement_return.expression = ast_parse_expression(pc, &token_index, false);
	883	static AstNode ast_parse_bin_xor_expr(ParseContext pc, int *token_index, bool mandatory) {
	884	AstNode *operand_1 = ast_parse_bin_and_expr(pc, token_index, mandatory);
	885	if (!operand_1)
	886	return nullptr;
508	887
509		Token *semicolon = &pc->tokens->at(token_index);
510		token_index += 1;
511		ast_expect_token(pc, semicolon, TokenIdSemicolon);
512		*new_token_index = token_index;
513		return node;
514		} else if (token->id == TokenIdSymbol \|\|
515		token->id == TokenIdStringLiteral \|\|
516		token->id == TokenIdKeywordUnreachable \|\|
517		token->id == TokenIdNumberLiteral)
518		{
519		AstNode *node = ast_parse_expression(pc, &token_index, true);
520
521		Token *semicolon = &pc->tokens->at(token_index);
522		token_index += 1;
523		ast_expect_token(pc, semicolon, TokenIdSemicolon);
524		*new_token_index = token_index;
	888	Token token = &pc->tokens->at(token_index);
	889	if (token->id != TokenIdBinXor)
	890	return operand_1;
	891	*token_index += 1;
	892
	893	AstNode *operand_2 = ast_parse_bin_and_expr(pc, token_index, true);
	894
	895	AstNode *node = ast_create_node(NodeTypeBinXorExpr, token);
	896	node->data.bin_xor_expr.op1 = operand_1;
	897	node->data.bin_xor_expr.op2 = operand_2;
	898
	899	return node;
	900	}
	901
	902	/*
	903	BinaryOrExpression : BinaryXorExpression token(BinOr) BinaryXorExpression \| BinaryXorExpression
	904	*/
	905	static AstNode ast_parse_bin_or_expr(ParseContext pc, int *token_index, bool mandatory) {
	906	AstNode *operand_1 = ast_parse_bin_xor_expr(pc, token_index, mandatory);
	907	if (!operand_1)
	908	return nullptr;
	909
	910	Token token = &pc->tokens->at(token_index);
	911	if (token->id != TokenIdBinOr)
	912	return operand_1;
	913	*token_index += 1;
	914
	915	AstNode *operand_2 = ast_parse_bin_xor_expr(pc, token_index, true);
	916
	917	AstNode *node = ast_create_node(NodeTypeBinOrExpr, token);
	918	node->data.bin_or_expr.op1 = operand_1;
	919	node->data.bin_or_expr.op2 = operand_2;
	920
	921	return node;
	922	}
	923
	924	static CmpOp tok_to_cmp_op(Token *token) {
	925	switch (token->id) {
	926	case TokenIdCmpEq: return CmpOpEq;
	927	case TokenIdCmpNotEq: return CmpOpNotEq;
	928	case TokenIdCmpLessThan: return CmpOpLessThan;
	929	case TokenIdCmpGreaterThan: return CmpOpGreaterThan;
	930	case TokenIdCmpLessOrEq: return CmpOpLessOrEq;
	931	case TokenIdCmpGreaterOrEq: return CmpOpGreaterOrEq;
	932	default: return CmpOpInvalid;
	933	}
	934	}
	935
	936	static CmpOp ast_parse_comparison_operator(ParseContext pc, int token_index, bool mandatory) {
	937	Token token = &pc->tokens->at(token_index);
	938	CmpOp result = tok_to_cmp_op(token);
	939	if (result == CmpOpInvalid) {
	940	if (mandatory) {
	941	ast_invalid_token_error(pc, token);
	942	} else {
	943	return CmpOpInvalid;
	944	}
	945	}
	946	*token_index += 1;
	947	return result;
	948	}
	949
	950	/*
	951	ComparisonExpression : BinaryOrExpression ComparisonOperator BinaryOrExpression \| BinaryOrExpression
	952	*/
	953	static AstNode ast_parse_comparison_expr(ParseContext pc, int *token_index, bool mandatory) {
	954	AstNode *operand_1 = ast_parse_bin_or_expr(pc, token_index, mandatory);
	955	if (!operand_1)
	956	return nullptr;
	957
	958	Token token = &pc->tokens->at(token_index);
	959	CmpOp cmp_op = ast_parse_comparison_operator(pc, token_index, false);
	960	if (cmp_op == CmpOpInvalid)
	961	return operand_1;
	962
	963	AstNode *operand_2 = ast_parse_bin_or_expr(pc, token_index, true);
	964
	965	AstNode *node = ast_create_node(NodeTypeComparisonExpr, token);
	966	node->data.comparison_expr.op1 = operand_1;
	967	node->data.comparison_expr.cmp_op = cmp_op;
	968	node->data.comparison_expr.op2 = operand_2;
	969
	970	return node;
	971	}
	972
	973	/*
	974	BoolAndExpression : ComparisonExpression token(BoolAnd) ComparisonExpression \| ComparisonExpression
	975	*/
	976	static AstNode ast_parse_bool_and_expr(ParseContext pc, int *token_index, bool mandatory) {
	977	AstNode *operand_1 = ast_parse_comparison_expr(pc, token_index, mandatory);
	978	if (!operand_1)
	979	return nullptr;
	980
	981	Token token = &pc->tokens->at(token_index);
	982	if (token->id != TokenIdBoolAnd)
	983	return operand_1;
	984	*token_index += 1;
	985
	986	AstNode *operand_2 = ast_parse_comparison_expr(pc, token_index, true);
	987
	988	AstNode *node = ast_create_node(NodeTypeBoolAndExpr, token);
	989	node->data.bool_and_expr.op1 = operand_1;
	990	node->data.bool_and_expr.op2 = operand_2;
	991
	992	return node;
	993	}
	994
	995	/*
	996	ReturnExpression : token(Return) option(Expression)
	997	*/
	998	static AstNode ast_parse_return_expr(ParseContext pc, int *token_index, bool mandatory) {
	999	Token return_tok = &pc->tokens->at(token_index);
	1000	if (return_tok->id == TokenIdKeywordReturn) {
	1001	*token_index += 1;
	1002	AstNode *node = ast_create_node(NodeTypeReturnExpr, return_tok);
	1003	node->data.return_expr.expr = ast_parse_expression(pc, token_index, false);
525	1004	return node;
	1005	} else if (mandatory) {
	1006	ast_invalid_token_error(pc, return_tok);
526	1007	} else {
527		ast_invalid_token_error(pc, token);
	1008	return nullptr;
528	1009	}
529	1010	}
530	1011
	1012	/*
	1013	BoolOrExpression : BoolAndExpression token(BoolOr) BoolAndExpression \| BoolAndExpression
	1014	*/
	1015	static AstNode ast_parse_bool_or_expr(ParseContext pc, int *token_index, bool mandatory) {
	1016	AstNode *operand_1 = ast_parse_bool_and_expr(pc, token_index, mandatory);
	1017	if (!operand_1)
	1018	return nullptr;
	1019
	1020	Token token = &pc->tokens->at(token_index);
	1021	if (token->id != TokenIdBoolOr)
	1022	return operand_1;
	1023	*token_index += 1;
	1024
	1025	AstNode *operand_2 = ast_parse_bool_and_expr(pc, token_index, true);
	1026
	1027	AstNode *node = ast_create_node(NodeTypeBoolOrExpr, token);
	1028	node->data.bool_or_expr.op1 = operand_1;
	1029	node->data.bool_or_expr.op2 = operand_2;
	1030
	1031	return node;
	1032	}
	1033
	1034	/*
	1035	Expression : BoolOrExpression \| ReturnExpression
	1036	*/
	1037	static AstNode ast_parse_expression(ParseContext pc, int *token_index, bool mandatory) {
	1038	Token token = &pc->tokens->at(token_index);
	1039
	1040	AstNode *return_expr = ast_parse_return_expr(pc, token_index, false);
	1041	if (return_expr)
	1042	return return_expr;
	1043
	1044	AstNode *bool_or_expr = ast_parse_bool_or_expr(pc, token_index, false);
	1045	if (bool_or_expr)
	1046	return bool_or_expr;
	1047
	1048	if (!mandatory)
	1049	return nullptr;
	1050
	1051	ast_invalid_token_error(pc, token);
	1052	}
	1053
	1054	/*
	1055	ExpressionStatement : Expression token(Semicolon)
	1056	*/
	1057	static AstNode ast_parse_expression_statement(ParseContext pc, int *token_index) {
	1058	AstNode *expr_node = ast_parse_expression(pc, token_index, true);
	1059
	1060	Token semicolon = &pc->tokens->at(token_index);
	1061	*token_index += 1;
	1062	ast_expect_token(pc, semicolon, TokenIdSemicolon);
	1063
	1064	return expr_node;
	1065	}
	1066
	1067	/*
	1068	Statement : ExpressionStatement
	1069	*/
	1070	static AstNode ast_parse_statement(ParseContext pc, int *token_index) {
	1071	return ast_parse_expression_statement(pc, token_index);
	1072	}
	1073
531	1074	/*
532	1075	Block : token(LBrace) many(Statement) token(RBrace);
533	1076	*/
534		static AstNode ast_parse_block(ParseContext pc, int token_index, int *new_token_index) {
535		Token *l_brace = &pc->tokens->at(token_index);
536		token_index += 1;
537		ast_expect_token(pc, l_brace, TokenIdLBrace);
	1077	static AstNode ast_parse_block(ParseContext pc, int *token_index, bool mandatory) {
	1078	Token l_brace = &pc->tokens->at(token_index);
538	1079
539		AstNode *node = ast_create_node(NodeTypeBlock, l_brace);
	1080	if (l_brace->id != TokenIdLBrace) {
	1081	if (mandatory) {
	1082	ast_invalid_token_error(pc, l_brace);
	1083	} else {
	1084	return nullptr;
	1085	}
	1086	}
	1087	*token_index += 1;
540	1088
	1089	AstNode *node = ast_create_node(NodeTypeBlock, l_brace);
541	1090
542	1091	for (;;) {
543		Token *token = &pc->tokens->at(token_index);
	1092	Token token = &pc->tokens->at(token_index);
544	1093	if (token->id == TokenIdRBrace) {
545		token_index += 1;
546		*new_token_index = token_index;
	1094	*token_index += 1;
547	1095	return node;
548	1096	} else {
549		AstNode *statement_node = ast_parse_statement(pc, token_index, &token_index);
	1097	AstNode *statement_node = ast_parse_statement(pc, token_index);
550	1098	node->data.block.statements.append(statement_node);
551	1099	}
552	1100	}
...	...	@@ -620,7 +1168,7 @@ static AstNode ast_parse_fn_def(ParseContext pc, int *token_index, bool mandat
620	1168	AstNode *node = ast_create_node_with_node(NodeTypeFnDef, fn_proto);
621	1169
622	1170	node->data.fn_def.fn_proto = fn_proto;
623		node->data.fn_def.body = ast_parse_block(pc, *token_index, token_index);
	1171	node->data.fn_def.body = ast_parse_block(pc, token_index, true);
624	1172
625	1173	return node;
626	1174	}

src/parser.hpp+144-20

...	...	@@ -24,11 +24,22 @@ enum NodeType {
24	24	NodeTypeParamDecl,
25	25	NodeTypeType,
26	26	NodeTypeBlock,
27		NodeTypeExpression,
28	27	NodeTypeFnCall,
29	28	NodeTypeExternBlock,
30	29	NodeTypeDirective,
31		NodeTypeStatementReturn,
	30	NodeTypeReturnExpr,
	31	NodeTypeBoolOrExpr,
	32	NodeTypeBoolAndExpr,
	33	NodeTypeComparisonExpr,
	34	NodeTypeBinOrExpr,
	35	NodeTypeBinXorExpr,
	36	NodeTypeBinAndExpr,
	37	NodeTypeBitShiftExpr,
	38	NodeTypeAddExpr,
	39	NodeTypeMultExpr,
	40	NodeTypeCastExpr,
	41	NodeTypePrimaryExpr,
	42	NodeTypeGroupedExpr,
32	43	};
33	44
34	45	struct AstNodeRoot {
...	...	@@ -80,24 +91,15 @@ struct AstNodeBlock {
80	91	ZigList<AstNode *> statements;
81	92	};
82	93
83		struct AstNodeStatementReturn {
84		AstNode *expression;
	94	struct AstNodeReturnExpr {
	95	// might be null in case of return void;
	96	AstNode *expr;
85	97	};
86	98
87		enum AstNodeExpressionType {
88		AstNodeExpressionTypeNumber,
89		AstNodeExpressionTypeString,
90		AstNodeExpressionTypeFnCall,
91		AstNodeExpressionTypeUnreachable,
92		};
93
94		struct AstNodeExpression {
95		AstNodeExpressionType type;
96		union {
97		Buf number;
98		Buf string;
99		AstNode *fn_call;
100		} data;
	99	struct AstNodeBoolOrExpr {
	100	AstNode *op1;
	101	// if op2 is non-null, do boolean or, otherwise nothing
	102	AstNode *op2;
101	103	};
102	104
103	105	struct AstNodeFnCall {
...	...	@@ -120,6 +122,117 @@ struct AstNodeRootExportDecl {
120	122	Buf name;
121	123	};
122	124
	125	struct AstNodeBoolAndExpr {
	126	AstNode *op1;
	127	// if op2 is non-null, do boolean and, otherwise nothing
	128	AstNode *op2;
	129	};
	130
	131	enum CmpOp {
	132	CmpOpInvalid,
	133	CmpOpEq,
	134	CmpOpNotEq,
	135	CmpOpLessThan,
	136	CmpOpGreaterThan,
	137	CmpOpLessOrEq,
	138	CmpOpGreaterOrEq,
	139	};
	140
	141	struct AstNodeComparisonExpr {
	142	AstNode *op1;
	143	CmpOp cmp_op;
	144	// if op2 is non-null, do cmp_op, otherwise nothing
	145	AstNode *op2;
	146	};
	147
	148	struct AstNodeBinOrExpr {
	149	AstNode *op1;
	150	// if op2 is non-null, do binary or, otherwise nothing
	151	AstNode *op2;
	152	};
	153
	154	struct AstNodeBinXorExpr {
	155	AstNode *op1;
	156	// if op2 is non-null, do binary xor, otherwise nothing
	157	AstNode *op2;
	158	};
	159
	160	struct AstNodeBinAndExpr {
	161	AstNode *op1;
	162	// if op2 is non-null, do binary and, otherwise nothing
	163	AstNode *op2;
	164	};
	165
	166	enum BitShiftOp {
	167	BitShiftOpInvalid,
	168	BitShiftOpLeft,
	169	BitShiftOpRight,
	170	};
	171
	172	struct AstNodeBitShiftExpr {
	173	AstNode *op1;
	174	BitShiftOp bit_shift_op;
	175	// if op2 is non-null, do bit_shift_op, otherwise nothing
	176	AstNode *op2;
	177	};
	178
	179	enum AddOp {
	180	AddOpInvalid,
	181	AddOpAdd,
	182	AddOpSub,
	183	};
	184
	185	struct AstNodeAddExpr {
	186	AstNode *op1;
	187	AddOp add_op;
	188	// if op2 is non-null, do add_op, otherwise nothing
	189	AstNode *op2;
	190	};
	191
	192	enum MultOp {
	193	MultOpInvalid,
	194	MultOpMult,
	195	MultOpDiv,
	196	MultOpMod,
	197	};
	198
	199	struct AstNodeMultExpr {
	200	AstNode *op1;
	201	MultOp mult_op;
	202	// if op2 is non-null, do mult_op, otherwise nothing
	203	AstNode *op2;
	204	};
	205
	206	struct AstNodeCastExpr {
	207	AstNode *primary_expr;
	208	// if type is non-null, do cast, otherwise nothing
	209	AstNode *type;
	210	};
	211
	212	enum PrimaryExprType {
	213	PrimaryExprTypeNumber,
	214	PrimaryExprTypeString,
	215	PrimaryExprTypeUnreachable,
	216	PrimaryExprTypeFnCall,
	217	PrimaryExprTypeGroupedExpr,
	218	PrimaryExprTypeBlock,
	219	};
	220
	221	struct AstNodePrimaryExpr {
	222	PrimaryExprType type;
	223	union {
	224	Buf number;
	225	Buf string;
	226	AstNode *fn_call;
	227	AstNode *grouped_expr;
	228	AstNode *block;
	229	} data;
	230	};
	231
	232	struct AstNodeGroupedExpr {
	233	AstNode *expr;
	234	};
	235
123	236	struct AstNode {
124	237	enum NodeType type;
125	238	AstNode *parent;
...	...	@@ -135,11 +248,22 @@ struct AstNode {
135	248	AstNodeType type;
136	249	AstNodeParamDecl param_decl;
137	250	AstNodeBlock block;
138		AstNodeStatementReturn statement_return;
139		AstNodeExpression expression;
	251	AstNodeReturnExpr return_expr;
	252	AstNodeBoolOrExpr bool_or_expr;
140	253	AstNodeFnCall fn_call;
141	254	AstNodeExternBlock extern_block;
142	255	AstNodeDirective directive;
	256	AstNodeBoolAndExpr bool_and_expr;
	257	AstNodeComparisonExpr comparison_expr;
	258	AstNodeBinOrExpr bin_or_expr;
	259	AstNodeBinXorExpr bin_xor_expr;
	260	AstNodeBinAndExpr bin_and_expr;
	261	AstNodeBitShiftExpr bit_shift_expr;
	262	AstNodeAddExpr add_expr;
	263	AstNodeMultExpr mult_expr;
	264	AstNodeCastExpr cast_expr;
	265	AstNodePrimaryExpr primary_expr;
	266	AstNodeGroupedExpr grouped_expr;
143	267	} data;
144	268	};
145	269

src/tokenizer.cpp+168-6

...	...	@@ -98,6 +98,12 @@ enum TokenizeState {
98	98	TokenizeStateMultiLineComment,
99	99	TokenizeStateMultiLineCommentSlash,
100	100	TokenizeStateMultiLineCommentStar,
	101	TokenizeStatePipe,
	102	TokenizeStateAmpersand,
	103	TokenizeStateEq,
	104	TokenizeStateBang,
	105	TokenizeStateLessThan,
	106	TokenizeStateGreaterThan,
101	107	};
102	108
103	109
...	...	@@ -144,6 +150,11 @@ static void begin_token(Tokenize *t, TokenId id) {
144	150	t->cur_tok = token;
145	151	}
146	152
	153	static void cancel_token(Tokenize *t) {
	154	t->tokens->pop();
	155	t->cur_tok = nullptr;
	156	}
	157
147	158	static void end_token(Tokenize *t) {
148	159	assert(t->cur_tok);
149	160	t->cur_tok->end_pos = t->pos + 1;
...	...	@@ -167,6 +178,8 @@ static void end_token(Tokenize *t) {
167	178	t->cur_tok->id = TokenIdKeywordPub;
168	179	} else if (mem_eql_str(token_mem, token_len, "export")) {
169	180	t->cur_tok->id = TokenIdKeywordExport;
	181	} else if (mem_eql_str(token_mem, token_len, "as")) {
	182	t->cur_tok->id = TokenIdKeywordAs;
170	183	}
171	184
172	185	t->cur_tok = nullptr;
...	...	@@ -212,6 +225,10 @@ ZigList<Token> tokenize(Buf buf) {
212	225	begin_token(&t, TokenIdStar);
213	226	end_token(&t);
214	227	break;
	228	case '%':
	229	begin_token(&t, TokenIdPercent);
	230	end_token(&t);
	231	break;
215	232	case '{':
216	233	begin_token(&t, TokenIdLBrace);
217	234	end_token(&t);
...	...	@@ -240,24 +257,149 @@ ZigList<Token> tokenize(Buf buf) {
240	257	begin_token(&t, TokenIdNumberSign);
241	258	end_token(&t);
242	259	break;
	260	case '^':
	261	begin_token(&t, TokenIdBinXor);
	262	end_token(&t);
	263	break;
243	264	case '/':
	265	begin_token(&t, TokenIdSlash);
244	266	t.state = TokenizeStateSawSlash;
245	267	break;
	268	case '\|':
	269	begin_token(&t, TokenIdBinOr);
	270	t.state = TokenizeStatePipe;
	271	break;
	272	case '&':
	273	begin_token(&t, TokenIdBinAnd);
	274	t.state = TokenizeStateAmpersand;
	275	break;
	276	case '=':
	277	begin_token(&t, TokenIdEq);
	278	t.state = TokenizeStateEq;
	279	break;
	280	case '!':
	281	begin_token(&t, TokenIdNot);
	282	t.state = TokenizeStateBang;
	283	break;
	284	case '<':
	285	begin_token(&t, TokenIdCmpLessThan);
	286	t.state = TokenizeStateLessThan;
	287	break;
	288	case '>':
	289	begin_token(&t, TokenIdCmpGreaterThan);
	290	t.state = TokenizeStateGreaterThan;
	291	break;
246	292	default:
247	293	tokenize_error(&t, "invalid character: '%c'", c);
248	294	}
249	295	break;
	296	case TokenizeStateGreaterThan:
	297	switch (c) {
	298	case '=':
	299	t.cur_tok->id = TokenIdCmpGreaterOrEq;
	300	end_token(&t);
	301	t.state = TokenizeStateStart;
	302	break;
	303	case '>':
	304	t.cur_tok->id = TokenIdBitShiftRight;
	305	end_token(&t);
	306	t.state = TokenizeStateStart;
	307	break;
	308	default:
	309	t.pos -= 1;
	310	end_token(&t);
	311	t.state = TokenizeStateStart;
	312	continue;
	313	}
	314	break;
	315	case TokenizeStateLessThan:
	316	switch (c) {
	317	case '=':
	318	t.cur_tok->id = TokenIdCmpLessOrEq;
	319	end_token(&t);
	320	t.state = TokenizeStateStart;
	321	case '<':
	322	t.cur_tok->id = TokenIdBitShiftLeft;
	323	end_token(&t);
	324	t.state = TokenizeStateStart;
	325	break;
	326	default:
	327	t.pos -= 1;
	328	end_token(&t);
	329	t.state = TokenizeStateStart;
	330	continue;
	331	}
	332	break;
	333	case TokenizeStateBang:
	334	switch (c) {
	335	case '=':
	336	t.cur_tok->id = TokenIdCmpNotEq;
	337	end_token(&t);
	338	t.state = TokenizeStateStart;
	339	break;
	340	default:
	341	t.pos -= 1;
	342	end_token(&t);
	343	t.state = TokenizeStateStart;
	344	continue;
	345	}
	346	break;
	347	case TokenizeStateEq:
	348	switch (c) {
	349	case '=':
	350	t.cur_tok->id = TokenIdCmpEq;
	351	end_token(&t);
	352	t.state = TokenizeStateStart;
	353	break;
	354	default:
	355	t.pos -= 1;
	356	end_token(&t);
	357	t.state = TokenizeStateStart;
	358	continue;
	359	}
	360	break;
	361	case TokenizeStateAmpersand:
	362	switch (c) {
	363	case '&':
	364	t.cur_tok->id = TokenIdBoolAnd;
	365	end_token(&t);
	366	t.state = TokenizeStateStart;
	367	break;
	368	default:
	369	t.pos -= 1;
	370	end_token(&t);
	371	t.state = TokenizeStateStart;
	372	continue;
	373	}
	374	break;
	375	case TokenizeStatePipe:
	376	switch (c) {
	377	case '\|':
	378	t.cur_tok->id = TokenIdBoolOr;
	379	end_token(&t);
	380	t.state = TokenizeStateStart;
	381	break;
	382	default:
	383	t.pos -= 1;
	384	end_token(&t);
	385	t.state = TokenizeStateStart;
	386	continue;
	387	}
	388	break;
250	389	case TokenizeStateSawSlash:
251	390	switch (c) {
252	391	case '/':
	392	cancel_token(&t);
253	393	t.state = TokenizeStateLineComment;
254	394	break;
255	395	case '*':
	396	cancel_token(&t);
256	397	t.state = TokenizeStateMultiLineComment;
257	398	t.multi_line_comment_count = 1;
258	399	break;
259	400	default:
260		tokenize_error(&t, "invalid character: '%c'", c);
	401	end_token(&t);
	402	t.state = TokenizeStateStart;
261	403	break;
262	404	}
263	405	break;
...	...	@@ -371,16 +513,18 @@ ZigList<Token> tokenize(Buf buf) {
371	513	switch (t.state) {
372	514	case TokenizeStateStart:
373	515	break;
374		case TokenizeStateSymbol:
375		end_token(&t);
376		break;
377	516	case TokenizeStateString:
378	517	tokenize_error(&t, "unterminated string");
379	518	break;
	519	case TokenizeStateSymbol:
380	520	case TokenizeStateNumber:
381		end_token(&t);
382		break;
383	521	case TokenizeStateSawDash:
	522	case TokenizeStatePipe:
	523	case TokenizeStateAmpersand:
	524	case TokenizeStateEq:
	525	case TokenizeStateBang:
	526	case TokenizeStateLessThan:
	527	case TokenizeStateGreaterThan:
384	528	end_token(&t);
385	529	break;
386	530	case TokenizeStateSawSlash:
...	...	@@ -413,6 +557,7 @@ static const char * token_name(Token *token) {
413	557	case TokenIdKeywordUnreachable: return "Unreachable";
414	558	case TokenIdKeywordPub: return "Pub";
415	559	case TokenIdKeywordExport: return "Export";
	560	case TokenIdKeywordAs: return "As";
416	561	case TokenIdLParen: return "LParen";
417	562	case TokenIdRParen: return "RParen";
418	563	case TokenIdComma: return "Comma";
...	...	@@ -427,6 +572,23 @@ static const char * token_name(Token *token) {
427	572	case TokenIdArrow: return "Arrow";
428	573	case TokenIdDash: return "Dash";
429	574	case TokenIdNumberSign: return "NumberSign";
	575	case TokenIdBinOr: return "BinOr";
	576	case TokenIdBinAnd: return "BinAnd";
	577	case TokenIdBinXor: return "BinXor";
	578	case TokenIdBoolOr: return "BoolOr";
	579	case TokenIdBoolAnd: return "BoolAnd";
	580	case TokenIdEq: return "Eq";
	581	case TokenIdNot: return "Not";
	582	case TokenIdCmpEq: return "CmpEq";
	583	case TokenIdCmpNotEq: return "CmpNotEq";
	584	case TokenIdCmpLessThan: return "CmpLessThan";
	585	case TokenIdCmpGreaterThan: return "CmpGreaterThan";
	586	case TokenIdCmpLessOrEq: return "CmpLessOrEq";
	587	case TokenIdCmpGreaterOrEq: return "CmpGreaterOrEq";
	588	case TokenIdBitShiftLeft: return "BitShiftLeft";
	589	case TokenIdBitShiftRight: return "BitShiftRight";
	590	case TokenIdSlash: return "Slash";
	591	case TokenIdPercent: return "Percent";
430	592	}
431	593	return "(invalid token)";
432	594	}

src/tokenizer.hpp+18

...	...	@@ -21,6 +21,7 @@ enum TokenId {
21	21	TokenIdKeywordUnreachable,
22	22	TokenIdKeywordPub,
23	23	TokenIdKeywordExport,
	24	TokenIdKeywordAs,
24	25	TokenIdLParen,
25	26	TokenIdRParen,
26	27	TokenIdComma,
...	...	@@ -35,6 +36,23 @@ enum TokenId {
35	36	TokenIdArrow,
36	37	TokenIdDash,
37	38	TokenIdNumberSign,
	39	TokenIdBoolOr,
	40	TokenIdBoolAnd,
	41	TokenIdBinOr,
	42	TokenIdBinAnd,
	43	TokenIdBinXor,
	44	TokenIdEq,
	45	TokenIdCmpEq,
	46	TokenIdNot,
	47	TokenIdCmpNotEq,
	48	TokenIdCmpLessThan,
	49	TokenIdCmpGreaterThan,
	50	TokenIdCmpLessOrEq,
	51	TokenIdCmpGreaterOrEq,
	52	TokenIdBitShiftLeft,
	53	TokenIdBitShiftRight,
	54	TokenIdSlash,
	55	TokenIdPercent,
38	56	};
39	57
40	58	struct Token {