authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:09:03-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:09:03-05:00
log798dbe487b678a85c7e10b78c2424c4b3508475b
treedd2a88feaef08d5f6da545dceb724f0cd1eb9b44
parent31d9dc35395d495fa896532bdffb4529c136ce41

simple tokenization

2 files changed, 152 insertions(+), 22 deletions(-)

src-self-hosted/main.zig+127-2
...@@ -3,18 +3,134 @@ const io = @import("std").io;...@@ -3,18 +3,134 @@ const io = @import("std").io;
3const os = @import("std").os;3const os = @import("std").os;
4const heap = @import("std").heap;4const heap = @import("std").heap;
5const warn = @import("std").debug.warn;5const warn = @import("std").debug.warn;
66const assert = @import("std").debug.assert;
7const mem = @import("std").mem;
78
8const Token = struct {9const Token = struct {
10 id: Id,
11 start: usize,
12 end: usize,
13
14 const Keyword = enum {
15 @"align",
16 @"and",
17 @"asm",
18 @"break",
19 @"coldcc",
20 @"comptime",
21 @"const",
22 @"continue",
23 @"defer",
24 @"else",
25 @"enum",
26 @"error",
27 @"export",
28 @"extern",
29 @"false",
30 @"fn",
31 @"for",
32 @"goto",
33 @"if",
34 @"inline",
35 @"nakedcc",
36 @"noalias",
37 @"null",
38 @"or",
39 @"packed",
40 @"pub",
41 @"return",
42 @"stdcallcc",
43 @"struct",
44 @"switch",
45 @"test",
46 @"this",
47 @"true",
48 @"undefined",
49 @"union",
50 @"unreachable",
51 @"use",
52 @"var",
53 @"volatile",
54 @"while",
55 };
956
57 fn getKeyword(bytes: []const u8) -> ?Keyword {
58 comptime var i = 0;
59 inline while (i < @memberCount(Keyword)) : (i += 1) {
60 if (mem.eql(u8, @memberName(Keyword, i), bytes)) {
61 return Keyword(i);
62 }
63 }
64 return null;
65 }
66
67
68 const Id = union(enum) {
69 Invalid,
70 Identifier,
71 Keyword: Keyword,
72 Eof,
73 };
10};74};
1175
12const Tokenizer = struct {76const Tokenizer = struct {
77 buffer: []const u8,
78 index: usize,
1379
14 pub fn next() -> Token {80 pub fn dump(self: &Tokenizer, token: &const Token) {
81 warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
82 }
1583
84 pub fn init(buffer: []const u8) -> Tokenizer {
85 return Tokenizer {
86 .buffer = buffer,
87 .index = 0,
88 };
16 }89 }
1790
91 const State = enum {
92 Start,
93 Identifier,
94 };
95
96 pub fn next(self: &Tokenizer) -> Token {
97 var state = State.Start;
98 var result = Token {
99 .id = Token.Id { .Eof = {} },
100 .start = self.index,
101 .end = undefined,
102 };
103 while (self.index < self.buffer.len) : (self.index += 1) {
104 const c = self.buffer[self.index];
105 switch (state) {
106 State.Start => switch (c) {
107 ' ', '\n' => {
108 result.start = self.index + 1;
109 },
110 'a'...'z', 'A'...'Z', '_' => {
111 state = State.Identifier;
112 result.id = Token.Id { .Identifier = {} };
113 },
114 else => {
115 result.id = Token.Id { .Invalid = {} };
116 self.index += 1;
117 break;
118 },
119 },
120 State.Identifier => switch (c) {
121 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
122 else => {
123 if (Token.getKeyword(self.buffer[result.start..self.index])) |keyword_id| {
124 result.id = Token.Id { .Keyword = keyword_id };
125 }
126 break;
127 },
128 },
129 }
130 }
131 result.end = self.index;
132 return result;
133 }
18};134};
19135
20136
...@@ -36,4 +152,13 @@ pub fn main2() -> %void {...@@ -36,4 +152,13 @@ pub fn main2() -> %void {
36 const target_file_buf = %return io.readFileAlloc(target_file, allocator);152 const target_file_buf = %return io.readFileAlloc(target_file, allocator);
37153
38 warn("{}", target_file_buf);154 warn("{}", target_file_buf);
155
156 var tokenizer = Tokenizer.init(target_file_buf);
157 while (true) {
158 const token = tokenizer.next();
159 tokenizer.dump(token);
160 if (@TagType(Token.Id)(token.id) == Token.Id.Eof) {
161 break;
162 }
163 }
39}164}
src/ir.cpp+25-20
...@@ -8422,16 +8422,6 @@ static IrInstruction *ir_analyze_int_to_enum(IrAnalyze *ira, IrInstruction *sour...@@ -8422,16 +8422,6 @@ static IrInstruction *ir_analyze_int_to_enum(IrAnalyze *ira, IrInstruction *sour
8422 if (type_is_invalid(wanted_type))8422 if (type_is_invalid(wanted_type))
8423 return ira->codegen->invalid_instruction;8423 return ira->codegen->invalid_instruction;
84248424
8425 if (actual_type != wanted_type->data.enumeration.tag_int_type) {
8426 ir_add_error(ira, source_instr,
8427 buf_sprintf("integer to enum cast from '%s' instead of its tag type, '%s'",
8428 buf_ptr(&actual_type->name),
8429 buf_ptr(&wanted_type->data.enumeration.tag_int_type->name)));
8430 return ira->codegen->invalid_instruction;
8431 }
8432
8433 assert(actual_type->id == TypeTableEntryIdInt);
8434
8435 if (instr_is_comptime(target)) {8425 if (instr_is_comptime(target)) {
8436 ConstExprValue *val = ir_resolve_const(ira, target, UndefBad);8426 ConstExprValue *val = ir_resolve_const(ira, target, UndefBad);
8437 if (!val)8427 if (!val)
...@@ -8453,6 +8443,17 @@ static IrInstruction *ir_analyze_int_to_enum(IrAnalyze *ira, IrInstruction *sour...@@ -8453,6 +8443,17 @@ static IrInstruction *ir_analyze_int_to_enum(IrAnalyze *ira, IrInstruction *sour
8453 return result;8443 return result;
8454 }8444 }
84558445
8446 if (actual_type != wanted_type->data.enumeration.tag_int_type) {
8447 ir_add_error(ira, source_instr,
8448 buf_sprintf("integer to enum cast from '%s' instead of its tag type, '%s'",
8449 buf_ptr(&actual_type->name),
8450 buf_ptr(&wanted_type->data.enumeration.tag_int_type->name)));
8451 return ira->codegen->invalid_instruction;
8452 }
8453
8454 assert(actual_type->id == TypeTableEntryIdInt);
8455
8456
8456 IrInstruction *result = ir_build_int_to_enum(&ira->new_irb, source_instr->scope,8457 IrInstruction *result = ir_build_int_to_enum(&ira->new_irb, source_instr->scope,
8457 source_instr->source_node, target);8458 source_instr->source_node, target);
8458 result->value.type = wanted_type;8459 result->value.type = wanted_type;
...@@ -8822,6 +8823,20 @@ static IrInstruction *ir_analyze_cast(IrAnalyze *ira, IrInstruction *source_inst...@@ -8822,6 +8823,20 @@ static IrInstruction *ir_analyze_cast(IrAnalyze *ira, IrInstruction *source_inst
8822 }8823 }
8823 }8824 }
88248825
8826 // explicit cast from integer to enum type with no payload
8827 if ((actual_type->id == TypeTableEntryIdInt || actual_type->id == TypeTableEntryIdNumLitInt) &&
8828 wanted_type->id == TypeTableEntryIdEnum)
8829 {
8830 return ir_analyze_int_to_enum(ira, source_instr, value, wanted_type);
8831 }
8832
8833 // explicit cast from enum type with no payload to integer
8834 if ((wanted_type->id == TypeTableEntryIdInt || wanted_type->id == TypeTableEntryIdNumLitInt) &&
8835 actual_type->id == TypeTableEntryIdEnum)
8836 {
8837 return ir_analyze_enum_to_int(ira, source_instr, value, wanted_type);
8838 }
8839
8825 // explicit cast from number literal to another type8840 // explicit cast from number literal to another type
8826 // explicit cast from number literal to &const integer8841 // explicit cast from number literal to &const integer
8827 if (actual_type->id == TypeTableEntryIdNumLitFloat ||8842 if (actual_type->id == TypeTableEntryIdNumLitFloat ||
...@@ -8886,16 +8901,6 @@ static IrInstruction *ir_analyze_cast(IrAnalyze *ira, IrInstruction *source_inst...@@ -8886,16 +8901,6 @@ static IrInstruction *ir_analyze_cast(IrAnalyze *ira, IrInstruction *source_inst
8886 return ir_analyze_int_to_err(ira, source_instr, value);8901 return ir_analyze_int_to_err(ira, source_instr, value);
8887 }8902 }
88888903
8889 // explicit cast from integer to enum type with no payload
8890 if (actual_type->id == TypeTableEntryIdInt && wanted_type->id == TypeTableEntryIdEnum) {
8891 return ir_analyze_int_to_enum(ira, source_instr, value, wanted_type);
8892 }
8893
8894 // explicit cast from enum type with no payload to integer
8895 if (wanted_type->id == TypeTableEntryIdInt && actual_type->id == TypeTableEntryIdEnum) {
8896 return ir_analyze_enum_to_int(ira, source_instr, value, wanted_type);
8897 }
8898
8899 // explicit cast from union to the enum type of the union8904 // explicit cast from union to the enum type of the union
8900 if (actual_type->id == TypeTableEntryIdUnion && wanted_type->id == TypeTableEntryIdEnum) {8905 if (actual_type->id == TypeTableEntryIdUnion && wanted_type->id == TypeTableEntryIdEnum) {
8901 type_ensure_zero_bits_known(ira->codegen, actual_type);8906 type_ensure_zero_bits_known(ira->codegen, actual_type);