authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:09:03-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:09:03-05:00
log798dbe487b678a85c7e10b78c2424c4b3508475b
treedd2a88feaef08d5f6da545dceb724f0cd1eb9b44
parent31d9dc35395d495fa896532bdffb4529c136ce41

simple tokenization

2 files changed, 152 insertions(+), 22 deletions(-)

src-self-hosted/main.zig+127-2
......@@ -3,18 +3,134 @@ const io = @import("std").io;
33const os = @import("std").os;
44const heap = @import("std").heap;
55const warn = @import("std").debug.warn;
6
6const assert = @import("std").debug.assert;
7const mem = @import("std").mem;
78
89const Token = struct {
10 id: Id,
11 start: usize,
12 end: usize,
13
14 const Keyword = enum {
15 @"align",
16 @"and",
17 @"asm",
18 @"break",
19 @"coldcc",
20 @"comptime",
21 @"const",
22 @"continue",
23 @"defer",
24 @"else",
25 @"enum",
26 @"error",
27 @"export",
28 @"extern",
29 @"false",
30 @"fn",
31 @"for",
32 @"goto",
33 @"if",
34 @"inline",
35 @"nakedcc",
36 @"noalias",
37 @"null",
38 @"or",
39 @"packed",
40 @"pub",
41 @"return",
42 @"stdcallcc",
43 @"struct",
44 @"switch",
45 @"test",
46 @"this",
47 @"true",
48 @"undefined",
49 @"union",
50 @"unreachable",
51 @"use",
52 @"var",
53 @"volatile",
54 @"while",
55 };
956
57 fn getKeyword(bytes: []const u8) -> ?Keyword {
58 comptime var i = 0;
59 inline while (i < @memberCount(Keyword)) : (i += 1) {
60 if (mem.eql(u8, @memberName(Keyword, i), bytes)) {
61 return Keyword(i);
62 }
63 }
64 return null;
65 }
66
67
68 const Id = union(enum) {
69 Invalid,
70 Identifier,
71 Keyword: Keyword,
72 Eof,
73 };
1074};
1175
1276const Tokenizer = struct {
77 buffer: []const u8,
78 index: usize,
1379
14 pub fn next() -> Token {
80 pub fn dump(self: &Tokenizer, token: &const Token) {
81 warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
82 }
1583
84 pub fn init(buffer: []const u8) -> Tokenizer {
85 return Tokenizer {
86 .buffer = buffer,
87 .index = 0,
88 };
1689 }
1790
91 const State = enum {
92 Start,
93 Identifier,
94 };
95
96 pub fn next(self: &Tokenizer) -> Token {
97 var state = State.Start;
98 var result = Token {
99 .id = Token.Id { .Eof = {} },
100 .start = self.index,
101 .end = undefined,
102 };
103 while (self.index < self.buffer.len) : (self.index += 1) {
104 const c = self.buffer[self.index];
105 switch (state) {
106 State.Start => switch (c) {
107 ' ', '\n' => {
108 result.start = self.index + 1;
109 },
110 'a'...'z', 'A'...'Z', '_' => {
111 state = State.Identifier;
112 result.id = Token.Id { .Identifier = {} };
113 },
114 else => {
115 result.id = Token.Id { .Invalid = {} };
116 self.index += 1;
117 break;
118 },
119 },
120 State.Identifier => switch (c) {
121 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
122 else => {
123 if (Token.getKeyword(self.buffer[result.start..self.index])) |keyword_id| {
124 result.id = Token.Id { .Keyword = keyword_id };
125 }
126 break;
127 },
128 },
129 }
130 }
131 result.end = self.index;
132 return result;
133 }
18134};
19135
20136
......@@ -36,4 +152,13 @@ pub fn main2() -> %void {
36152 const target_file_buf = %return io.readFileAlloc(target_file, allocator);
37153
38154 warn("{}", target_file_buf);
155
156 var tokenizer = Tokenizer.init(target_file_buf);
157 while (true) {
158 const token = tokenizer.next();
159 tokenizer.dump(token);
160 if (@TagType(Token.Id)(token.id) == Token.Id.Eof) {
161 break;
162 }
163 }
39164}
src/ir.cpp+25-20
......@@ -8422,16 +8422,6 @@ static IrInstruction *ir_analyze_int_to_enum(IrAnalyze *ira, IrInstruction *sour
84228422 if (type_is_invalid(wanted_type))
84238423 return ira->codegen->invalid_instruction;
84248424
8425 if (actual_type != wanted_type->data.enumeration.tag_int_type) {
8426 ir_add_error(ira, source_instr,
8427 buf_sprintf("integer to enum cast from '%s' instead of its tag type, '%s'",
8428 buf_ptr(&actual_type->name),
8429 buf_ptr(&wanted_type->data.enumeration.tag_int_type->name)));
8430 return ira->codegen->invalid_instruction;
8431 }
8432
8433 assert(actual_type->id == TypeTableEntryIdInt);
8434
84358425 if (instr_is_comptime(target)) {
84368426 ConstExprValue *val = ir_resolve_const(ira, target, UndefBad);
84378427 if (!val)
......@@ -8453,6 +8443,17 @@ static IrInstruction *ir_analyze_int_to_enum(IrAnalyze *ira, IrInstruction *sour
84538443 return result;
84548444 }
84558445
8446 if (actual_type != wanted_type->data.enumeration.tag_int_type) {
8447 ir_add_error(ira, source_instr,
8448 buf_sprintf("integer to enum cast from '%s' instead of its tag type, '%s'",
8449 buf_ptr(&actual_type->name),
8450 buf_ptr(&wanted_type->data.enumeration.tag_int_type->name)));
8451 return ira->codegen->invalid_instruction;
8452 }
8453
8454 assert(actual_type->id == TypeTableEntryIdInt);
8455
8456
84568457 IrInstruction *result = ir_build_int_to_enum(&ira->new_irb, source_instr->scope,
84578458 source_instr->source_node, target);
84588459 result->value.type = wanted_type;
......@@ -8822,6 +8823,20 @@ static IrInstruction *ir_analyze_cast(IrAnalyze *ira, IrInstruction *source_inst
88228823 }
88238824 }
88248825
8826 // explicit cast from integer to enum type with no payload
8827 if ((actual_type->id == TypeTableEntryIdInt || actual_type->id == TypeTableEntryIdNumLitInt) &&
8828 wanted_type->id == TypeTableEntryIdEnum)
8829 {
8830 return ir_analyze_int_to_enum(ira, source_instr, value, wanted_type);
8831 }
8832
8833 // explicit cast from enum type with no payload to integer
8834 if ((wanted_type->id == TypeTableEntryIdInt || wanted_type->id == TypeTableEntryIdNumLitInt) &&
8835 actual_type->id == TypeTableEntryIdEnum)
8836 {
8837 return ir_analyze_enum_to_int(ira, source_instr, value, wanted_type);
8838 }
8839
88258840 // explicit cast from number literal to another type
88268841 // explicit cast from number literal to &const integer
88278842 if (actual_type->id == TypeTableEntryIdNumLitFloat ||
......@@ -8886,16 +8901,6 @@ static IrInstruction *ir_analyze_cast(IrAnalyze *ira, IrInstruction *source_inst
88868901 return ir_analyze_int_to_err(ira, source_instr, value);
88878902 }
88888903
8889 // explicit cast from integer to enum type with no payload
8890 if (actual_type->id == TypeTableEntryIdInt && wanted_type->id == TypeTableEntryIdEnum) {
8891 return ir_analyze_int_to_enum(ira, source_instr, value, wanted_type);
8892 }
8893
8894 // explicit cast from enum type with no payload to integer
8895 if (wanted_type->id == TypeTableEntryIdInt && actual_type->id == TypeTableEntryIdEnum) {
8896 return ir_analyze_enum_to_int(ira, source_instr, value, wanted_type);
8897 }
8898
88998904 // explicit cast from union to the enum type of the union
89008905 if (actual_type->id == TypeTableEntryIdUnion && wanted_type->id == TypeTableEntryIdEnum) {
89018906 type_ensure_zero_bits_known(ira->codegen, actual_type);