| author | |
| committer | |
| log | 75218d4765bdf0dbdf97581b7dd05b45570ab940 |
| tree | 38782cf017e2d9fd3a3e6e8af802213a31f943db |
| parent | c3724a6e723dfb5ec78c6ca87e2f02e121d39bc2 |
| signature | Commit is signed but in an unrecognized format. |
4 files changed, 542 insertions(+), 81 deletions(-)
src-self-hosted/c_tokenizer.zig+157-57| ... | @@ -1,4 +1,5 @@ | ... | @@ -1,4 +1,5 @@ |
| 1 | const std = @import("std"); | 1 | const std = @import("std"); |
| 2 | const expect = std.testing.expect; | ||
| 2 | 3 | ||
| 3 | pub const TokenList = std.SegmentedList(CToken, 32); | 4 | pub const TokenList = std.SegmentedList(CToken, 32); |
| 4 | 5 | ||
| ... | @@ -28,6 +29,7 @@ pub const CToken = struct { | ... | @@ -28,6 +29,7 @@ pub const CToken = struct { |
| 28 | 29 | ||
| 29 | pub const NumLitSuffix = enum { | 30 | pub const NumLitSuffix = enum { |
| 30 | None, | 31 | None, |
| 32 | F, | ||
| 31 | L, | 33 | L, |
| 32 | U, | 34 | U, |
| 33 | LU, | 35 | LU, |
| ... | @@ -39,19 +41,18 @@ pub const CToken = struct { | ... | @@ -39,19 +41,18 @@ pub const CToken = struct { |
| 39 | pub fn tokenizeCMacro(tl: *TokenList, chars: [*]const u8) !void { | 41 | pub fn tokenizeCMacro(tl: *TokenList, chars: [*]const u8) !void { |
| 40 | var index: usize = 0; | 42 | var index: usize = 0; |
| 41 | while (true) { | 43 | while (true) { |
| 42 | const tok = try next(chars[index..], &index); | 44 | const tok = try next(chars, &index); |
| 43 | tl.push(tok); | 45 | try tl.push(tok); |
| 44 | if (tok.id == .Eof) | 46 | if (tok.id == .Eof) |
| 45 | return; | 47 | return; |
| 46 | } | 48 | } |
| 47 | } | 49 | } |
| 48 | 50 | ||
| 49 | fn next(chars: [*]const u8, index: *usize) !CToken { | 51 | fn next(chars: [*]const u8, i: *usize) !CToken { |
| 50 | var state: enum { | 52 | var state: enum { |
| 51 | Start, | 53 | Start, |
| 52 | GotLt, | 54 | GotLt, |
| 53 | ExpectChar, | 55 | CharLit, |
| 54 | ExpectEndQuot, | ||
| 55 | OpenComment, | 56 | OpenComment, |
| 56 | Comment, | 57 | Comment, |
| 57 | CommentStar, | 58 | CommentStar, |
| ... | @@ -62,6 +63,7 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -62,6 +63,7 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 62 | Octal, | 63 | Octal, |
| 63 | GotZero, | 64 | GotZero, |
| 64 | Hex, | 65 | Hex, |
| 66 | Bin, | ||
| 65 | Float, | 67 | Float, |
| 66 | ExpSign, | 68 | ExpSign, |
| 67 | FloatExp, | 69 | FloatExp, |
| ... | @@ -70,7 +72,6 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -70,7 +72,6 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 70 | NumLitIntSuffixL, | 72 | NumLitIntSuffixL, |
| 71 | NumLitIntSuffixLL, | 73 | NumLitIntSuffixLL, |
| 72 | NumLitIntSuffixUL, | 74 | NumLitIntSuffixUL, |
| 73 | GotLt, | ||
| 74 | } = .Start; | 75 | } = .Start; |
| 75 | 76 | ||
| 76 | var result = CToken{ | 77 | var result = CToken{ |
| ... | @@ -79,9 +80,10 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -79,9 +80,10 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 79 | }; | 80 | }; |
| 80 | var begin_index: usize = 0; | 81 | var begin_index: usize = 0; |
| 81 | var digits: u8 = 0; | 82 | var digits: u8 = 0; |
| 82 | var pre_escape = .Start; | 83 | var pre_escape = state; |
| 83 | 84 | ||
| 84 | for (chars[begin_index..]) |c, i| { | 85 | while (true) { |
| 86 | const c = chars[i.*]; | ||
| 85 | if (c == 0) { | 87 | if (c == 0) { |
| 86 | switch (state) { | 88 | switch (state) { |
| 87 | .Start => { | 89 | .Start => { |
| ... | @@ -90,22 +92,25 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -90,22 +92,25 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 90 | .Identifier, | 92 | .Identifier, |
| 91 | .Decimal, | 93 | .Decimal, |
| 92 | .Hex, | 94 | .Hex, |
| 95 | .Bin, | ||
| 93 | .Octal, | 96 | .Octal, |
| 94 | .GotZero, | 97 | .GotZero, |
| 98 | .Float, | ||
| 99 | .FloatExp, | ||
| 100 | => { | ||
| 101 | result.bytes = chars[begin_index..i.*]; | ||
| 102 | return result; | ||
| 103 | }, | ||
| 95 | .NumLitIntSuffixU, | 104 | .NumLitIntSuffixU, |
| 96 | .NumLitIntSuffixL, | 105 | .NumLitIntSuffixL, |
| 97 | .NumLitIntSuffixUL, | 106 | .NumLitIntSuffixUL, |
| 98 | .NumLitIntSuffixLL, | 107 | .NumLitIntSuffixLL, |
| 99 | .Float, | ||
| 100 | .FloatExp, | ||
| 101 | .GotLt, | 108 | .GotLt, |
| 102 | => { | 109 | => { |
| 103 | return result; | 110 | return result; |
| 104 | }, | 111 | }, |
| 105 | .ExpectChar, | 112 | .CharLit, |
| 106 | .ExpectEndQuot, | ||
| 107 | .OpenComment, | 113 | .OpenComment, |
| 108 | .LineComment, | ||
| 109 | .Comment, | 114 | .Comment, |
| 110 | .CommentStar, | 115 | .CommentStar, |
| 111 | .Backslash, | 116 | .Backslash, |
| ... | @@ -115,20 +120,20 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -115,20 +120,20 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 115 | => return error.TokenizingFailed, | 120 | => return error.TokenizingFailed, |
| 116 | } | 121 | } |
| 117 | } | 122 | } |
| 118 | index.* += 1; | 123 | i.* += 1; |
| 119 | switch (state) { | 124 | switch (state) { |
| 120 | .Start => { | 125 | .Start => { |
| 121 | switch (c) { | 126 | switch (c) { |
| 122 | ' ', '\t', '\x0B', '\x0C' => {}, | 127 | ' ', '\t', '\x0B', '\x0C' => {}, |
| 123 | '\'' => { | 128 | '\'' => { |
| 124 | state = .ExpectChar; | 129 | state = .CharLit; |
| 125 | result.id = .CharLit; | 130 | result.id = .CharLit; |
| 126 | begin_index = i; | 131 | begin_index = i.* - 1; |
| 127 | }, | 132 | }, |
| 128 | '\"' => { | 133 | '\"' => { |
| 129 | state = .String; | 134 | state = .String; |
| 130 | result.id = .StrLit; | 135 | result.id = .StrLit; |
| 131 | begin_index = i; | 136 | begin_index = i.* - 1; |
| 132 | }, | 137 | }, |
| 133 | '/' => { | 138 | '/' => { |
| 134 | state = .OpenComment; | 139 | state = .OpenComment; |
| ... | @@ -142,17 +147,17 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -142,17 +147,17 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 142 | 'a'...'z', 'A'...'Z', '_' => { | 147 | 'a'...'z', 'A'...'Z', '_' => { |
| 143 | state = .Identifier; | 148 | state = .Identifier; |
| 144 | result.id = .Identifier; | 149 | result.id = .Identifier; |
| 145 | begin_index = i; | 150 | begin_index = i.* - 1; |
| 146 | }, | 151 | }, |
| 147 | '1'...'9' => { | 152 | '1'...'9' => { |
| 148 | state = .Decimal; | 153 | state = .Decimal; |
| 149 | result.id = .NumLitInt; | 154 | result.id = .NumLitInt; |
| 150 | begin_index = i; | 155 | begin_index = i.* - 1; |
| 151 | }, | 156 | }, |
| 152 | '0' => { | 157 | '0' => { |
| 153 | state = .GotZero; | 158 | state = .GotZero; |
| 154 | result.id = .NumLitInt; | 159 | result.id = .NumLitInt; |
| 155 | begin_index = i; | 160 | begin_index = i.* - 1; |
| 156 | }, | 161 | }, |
| 157 | '.' => { | 162 | '.' => { |
| 158 | result.id = .Dot; | 163 | result.id = .Dot; |
| ... | @@ -206,12 +211,23 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -206,12 +211,23 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 206 | 'e', 'E' => { | 211 | 'e', 'E' => { |
| 207 | state = .ExpSign; | 212 | state = .ExpSign; |
| 208 | }, | 213 | }, |
| 209 | 'f', 'F', 'l', 'L' => { | 214 | 'f', |
| 210 | result.bytes = chars[begin_index..i]; | 215 | 'F', |
| 216 | => { | ||
| 217 | i.* -= 1; | ||
| 218 | result.num_lit_suffix = .F; | ||
| 219 | result.bytes = chars[begin_index..i.*]; | ||
| 220 | return result; | ||
| 221 | }, | ||
| 222 | 'l', 'L' => { | ||
| 223 | i.* -= 1; | ||
| 224 | result.num_lit_suffix = .L; | ||
| 225 | result.bytes = chars[begin_index..i.*]; | ||
| 211 | return result; | 226 | return result; |
| 212 | }, | 227 | }, |
| 213 | else => { | 228 | else => { |
| 214 | result.bytes = chars[begin_index..i]; | 229 | i.* -= 1; |
| 230 | result.bytes = chars[begin_index..i.*]; | ||
| 215 | return result; | 231 | return result; |
| 216 | }, | 232 | }, |
| 217 | } | 233 | } |
| ... | @@ -238,12 +254,19 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -238,12 +254,19 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 238 | .FloatExp => { | 254 | .FloatExp => { |
| 239 | switch (c) { | 255 | switch (c) { |
| 240 | '0'...'9' => {}, | 256 | '0'...'9' => {}, |
| 241 | 'f', 'F', 'l', 'L' => { | 257 | 'f', 'F' => { |
| 242 | result.bytes = chars[begin_index..i]; | 258 | result.num_lit_suffix = .F; |
| 259 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 260 | return result; | ||
| 261 | }, | ||
| 262 | 'l', 'L' => { | ||
| 263 | result.num_lit_suffix = .L; | ||
| 264 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 243 | return result; | 265 | return result; |
| 244 | }, | 266 | }, |
| 245 | else => { | 267 | else => { |
| 246 | result.bytes = chars[begin_index..i]; | 268 | i.* -= 1; |
| 269 | result.bytes = chars[begin_index..i.*]; | ||
| 247 | return result; | 270 | return result; |
| 248 | }, | 271 | }, |
| 249 | } | 272 | } |
| ... | @@ -255,17 +278,20 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -255,17 +278,20 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 255 | 'u', 'U' => { | 278 | 'u', 'U' => { |
| 256 | state = .NumLitIntSuffixU; | 279 | state = .NumLitIntSuffixU; |
| 257 | result.num_lit_suffix = .U; | 280 | result.num_lit_suffix = .U; |
| 281 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 258 | }, | 282 | }, |
| 259 | 'l', 'L' => { | 283 | 'l', 'L' => { |
| 260 | state = .NumLitIntSuffixL; | 284 | state = .NumLitIntSuffixL; |
| 261 | result.num_lit_suffix = .L; | 285 | result.num_lit_suffix = .L; |
| 286 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 262 | }, | 287 | }, |
| 263 | '.' => { | 288 | '.' => { |
| 264 | result.id = .NumLitFloat; | 289 | result.id = .NumLitFloat; |
| 265 | state = .Float; | 290 | state = .Float; |
| 266 | }, | 291 | }, |
| 267 | else => { | 292 | else => { |
| 268 | result.bytes = chars[begin_index..i]; | 293 | i.* -= 1; |
| 294 | result.bytes = chars[begin_index..i.*]; | ||
| 269 | return result; | 295 | return result; |
| 270 | }, | 296 | }, |
| 271 | } | 297 | } |
| ... | @@ -275,15 +301,25 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -275,15 +301,25 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 275 | 'x', 'X' => { | 301 | 'x', 'X' => { |
| 276 | state = .Hex; | 302 | state = .Hex; |
| 277 | }, | 303 | }, |
| 304 | 'b', 'B' => { | ||
| 305 | state = .Bin; | ||
| 306 | }, | ||
| 278 | '.' => { | 307 | '.' => { |
| 279 | state = .Float; | 308 | state = .Float; |
| 280 | result.id = .NumLitFloat; | 309 | result.id = .NumLitFloat; |
| 281 | }, | 310 | }, |
| 282 | 'l', 'L', 'u', 'U' => { | 311 | 'u', 'U' => { |
| 283 | c -= 1; | 312 | state = .NumLitIntSuffixU; |
| 284 | state = .Decimal; | 313 | result.num_lit_suffix = .U; |
| 314 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 315 | }, | ||
| 316 | 'l', 'L' => { | ||
| 317 | state = .NumLitIntSuffixL; | ||
| 318 | result.num_lit_suffix = .L; | ||
| 319 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 285 | }, | 320 | }, |
| 286 | else => { | 321 | else => { |
| 322 | i.* -= 1; | ||
| 287 | state = .Octal; | 323 | state = .Octal; |
| 288 | }, | 324 | }, |
| 289 | } | 325 | } |
| ... | @@ -293,7 +329,8 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -293,7 +329,8 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 293 | '0'...'7' => {}, | 329 | '0'...'7' => {}, |
| 294 | '8', '9' => return error.TokenizingFailed, | 330 | '8', '9' => return error.TokenizingFailed, |
| 295 | else => { | 331 | else => { |
| 296 | result.bytes = chars[begin_index..i]; | 332 | i.* -= 1; |
| 333 | result.bytes = chars[begin_index..i.*]; | ||
| 297 | return result; | 334 | return result; |
| 298 | }, | 335 | }, |
| 299 | } | 336 | } |
| ... | @@ -301,23 +338,44 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -301,23 +338,44 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 301 | .Hex => { | 338 | .Hex => { |
| 302 | switch (c) { | 339 | switch (c) { |
| 303 | '0'...'9', 'a'...'f', 'A'...'F' => {}, | 340 | '0'...'9', 'a'...'f', 'A'...'F' => {}, |
| 304 | 341 | 'u', 'U' => { | |
| 305 | 'p', 'P' => { | 342 | // marks the number literal as unsigned |
| 306 | result.id = .NumLitFloat; | 343 | state = .NumLitIntSuffixU; |
| 307 | state = .ExpSign; | 344 | result.num_lit_suffix = .U; |
| 345 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 308 | }, | 346 | }, |
| 347 | 'l', 'L' => { | ||
| 348 | // marks the number literal as long | ||
| 349 | state = .NumLitIntSuffixL; | ||
| 350 | result.num_lit_suffix = .L; | ||
| 351 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 352 | }, | ||
| 353 | else => { | ||
| 354 | i.* -= 1; | ||
| 355 | result.bytes = chars[begin_index..i.*]; | ||
| 356 | return result; | ||
| 357 | }, | ||
| 358 | } | ||
| 359 | }, | ||
| 360 | .Bin => { | ||
| 361 | switch (c) { | ||
| 362 | '0'...'1' => {}, | ||
| 363 | '2'...'9' => return error.TokenizingFailed, | ||
| 309 | 'u', 'U' => { | 364 | 'u', 'U' => { |
| 310 | // marks the number literal as unsigned | 365 | // marks the number literal as unsigned |
| 311 | state = .NumLitIntSuffixU; | 366 | state = .NumLitIntSuffixU; |
| 312 | result.num_lit_suffix = .U; | 367 | result.num_lit_suffix = .U; |
| 368 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 313 | }, | 369 | }, |
| 314 | 'l', 'L' => { | 370 | 'l', 'L' => { |
| 315 | // marks the number literal as long | 371 | // marks the number literal as long |
| 316 | state = .NumLitIntSuffixL; | 372 | state = .NumLitIntSuffixL; |
| 317 | result.num_lit_suffix = .L; | 373 | result.num_lit_suffix = .L; |
| 374 | result.bytes = chars[begin_index .. i.* - 1]; | ||
| 318 | }, | 375 | }, |
| 319 | else => { | 376 | else => { |
| 320 | result.bytes = chars[begin_index..i]; | 377 | i.* -= 1; |
| 378 | result.bytes = chars[begin_index..i.*]; | ||
| 321 | return result; | 379 | return result; |
| 322 | }, | 380 | }, |
| 323 | } | 381 | } |
| ... | @@ -329,7 +387,7 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -329,7 +387,7 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 329 | state = .NumLitIntSuffixUL; | 387 | state = .NumLitIntSuffixUL; |
| 330 | }, | 388 | }, |
| 331 | else => { | 389 | else => { |
| 332 | result.bytes = chars[begin_index..i - 1]; | 390 | i.* -= 1; |
| 333 | return result; | 391 | return result; |
| 334 | }, | 392 | }, |
| 335 | } | 393 | } |
| ... | @@ -342,11 +400,10 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -342,11 +400,10 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 342 | }, | 400 | }, |
| 343 | 'u', 'U' => { | 401 | 'u', 'U' => { |
| 344 | result.num_lit_suffix = .LU; | 402 | result.num_lit_suffix = .LU; |
| 345 | result.bytes = chars[begin_index..i - 2]; | ||
| 346 | return result; | 403 | return result; |
| 347 | }, | 404 | }, |
| 348 | else => { | 405 | else => { |
| 349 | result.bytes = chars[begin_index..i - 1]; | 406 | i.* -= 1; |
| 350 | return result; | 407 | return result; |
| 351 | }, | 408 | }, |
| 352 | } | 409 | } |
| ... | @@ -355,11 +412,10 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -355,11 +412,10 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 355 | switch (c) { | 412 | switch (c) { |
| 356 | 'u', 'U' => { | 413 | 'u', 'U' => { |
| 357 | result.num_lit_suffix = .LLU; | 414 | result.num_lit_suffix = .LLU; |
| 358 | result.bytes = chars[begin_index..i - 3]; | ||
| 359 | return result; | 415 | return result; |
| 360 | }, | 416 | }, |
| 361 | else => { | 417 | else => { |
| 362 | result.bytes = chars[begin_index..i - 2]; | 418 | i.* -= 1; |
| 363 | return result; | 419 | return result; |
| 364 | }, | 420 | }, |
| 365 | } | 421 | } |
| ... | @@ -368,11 +424,10 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -368,11 +424,10 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 368 | switch (c) { | 424 | switch (c) { |
| 369 | 'l', 'L' => { | 425 | 'l', 'L' => { |
| 370 | result.num_lit_suffix = .LLU; | 426 | result.num_lit_suffix = .LLU; |
| 371 | result.bytes = chars[begin_index..i - 3]; | ||
| 372 | return result; | 427 | return result; |
| 373 | }, | 428 | }, |
| 374 | else => { | 429 | else => { |
| 375 | result.bytes = chars[begin_index..i - 2]; | 430 | i.* -= 1; |
| 376 | return result; | 431 | return result; |
| 377 | }, | 432 | }, |
| 378 | } | 433 | } |
| ... | @@ -381,35 +436,28 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -381,35 +436,28 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 381 | switch (c) { | 436 | switch (c) { |
| 382 | '_', 'a'...'z', 'A'...'Z', '0'...'9' => {}, | 437 | '_', 'a'...'z', 'A'...'Z', '0'...'9' => {}, |
| 383 | else => { | 438 | else => { |
| 384 | result.bytes = chars[begin_index..i]; | 439 | i.* -= 1; |
| 440 | result.bytes = chars[begin_index..i.*]; | ||
| 385 | return result; | 441 | return result; |
| 386 | }, | 442 | }, |
| 387 | } | 443 | } |
| 388 | }, | 444 | }, |
| 389 | .String => { | 445 | .String => { // TODO char escapes |
| 390 | switch (c) { | 446 | switch (c) { |
| 391 | '\"' => { | 447 | '\"' => { |
| 392 | result.bytes = chars[begin_index + 1 .. i]; | 448 | result.bytes = chars[begin_index + 1 .. i.* - 1]; |
| 393 | return result; | 449 | return result; |
| 394 | }, | 450 | }, |
| 395 | else => {}, | 451 | else => {}, |
| 396 | } | 452 | } |
| 397 | }, | 453 | }, |
| 398 | .ExpectChar => { | 454 | .CharLit => { |
| 399 | switch (c) { | ||
| 400 | '\'' => return error.TokenizingFailed, | ||
| 401 | else => { | ||
| 402 | state = .ExpectEndQuot; | ||
| 403 | }, | ||
| 404 | } | ||
| 405 | }, | ||
| 406 | .ExpectEndQuot => { | ||
| 407 | switch (c) { | 455 | switch (c) { |
| 408 | '\'' => { | 456 | '\'' => { |
| 409 | result.bytes = chars[begin_index + 1 .. i]; | 457 | result.bytes = chars[begin_index + 1 .. i.* - 1]; |
| 410 | return result; | 458 | return result; |
| 411 | }, | 459 | }, |
| 412 | else => return error.TokenizingFailed, | 460 | else => {}, |
| 413 | } | 461 | } |
| 414 | }, | 462 | }, |
| 415 | .OpenComment => { | 463 | .OpenComment => { |
| ... | @@ -455,4 +503,56 @@ fn next(chars: [*]const u8, index: *usize) !CToken { | ... | @@ -455,4 +503,56 @@ fn next(chars: [*]const u8, index: *usize) !CToken { |
| 455 | }, | 503 | }, |
| 456 | } | 504 | } |
| 457 | } | 505 | } |
| 506 | unreachable; | ||
| 507 | } | ||
| 508 | |||
| 509 | test "tokenize macro" { | ||
| 510 | var tl = TokenList.init(std.heap.page_allocator); | ||
| 511 | defer tl.deinit(); | ||
| 512 | |||
| 513 | const src = "TEST 0\n"; | ||
| 514 | try tokenizeCMacro(&tl, src); | ||
| 515 | var it = tl.iterator(0); | ||
| 516 | expect(it.next().?.id == .Identifier); | ||
| 517 | expect(std.mem.eql(u8, it.next().?.bytes, "0")); | ||
| 518 | expect(it.next().?.id == .Eof); | ||
| 519 | expect(it.next() == null); | ||
| 520 | tl.shrink(0); | ||
| 521 | |||
| 522 | const src2 = "__FLT_MIN_10_EXP__ -37\n"; | ||
| 523 | try tokenizeCMacro(&tl, src2); | ||
| 524 | it = tl.iterator(0); | ||
| 525 | expect(std.mem.eql(u8, it.next().?.bytes, "__FLT_MIN_10_EXP__")); | ||
| 526 | expect(it.next().?.id == .Minus); | ||
| 527 | expect(std.mem.eql(u8, it.next().?.bytes, "37")); | ||
| 528 | expect(it.next().?.id == .Eof); | ||
| 529 | expect(it.next() == null); | ||
| 530 | tl.shrink(0); | ||
| 531 | |||
| 532 | const src3 = "__llvm__ 1\n#define"; | ||
| 533 | try tokenizeCMacro(&tl, src3); | ||
| 534 | it = tl.iterator(0); | ||
| 535 | expect(std.mem.eql(u8, it.next().?.bytes, "__llvm__")); | ||
| 536 | expect(std.mem.eql(u8, it.next().?.bytes, "1")); | ||
| 537 | expect(it.next().?.id == .Eof); | ||
| 538 | expect(it.next() == null); | ||
| 539 | tl.shrink(0); | ||
| 540 | |||
| 541 | const src4 = "TEST 2"; | ||
| 542 | try tokenizeCMacro(&tl, src4); | ||
| 543 | it = tl.iterator(0); | ||
| 544 | expect(it.next().?.id == .Identifier); | ||
| 545 | expect(std.mem.eql(u8, it.next().?.bytes, "2")); | ||
| 546 | expect(it.next().?.id == .Eof); | ||
| 547 | expect(it.next() == null); | ||
| 548 | tl.shrink(0); | ||
| 549 | |||
| 550 | const src5 = "FOO 0l"; | ||
| 551 | try tokenizeCMacro(&tl, src5); | ||
| 552 | it = tl.iterator(0); | ||
| 553 | expect(it.next().?.id == .Identifier); | ||
| 554 | expect(std.mem.eql(u8, it.next().?.bytes, "0")); | ||
| 555 | expect(it.next().?.id == .Eof); | ||
| 556 | expect(it.next() == null); | ||
| 557 | tl.shrink(0); | ||
| 458 | } | 558 | } |
src-self-hosted/clang.zig+22| ... | @@ -75,6 +75,7 @@ pub const struct_ZigClangWhileStmt = @OpaqueType(); | ... | @@ -75,6 +75,7 @@ pub const struct_ZigClangWhileStmt = @OpaqueType(); |
| 75 | pub const struct_ZigClangFunctionType = @OpaqueType(); | 75 | pub const struct_ZigClangFunctionType = @OpaqueType(); |
| 76 | pub const struct_ZigClangPredefinedExpr = @OpaqueType(); | 76 | pub const struct_ZigClangPredefinedExpr = @OpaqueType(); |
| 77 | pub const struct_ZigClangInitListExpr = @OpaqueType(); | 77 | pub const struct_ZigClangInitListExpr = @OpaqueType(); |
| 78 | pub const ZigClangPreprocessingRecord = @OpaqueType(); | ||
| 78 | 79 | ||
| 79 | pub const ZigClangBO = extern enum { | 80 | pub const ZigClangBO = extern enum { |
| 80 | PtrMemD, | 81 | PtrMemD, |
| ... | @@ -717,6 +718,18 @@ pub const ZigClangEnumDecl_enumerator_iterator = extern struct { | ... | @@ -717,6 +718,18 @@ pub const ZigClangEnumDecl_enumerator_iterator = extern struct { |
| 717 | opaque: *c_void, | 718 | opaque: *c_void, |
| 718 | }; | 719 | }; |
| 719 | 720 | ||
| 721 | pub const ZigClangPreprocessingRecord_iterator = extern struct { | ||
| 722 | I: c_int, | ||
| 723 | Self: *ZigClangPreprocessingRecord, | ||
| 724 | }; | ||
| 725 | |||
| 726 | pub const ZigClangPreprocessedEntity_EntityKind = extern enum { | ||
| 727 | InvalidKind, | ||
| 728 | MacroExpansionKind, | ||
| 729 | MacroDefinitionKind, | ||
| 730 | InclusionDirectiveKind, | ||
| 731 | }; | ||
| 732 | |||
| 720 | pub extern fn ZigClangSourceManager_getSpellingLoc(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) struct_ZigClangSourceLocation; | 733 | pub extern fn ZigClangSourceManager_getSpellingLoc(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) struct_ZigClangSourceLocation; |
| 721 | pub extern fn ZigClangSourceManager_getFilename(self: *const struct_ZigClangSourceManager, SpellingLoc: struct_ZigClangSourceLocation) ?[*:0]const u8; | 734 | pub extern fn ZigClangSourceManager_getFilename(self: *const struct_ZigClangSourceManager, SpellingLoc: struct_ZigClangSourceLocation) ?[*:0]const u8; |
| 722 | pub extern fn ZigClangSourceManager_getSpellingLineNumber(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) c_uint; | 735 | pub extern fn ZigClangSourceManager_getSpellingLineNumber(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) c_uint; |
| ... | @@ -1014,3 +1027,12 @@ pub extern fn ZigClangFieldDecl_getLocation(*const struct_ZigClangFieldDecl) str | ... | @@ -1014,3 +1027,12 @@ pub extern fn ZigClangFieldDecl_getLocation(*const struct_ZigClangFieldDecl) str |
| 1014 | 1027 | ||
| 1015 | pub extern fn ZigClangEnumConstantDecl_getInitExpr(*const ZigClangEnumConstantDecl) ?*const ZigClangExpr; | 1028 | pub extern fn ZigClangEnumConstantDecl_getInitExpr(*const ZigClangEnumConstantDecl) ?*const ZigClangExpr; |
| 1016 | pub extern fn ZigClangEnumConstantDecl_getInitVal(*const ZigClangEnumConstantDecl) *const ZigClangAPSInt; | 1029 | pub extern fn ZigClangEnumConstantDecl_getInitVal(*const ZigClangEnumConstantDecl) *const ZigClangAPSInt; |
| 1030 | |||
| 1031 | pub extern fn ZigClangASTUnit_getLocalPreprocessingEntities_begin(*ZigClangASTUnit) ZigClangPreprocessingRecord_iterator; | ||
| 1032 | pub extern fn ZigClangASTUnit_getLocalPreprocessingEntities_end(*ZigClangASTUnit) ZigClangPreprocessingRecord_iterator; | ||
| 1033 | pub extern fn ZigClangPreprocessingRecord_iterator_deref(ZigClangPreprocessingRecord_iterator) *ZigClangPreprocessedEntity; | ||
| 1034 | pub extern fn ZigClangPreprocessedEntity_getKind(*const ZigClangPreprocessedEntity) ZigClangPreprocessedEntity_EntityKind; | ||
| 1035 | |||
| 1036 | pub extern fn ZigClangMacroDefinitionRecord_getName_getNameStart(*const ZigClangMacroDefinitionRecord) [*:0]const u8; | ||
| 1037 | pub extern fn ZigClangMacroDefinitionRecord_getSourceRange_getBegin(*const ZigClangMacroDefinitionRecord) ZigClangSourceLocation; | ||
| 1038 | pub extern fn ZigClangMacroDefinitionRecord_getSourceRange_getEnd(*const ZigClangMacroDefinitionRecord) ZigClangSourceLocation; |
src-self-hosted/translate_c.zig+294-11| ... | @@ -6,6 +6,8 @@ const assert = std.debug.assert; | ... | @@ -6,6 +6,8 @@ const assert = std.debug.assert; |
| 6 | const ast = std.zig.ast; | 6 | const ast = std.zig.ast; |
| 7 | const Token = std.zig.Token; | 7 | const Token = std.zig.Token; |
| 8 | usingnamespace @import("clang.zig"); | 8 | usingnamespace @import("clang.zig"); |
| 9 | const ctok = @import("c_tokenizer.zig"); | ||
| 10 | const CToken = ctok.CToken; | ||
| 9 | 11 | ||
| 10 | const CallingConvention = std.builtin.TypeInfo.CallingConvention; | 12 | const CallingConvention = std.builtin.TypeInfo.CallingConvention; |
| 11 | 13 | ||
| ... | @@ -31,6 +33,7 @@ fn addrEql(a: usize, b: usize) bool { | ... | @@ -31,6 +33,7 @@ fn addrEql(a: usize, b: usize) bool { |
| 31 | return a == b; | 33 | return a == b; |
| 32 | } | 34 | } |
| 33 | 35 | ||
| 36 | const MacroTable = std.StringHashMap(*ast.Node); | ||
| 34 | const SymbolTable = std.StringHashMap(void); | 37 | const SymbolTable = std.StringHashMap(void); |
| 35 | const AliasList = std.SegmentedList(struct { | 38 | const AliasList = std.SegmentedList(struct { |
| 36 | alias: []const u8, | 39 | alias: []const u8, |
| ... | @@ -106,6 +109,7 @@ const Context = struct { | ... | @@ -106,6 +109,7 @@ const Context = struct { |
| 106 | decl_table: DeclTable, | 109 | decl_table: DeclTable, |
| 107 | alias_list: AliasList, | 110 | alias_list: AliasList, |
| 108 | sym_table: SymbolTable, | 111 | sym_table: SymbolTable, |
| 112 | macro_table: MacroTable, | ||
| 109 | global_scope: *Scope.Root, | 113 | global_scope: *Scope.Root, |
| 110 | ptr_params: std.BufSet, | 114 | ptr_params: std.BufSet, |
| 111 | clang_context: *ZigClangASTContext, | 115 | clang_context: *ZigClangASTContext, |
| ... | @@ -193,6 +197,7 @@ pub fn translate( | ... | @@ -193,6 +197,7 @@ pub fn translate( |
| 193 | .decl_table = DeclTable.init(arena), | 197 | .decl_table = DeclTable.init(arena), |
| 194 | .alias_list = AliasList.init(arena), | 198 | .alias_list = AliasList.init(arena), |
| 195 | .sym_table = SymbolTable.init(arena), | 199 | .sym_table = SymbolTable.init(arena), |
| 200 | .macro_table = MacroTable.init(arena), | ||
| 196 | .global_scope = try arena.create(Scope.Root), | 201 | .global_scope = try arena.create(Scope.Root), |
| 197 | .ptr_params = std.BufSet.init(arena), | 202 | .ptr_params = std.BufSet.init(arena), |
| 198 | .clang_context = ZigClangASTUnit_getASTContext(ast_unit).?, | 203 | .clang_context = ZigClangASTUnit_getASTContext(ast_unit).?, |
| ... | @@ -207,6 +212,14 @@ pub fn translate( | ... | @@ -207,6 +212,14 @@ pub fn translate( |
| 207 | if (!ZigClangASTUnit_visitLocalTopLevelDecls(ast_unit, &context, declVisitorC)) { | 212 | if (!ZigClangASTUnit_visitLocalTopLevelDecls(ast_unit, &context, declVisitorC)) { |
| 208 | return context.err; | 213 | return context.err; |
| 209 | } | 214 | } |
| 215 | |||
| 216 | try transPreprocessorEntities(&context, ast_unit); | ||
| 217 | |||
| 218 | var macro_it = context.macro_table.iterator(); | ||
| 219 | while (macro_it.next()) |kv| { | ||
| 220 | try addTopLevelDecl(&context, kv.key, kv.value); | ||
| 221 | } | ||
| 222 | |||
| 210 | var it = context.alias_list.iterator(0); | 223 | var it = context.alias_list.iterator(0); |
| 211 | while (it.next()) |alias| { | 224 | while (it.next()) |alias| { |
| 212 | if (!context.sym_table.contains(alias.alias)) { | 225 | if (!context.sym_table.contains(alias.alias)) { |
| ... | @@ -1931,18 +1944,18 @@ fn transCreateNodeInt(c: *Context, int: var) !*ast.Node { | ... | @@ -1931,18 +1944,18 @@ fn transCreateNodeInt(c: *Context, int: var) !*ast.Node { |
| 1931 | return &node.base; | 1944 | return &node.base; |
| 1932 | } | 1945 | } |
| 1933 | 1946 | ||
| 1934 | fn transCreateNodeOpaqueType(c: *Context) !*ast.Node { | 1947 | fn transCreateNodeFloat(c: *Context, int: var) !*ast.Node { |
| 1935 | const builtin_tok = try appendToken(c, .Builtin, "@OpaqueType"); | 1948 | const token = try appendTokenFmt(c, .FloatLiteral, "{}", .{int}); |
| 1936 | _ = try appendToken(c, .LParen, "("); | 1949 | const node = try c.a().create(ast.Node.FloatLiteral); |
| 1937 | const rparen_tok = try appendToken(c, .RParen, ")"); | 1950 | node.* = .{ |
| 1938 | 1951 | .token = token, | |
| 1939 | const call_node = try c.a().create(ast.Node.BuiltinCall); | ||
| 1940 | call_node.* = ast.Node.BuiltinCall{ | ||
| 1941 | .base = ast.Node{ .id = ast.Node.Id.BuiltinCall }, | ||
| 1942 | .builtin_token = builtin_tok, | ||
| 1943 | .params = ast.Node.BuiltinCall.ParamList.init(c.a()), | ||
| 1944 | .rparen_token = rparen_tok, | ||
| 1945 | }; | 1952 | }; |
| 1953 | return &node.base; | ||
| 1954 | } | ||
| 1955 | |||
| 1956 | fn transCreateNodeOpaqueType(c: *Context) !*ast.Node { | ||
| 1957 | const call_node = try transCreateNodeBuiltinFnCall(c, "@OpaqueType"); | ||
| 1958 | call_node.rparen_token = try appendToken(c, .RParen, ")"); | ||
| 1946 | return &call_node.base; | 1959 | return &call_node.base; |
| 1947 | } | 1960 | } |
| 1948 | 1961 | ||
| ... | @@ -2441,3 +2454,273 @@ fn transCreateNodeIdentifier(c: *Context, name: []const u8) !*ast.Node { | ... | @@ -2441,3 +2454,273 @@ fn transCreateNodeIdentifier(c: *Context, name: []const u8) !*ast.Node { |
| 2441 | pub fn freeErrors(errors: []ClangErrMsg) void { | 2454 | pub fn freeErrors(errors: []ClangErrMsg) void { |
| 2442 | ZigClangErrorMsg_delete(errors.ptr, errors.len); | 2455 | ZigClangErrorMsg_delete(errors.ptr, errors.len); |
| 2443 | } | 2456 | } |
| 2457 | |||
| 2458 | fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void { | ||
| 2459 | // TODO if we see #undef, delete it from the table | ||
| 2460 | var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit); | ||
| 2461 | const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit); | ||
| 2462 | var tok_list = ctok.TokenList.init(c.a()); | ||
| 2463 | |||
| 2464 | while (it.I != it_end.I) : (it.I += 1) { | ||
| 2465 | const entity = ZigClangPreprocessingRecord_iterator_deref(it); | ||
| 2466 | tok_list.shrink(0); | ||
| 2467 | |||
| 2468 | switch (ZigClangPreprocessedEntity_getKind(entity)) { | ||
| 2469 | .MacroExpansionKind => { | ||
| 2470 | // TODO | ||
| 2471 | }, | ||
| 2472 | .MacroDefinitionKind => { | ||
| 2473 | const macro = @ptrCast(*ZigClangMacroDefinitionRecord, entity); | ||
| 2474 | const raw_name = ZigClangMacroDefinitionRecord_getName_getNameStart(macro); | ||
| 2475 | const begin_loc = ZigClangMacroDefinitionRecord_getSourceRange_getBegin(macro); | ||
| 2476 | |||
| 2477 | const name = try c.str(raw_name); | ||
| 2478 | // if (name_exists_global(c, name)) { // TODO | ||
| 2479 | // continue; | ||
| 2480 | // } | ||
| 2481 | |||
| 2482 | const begin_c = ZigClangSourceManager_getCharacterData(c.source_manager, begin_loc); | ||
| 2483 | try transMacroDefine(c, &tok_list, name, begin_c, begin_loc); | ||
| 2484 | }, | ||
| 2485 | else => {}, | ||
| 2486 | } | ||
| 2487 | } | ||
| 2488 | } | ||
| 2489 | |||
| 2490 | fn transMacroDefine(c: *Context, tok_list: *ctok.TokenList, name: []const u8, char_ptr: [*]const u8, source_loc: ZigClangSourceLocation) Error!void { | ||
| 2491 | ctok.tokenizeCMacro(tok_list, char_ptr) catch |err| switch (err) { | ||
| 2492 | error.OutOfMemory => |e| return e, | ||
| 2493 | else => return failDecl(c, source_loc, name, "unable to tokenize macro definition", .{}), | ||
| 2494 | }; | ||
| 2495 | const rp = makeRestorePoint(c); | ||
| 2496 | |||
| 2497 | var it = tok_list.iterator(0); | ||
| 2498 | const first_tok = it.next().?; | ||
| 2499 | assert(first_tok.id == .Identifier and std.mem.eql(u8, first_tok.bytes, name)); | ||
| 2500 | const next = it.peek().?; | ||
| 2501 | switch (next.id) { | ||
| 2502 | .Identifier => { | ||
| 2503 | // if it equals itself, ignore. for example, from stdio.h: | ||
| 2504 | // #define stdin stdin | ||
| 2505 | if (std.mem.eql(u8, name, next.bytes)) { | ||
| 2506 | return; | ||
| 2507 | } | ||
| 2508 | }, | ||
| 2509 | .Eof => { | ||
| 2510 | // this means it is a macro without a value | ||
| 2511 | // we don't care about such things | ||
| 2512 | return; | ||
| 2513 | }, | ||
| 2514 | else => {}, | ||
| 2515 | } | ||
| 2516 | |||
| 2517 | const visib_tok = try appendToken(c, .Keyword_pub, "pub"); | ||
| 2518 | const mut_tok = try appendToken(c, .Keyword_const, "const"); | ||
| 2519 | const name_tok = try appendIdentifier(c, name); | ||
| 2520 | |||
| 2521 | const eq_tok = try appendToken(c, .Equal, "="); | ||
| 2522 | |||
| 2523 | const init_node = parseCExpr(rp, &it, source_loc) catch |err| switch (err) { | ||
| 2524 | error.UnsupportedTranslation, | ||
| 2525 | error.ParseError, | ||
| 2526 | => return failDecl(c, source_loc, name, "unable to translate macro", .{}), | ||
| 2527 | error.OutOfMemory => |e| return e, | ||
| 2528 | }; | ||
| 2529 | |||
| 2530 | const node = try c.a().create(ast.Node.VarDecl); | ||
| 2531 | node.* = ast.Node.VarDecl{ | ||
| 2532 | .doc_comments = null, | ||
| 2533 | .visib_token = visib_tok, | ||
| 2534 | .thread_local_token = null, | ||
| 2535 | .name_token = name_tok, | ||
| 2536 | .eq_token = eq_tok, | ||
| 2537 | .mut_token = mut_tok, | ||
| 2538 | .comptime_token = null, | ||
| 2539 | .extern_export_token = null, | ||
| 2540 | .lib_name = null, | ||
| 2541 | .type_node = null, | ||
| 2542 | .align_node = null, | ||
| 2543 | .section_node = null, | ||
| 2544 | .init_node = init_node, | ||
| 2545 | .semicolon_token = try appendToken(c, .Semicolon, ";"), | ||
| 2546 | }; | ||
| 2547 | _ = try c.macro_table.put(name, &node.base); | ||
| 2548 | } | ||
| 2549 | |||
| 2550 | const ParseError = Error || error{ | ||
| 2551 | ParseError, | ||
| 2552 | UnsupportedTranslation, | ||
| 2553 | }; | ||
| 2554 | |||
| 2555 | fn parseCExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation) ParseError!*ast.Node { | ||
| 2556 | return parseCPrefixOpExpr(rp, it, source_loc); | ||
| 2557 | } | ||
| 2558 | |||
| 2559 | fn parseCNumLit(rp: RestorePoint, tok: *CToken, source_loc: ZigClangSourceLocation) ParseError!*ast.Node { | ||
| 2560 | if (tok.id == .NumLitInt) { | ||
| 2561 | if (tok.num_lit_suffix == .None) { | ||
| 2562 | if (tok.bytes.len > 2 and tok.bytes[0] == '0') { | ||
| 2563 | switch (tok.bytes[1]) { | ||
| 2564 | '0'...'7' => { | ||
| 2565 | // octal | ||
| 2566 | return transCreateNodeInt(rp.c, try std.fmt.allocPrint(rp.c.a(), "0o{}", .{tok.bytes})); | ||
| 2567 | }, | ||
| 2568 | else => {}, | ||
| 2569 | } | ||
| 2570 | } | ||
| 2571 | return transCreateNodeInt(rp.c, tok.bytes); | ||
| 2572 | } | ||
| 2573 | const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@as"); | ||
| 2574 | try cast_node.params.push(try transCreateNodeIdentifier(rp.c, switch (tok.num_lit_suffix) { | ||
| 2575 | .U => "c_uint", | ||
| 2576 | .L => "c_long", | ||
| 2577 | .LU => "c_ulong", | ||
| 2578 | .LL => "c_longlong", | ||
| 2579 | .LLU => "c_ulonglong", | ||
| 2580 | else => unreachable, | ||
| 2581 | })); | ||
| 2582 | _ = try appendToken(rp.c, .Comma, ","); | ||
| 2583 | try cast_node.params.push(try transCreateNodeInt(rp.c, tok.bytes)); | ||
| 2584 | cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); | ||
| 2585 | return &cast_node.base; | ||
| 2586 | } else if (tok.id == .NumLitFloat) { | ||
| 2587 | if (tok.num_lit_suffix == .None) { | ||
| 2588 | return transCreateNodeFloat(rp.c, tok.bytes); | ||
| 2589 | } | ||
| 2590 | const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@as"); | ||
| 2591 | try cast_node.params.push(try transCreateNodeIdentifier(rp.c, switch (tok.num_lit_suffix) { | ||
| 2592 | .F => "f32", | ||
| 2593 | .L => "f64", | ||
| 2594 | else => unreachable, | ||
| 2595 | })); | ||
| 2596 | _ = try appendToken(rp.c, .Comma, ","); | ||
| 2597 | try cast_node.params.push(try transCreateNodeFloat(rp.c, tok.bytes)); | ||
| 2598 | cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); | ||
| 2599 | return &cast_node.base; | ||
| 2600 | } else | ||
| 2601 | return revertAndWarn( | ||
| 2602 | rp, | ||
| 2603 | error.ParseError, | ||
| 2604 | source_loc, | ||
| 2605 | "expected number literal", | ||
| 2606 | .{}, | ||
| 2607 | ); | ||
| 2608 | } | ||
| 2609 | |||
| 2610 | fn parseCPrimaryExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation) ParseError!*ast.Node { | ||
| 2611 | const tok = it.next().?; | ||
| 2612 | switch (tok.id) { | ||
| 2613 | .CharLit => { | ||
| 2614 | const buf = try rp.c.a().alloc(u8, tok.bytes.len + "''".len); | ||
| 2615 | buf[0] = '\''; | ||
| 2616 | writeEscapedString(buf[1..], tok.bytes); | ||
| 2617 | buf[buf.len - 1] = '\''; | ||
| 2618 | const token = try appendToken(rp.c, .CharLiteral, buf); | ||
| 2619 | const node = try rp.c.a().create(ast.Node.CharLiteral); | ||
| 2620 | node.* = ast.Node.CharLiteral{ | ||
| 2621 | .token = token, | ||
| 2622 | }; | ||
| 2623 | return &node.base; | ||
| 2624 | }, | ||
| 2625 | .StrLit => { | ||
| 2626 | const buf = try rp.c.a().alloc(u8, tok.bytes.len + "\"\"".len); | ||
| 2627 | buf[0] = '"'; | ||
| 2628 | writeEscapedString(buf[1..], tok.bytes); | ||
| 2629 | buf[buf.len - 1] = '"'; | ||
| 2630 | const token = try appendToken(rp.c, .StringLiteral, buf); | ||
| 2631 | const node = try rp.c.a().create(ast.Node.StringLiteral); | ||
| 2632 | node.* = ast.Node.StringLiteral{ | ||
| 2633 | .token = token, | ||
| 2634 | }; | ||
| 2635 | return &node.base; | ||
| 2636 | }, | ||
| 2637 | .Minus => { | ||
| 2638 | const node = try transCreateNodePrefixOp( | ||
| 2639 | rp.c, | ||
| 2640 | .Negation, | ||
| 2641 | .Minus, | ||
| 2642 | "-", | ||
| 2643 | ); | ||
| 2644 | node.rhs = try parseCNumLit(rp, it.next().?, source_loc); | ||
| 2645 | return &node.base; | ||
| 2646 | }, | ||
| 2647 | .NumLitInt, .NumLitFloat => { | ||
| 2648 | return parseCNumLit(rp, tok, source_loc); | ||
| 2649 | }, | ||
| 2650 | .Identifier => return transCreateNodeIdentifier(rp.c, tok.bytes), | ||
| 2651 | .LParen => { | ||
| 2652 | _ = try appendToken(rp.c, .LParen, "("); | ||
| 2653 | const inner_node = try parseCExpr(rp, it, source_loc); | ||
| 2654 | _ = try appendToken(rp.c, .RParen, ")"); | ||
| 2655 | |||
| 2656 | return inner_node; // TODO | ||
| 2657 | }, | ||
| 2658 | else => return revertAndWarn( | ||
| 2659 | rp, | ||
| 2660 | error.UnsupportedTranslation, | ||
| 2661 | source_loc, | ||
| 2662 | "unable to translate C expr", | ||
| 2663 | .{}, | ||
| 2664 | ), | ||
| 2665 | } | ||
| 2666 | } | ||
| 2667 | |||
| 2668 | fn parseCSuffixOpExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation) ParseError!*ast.Node { | ||
| 2669 | var node = try parseCPrimaryExpr(rp, it, source_loc); | ||
| 2670 | while (true) { | ||
| 2671 | const tok = it.next().?; | ||
| 2672 | switch (tok.id) { | ||
| 2673 | .Dot => { | ||
| 2674 | const name_tok = it.next().?; | ||
| 2675 | if (name_tok.id != .Identifier) | ||
| 2676 | return revertAndWarn( | ||
| 2677 | rp, | ||
| 2678 | error.ParseError, | ||
| 2679 | source_loc, | ||
| 2680 | "unable to translate C expr", | ||
| 2681 | .{}, | ||
| 2682 | ); | ||
| 2683 | |||
| 2684 | const op_token = try appendToken(rp.c, .Period, "."); | ||
| 2685 | const rhs = try transCreateNodeIdentifier(rp.c, tok.bytes); | ||
| 2686 | const access_node = try rp.c.a().create(ast.Node.InfixOp); | ||
| 2687 | access_node.* = .{ | ||
| 2688 | .op_token = op_token, | ||
| 2689 | .lhs = node, | ||
| 2690 | .op = .Period, | ||
| 2691 | .rhs = rhs, | ||
| 2692 | }; | ||
| 2693 | node = &access_node.base; | ||
| 2694 | }, | ||
| 2695 | .Shl => { | ||
| 2696 | const rhs_node = try parseCPrimaryExpr(rp, it, source_loc); | ||
| 2697 | |||
| 2698 | const op_token = try appendToken(rp.c, .AngleBracketAngleBracketLeft, "<<"); | ||
| 2699 | const rhs = try parseCPrimaryExpr(rp, it, source_loc); | ||
| 2700 | const bitshift_node = try rp.c.a().create(ast.Node.InfixOp); | ||
| 2701 | bitshift_node.* = .{ | ||
| 2702 | .op_token = op_token, | ||
| 2703 | .lhs = node, | ||
| 2704 | .op = .BitShiftLeft, | ||
| 2705 | .rhs = rhs, | ||
| 2706 | }; | ||
| 2707 | node = &bitshift_node.base; | ||
| 2708 | }, | ||
| 2709 | else => { | ||
| 2710 | _ = it.prev(); | ||
| 2711 | return node; | ||
| 2712 | }, | ||
| 2713 | } | ||
| 2714 | } | ||
| 2715 | } | ||
| 2716 | |||
| 2717 | fn parseCPrefixOpExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation) ParseError!*ast.Node { | ||
| 2718 | const op_tok = it.next().?; | ||
| 2719 | |||
| 2720 | switch (op_tok.id) { | ||
| 2721 | else => { | ||
| 2722 | _ = it.prev(); | ||
| 2723 | return try parseCSuffixOpExpr(rp, it, source_loc); | ||
| 2724 | }, | ||
| 2725 | } | ||
| 2726 | } |
test/translate_c.zig+69-13| ... | @@ -214,6 +214,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -214,6 +214,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 214 | \\ Clear: c_int, | 214 | \\ Clear: c_int, |
| 215 | \\ }, | 215 | \\ }, |
| 216 | \\}; | 216 | \\}; |
| 217 | , | ||
| 217 | \\pub const OpenGLProcs = union_OpenGLProcs; | 218 | \\pub const OpenGLProcs = union_OpenGLProcs; |
| 218 | }); | 219 | }); |
| 219 | 220 | ||
| ... | @@ -280,9 +281,64 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -280,9 +281,64 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 280 | \\ o, | 281 | \\ o, |
| 281 | \\ p, | 282 | \\ p, |
| 282 | \\}; | 283 | \\}; |
| 284 | , | ||
| 283 | \\pub const Baz = struct_Baz; | 285 | \\pub const Baz = struct_Baz; |
| 284 | }); | 286 | }); |
| 285 | 287 | ||
| 288 | cases.add_2("#define a char literal", | ||
| 289 | \\#define A_CHAR 'a' | ||
| 290 | , &[_][]const u8{ | ||
| 291 | \\pub const A_CHAR = 'a'; | ||
| 292 | }); | ||
| 293 | |||
| 294 | cases.add_2("comment after integer literal", | ||
| 295 | \\#define SDL_INIT_VIDEO 0x00000020 /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */ | ||
| 296 | , &[_][]const u8{ | ||
| 297 | \\pub const SDL_INIT_VIDEO = 0x00000020; | ||
| 298 | }); | ||
| 299 | |||
| 300 | cases.add_2("u integer suffix after hex literal", | ||
| 301 | \\#define SDL_INIT_VIDEO 0x00000020u /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */ | ||
| 302 | , &[_][]const u8{ | ||
| 303 | \\pub const SDL_INIT_VIDEO = @as(c_uint, 0x00000020); | ||
| 304 | }); | ||
| 305 | |||
| 306 | cases.add_2("l integer suffix after hex literal", | ||
| 307 | \\#define SDL_INIT_VIDEO 0x00000020l /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */ | ||
| 308 | , &[_][]const u8{ | ||
| 309 | \\pub const SDL_INIT_VIDEO = @as(c_long, 0x00000020); | ||
| 310 | }); | ||
| 311 | |||
| 312 | cases.add_2("ul integer suffix after hex literal", | ||
| 313 | \\#define SDL_INIT_VIDEO 0x00000020ul /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */ | ||
| 314 | , &[_][]const u8{ | ||
| 315 | \\pub const SDL_INIT_VIDEO = @as(c_ulong, 0x00000020); | ||
| 316 | }); | ||
| 317 | |||
| 318 | cases.add_2("lu integer suffix after hex literal", | ||
| 319 | \\#define SDL_INIT_VIDEO 0x00000020lu /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */ | ||
| 320 | , &[_][]const u8{ | ||
| 321 | \\pub const SDL_INIT_VIDEO = @as(c_ulong, 0x00000020); | ||
| 322 | }); | ||
| 323 | |||
| 324 | cases.add_2("ll integer suffix after hex literal", | ||
| 325 | \\#define SDL_INIT_VIDEO 0x00000020ll /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */ | ||
| 326 | , &[_][]const u8{ | ||
| 327 | \\pub const SDL_INIT_VIDEO = @as(c_longlong, 0x00000020); | ||
| 328 | }); | ||
| 329 | |||
| 330 | cases.add_2("ull integer suffix after hex literal", | ||
| 331 | \\#define SDL_INIT_VIDEO 0x00000020ull /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */ | ||
| 332 | , &[_][]const u8{ | ||
| 333 | \\pub const SDL_INIT_VIDEO = @as(c_ulonglong, 0x00000020); | ||
| 334 | }); | ||
| 335 | |||
| 336 | cases.add_2("llu integer suffix after hex literal", | ||
| 337 | \\#define SDL_INIT_VIDEO 0x00000020llu /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */ | ||
| 338 | , &[_][]const u8{ | ||
| 339 | \\pub const SDL_INIT_VIDEO = @as(c_ulonglong, 0x00000020); | ||
| 340 | }); | ||
| 341 | |||
| 286 | /////////////// Cases for only stage1 which are TODO items for stage2 //////////////// | 342 | /////////////// Cases for only stage1 which are TODO items for stage2 //////////////// |
| 287 | 343 | ||
| 288 | cases.add_both("typedef of function in struct field", | 344 | cases.add_both("typedef of function in struct field", |
| ... | @@ -314,7 +370,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -314,7 +370,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 314 | \\}; | 370 | \\}; |
| 315 | }); | 371 | }); |
| 316 | 372 | ||
| 317 | cases.add("macro with left shift", | 373 | cases.add_both("macro with left shift", |
| 318 | \\#define REDISMODULE_READ (1<<0) | 374 | \\#define REDISMODULE_READ (1<<0) |
| 319 | , &[_][]const u8{ | 375 | , &[_][]const u8{ |
| 320 | \\pub const REDISMODULE_READ = 1 << 0; | 376 | \\pub const REDISMODULE_READ = 1 << 0; |
| ... | @@ -637,13 +693,13 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -637,13 +693,13 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 637 | \\pub const A_CHAR = 97; | 693 | \\pub const A_CHAR = 97; |
| 638 | }); | 694 | }); |
| 639 | 695 | ||
| 640 | cases.add("#define an unsigned integer literal", | 696 | cases.add_both("#define an unsigned integer literal", |
| 641 | \\#define CHANNEL_COUNT 24 | 697 | \\#define CHANNEL_COUNT 24 |
| 642 | , &[_][]const u8{ | 698 | , &[_][]const u8{ |
| 643 | \\pub const CHANNEL_COUNT = 24; | 699 | \\pub const CHANNEL_COUNT = 24; |
| 644 | }); | 700 | }); |
| 645 | 701 | ||
| 646 | cases.add("#define referencing another #define", | 702 | cases.add_both("#define referencing another #define", |
| 647 | \\#define THING2 THING1 | 703 | \\#define THING2 THING1 |
| 648 | \\#define THING1 1234 | 704 | \\#define THING1 1234 |
| 649 | , &[_][]const u8{ | 705 | , &[_][]const u8{ |
| ... | @@ -692,7 +748,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -692,7 +748,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 692 | \\} | 748 | \\} |
| 693 | }); | 749 | }); |
| 694 | 750 | ||
| 695 | cases.add("#define string", | 751 | cases.add_both("#define string", |
| 696 | \\#define foo "a string" | 752 | \\#define foo "a string" |
| 697 | , &[_][]const u8{ | 753 | , &[_][]const u8{ |
| 698 | \\pub const foo = "a string"; | 754 | \\pub const foo = "a string"; |
| ... | @@ -788,7 +844,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -788,7 +844,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 788 | \\pub const FOO_CHAR = 63; | 844 | \\pub const FOO_CHAR = 63; |
| 789 | }); | 845 | }); |
| 790 | 846 | ||
| 791 | cases.add("macro with parens around negative number", | 847 | cases.add_both("macro with parens around negative number", |
| 792 | \\#define LUA_GLOBALSINDEX (-10002) | 848 | \\#define LUA_GLOBALSINDEX (-10002) |
| 793 | , &[_][]const u8{ | 849 | , &[_][]const u8{ |
| 794 | \\pub const LUA_GLOBALSINDEX = -10002; | 850 | \\pub const LUA_GLOBALSINDEX = -10002; |
| ... | @@ -1732,7 +1788,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -1732,7 +1788,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 1732 | \\} | 1788 | \\} |
| 1733 | }); | 1789 | }); |
| 1734 | 1790 | ||
| 1735 | cases.addC( | 1791 | cases.add_both( |
| 1736 | "u integer suffix after 0 (zero) in macro definition", | 1792 | "u integer suffix after 0 (zero) in macro definition", |
| 1737 | "#define ZERO 0U", | 1793 | "#define ZERO 0U", |
| 1738 | &[_][]const u8{ | 1794 | &[_][]const u8{ |
| ... | @@ -1740,7 +1796,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -1740,7 +1796,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 1740 | }, | 1796 | }, |
| 1741 | ); | 1797 | ); |
| 1742 | 1798 | ||
| 1743 | cases.addC( | 1799 | cases.add_both( |
| 1744 | "l integer suffix after 0 (zero) in macro definition", | 1800 | "l integer suffix after 0 (zero) in macro definition", |
| 1745 | "#define ZERO 0L", | 1801 | "#define ZERO 0L", |
| 1746 | &[_][]const u8{ | 1802 | &[_][]const u8{ |
| ... | @@ -1748,7 +1804,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -1748,7 +1804,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 1748 | }, | 1804 | }, |
| 1749 | ); | 1805 | ); |
| 1750 | 1806 | ||
| 1751 | cases.addC( | 1807 | cases.add_both( |
| 1752 | "ul integer suffix after 0 (zero) in macro definition", | 1808 | "ul integer suffix after 0 (zero) in macro definition", |
| 1753 | "#define ZERO 0UL", | 1809 | "#define ZERO 0UL", |
| 1754 | &[_][]const u8{ | 1810 | &[_][]const u8{ |
| ... | @@ -1756,7 +1812,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -1756,7 +1812,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 1756 | }, | 1812 | }, |
| 1757 | ); | 1813 | ); |
| 1758 | 1814 | ||
| 1759 | cases.addC( | 1815 | cases.add_both( |
| 1760 | "lu integer suffix after 0 (zero) in macro definition", | 1816 | "lu integer suffix after 0 (zero) in macro definition", |
| 1761 | "#define ZERO 0LU", | 1817 | "#define ZERO 0LU", |
| 1762 | &[_][]const u8{ | 1818 | &[_][]const u8{ |
| ... | @@ -1764,7 +1820,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -1764,7 +1820,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 1764 | }, | 1820 | }, |
| 1765 | ); | 1821 | ); |
| 1766 | 1822 | ||
| 1767 | cases.addC( | 1823 | cases.add_both( |
| 1768 | "ll integer suffix after 0 (zero) in macro definition", | 1824 | "ll integer suffix after 0 (zero) in macro definition", |
| 1769 | "#define ZERO 0LL", | 1825 | "#define ZERO 0LL", |
| 1770 | &[_][]const u8{ | 1826 | &[_][]const u8{ |
| ... | @@ -1772,7 +1828,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -1772,7 +1828,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 1772 | }, | 1828 | }, |
| 1773 | ); | 1829 | ); |
| 1774 | 1830 | ||
| 1775 | cases.addC( | 1831 | cases.add_both( |
| 1776 | "ull integer suffix after 0 (zero) in macro definition", | 1832 | "ull integer suffix after 0 (zero) in macro definition", |
| 1777 | "#define ZERO 0ULL", | 1833 | "#define ZERO 0ULL", |
| 1778 | &[_][]const u8{ | 1834 | &[_][]const u8{ |
| ... | @@ -1780,7 +1836,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -1780,7 +1836,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 1780 | }, | 1836 | }, |
| 1781 | ); | 1837 | ); |
| 1782 | 1838 | ||
| 1783 | cases.addC( | 1839 | cases.add_both( |
| 1784 | "llu integer suffix after 0 (zero) in macro definition", | 1840 | "llu integer suffix after 0 (zero) in macro definition", |
| 1785 | "#define ZERO 0LLU", | 1841 | "#define ZERO 0LLU", |
| 1786 | &[_][]const u8{ | 1842 | &[_][]const u8{ |
| ... | @@ -1788,7 +1844,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { | ... | @@ -1788,7 +1844,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void { |
| 1788 | }, | 1844 | }, |
| 1789 | ); | 1845 | ); |
| 1790 | 1846 | ||
| 1791 | cases.addC( | 1847 | cases.addC(//todo |
| 1792 | "bitwise not on u-suffixed 0 (zero) in macro definition", | 1848 | "bitwise not on u-suffixed 0 (zero) in macro definition", |
| 1793 | "#define NOT_ZERO (~0U)", | 1849 | "#define NOT_ZERO (~0U)", |
| 1794 | &[_][]const u8{ | 1850 | &[_][]const u8{ |