authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-05-30 12:07:55-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-05-30 12:07:55-04:00
log7878f9660fcb8fa64fda0b2b67320fe9ddc2727b
treee9286a58127e707c0c06b302347e7d49079e343e
parent5954d5235fd0d029034d68f82bb7831ff47506f8
signaturelock-open Commit is signed but in an unrecognized format.

dep tokenizer: run zig fmt and move exports to canonical location

2 files changed, 75 insertions(+), 102 deletions(-)

src-self-hosted/dep_tokenizer.zig+16-101
...@@ -359,63 +359,6 @@ pub const Tokenizer = struct {...@@ -359,63 +359,6 @@ pub const Tokenizer = struct {
359 };359 };
360};360};
361361
362export fn stage2_DepTokenizer_init(input: [*]const u8, len: usize) stage2_DepTokenizer {
363 const t = std.heap.c_allocator.create(Tokenizer) catch @panic("failed to create .d tokenizer");
364 t.* = Tokenizer.init(std.heap.c_allocator, input[0..len]);
365 return stage2_DepTokenizer{
366 .handle = t,
367 };
368}
369
370export fn stage2_DepTokenizer_deinit(self: *stage2_DepTokenizer) void {
371 self.handle.deinit();
372}
373
374export fn stage2_DepTokenizer_next(self: *stage2_DepTokenizer) stage2_DepNextResult {
375 const otoken = self.handle.next() catch {
376 const textz = std.Buffer.init(&self.handle.arena.allocator, self.handle.error_text) catch @panic("failed to create .d tokenizer error text");
377 return stage2_DepNextResult{
378 .type_id = .error_,
379 .textz = textz.toSlice().ptr,
380 };
381 };
382 const token = otoken orelse {
383 return stage2_DepNextResult{
384 .type_id = .null_,
385 .textz = undefined,
386 };
387 };
388 const textz = std.Buffer.init(&self.handle.arena.allocator, token.bytes) catch @panic("failed to create .d tokenizer token text");
389 return stage2_DepNextResult{
390 .type_id = switch (token.id) {
391 .target => stage2_DepNextResult.TypeId.target,
392 .prereq => stage2_DepNextResult.TypeId.prereq,
393 },
394 .textz = textz.toSlice().ptr,
395 };
396}
397
398export const stage2_DepTokenizer = extern struct {
399 handle: *Tokenizer,
400};
401
402export const stage2_DepNextResult = extern struct {
403 type_id: TypeId,
404
405 // when type_id == error --> error text
406 // when type_id == null --> undefined
407 // when type_id == target --> target pathname
408 // when type_id == prereq --> prereq pathname
409 textz: [*]const u8,
410
411 export const TypeId = extern enum {
412 error_,
413 null_,
414 target,
415 prereq,
416 };
417};
418
419test "empty file" {362test "empty file" {
420 try depTokenizer("", "");363 try depTokenizer("", "");
421}364}
...@@ -469,78 +412,54 @@ test "empty target linefeeds" {...@@ -469,78 +412,54 @@ test "empty target linefeeds" {
469 const expect = "target = {foo.o}";412 const expect = "target = {foo.o}";
470 try depTokenizer(413 try depTokenizer(
471 \\foo.o:414 \\foo.o:
472 ,415 , expect);
473 expect
474 );
475 try depTokenizer(416 try depTokenizer(
476 \\foo.o:417 \\foo.o:
477 \\418 \\
478 ,419 , expect);
479 expect
480 );
481 try depTokenizer(420 try depTokenizer(
482 \\foo.o:421 \\foo.o:
483 ,422 , expect);
484 expect
485 );
486 try depTokenizer(423 try depTokenizer(
487 \\foo.o:424 \\foo.o:
488 \\425 \\
489 ,426 , expect);
490 expect
491 );
492}427}
493428
494test "empty target linefeeds + continuations" {429test "empty target linefeeds + continuations" {
495 const expect = "target = {foo.o}";430 const expect = "target = {foo.o}";
496 try depTokenizer(431 try depTokenizer(
497 \\foo.o:\432 \\foo.o:\
498 ,433 , expect);
499 expect
500 );
501 try depTokenizer(434 try depTokenizer(
502 \\foo.o:\435 \\foo.o:\
503 \\436 \\
504 ,437 , expect);
505 expect
506 );
507 try depTokenizer(438 try depTokenizer(
508 \\foo.o:\439 \\foo.o:\
509 ,440 , expect);
510 expect
511 );
512 try depTokenizer(441 try depTokenizer(
513 \\foo.o:\442 \\foo.o:\
514 \\443 \\
515 ,444 , expect);
516 expect
517 );
518}445}
519446
520test "empty target linefeeds + hspace + continuations" {447test "empty target linefeeds + hspace + continuations" {
521 const expect = "target = {foo.o}";448 const expect = "target = {foo.o}";
522 try depTokenizer(449 try depTokenizer(
523 \\foo.o: \450 \\foo.o: \
524 ,451 , expect);
525 expect
526 );
527 try depTokenizer(452 try depTokenizer(
528 \\foo.o: \453 \\foo.o: \
529 \\454 \\
530 ,455 , expect);
531 expect
532 );
533 try depTokenizer(456 try depTokenizer(
534 \\foo.o: \457 \\foo.o: \
535 ,458 , expect);
536 expect
537 );
538 try depTokenizer(459 try depTokenizer(
539 \\foo.o: \460 \\foo.o: \
540 \\461 \\
541 ,462 , expect);
542 expect
543 );
544}463}
545464
546test "prereq" {465test "prereq" {
...@@ -572,15 +491,11 @@ test "prereq continuation" {...@@ -572,15 +491,11 @@ test "prereq continuation" {
572 try depTokenizer(491 try depTokenizer(
573 \\foo.o: foo.h\492 \\foo.o: foo.h\
574 \\bar.h493 \\bar.h
575 ,494 , expect);
576 expect
577 );
578 try depTokenizer(495 try depTokenizer(
579 \\foo.o: foo.h\496 \\foo.o: foo.h\
580 \\bar.h497 \\bar.h
581 ,498 , expect);
582 expect
583 );
584}499}
585500
586test "multiple prereqs" {501test "multiple prereqs" {
...@@ -907,14 +822,14 @@ test "error target - continuation expecting end-of-line" {...@@ -907,14 +822,14 @@ test "error target - continuation expecting end-of-line" {
907 \\target = {foo.o}822 \\target = {foo.o}
908 \\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line823 \\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
909 );824 );
910 try depTokenizer("foo.o: \\ x",825 try depTokenizer("foo.o: \\\x0dx",
911 \\target = {foo.o}826 \\target = {foo.o}
912 \\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line827 \\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
913 );828 );
914}829}
915830
916test "error prereq - continuation expecting end-of-line" {831test "error prereq - continuation expecting end-of-line" {
917 try depTokenizer("foo.o: foo.h\\ x",832 try depTokenizer("foo.o: foo.h\\\x0dx",
918 \\target = {foo.o}833 \\target = {foo.o}
919 \\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line834 \\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
920 );835 );
src-self-hosted/stage1.zig+59-1
...@@ -15,13 +15,14 @@ const self_hosted_main = @import("main.zig");...@@ -15,13 +15,14 @@ const self_hosted_main = @import("main.zig");
15const Args = arg.Args;15const Args = arg.Args;
16const Flag = arg.Flag;16const Flag = arg.Flag;
17const errmsg = @import("errmsg.zig");17const errmsg = @import("errmsg.zig");
18const DepTokenizer = @import("dep_tokenizer.zig").Tokenizer;
1819
19var stderr_file: fs.File = undefined;20var stderr_file: fs.File = undefined;
20var stderr: *io.OutStream(fs.File.WriteError) = undefined;21var stderr: *io.OutStream(fs.File.WriteError) = undefined;
21var stdout: *io.OutStream(fs.File.WriteError) = undefined;22var stdout: *io.OutStream(fs.File.WriteError) = undefined;
2223
23comptime {24comptime {
24 _ = @import("dep_tokenizer.zig");25 _ = @import("dep_tokenizer.zig");
25}26}
2627
27// ABI warning28// ABI warning
...@@ -397,3 +398,60 @@ fn printErrMsgToFile(...@@ -397,3 +398,60 @@ fn printErrMsgToFile(
397 try stream.writeByteNTimes('~', last_token.end - first_token.start);398 try stream.writeByteNTimes('~', last_token.end - first_token.start);
398 try stream.write("\n");399 try stream.write("\n");
399}400}
401
402export fn stage2_DepTokenizer_init(input: [*]const u8, len: usize) stage2_DepTokenizer {
403 const t = std.heap.c_allocator.create(DepTokenizer) catch @panic("failed to create .d tokenizer");
404 t.* = DepTokenizer.init(std.heap.c_allocator, input[0..len]);
405 return stage2_DepTokenizer{
406 .handle = t,
407 };
408}
409
410export fn stage2_DepTokenizer_deinit(self: *stage2_DepTokenizer) void {
411 self.handle.deinit();
412}
413
414export fn stage2_DepTokenizer_next(self: *stage2_DepTokenizer) stage2_DepNextResult {
415 const otoken = self.handle.next() catch {
416 const textz = std.Buffer.init(&self.handle.arena.allocator, self.handle.error_text) catch @panic("failed to create .d tokenizer error text");
417 return stage2_DepNextResult{
418 .type_id = .error_,
419 .textz = textz.toSlice().ptr,
420 };
421 };
422 const token = otoken orelse {
423 return stage2_DepNextResult{
424 .type_id = .null_,
425 .textz = undefined,
426 };
427 };
428 const textz = std.Buffer.init(&self.handle.arena.allocator, token.bytes) catch @panic("failed to create .d tokenizer token text");
429 return stage2_DepNextResult{
430 .type_id = switch (token.id) {
431 .target => stage2_DepNextResult.TypeId.target,
432 .prereq => stage2_DepNextResult.TypeId.prereq,
433 },
434 .textz = textz.toSlice().ptr,
435 };
436}
437
438export const stage2_DepTokenizer = extern struct {
439 handle: *DepTokenizer,
440};
441
442export const stage2_DepNextResult = extern struct {
443 type_id: TypeId,
444
445 // when type_id == error --> error text
446 // when type_id == null --> undefined
447 // when type_id == target --> target pathname
448 // when type_id == prereq --> prereq pathname
449 textz: [*]const u8,
450
451 export const TypeId = extern enum {
452 error_,
453 null_,
454 target,
455 prereq,
456 };
457};