authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-31 19:52:34-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-07-31 19:52:34-07:00
logeb1a199dff2b54271bd275c2528bdd898bf1d4eb
tree0cef5904cef4e638dc67248384663c6779b5e293
parent059856acfc9f87d723a90af6a4214e128b8cae2e
parentc2b8afcac9e427102370dc5bac8c3d9621eee6d8
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #20885 from ziglang/simplify-tokenizer

std.zig.tokenizer: simplification and spec conformance

19 files changed, 418 insertions(+), 529 deletions(-)

lib/std/crypto/ml_kem.zig+3-3
...@@ -677,10 +677,10 @@ fn montReduce(x: i32) i16 {...@@ -677,10 +677,10 @@ fn montReduce(x: i32) i16 {
677 // Note gcd(2¹⁶, q) = 1 as q is prime. Write q' := 62209 = q⁻¹ mod R.677 // Note gcd(2¹⁶, q) = 1 as q is prime. Write q' := 62209 = q⁻¹ mod R.
678 // First we compute678 // First we compute
679 //679 //
680 // m := ((x mod R) q') mod R680 // m := ((x mod R) q') mod R
681 // = x q' mod R681 // = x q' mod R
682 // = int16(x q')682 // = int16(x q')
683 // = int16(int32(x) * int32(q'))683 // = int16(int32(x) * int32(q'))
684 //684 //
685 // Note that x q' might be as big as 2³² and could overflow the int32685 // Note that x q' might be as big as 2³² and could overflow the int32
686 // multiplication in the last line. However for any int32s a and b,686 // multiplication in the last line. However for any int32s a and b,
lib/std/macho.zig+4-5
...@@ -203,8 +203,7 @@ pub const symtab_command = extern struct {...@@ -203,8 +203,7 @@ pub const symtab_command = extern struct {
203/// local symbols (static and debugging symbols) - grouped by module203/// local symbols (static and debugging symbols) - grouped by module
204/// defined external symbols - grouped by module (sorted by name if not lib)204/// defined external symbols - grouped by module (sorted by name if not lib)
205/// undefined external symbols (sorted by name if MH_BINDATLOAD is not set,205/// undefined external symbols (sorted by name if MH_BINDATLOAD is not set,
206/// and in order the were seen by the static206/// and in order the were seen by the static linker if MH_BINDATLOAD is set)
207/// linker if MH_BINDATLOAD is set)
208/// In this load command there are offsets and counts to each of the three groups207/// In this load command there are offsets and counts to each of the three groups
209/// of symbols.208/// of symbols.
210///209///
...@@ -219,9 +218,9 @@ pub const symtab_command = extern struct {...@@ -219,9 +218,9 @@ pub const symtab_command = extern struct {
219/// shared library. For executable and object modules, which are files218/// shared library. For executable and object modules, which are files
220/// containing only one module, the information that would be in these three219/// containing only one module, the information that would be in these three
221/// tables is determined as follows:220/// tables is determined as follows:
222/// table of contents - the defined external symbols are sorted by name221/// table of contents - the defined external symbols are sorted by name
223/// module table - the file contains only one module so everything in the222/// module table - the file contains only one module so everything in the file
224/// file is part of the module.223/// is part of the module.
225/// reference symbol table - is the defined and undefined external symbols224/// reference symbol table - is the defined and undefined external symbols
226///225///
227/// For dynamically linked shared library files this load command also contains226/// For dynamically linked shared library files this load command also contains
lib/std/unicode.zig+14-19
...@@ -95,16 +95,13 @@ pub inline fn utf8EncodeComptime(comptime c: u21) [...@@ -95,16 +95,13 @@ pub inline fn utf8EncodeComptime(comptime c: u21) [
9595
96const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error;96const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error;
9797
98/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.98/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
99/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.
100/// If you already know the length at comptime, you can call one of
101/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.
102pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {99pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {
103 return switch (bytes.len) {100 return switch (bytes.len) {
104 1 => @as(u21, bytes[0]),101 1 => bytes[0],
105 2 => utf8Decode2(bytes),102 2 => utf8Decode2(bytes[0..2].*),
106 3 => utf8Decode3(bytes),103 3 => utf8Decode3(bytes[0..3].*),
107 4 => utf8Decode4(bytes),104 4 => utf8Decode4(bytes[0..4].*),
108 else => unreachable,105 else => unreachable,
109 };106 };
110}107}
...@@ -113,8 +110,7 @@ const Utf8Decode2Error = error{...@@ -113,8 +110,7 @@ const Utf8Decode2Error = error{
113 Utf8ExpectedContinuation,110 Utf8ExpectedContinuation,
114 Utf8OverlongEncoding,111 Utf8OverlongEncoding,
115};112};
116pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {113pub fn utf8Decode2(bytes: [2]u8) Utf8Decode2Error!u21 {
117 assert(bytes.len == 2);
118 assert(bytes[0] & 0b11100000 == 0b11000000);114 assert(bytes[0] & 0b11100000 == 0b11000000);
119 var value: u21 = bytes[0] & 0b00011111;115 var value: u21 = bytes[0] & 0b00011111;
120116
...@@ -130,7 +126,7 @@ pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {...@@ -130,7 +126,7 @@ pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
130const Utf8Decode3Error = Utf8Decode3AllowSurrogateHalfError || error{126const Utf8Decode3Error = Utf8Decode3AllowSurrogateHalfError || error{
131 Utf8EncodesSurrogateHalf,127 Utf8EncodesSurrogateHalf,
132};128};
133pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {129pub fn utf8Decode3(bytes: [3]u8) Utf8Decode3Error!u21 {
134 const value = try utf8Decode3AllowSurrogateHalf(bytes);130 const value = try utf8Decode3AllowSurrogateHalf(bytes);
135131
136 if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;132 if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
...@@ -142,8 +138,7 @@ const Utf8Decode3AllowSurrogateHalfError = error{...@@ -142,8 +138,7 @@ const Utf8Decode3AllowSurrogateHalfError = error{
142 Utf8ExpectedContinuation,138 Utf8ExpectedContinuation,
143 Utf8OverlongEncoding,139 Utf8OverlongEncoding,
144};140};
145pub fn utf8Decode3AllowSurrogateHalf(bytes: []const u8) Utf8Decode3AllowSurrogateHalfError!u21 {141pub fn utf8Decode3AllowSurrogateHalf(bytes: [3]u8) Utf8Decode3AllowSurrogateHalfError!u21 {
146 assert(bytes.len == 3);
147 assert(bytes[0] & 0b11110000 == 0b11100000);142 assert(bytes[0] & 0b11110000 == 0b11100000);
148 var value: u21 = bytes[0] & 0b00001111;143 var value: u21 = bytes[0] & 0b00001111;
149144
...@@ -165,8 +160,7 @@ const Utf8Decode4Error = error{...@@ -165,8 +160,7 @@ const Utf8Decode4Error = error{
165 Utf8OverlongEncoding,160 Utf8OverlongEncoding,
166 Utf8CodepointTooLarge,161 Utf8CodepointTooLarge,
167};162};
168pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {163pub fn utf8Decode4(bytes: [4]u8) Utf8Decode4Error!u21 {
169 assert(bytes.len == 4);
170 assert(bytes[0] & 0b11111000 == 0b11110000);164 assert(bytes[0] & 0b11111000 == 0b11110000);
171 var value: u21 = bytes[0] & 0b00000111;165 var value: u21 = bytes[0] & 0b00000111;
172166
...@@ -1637,12 +1631,13 @@ pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3 {...@@ -1637,12 +1631,13 @@ pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3 {
16371631
1638const Wtf8DecodeError = Utf8Decode2Error || Utf8Decode3AllowSurrogateHalfError || Utf8Decode4Error;1632const Wtf8DecodeError = Utf8Decode2Error || Utf8Decode3AllowSurrogateHalfError || Utf8Decode4Error;
16391633
1634/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
1640pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21 {1635pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21 {
1641 return switch (bytes.len) {1636 return switch (bytes.len) {
1642 1 => @as(u21, bytes[0]),1637 1 => bytes[0],
1643 2 => utf8Decode2(bytes),1638 2 => utf8Decode2(bytes[0..2].*),
1644 3 => utf8Decode3AllowSurrogateHalf(bytes),1639 3 => utf8Decode3AllowSurrogateHalf(bytes[0..3].*),
1645 4 => utf8Decode4(bytes),1640 4 => utf8Decode4(bytes[0..4].*),
1646 else => unreachable,1641 else => unreachable,
1647 };1642 };
1648}1643}
lib/std/zig/Ast.zig+1-1
...@@ -69,7 +69,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!A...@@ -69,7 +69,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!A
69 const token = tokenizer.next();69 const token = tokenizer.next();
70 try tokens.append(gpa, .{70 try tokens.append(gpa, .{
71 .tag = token.tag,71 .tag = token.tag,
72 .start = @as(u32, @intCast(token.loc.start)),72 .start = @intCast(token.loc.start),
73 });73 });
74 if (token.tag == .eof) break;74 if (token.tag == .eof) break;
75 }75 }
lib/std/zig/AstGen.zig+3-12
...@@ -11351,6 +11351,9 @@ fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token...@@ -11351,6 +11351,9 @@ fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token
11351 .{raw_string[bad_index]},11351 .{raw_string[bad_index]},
11352 );11352 );
11353 },11353 },
11354 .empty_char_literal => {
11355 return astgen.failOff(token, offset, "empty character literal", .{});
11356 },
11354 }11357 }
11355}11358}
1135611359
...@@ -13820,21 +13823,9 @@ fn lowerAstErrors(astgen: *AstGen) !void {...@@ -13820,21 +13823,9 @@ fn lowerAstErrors(astgen: *AstGen) !void {
13820 var msg: std.ArrayListUnmanaged(u8) = .{};13823 var msg: std.ArrayListUnmanaged(u8) = .{};
13821 defer msg.deinit(gpa);13824 defer msg.deinit(gpa);
1382213825
13823 const token_starts = tree.tokens.items(.start);
13824 const token_tags = tree.tokens.items(.tag);
13825
13826 var notes: std.ArrayListUnmanaged(u32) = .{};13826 var notes: std.ArrayListUnmanaged(u32) = .{};
13827 defer notes.deinit(gpa);13827 defer notes.deinit(gpa);
1382813828
13829 const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
13830 if (token_tags[tok] == .invalid) {
13831 const bad_off: u32 = @intCast(tree.tokenSlice(tok).len);
13832 const byte_abs = token_starts[tok] + bad_off;
13833 try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
13834 std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
13835 }));
13836 }
13837
13838 for (tree.errors[1..]) |note| {13829 for (tree.errors[1..]) |note| {
13839 if (!note.is_note) break;13830 if (!note.is_note) break;
1384013831
lib/std/zig/parser_test.zig-1
...@@ -6061,7 +6061,6 @@ test "recovery: invalid container members" {...@@ -6061,7 +6061,6 @@ test "recovery: invalid container members" {
6061 , &[_]Error{6061 , &[_]Error{
6062 .expected_expr,6062 .expected_expr,
6063 .expected_comma_after_field,6063 .expected_comma_after_field,
6064 .expected_type_expr,
6065 .expected_semi_after_stmt,6064 .expected_semi_after_stmt,
6066 });6065 });
6067}6066}
lib/std/zig/string_literal.zig+19-5
...@@ -1,6 +1,5 @@...@@ -1,6 +1,5 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const utf8Decode = std.unicode.utf8Decode;
4const utf8Encode = std.unicode.utf8Encode;3const utf8Encode = std.unicode.utf8Encode;
54
6pub const ParseError = error{5pub const ParseError = error{
...@@ -37,12 +36,16 @@ pub const Error = union(enum) {...@@ -37,12 +36,16 @@ pub const Error = union(enum) {
37 expected_single_quote: usize,36 expected_single_quote: usize,
38 /// The character at this index cannot be represented without an escape sequence.37 /// The character at this index cannot be represented without an escape sequence.
39 invalid_character: usize,38 invalid_character: usize,
39 /// `''`. Not returned for string literals.
40 empty_char_literal,
40};41};
4142
42/// Only validates escape sequence characters.43/// Asserts the slice starts and ends with single-quotes.
43/// Slice must be valid utf8 starting and ending with "'" and exactly one codepoint in between.44/// Returns an error if there is not exactly one UTF-8 codepoint in between.
44pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {45pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
45 assert(slice.len >= 3 and slice[0] == '\'' and slice[slice.len - 1] == '\'');46 if (slice.len < 3) return .{ .failure = .empty_char_literal };
47 assert(slice[0] == '\'');
48 assert(slice[slice.len - 1] == '\'');
4649
47 switch (slice[1]) {50 switch (slice[1]) {
48 '\\' => {51 '\\' => {
...@@ -55,7 +58,18 @@ pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {...@@ -55,7 +58,18 @@ pub fn parseCharLiteral(slice: []const u8) ParsedCharLiteral {
55 },58 },
56 0 => return .{ .failure = .{ .invalid_character = 1 } },59 0 => return .{ .failure = .{ .invalid_character = 1 } },
57 else => {60 else => {
58 const codepoint = utf8Decode(slice[1 .. slice.len - 1]) catch unreachable;61 const inner = slice[1 .. slice.len - 1];
62 const n = std.unicode.utf8ByteSequenceLength(inner[0]) catch return .{
63 .failure = .{ .invalid_unicode_codepoint = 1 },
64 };
65 if (inner.len > n) return .{ .failure = .{ .expected_single_quote = 1 + n } };
66 const codepoint = switch (n) {
67 1 => inner[0],
68 2 => std.unicode.utf8Decode2(inner[0..2].*),
69 3 => std.unicode.utf8Decode3(inner[0..3].*),
70 4 => std.unicode.utf8Decode4(inner[0..4].*),
71 else => unreachable,
72 } catch return .{ .failure = .{ .invalid_unicode_codepoint = 1 } };
59 return .{ .success = codepoint };73 return .{ .success = codepoint };
60 },74 },
61 }75 }
lib/std/zig/system/darwin/macos.zig+46-46
...@@ -303,16 +303,16 @@ test "detect" {...@@ -303,16 +303,16 @@ test "detect" {
303 \\<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">303 \\<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
304 \\<plist version="1.0">304 \\<plist version="1.0">
305 \\<dict>305 \\<dict>
306 \\ <key>ProductBuildVersion</key>306 \\ <key>ProductBuildVersion</key>
307 \\ <string>7W98</string>307 \\ <string>7W98</string>
308 \\ <key>ProductCopyright</key>308 \\ <key>ProductCopyright</key>
309 \\ <string>Apple Computer, Inc. 1983-2004</string>309 \\ <string>Apple Computer, Inc. 1983-2004</string>
310 \\ <key>ProductName</key>310 \\ <key>ProductName</key>
311 \\ <string>Mac OS X</string>311 \\ <string>Mac OS X</string>
312 \\ <key>ProductUserVisibleVersion</key>312 \\ <key>ProductUserVisibleVersion</key>
313 \\ <string>10.3.9</string>313 \\ <string>10.3.9</string>
314 \\ <key>ProductVersion</key>314 \\ <key>ProductVersion</key>
315 \\ <string>10.3.9</string>315 \\ <string>10.3.9</string>
316 \\</dict>316 \\</dict>
317 \\</plist>317 \\</plist>
318 ,318 ,
...@@ -323,18 +323,18 @@ test "detect" {...@@ -323,18 +323,18 @@ test "detect" {
323 \\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">323 \\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
324 \\<plist version="1.0">324 \\<plist version="1.0">
325 \\<dict>325 \\<dict>
326 \\ <key>ProductBuildVersion</key>326 \\ <key>ProductBuildVersion</key>
327 \\ <string>19G68</string>327 \\ <string>19G68</string>
328 \\ <key>ProductCopyright</key>328 \\ <key>ProductCopyright</key>
329 \\ <string>1983-2020 Apple Inc.</string>329 \\ <string>1983-2020 Apple Inc.</string>
330 \\ <key>ProductName</key>330 \\ <key>ProductName</key>
331 \\ <string>Mac OS X</string>331 \\ <string>Mac OS X</string>
332 \\ <key>ProductUserVisibleVersion</key>332 \\ <key>ProductUserVisibleVersion</key>
333 \\ <string>10.15.6</string>333 \\ <string>10.15.6</string>
334 \\ <key>ProductVersion</key>334 \\ <key>ProductVersion</key>
335 \\ <string>10.15.6</string>335 \\ <string>10.15.6</string>
336 \\ <key>iOSSupportVersion</key>336 \\ <key>iOSSupportVersion</key>
337 \\ <string>13.6</string>337 \\ <string>13.6</string>
338 \\</dict>338 \\</dict>
339 \\</plist>339 \\</plist>
340 ,340 ,
...@@ -345,18 +345,18 @@ test "detect" {...@@ -345,18 +345,18 @@ test "detect" {
345 \\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">345 \\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
346 \\<plist version="1.0">346 \\<plist version="1.0">
347 \\<dict>347 \\<dict>
348 \\ <key>ProductBuildVersion</key>348 \\ <key>ProductBuildVersion</key>
349 \\ <string>20A2408</string>349 \\ <string>20A2408</string>
350 \\ <key>ProductCopyright</key>350 \\ <key>ProductCopyright</key>
351 \\ <string>1983-2020 Apple Inc.</string>351 \\ <string>1983-2020 Apple Inc.</string>
352 \\ <key>ProductName</key>352 \\ <key>ProductName</key>
353 \\ <string>macOS</string>353 \\ <string>macOS</string>
354 \\ <key>ProductUserVisibleVersion</key>354 \\ <key>ProductUserVisibleVersion</key>
355 \\ <string>11.0</string>355 \\ <string>11.0</string>
356 \\ <key>ProductVersion</key>356 \\ <key>ProductVersion</key>
357 \\ <string>11.0</string>357 \\ <string>11.0</string>
358 \\ <key>iOSSupportVersion</key>358 \\ <key>iOSSupportVersion</key>
359 \\ <string>14.2</string>359 \\ <string>14.2</string>
360 \\</dict>360 \\</dict>
361 \\</plist>361 \\</plist>
362 ,362 ,
...@@ -367,18 +367,18 @@ test "detect" {...@@ -367,18 +367,18 @@ test "detect" {
367 \\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">367 \\<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
368 \\<plist version="1.0">368 \\<plist version="1.0">
369 \\<dict>369 \\<dict>
370 \\ <key>ProductBuildVersion</key>370 \\ <key>ProductBuildVersion</key>
371 \\ <string>20C63</string>371 \\ <string>20C63</string>
372 \\ <key>ProductCopyright</key>372 \\ <key>ProductCopyright</key>
373 \\ <string>1983-2020 Apple Inc.</string>373 \\ <string>1983-2020 Apple Inc.</string>
374 \\ <key>ProductName</key>374 \\ <key>ProductName</key>
375 \\ <string>macOS</string>375 \\ <string>macOS</string>
376 \\ <key>ProductUserVisibleVersion</key>376 \\ <key>ProductUserVisibleVersion</key>
377 \\ <string>11.1</string>377 \\ <string>11.1</string>
378 \\ <key>ProductVersion</key>378 \\ <key>ProductVersion</key>
379 \\ <string>11.1</string>379 \\ <string>11.1</string>
380 \\ <key>iOSSupportVersion</key>380 \\ <key>iOSSupportVersion</key>
381 \\ <string>14.3</string>381 \\ <string>14.3</string>
382 \\</dict>382 \\</dict>
383 \\</plist>383 \\</plist>
384 ,384 ,
lib/std/zig/system/linux.zig+30-30
...@@ -109,12 +109,12 @@ const RiscvCpuinfoParser = CpuinfoParser(RiscvCpuinfoImpl);...@@ -109,12 +109,12 @@ const RiscvCpuinfoParser = CpuinfoParser(RiscvCpuinfoImpl);
109109
110test "cpuinfo: RISC-V" {110test "cpuinfo: RISC-V" {
111 try testParser(RiscvCpuinfoParser, .riscv64, &Target.riscv.cpu.sifive_u74,111 try testParser(RiscvCpuinfoParser, .riscv64, &Target.riscv.cpu.sifive_u74,
112 \\processor : 0112 \\processor : 0
113 \\hart : 1113 \\hart : 1
114 \\isa : rv64imafdc114 \\isa : rv64imafdc
115 \\mmu : sv39115 \\mmu : sv39
116 \\isa-ext :116 \\isa-ext :
117 \\uarch : sifive,u74-mc117 \\uarch : sifive,u74-mc
118 );118 );
119}119}
120120
...@@ -177,16 +177,16 @@ const PowerpcCpuinfoParser = CpuinfoParser(PowerpcCpuinfoImpl);...@@ -177,16 +177,16 @@ const PowerpcCpuinfoParser = CpuinfoParser(PowerpcCpuinfoImpl);
177177
178test "cpuinfo: PowerPC" {178test "cpuinfo: PowerPC" {
179 try testParser(PowerpcCpuinfoParser, .powerpc, &Target.powerpc.cpu.@"970",179 try testParser(PowerpcCpuinfoParser, .powerpc, &Target.powerpc.cpu.@"970",
180 \\processor : 0180 \\processor : 0
181 \\cpu : PPC970MP, altivec supported181 \\cpu : PPC970MP, altivec supported
182 \\clock : 1250.000000MHz182 \\clock : 1250.000000MHz
183 \\revision : 1.1 (pvr 0044 0101)183 \\revision : 1.1 (pvr 0044 0101)
184 );184 );
185 try testParser(PowerpcCpuinfoParser, .powerpc64le, &Target.powerpc.cpu.pwr8,185 try testParser(PowerpcCpuinfoParser, .powerpc64le, &Target.powerpc.cpu.pwr8,
186 \\processor : 0186 \\processor : 0
187 \\cpu : POWER8 (raw), altivec supported187 \\cpu : POWER8 (raw), altivec supported
188 \\clock : 2926.000000MHz188 \\clock : 2926.000000MHz
189 \\revision : 2.0 (pvr 004d 0200)189 \\revision : 2.0 (pvr 004d 0200)
190 );190 );
191}191}
192192
...@@ -304,25 +304,25 @@ test "cpuinfo: ARM" {...@@ -304,25 +304,25 @@ test "cpuinfo: ARM" {
304 \\CPU revision : 7304 \\CPU revision : 7
305 );305 );
306 try testParser(ArmCpuinfoParser, .arm, &Target.arm.cpu.cortex_a7,306 try testParser(ArmCpuinfoParser, .arm, &Target.arm.cpu.cortex_a7,
307 \\processor : 0307 \\processor : 0
308 \\model name : ARMv7 Processor rev 3 (v7l)308 \\model name : ARMv7 Processor rev 3 (v7l)
309 \\BogoMIPS : 18.00309 \\BogoMIPS : 18.00
310 \\Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae310 \\Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae
311 \\CPU implementer : 0x41311 \\CPU implementer : 0x41
312 \\CPU architecture: 7312 \\CPU architecture: 7
313 \\CPU variant : 0x0313 \\CPU variant : 0x0
314 \\CPU part : 0xc07314 \\CPU part : 0xc07
315 \\CPU revision : 3315 \\CPU revision : 3
316 \\316 \\
317 \\processor : 4317 \\processor : 4
318 \\model name : ARMv7 Processor rev 3 (v7l)318 \\model name : ARMv7 Processor rev 3 (v7l)
319 \\BogoMIPS : 90.00319 \\BogoMIPS : 90.00
320 \\Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae320 \\Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae
321 \\CPU implementer : 0x41321 \\CPU implementer : 0x41
322 \\CPU architecture: 7322 \\CPU architecture: 7
323 \\CPU variant : 0x2323 \\CPU variant : 0x2
324 \\CPU part : 0xc0f324 \\CPU part : 0xc0f
325 \\CPU revision : 3325 \\CPU revision : 3
326 );326 );
327 try testParser(ArmCpuinfoParser, .aarch64, &Target.aarch64.cpu.cortex_a72,327 try testParser(ArmCpuinfoParser, .aarch64, &Target.aarch64.cpu.cortex_a72,
328 \\processor : 0328 \\processor : 0
lib/std/zig/tokenizer.zig+254-355
...@@ -320,7 +320,7 @@ pub const Token = struct {...@@ -320,7 +320,7 @@ pub const Token = struct {
320320
321 pub fn symbol(tag: Tag) []const u8 {321 pub fn symbol(tag: Tag) []const u8 {
322 return tag.lexeme() orelse switch (tag) {322 return tag.lexeme() orelse switch (tag) {
323 .invalid => "invalid bytes",323 .invalid => "invalid token",
324 .identifier => "an identifier",324 .identifier => "an identifier",
325 .string_literal, .multiline_string_literal_line => "a string literal",325 .string_literal, .multiline_string_literal_line => "a string literal",
326 .char_literal => "a character literal",326 .char_literal => "a character literal",
...@@ -338,22 +338,22 @@ pub const Tokenizer = struct {...@@ -338,22 +338,22 @@ pub const Tokenizer = struct {
338 buffer: [:0]const u8,338 buffer: [:0]const u8,
339 index: usize,339 index: usize,
340340
341 /// For debugging purposes341 /// For debugging purposes.
342 pub fn dump(self: *Tokenizer, token: *const Token) void {342 pub fn dump(self: *Tokenizer, token: *const Token) void {
343 std.debug.print("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] });343 std.debug.print("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] });
344 }344 }
345345
346 pub fn init(buffer: [:0]const u8) Tokenizer {346 pub fn init(buffer: [:0]const u8) Tokenizer {
347 // Skip the UTF-8 BOM if present347 // Skip the UTF-8 BOM if present.
348 const src_start: usize = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0;348 return .{
349 return Tokenizer{
350 .buffer = buffer,349 .buffer = buffer,
351 .index = src_start,350 .index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0,
352 };351 };
353 }352 }
354353
355 const State = enum {354 const State = enum {
356 start,355 start,
356 expect_newline,
357 identifier,357 identifier,
358 builtin,358 builtin,
359 string_literal,359 string_literal,
...@@ -361,10 +361,6 @@ pub const Tokenizer = struct {...@@ -361,10 +361,6 @@ pub const Tokenizer = struct {
361 multiline_string_literal_line,361 multiline_string_literal_line,
362 char_literal,362 char_literal,
363 char_literal_backslash,363 char_literal_backslash,
364 char_literal_hex_escape,
365 char_literal_unicode_escape_saw_u,
366 char_literal_unicode_escape,
367 char_literal_end,
368 backslash,364 backslash,
369 equal,365 equal,
370 bang,366 bang,
...@@ -400,30 +396,33 @@ pub const Tokenizer = struct {...@@ -400,30 +396,33 @@ pub const Tokenizer = struct {
400 period_2,396 period_2,
401 period_asterisk,397 period_asterisk,
402 saw_at_sign,398 saw_at_sign,
399 invalid,
403 };400 };
404401
402 /// After this returns invalid, it will reset on the next newline, returning tokens starting from there.
403 /// An eof token will always be returned at the end.
405 pub fn next(self: *Tokenizer) Token {404 pub fn next(self: *Tokenizer) Token {
406 var state: State = .start;405 var state: State = .start;
407 var result = Token{406 var result: Token = .{
408 .tag = .eof,407 .tag = undefined,
409 .loc = .{408 .loc = .{
410 .start = self.index,409 .start = self.index,
411 .end = undefined,410 .end = undefined,
412 },411 },
413 };412 };
414 var seen_escape_digits: usize = undefined;
415 while (true) : (self.index += 1) {413 while (true) : (self.index += 1) {
416 const c = self.buffer[self.index];414 const c = self.buffer[self.index];
417 switch (state) {415 switch (state) {
418 .start => switch (c) {416 .start => switch (c) {
419 0 => {417 0 => {
420 if (self.index != self.buffer.len) {418 if (self.index == self.buffer.len) return .{
421 result.tag = .invalid;419 .tag = .eof,
422 result.loc.end = self.index;420 .loc = .{
423 self.index += 1;421 .start = self.index,
424 return result;422 .end = self.index,
425 }423 },
426 break;424 };
425 state = .invalid;
427 },426 },
428 ' ', '\n', '\t', '\r' => {427 ' ', '\n', '\t', '\r' => {
429 result.loc.start = self.index + 1;428 result.loc.start = self.index + 1;
...@@ -434,6 +433,7 @@ pub const Tokenizer = struct {...@@ -434,6 +433,7 @@ pub const Tokenizer = struct {
434 },433 },
435 '\'' => {434 '\'' => {
436 state = .char_literal;435 state = .char_literal;
436 result.tag = .char_literal;
437 },437 },
438 'a'...'z', 'A'...'Z', '_' => {438 'a'...'z', 'A'...'Z', '_' => {
439 state = .identifier;439 state = .identifier;
...@@ -545,14 +545,44 @@ pub const Tokenizer = struct {...@@ -545,14 +545,44 @@ pub const Tokenizer = struct {
545 result.tag = .number_literal;545 result.tag = .number_literal;
546 },546 },
547 else => {547 else => {
548 state = .invalid;
549 },
550 },
551
552 .expect_newline => switch (c) {
553 0 => {
554 if (self.index == self.buffer.len) {
555 result.tag = .invalid;
556 break;
557 }
558 state = .invalid;
559 },
560 '\n' => {
561 result.loc.start = self.index + 1;
562 state = .start;
563 },
564 else => {
565 state = .invalid;
566 },
567 },
568
569 .invalid => switch (c) {
570 0 => if (self.index == self.buffer.len) {
548 result.tag = .invalid;571 result.tag = .invalid;
549 result.loc.end = self.index;572 break;
550 self.index += std.unicode.utf8ByteSequenceLength(c) catch 1;
551 return result;
552 },573 },
574 '\n' => {
575 result.tag = .invalid;
576 break;
577 },
578 else => continue,
553 },579 },
554580
555 .saw_at_sign => switch (c) {581 .saw_at_sign => switch (c) {
582 0, '\n' => {
583 result.tag = .invalid;
584 break;
585 },
556 '"' => {586 '"' => {
557 result.tag = .identifier;587 result.tag = .identifier;
558 state = .string_literal;588 state = .string_literal;
...@@ -562,8 +592,7 @@ pub const Tokenizer = struct {...@@ -562,8 +592,7 @@ pub const Tokenizer = struct {
562 result.tag = .builtin;592 result.tag = .builtin;
563 },593 },
564 else => {594 else => {
565 result.tag = .invalid;595 state = .invalid;
566 break;
567 },596 },
568 },597 },
569598
...@@ -698,7 +727,7 @@ pub const Tokenizer = struct {...@@ -698,7 +727,7 @@ pub const Tokenizer = struct {
698 },727 },
699728
700 .identifier => switch (c) {729 .identifier => switch (c) {
701 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},730 'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
702 else => {731 else => {
703 if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {732 if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {
704 result.tag = tag;733 result.tag = tag;
...@@ -707,26 +736,37 @@ pub const Tokenizer = struct {...@@ -707,26 +736,37 @@ pub const Tokenizer = struct {
707 },736 },
708 },737 },
709 .builtin => switch (c) {738 .builtin => switch (c) {
710 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},739 'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
711 else => break,740 else => break,
712 },741 },
713 .backslash => switch (c) {742 .backslash => switch (c) {
743 0 => {
744 result.tag = .invalid;
745 break;
746 },
714 '\\' => {747 '\\' => {
715 state = .multiline_string_literal_line;748 state = .multiline_string_literal_line;
716 },749 },
717 else => {750 '\n' => {
718 result.tag = .invalid;751 result.tag = .invalid;
719 break;752 break;
720 },753 },
754 else => {
755 state = .invalid;
756 },
721 },757 },
722 .string_literal => switch (c) {758 .string_literal => switch (c) {
723 0, '\n' => {759 0 => {
724 result.tag = .invalid;
725 result.loc.end = self.index;
726 if (self.index != self.buffer.len) {760 if (self.index != self.buffer.len) {
727 self.index += 1;761 state = .invalid;
762 continue;
728 }763 }
729 return result;764 result.tag = .invalid;
765 break;
766 },
767 '\n' => {
768 result.tag = .invalid;
769 break;
730 },770 },
731 '\\' => {771 '\\' => {
732 state = .string_literal_backslash;772 state = .string_literal_backslash;
...@@ -735,150 +775,74 @@ pub const Tokenizer = struct {...@@ -735,150 +775,74 @@ pub const Tokenizer = struct {
735 self.index += 1;775 self.index += 1;
736 break;776 break;
737 },777 },
738 else => {778 0x01...0x09, 0x0b...0x1f, 0x7f => {
739 if (self.invalidCharacterLength()) |len| {779 state = .invalid;
740 result.tag = .invalid;
741 result.loc.end = self.index;
742 self.index += len;
743 return result;
744 }
745
746 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
747 },780 },
781 else => continue,
748 },782 },
749783
750 .string_literal_backslash => switch (c) {784 .string_literal_backslash => switch (c) {
751 0, '\n' => {785 0, '\n' => {
752 result.tag = .invalid;786 result.tag = .invalid;
753 result.loc.end = self.index;787 break;
754 if (self.index != self.buffer.len) {
755 self.index += 1;
756 }
757 return result;
758 },788 },
759 else => {789 else => {
760 state = .string_literal;790 state = .string_literal;
761
762 if (self.invalidCharacterLength()) |len| {
763 result.tag = .invalid;
764 result.loc.end = self.index;
765 self.index += len;
766 return result;
767 }
768
769 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
770 },791 },
771 },792 },
772793
773 .char_literal => switch (c) {794 .char_literal => switch (c) {
774 0, '\n', '\'' => {795 0 => {
775 result.tag = .invalid;
776 result.loc.end = self.index;
777 if (self.index != self.buffer.len) {796 if (self.index != self.buffer.len) {
778 self.index += 1;797 state = .invalid;
798 continue;
779 }799 }
780 return result;800 result.tag = .invalid;
801 break;
802 },
803 '\n' => {
804 result.tag = .invalid;
805 break;
781 },806 },
782 '\\' => {807 '\\' => {
783 state = .char_literal_backslash;808 state = .char_literal_backslash;
784 },809 },
785 else => {810 '\'' => {
786 state = .char_literal_end;811 self.index += 1;
787812 break;
788 if (self.invalidCharacterLength()) |len| {
789 result.tag = .invalid;
790 result.loc.end = self.index;
791 self.index += len;
792 return result;
793 }
794
795 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
796 },813 },
814 0x01...0x09, 0x0b...0x1f, 0x7f => {
815 state = .invalid;
816 },
817 else => continue,
797 },818 },
798819
799 .char_literal_backslash => switch (c) {820 .char_literal_backslash => switch (c) {
800 0, '\n' => {821 0 => {
801 result.tag = .invalid;
802 result.loc.end = self.index;
803 if (self.index != self.buffer.len) {822 if (self.index != self.buffer.len) {
804 self.index += 1;823 state = .invalid;
805 }824 continue;
806 return result;
807 },
808 'x' => {
809 state = .char_literal_hex_escape;
810 seen_escape_digits = 0;
811 },
812 'u' => {
813 state = .char_literal_unicode_escape_saw_u;
814 },
815 else => {
816 state = .char_literal_end;
817
818 if (self.invalidCharacterLength()) |len| {
819 result.tag = .invalid;
820 result.loc.end = self.index;
821 self.index += len;
822 return result;
823 }825 }
824
825 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
826 },
827 },
828
829 .char_literal_hex_escape => switch (c) {
830 '0'...'9', 'a'...'f', 'A'...'F' => {
831 seen_escape_digits += 1;
832 if (seen_escape_digits == 2) {
833 state = .char_literal_end;
834 }
835 },
836 else => {
837 result.tag = .invalid;826 result.tag = .invalid;
838 break;827 break;
839 },828 },
840 },829 '\n' => {
841
842 .char_literal_unicode_escape_saw_u => switch (c) {
843 '{' => {
844 state = .char_literal_unicode_escape;
845 },
846 else => {
847 result.tag = .invalid;
848 break;
849 },
850 },
851
852 .char_literal_unicode_escape => switch (c) {
853 '0'...'9', 'a'...'f', 'A'...'F' => {},
854 '}' => {
855 state = .char_literal_end; // too many/few digits handled later
856 },
857 else => {
858 result.tag = .invalid;830 result.tag = .invalid;
859 break;831 break;
860 },832 },
861 },833 0x01...0x09, 0x0b...0x1f, 0x7f => {
862834 state = .invalid;
863 .char_literal_end => switch (c) {
864 '\'' => {
865 result.tag = .char_literal;
866 self.index += 1;
867 break;
868 },835 },
869 else => {836 else => {
870 result.tag = .invalid;837 state = .char_literal;
871 break;
872 },838 },
873 },839 },
874840
875 .multiline_string_literal_line => switch (c) {841 .multiline_string_literal_line => switch (c) {
876 0 => {842 0 => {
877 if (self.index != self.buffer.len) {843 if (self.index != self.buffer.len) {
878 result.tag = .invalid;844 state = .invalid;
879 result.loc.end = self.index;845 continue;
880 self.index += 1;
881 return result;
882 }846 }
883 break;847 break;
884 },848 },
...@@ -886,17 +850,18 @@ pub const Tokenizer = struct {...@@ -886,17 +850,18 @@ pub const Tokenizer = struct {
886 self.index += 1;850 self.index += 1;
887 break;851 break;
888 },852 },
889 '\t' => {},853 '\r' => {
890 else => {854 if (self.buffer[self.index + 1] == '\n') {
891 if (self.invalidCharacterLength()) |len| {855 self.index += 2;
892 result.tag = .invalid;856 break;
893 result.loc.end = self.index;857 } else {
894 self.index += len;858 state = .invalid;
895 return result;
896 }859 }
897
898 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
899 },860 },
861 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
862 state = .invalid;
863 },
864 else => continue,
900 },865 },
901866
902 .bang => switch (c) {867 .bang => switch (c) {
...@@ -1113,12 +1078,16 @@ pub const Tokenizer = struct {...@@ -1113,12 +1078,16 @@ pub const Tokenizer = struct {
1113 .line_comment_start => switch (c) {1078 .line_comment_start => switch (c) {
1114 0 => {1079 0 => {
1115 if (self.index != self.buffer.len) {1080 if (self.index != self.buffer.len) {
1116 result.tag = .invalid;1081 state = .invalid;
1117 result.loc.end = self.index;1082 continue;
1118 self.index += 1;
1119 return result;
1120 }1083 }
1121 break;1084 return .{
1085 .tag = .eof,
1086 .loc = .{
1087 .start = self.index,
1088 .end = self.index,
1089 },
1090 };
1122 },1091 },
1123 '/' => {1092 '/' => {
1124 state = .doc_comment_start;1093 state = .doc_comment_start;
...@@ -1127,105 +1096,91 @@ pub const Tokenizer = struct {...@@ -1127,105 +1096,91 @@ pub const Tokenizer = struct {
1127 result.tag = .container_doc_comment;1096 result.tag = .container_doc_comment;
1128 state = .doc_comment;1097 state = .doc_comment;
1129 },1098 },
1099 '\r' => {
1100 state = .expect_newline;
1101 },
1130 '\n' => {1102 '\n' => {
1131 state = .start;1103 state = .start;
1132 result.loc.start = self.index + 1;1104 result.loc.start = self.index + 1;
1133 },1105 },
1134 '\t' => {1106 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1135 state = .line_comment;1107 state = .invalid;
1136 },1108 },
1137 else => {1109 else => {
1138 state = .line_comment;1110 state = .line_comment;
1139
1140 if (self.invalidCharacterLength()) |len| {
1141 result.tag = .invalid;
1142 result.loc.end = self.index;
1143 self.index += len;
1144 return result;
1145 }
1146
1147 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1148 },1111 },
1149 },1112 },
1150 .doc_comment_start => switch (c) {1113 .doc_comment_start => switch (c) {
1151 '/' => {1114 0, '\n' => {
1152 state = .line_comment;1115 result.tag = .doc_comment;
1116 break;
1153 },1117 },
1154 0 => {1118 '\r' => {
1155 if (self.index != self.buffer.len) {1119 if (self.buffer[self.index + 1] == '\n') {
1156 result.tag = .invalid;
1157 result.loc.end = self.index;
1158 self.index += 1;1120 self.index += 1;
1159 return result;1121 result.tag = .doc_comment;
1122 break;
1123 } else {
1124 state = .invalid;
1160 }1125 }
1161 result.tag = .doc_comment;
1162 break;
1163 },1126 },
1164 '\n' => {1127 '/' => {
1165 result.tag = .doc_comment;1128 state = .line_comment;
1166 break;
1167 },1129 },
1168 '\t' => {1130 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1169 state = .doc_comment;1131 state = .invalid;
1170 result.tag = .doc_comment;
1171 },1132 },
1172 else => {1133 else => {
1173 state = .doc_comment;1134 state = .doc_comment;
1174 result.tag = .doc_comment;1135 result.tag = .doc_comment;
1175
1176 if (self.invalidCharacterLength()) |len| {
1177 result.tag = .invalid;
1178 result.loc.end = self.index;
1179 self.index += len;
1180 return result;
1181 }
1182
1183 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1184 },1136 },
1185 },1137 },
1186 .line_comment => switch (c) {1138 .line_comment => switch (c) {
1187 0 => {1139 0 => {
1188 if (self.index != self.buffer.len) {1140 if (self.index != self.buffer.len) {
1189 result.tag = .invalid;1141 state = .invalid;
1190 result.loc.end = self.index;1142 continue;
1191 self.index += 1;
1192 return result;
1193 }1143 }
1194 break;1144 return .{
1145 .tag = .eof,
1146 .loc = .{
1147 .start = self.index,
1148 .end = self.index,
1149 },
1150 };
1151 },
1152 '\r' => {
1153 state = .expect_newline;
1195 },1154 },
1196 '\n' => {1155 '\n' => {
1197 state = .start;1156 state = .start;
1198 result.loc.start = self.index + 1;1157 result.loc.start = self.index + 1;
1199 },1158 },
1200 '\t' => {},1159 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1201 else => {1160 state = .invalid;
1202 if (self.invalidCharacterLength()) |len| {
1203 result.tag = .invalid;
1204 result.loc.end = self.index;
1205 self.index += len;
1206 return result;
1207 }
1208
1209 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1210 },1161 },
1162 else => continue,
1211 },1163 },
1212 .doc_comment => switch (c) {1164 .doc_comment => switch (c) {
1213 0, '\n' => break,1165 0, '\n' => {
1214 '\t' => {},1166 break;
1215 else => {1167 },
1216 if (self.invalidCharacterLength()) |len| {1168 '\r' => {
1217 result.tag = .invalid;1169 if (self.buffer[self.index + 1] == '\n') {
1218 result.loc.end = self.index;1170 self.index += 1;
1219 self.index += len;1171 break;
1220 return result;1172 } else {
1173 state = .invalid;
1221 }1174 }
1222
1223 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1224 },1175 },
1176 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1177 state = .invalid;
1178 },
1179 else => continue,
1225 },1180 },
1226 .int => switch (c) {1181 .int => switch (c) {
1227 '.' => state = .int_period,1182 '.' => state = .int_period,
1228 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},1183 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1229 'e', 'E', 'p', 'P' => state = .int_exponent,1184 'e', 'E', 'p', 'P' => state = .int_exponent,
1230 else => break,1185 else => break,
1231 },1186 },
...@@ -1249,7 +1204,7 @@ pub const Tokenizer = struct {...@@ -1249,7 +1204,7 @@ pub const Tokenizer = struct {
1249 },1204 },
1250 },1205 },
1251 .float => switch (c) {1206 .float => switch (c) {
1252 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},1207 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1253 'e', 'E', 'p', 'P' => state = .float_exponent,1208 'e', 'E', 'p', 'P' => state = .float_exponent,
1254 else => break,1209 else => break,
1255 },1210 },
...@@ -1263,57 +1218,9 @@ pub const Tokenizer = struct {...@@ -1263,57 +1218,9 @@ pub const Tokenizer = struct {
1263 }1218 }
1264 }1219 }
12651220
1266 if (result.tag == .eof) {
1267 result.loc.start = self.index;
1268 }
1269
1270 result.loc.end = self.index;1221 result.loc.end = self.index;
1271 return result;1222 return result;
1272 }1223 }
1273
1274 fn invalidCharacterLength(self: *Tokenizer) ?u3 {
1275 const c0 = self.buffer[self.index];
1276 if (std.ascii.isAscii(c0)) {
1277 if (c0 == '\r') {
1278 if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {
1279 // Carriage returns are *only* allowed just before a linefeed as part of a CRLF pair, otherwise
1280 // they constitute an illegal byte!
1281 return null;
1282 } else {
1283 return 1;
1284 }
1285 } else if (std.ascii.isControl(c0)) {
1286 // ascii control codes are never allowed
1287 // (note that \n was checked before we got here)
1288 return 1;
1289 }
1290 // looks fine to me.
1291 return null;
1292 } else {
1293 // check utf8-encoded character.
1294 const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
1295 if (self.index + length > self.buffer.len) {
1296 return @as(u3, @intCast(self.buffer.len - self.index));
1297 }
1298 const bytes = self.buffer[self.index .. self.index + length];
1299 switch (length) {
1300 2 => {
1301 const value = std.unicode.utf8Decode2(bytes) catch return length;
1302 if (value == 0x85) return length; // U+0085 (NEL)
1303 },
1304 3 => {
1305 const value = std.unicode.utf8Decode3(bytes) catch return length;
1306 if (value == 0x2028) return length; // U+2028 (LS)
1307 if (value == 0x2029) return length; // U+2029 (PS)
1308 },
1309 4 => {
1310 _ = std.unicode.utf8Decode4(bytes) catch return length;
1311 },
1312 else => unreachable,
1313 }
1314 return null;
1315 }
1316 }
1317};1224};
13181225
1319test "keywords" {1226test "keywords" {
...@@ -1355,7 +1262,7 @@ test "code point literal with hex escape" {...@@ -1355,7 +1262,7 @@ test "code point literal with hex escape" {
1355 , &.{.char_literal});1262 , &.{.char_literal});
1356 try testTokenize(1263 try testTokenize(
1357 \\'\x1'1264 \\'\x1'
1358 , &.{ .invalid, .invalid });1265 , &.{.char_literal});
1359}1266}
13601267
1361test "newline in char literal" {1268test "newline in char literal" {
...@@ -1396,40 +1303,30 @@ test "code point literal with unicode escapes" {...@@ -1396,40 +1303,30 @@ test "code point literal with unicode escapes" {
1396 // Invalid unicode escapes1303 // Invalid unicode escapes
1397 try testTokenize(1304 try testTokenize(
1398 \\'\u'1305 \\'\u'
1399 , &.{ .invalid, .invalid });1306 , &.{.char_literal});
1400 try testTokenize(1307 try testTokenize(
1401 \\'\u{{'1308 \\'\u{{'
1402 , &.{ .invalid, .l_brace, .invalid });1309 , &.{.char_literal});
1403 try testTokenize(1310 try testTokenize(
1404 \\'\u{}'1311 \\'\u{}'
1405 , &.{.char_literal});1312 , &.{.char_literal});
1406 try testTokenize(1313 try testTokenize(
1407 \\'\u{s}'1314 \\'\u{s}'
1408 , &.{1315 , &.{.char_literal});
1409 .invalid,
1410 .identifier,
1411 .r_brace,
1412 .invalid,
1413 });
1414 try testTokenize(1316 try testTokenize(
1415 \\'\u{2z}'1317 \\'\u{2z}'
1416 , &.{1318 , &.{.char_literal});
1417 .invalid,
1418 .identifier,
1419 .r_brace,
1420 .invalid,
1421 });
1422 try testTokenize(1319 try testTokenize(
1423 \\'\u{4a'1320 \\'\u{4a'
1424 , &.{ .invalid, .invalid }); // 4a is valid1321 , &.{.char_literal});
14251322
1426 // Test old-style unicode literals1323 // Test old-style unicode literals
1427 try testTokenize(1324 try testTokenize(
1428 \\'\u0333'1325 \\'\u0333'
1429 , &.{ .invalid, .number_literal, .invalid });1326 , &.{.char_literal});
1430 try testTokenize(1327 try testTokenize(
1431 \\'\U0333'1328 \\'\U0333'
1432 , &.{ .invalid, .number_literal, .invalid });1329 , &.{.char_literal});
1433}1330}
14341331
1435test "code point literal with unicode code point" {1332test "code point literal with unicode code point" {
...@@ -1465,24 +1362,15 @@ test "invalid token characters" {...@@ -1465,24 +1362,15 @@ test "invalid token characters" {
1465 try testTokenize("`", &.{.invalid});1362 try testTokenize("`", &.{.invalid});
1466 try testTokenize("'c", &.{.invalid});1363 try testTokenize("'c", &.{.invalid});
1467 try testTokenize("'", &.{.invalid});1364 try testTokenize("'", &.{.invalid});
1468 try testTokenize("''", &.{.invalid});1365 try testTokenize("''", &.{.char_literal});
1469 try testTokenize("'\n'", &.{ .invalid, .invalid });1366 try testTokenize("'\n'", &.{ .invalid, .invalid });
1470}1367}
14711368
1472test "invalid literal/comment characters" {1369test "invalid literal/comment characters" {
1473 try testTokenize("\"\x00\"", &.{1370 try testTokenize("\"\x00\"", &.{.invalid});
1474 .invalid,1371 try testTokenize("//\x00", &.{.invalid});
1475 .invalid, // Incomplete string literal starting after invalid1372 try testTokenize("//\x1f", &.{.invalid});
1476 });1373 try testTokenize("//\x7f", &.{.invalid});
1477 try testTokenize("//\x00", &.{
1478 .invalid,
1479 });
1480 try testTokenize("//\x1f", &.{
1481 .invalid,
1482 });
1483 try testTokenize("//\x7f", &.{
1484 .invalid,
1485 });
1486}1374}
14871375
1488test "utf8" {1376test "utf8" {
...@@ -1491,46 +1379,24 @@ test "utf8" {...@@ -1491,46 +1379,24 @@ test "utf8" {
1491}1379}
14921380
1493test "invalid utf8" {1381test "invalid utf8" {
1494 try testTokenize("//\x80", &.{1382 try testTokenize("//\x80", &.{});
1495 .invalid,1383 try testTokenize("//\xbf", &.{});
1496 });1384 try testTokenize("//\xf8", &.{});
1497 try testTokenize("//\xbf", &.{1385 try testTokenize("//\xff", &.{});
1498 .invalid,1386 try testTokenize("//\xc2\xc0", &.{});
1499 });1387 try testTokenize("//\xe0", &.{});
1500 try testTokenize("//\xf8", &.{1388 try testTokenize("//\xf0", &.{});
1501 .invalid,1389 try testTokenize("//\xf0\x90\x80\xc0", &.{});
1502 });
1503 try testTokenize("//\xff", &.{
1504 .invalid,
1505 });
1506 try testTokenize("//\xc2\xc0", &.{
1507 .invalid,
1508 });
1509 try testTokenize("//\xe0", &.{
1510 .invalid,
1511 });
1512 try testTokenize("//\xf0", &.{
1513 .invalid,
1514 });
1515 try testTokenize("//\xf0\x90\x80\xc0", &.{
1516 .invalid,
1517 });
1518}1390}
15191391
1520test "illegal unicode codepoints" {1392test "illegal unicode codepoints" {
1521 // unicode newline characters.U+0085, U+2028, U+20291393 // unicode newline characters.U+0085, U+2028, U+2029
1522 try testTokenize("//\xc2\x84", &.{});1394 try testTokenize("//\xc2\x84", &.{});
1523 try testTokenize("//\xc2\x85", &.{1395 try testTokenize("//\xc2\x85", &.{});
1524 .invalid,
1525 });
1526 try testTokenize("//\xc2\x86", &.{});1396 try testTokenize("//\xc2\x86", &.{});
1527 try testTokenize("//\xe2\x80\xa7", &.{});1397 try testTokenize("//\xe2\x80\xa7", &.{});
1528 try testTokenize("//\xe2\x80\xa8", &.{1398 try testTokenize("//\xe2\x80\xa8", &.{});
1529 .invalid,1399 try testTokenize("//\xe2\x80\xa9", &.{});
1530 });
1531 try testTokenize("//\xe2\x80\xa9", &.{
1532 .invalid,
1533 });
1534 try testTokenize("//\xe2\x80\xaa", &.{});1400 try testTokenize("//\xe2\x80\xaa", &.{});
1535}1401}
15361402
...@@ -1549,30 +1415,6 @@ test "string identifier and builtin fns" {...@@ -1549,30 +1415,6 @@ test "string identifier and builtin fns" {
1549 });1415 });
1550}1416}
15511417
1552test "multiline string literal with literal tab" {
1553 try testTokenize(
1554 \\\\foo bar
1555 , &.{
1556 .multiline_string_literal_line,
1557 });
1558}
1559
1560test "comments with literal tab" {
1561 try testTokenize(
1562 \\//foo bar
1563 \\//!foo bar
1564 \\///foo bar
1565 \\// foo
1566 \\/// foo
1567 \\/// /foo
1568 , &.{
1569 .container_doc_comment,
1570 .doc_comment,
1571 .doc_comment,
1572 .doc_comment,
1573 });
1574}
1575
1576test "pipe and then invalid" {1418test "pipe and then invalid" {
1577 try testTokenize("||=", &.{1419 try testTokenize("||=", &.{
1578 .pipe_pipe,1420 .pipe_pipe,
...@@ -1892,8 +1734,8 @@ test "multi line string literal with only 1 backslash" {...@@ -1892,8 +1734,8 @@ test "multi line string literal with only 1 backslash" {
1892}1734}
18931735
1894test "invalid builtin identifiers" {1736test "invalid builtin identifiers" {
1895 try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });1737 try testTokenize("@()", &.{.invalid});
1896 try testTokenize("@0()", &.{ .invalid, .number_literal, .l_paren, .r_paren });1738 try testTokenize("@0()", &.{.invalid});
1897}1739}
18981740
1899test "invalid token with unfinished escape right before eof" {1741test "invalid token with unfinished escape right before eof" {
...@@ -1921,21 +1763,78 @@ test "saturating operators" {...@@ -1921,21 +1763,78 @@ test "saturating operators" {
1921}1763}
19221764
1923test "null byte before eof" {1765test "null byte before eof" {
1924 try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });1766 try testTokenize("123 \x00 456", &.{ .number_literal, .invalid });
1925 try testTokenize("//\x00", &.{.invalid});1767 try testTokenize("//\x00", &.{.invalid});
1926 try testTokenize("\\\\\x00", &.{.invalid});1768 try testTokenize("\\\\\x00", &.{.invalid});
1927 try testTokenize("\x00", &.{.invalid});1769 try testTokenize("\x00", &.{.invalid});
1928 try testTokenize("// NUL\x00\n", &.{.invalid});1770 try testTokenize("// NUL\x00\n", &.{.invalid});
1929 try testTokenize("///\x00\n", &.{.invalid});1771 try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
1930 try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });1772 try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
1931}1773}
19321774
1775test "invalid tabs and carriage returns" {
1776 // "Inside Line Comments and Documentation Comments, Any TAB is rejected by
1777 // the grammar since it is ambiguous how it should be rendered."
1778 // https://github.com/ziglang/zig-spec/issues/38
1779 try testTokenize("//\t", &.{.invalid});
1780 try testTokenize("// \t", &.{.invalid});
1781 try testTokenize("///\t", &.{.invalid});
1782 try testTokenize("/// \t", &.{.invalid});
1783 try testTokenize("//!\t", &.{.invalid});
1784 try testTokenize("//! \t", &.{.invalid});
1785
1786 // "Inside Line Comments and Documentation Comments, CR directly preceding
1787 // NL is unambiguously part of the newline sequence. It is accepted by the
1788 // grammar and removed by zig fmt, leaving only NL. CR anywhere else is
1789 // rejected by the grammar."
1790 // https://github.com/ziglang/zig-spec/issues/38
1791 try testTokenize("//\r", &.{.invalid});
1792 try testTokenize("// \r", &.{.invalid});
1793 try testTokenize("///\r", &.{.invalid});
1794 try testTokenize("/// \r", &.{.invalid});
1795 try testTokenize("//\r ", &.{.invalid});
1796 try testTokenize("// \r ", &.{.invalid});
1797 try testTokenize("///\r ", &.{.invalid});
1798 try testTokenize("/// \r ", &.{.invalid});
1799 try testTokenize("//\r\n", &.{});
1800 try testTokenize("// \r\n", &.{});
1801 try testTokenize("///\r\n", &.{.doc_comment});
1802 try testTokenize("/// \r\n", &.{.doc_comment});
1803 try testTokenize("//!\r", &.{.invalid});
1804 try testTokenize("//! \r", &.{.invalid});
1805 try testTokenize("//!\r ", &.{.invalid});
1806 try testTokenize("//! \r ", &.{.invalid});
1807 try testTokenize("//!\r\n", &.{.container_doc_comment});
1808 try testTokenize("//! \r\n", &.{.container_doc_comment});
1809
1810 // The control characters TAB and CR are rejected by the grammar inside multi-line string literals,
1811 // except if CR is directly before NL.
1812 // https://github.com/ziglang/zig-spec/issues/38
1813 try testTokenize("\\\\\r", &.{.invalid});
1814 try testTokenize("\\\\\r ", &.{.invalid});
1815 try testTokenize("\\\\ \r", &.{.invalid});
1816 try testTokenize("\\\\\t", &.{.invalid});
1817 try testTokenize("\\\\\t ", &.{.invalid});
1818 try testTokenize("\\\\ \t", &.{.invalid});
1819 try testTokenize("\\\\\r\n", &.{.multiline_string_literal_line});
1820
1821 // "TAB used as whitespace is...accepted by the grammar. CR used as
1822 // whitespace, whether directly preceding NL or stray, is...accepted by the
1823 // grammar."
1824 // https://github.com/ziglang/zig-spec/issues/38
1825 try testTokenize("\tpub\tswitch\t", &.{ .keyword_pub, .keyword_switch });
1826 try testTokenize("\rpub\rswitch\r", &.{ .keyword_pub, .keyword_switch });
1827}
1828
1933fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {1829fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
1934 var tokenizer = Tokenizer.init(source);1830 var tokenizer = Tokenizer.init(source);
1935 for (expected_token_tags) |expected_token_tag| {1831 for (expected_token_tags) |expected_token_tag| {
1936 const token = tokenizer.next();1832 const token = tokenizer.next();
1937 try std.testing.expectEqual(expected_token_tag, token.tag);1833 try std.testing.expectEqual(expected_token_tag, token.tag);
1938 }1834 }
1835 // Last token should always be eof, even when the last token was invalid,
1836 // in which case the tokenizer is in an invalid state, which can only be
1837 // recovered by opinionated means outside the scope of this implementation.
1939 const last_token = tokenizer.next();1838 const last_token = tokenizer.next();
1940 try std.testing.expectEqual(Token.Tag.eof, last_token.tag);1839 try std.testing.expectEqual(Token.Tag.eof, last_token.tag);
1941 try std.testing.expectEqual(source.len, last_token.loc.start);1840 try std.testing.expectEqual(source.len, last_token.loc.start);
src/Package/Manifest.zig+3
...@@ -549,6 +549,9 @@ const Parse = struct {...@@ -549,6 +549,9 @@ const Parse = struct {
549 .{raw_string[bad_index]},549 .{raw_string[bad_index]},
550 );550 );
551 },551 },
552 .empty_char_literal => {
553 try p.appendErrorOff(token, offset, "empty character literal", .{});
554 },
552 }555 }
553 }556 }
554557
src/link/tapi/yaml/test.zig+1-4
...@@ -237,10 +237,7 @@ test "double quoted string" {...@@ -237,10 +237,7 @@ test "double quoted string" {
237 try testing.expectEqualStrings(237 try testing.expectEqualStrings(
238 \\"here" are some escaped quotes238 \\"here" are some escaped quotes
239 , arr[1]);239 , arr[1]);
240 try testing.expectEqualStrings(240 try testing.expectEqualStrings("newlines and tabs\nare\tsupported", arr[2]);
241 \\newlines and tabs
242 \\are supported
243 , arr[2]);
244 try testing.expectEqualStrings(241 try testing.expectEqualStrings(
245 \\let's have242 \\let's have
246 \\some fun!243 \\some fun!
test/cases/compile_errors/empty_char_lit.zig created+9
...@@ -0,0 +1,9 @@
1export fn entry() u8 {
2 return '';
3}
4
5// error
6// backend=stage2
7// target=native
8//
9// :2:12: error: empty character literal
test/cases/compile_errors/invalid_legacy_unicode_escape.zig+1-2
...@@ -6,5 +6,4 @@ export fn entry() void {...@@ -6,5 +6,4 @@ export fn entry() void {
6// backend=stage26// backend=stage2
7// target=native7// target=native
8//8//
9// :2:15: error: expected expression, found 'invalid bytes'9// :2:17: error: invalid escape character: 'U'
10// :2:18: note: invalid byte: '1'
test/cases/compile_errors/invalid_unicode_escape.zig+1-2
...@@ -6,6 +6,5 @@ export fn entry() void {...@@ -6,6 +6,5 @@ export fn entry() void {
6// backend=stage26// backend=stage2
7// target=native7// target=native
8//8//
9// :2:15: error: expected expression, found 'invalid bytes'9// :2:21: error: expected hex digit or '}', found 'z'
10// :2:21: note: invalid byte: 'z'
1110
test/cases/compile_errors/normal_string_with_newline.zig+1-2
...@@ -5,5 +5,4 @@ b";...@@ -5,5 +5,4 @@ b";
5// backend=stage25// backend=stage2
6// target=native6// target=native
7//7//
8// :1:13: error: expected expression, found 'invalid bytes'8// :1:13: error: expected expression, found 'invalid token'
9// :1:15: note: invalid byte: '\n'
test/compile_errors.zig+5-19
...@@ -38,15 +38,6 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -38,15 +38,6 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
38 });38 });
39 }39 }
4040
41 {
42 const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);
43
44 case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{
45 ":1:13: error: expected expression, found 'invalid bytes'",
46 ":1:19: note: invalid byte: '\\r'",
47 });
48 }
49
50 {41 {
51 const case = ctx.obj("missing semicolon at EOF", b.graph.host);42 const case = ctx.obj("missing semicolon at EOF", b.graph.host);
52 case.addError(43 case.addError(
...@@ -179,8 +170,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -179,8 +170,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
179 \\ return true;170 \\ return true;
180 \\}171 \\}
181 , &[_][]const u8{172 , &[_][]const u8{
182 ":1:1: error: expected type expression, found 'invalid bytes'",173 ":1:1: error: expected type expression, found 'invalid token'",
183 ":1:1: note: invalid byte: '\\xff'",
184 });174 });
185 }175 }
186176
...@@ -222,8 +212,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -222,8 +212,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
222 const case = ctx.obj("invalid byte in string", b.graph.host);212 const case = ctx.obj("invalid byte in string", b.graph.host);
223213
224 case.addError("_ = \"\x01Q\";", &[_][]const u8{214 case.addError("_ = \"\x01Q\";", &[_][]const u8{
225 ":1:5: error: expected expression, found 'invalid bytes'",215 ":1:5: error: expected expression, found 'invalid token'",
226 ":1:6: note: invalid byte: '\\x01'",
227 });216 });
228 }217 }
229218
...@@ -231,8 +220,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -231,8 +220,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
231 const case = ctx.obj("invalid byte in comment", b.graph.host);220 const case = ctx.obj("invalid byte in comment", b.graph.host);
232221
233 case.addError("//\x01Q", &[_][]const u8{222 case.addError("//\x01Q", &[_][]const u8{
234 ":1:1: error: expected type expression, found 'invalid bytes'",223 ":1:1: error: expected type expression, found 'invalid token'",
235 ":1:3: note: invalid byte: '\\x01'",
236 });224 });
237 }225 }
238226
...@@ -240,8 +228,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -240,8 +228,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
240 const case = ctx.obj("control character in character literal", b.graph.host);228 const case = ctx.obj("control character in character literal", b.graph.host);
241229
242 case.addError("const c = '\x01';", &[_][]const u8{230 case.addError("const c = '\x01';", &[_][]const u8{
243 ":1:11: error: expected expression, found 'invalid bytes'",231 ":1:11: error: expected expression, found 'invalid token'",
244 ":1:12: note: invalid byte: '\\x01'",
245 });232 });
246 }233 }
247234
...@@ -249,8 +236,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -249,8 +236,7 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
249 const case = ctx.obj("invalid byte at start of token", b.graph.host);236 const case = ctx.obj("invalid byte at start of token", b.graph.host);
250237
251 case.addError("x = \x00Q", &[_][]const u8{238 case.addError("x = \x00Q", &[_][]const u8{
252 ":1:5: error: expected expression, found 'invalid bytes'",239 ":1:5: error: expected expression, found 'invalid token'",
253 ":1:5: note: invalid byte: '\\x00'",
254 });240 });
255 }241 }
256}242}
test/run_translated_c.zig+6-6
...@@ -26,17 +26,17 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {...@@ -26,17 +26,17 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {
26 \\void baz(void);26 \\void baz(void);
27 \\struct foo { int x; };27 \\struct foo { int x; };
28 \\void bar() {28 \\void bar() {
29 \\ struct foo tmp;29 \\ struct foo tmp;
30 \\}30 \\}
31 \\31 \\
32 \\void baz() {32 \\void baz() {
33 \\ struct foo tmp;33 \\ struct foo tmp;
34 \\}34 \\}
35 \\35 \\
36 \\int main(void) {36 \\int main(void) {
37 \\ bar();37 \\ bar();
38 \\ baz();38 \\ baz();
39 \\ return 0;39 \\ return 0;
40 \\}40 \\}
41 , "");41 , "");
4242
...@@ -53,7 +53,7 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {...@@ -53,7 +53,7 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {
53 cases.add("parenthesized string literal",53 cases.add("parenthesized string literal",
54 \\void foo(const char *s) {}54 \\void foo(const char *s) {}
55 \\int main(void) {55 \\int main(void) {
56 \\ foo(("bar"));56 \\ foo(("bar"));
57 \\}57 \\}
58 , "");58 , "");
5959
test/translate_c.zig+17-17
...@@ -133,20 +133,20 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -133,20 +133,20 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
133133
134 cases.add("scoped typedef",134 cases.add("scoped typedef",
135 \\void foo() {135 \\void foo() {
136 \\ typedef union {136 \\ typedef union {
137 \\ int A;137 \\ int A;
138 \\ int B;138 \\ int B;
139 \\ int C;139 \\ int C;
140 \\ } Foo;140 \\ } Foo;
141 \\ Foo a = {0};141 \\ Foo a = {0};
142 \\ {142 \\ {
143 \\ typedef union {143 \\ typedef union {
144 \\ int A;144 \\ int A;
145 \\ int B;145 \\ int B;
146 \\ int C;146 \\ int C;
147 \\ } Foo;147 \\ } Foo;
148 \\ Foo a = {0};148 \\ Foo a = {0};
149 \\ }149 \\ }
150 \\}150 \\}
151 , &[_][]const u8{151 , &[_][]const u8{
152 \\pub export fn foo() void {152 \\pub export fn foo() void {
...@@ -2004,18 +2004,18 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -2004,18 +2004,18 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
2004 \\ break;2004 \\ break;
2005 \\ }2005 \\ }
2006 \\ case 4:2006 \\ case 4:
2007 \\ case 5:2007 \\ case 5:
2008 \\ res = 69;2008 \\ res = 69;
2009 \\ {2009 \\ {
2010 \\ res = 5;2010 \\ res = 5;
2011 \\ return;2011 \\ return;
2012 \\ }2012 \\ }
2013 \\ case 6:2013 \\ case 6:
2014 \\ switch (res) {2014 \\ switch (res) {
2015 \\ case 9: break;2015 \\ case 9: break;
2016 \\ }2016 \\ }
2017 \\ res = 1;2017 \\ res = 1;
2018 \\ return;2018 \\ return;
2019 \\ }2019 \\ }
2020 \\}2020 \\}
2021 , &[_][]const u8{2021 , &[_][]const u8{