authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-06-02 04:03:25-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-06-02 04:04:23-04:00
logf06bce5ddaea368040560f584170aee2864fa399
tree61937f83c19f0090cd9daadfa570504801dc24f8
parent7b386ea24285db2fbc40233a232cb294dddf9879

introduce [*] for unknown length pointers

See #770 Currently it does not have any different behavior than `*` but it is now recommended to use `[*]` for unknown length pointers to be future-proof. Instead of [ * ] being separate tokens as the proposal suggested, this commit implements `[*]` as a single token.

9 files changed, 87 insertions(+), 12 deletions(-)

doc/langref.html.in+1-1
...@@ -6450,7 +6450,7 @@ ContainerInitBody = list(StructLiteralField, ",") | list(Expression, ",")...@@ -6450,7 +6450,7 @@ ContainerInitBody = list(StructLiteralField, ",") | list(Expression, ",")
64506450
6451StructLiteralField = "." Symbol "=" Expression6451StructLiteralField = "." Symbol "=" Expression
64526452
6453PrefixOp = "!" | "-" | "~" | ("*" option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await"6453PrefixOp = "!" | "-" | "~" | (("*" | "[*]") option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await"
64546454
6455PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType6455PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType
64566456
src/all_types.hpp+1
...@@ -625,6 +625,7 @@ struct AstNodePrefixOpExpr {...@@ -625,6 +625,7 @@ struct AstNodePrefixOpExpr {
625};625};
626626
627struct AstNodePointerType {627struct AstNodePointerType {
628 Token *star_token;
628 AstNode *align_expr;629 AstNode *align_expr;
629 BigInt *bit_offset_start;630 BigInt *bit_offset_start;
630 BigInt *bit_offset_end;631 BigInt *bit_offset_end;
src/parser.cpp+3-2
...@@ -1174,6 +1174,7 @@ static PrefixOp tok_to_prefix_op(Token *token) {...@@ -1174,6 +1174,7 @@ static PrefixOp tok_to_prefix_op(Token *token) {
11741174
1175static AstNode *ast_parse_pointer_type(ParseContext *pc, size_t *token_index, Token *star_tok) {1175static AstNode *ast_parse_pointer_type(ParseContext *pc, size_t *token_index, Token *star_tok) {
1176 AstNode *node = ast_create_node(pc, NodeTypePointerType, star_tok);1176 AstNode *node = ast_create_node(pc, NodeTypePointerType, star_tok);
1177 node->data.pointer_type.star_token = star_tok;
11771178
1178 Token *token = &pc->tokens->at(*token_index);1179 Token *token = &pc->tokens->at(*token_index);
1179 if (token->id == TokenIdKeywordAlign) {1180 if (token->id == TokenIdKeywordAlign) {
...@@ -1211,11 +1212,11 @@ static AstNode *ast_parse_pointer_type(ParseContext *pc, size_t *token_index, To...@@ -1211,11 +1212,11 @@ static AstNode *ast_parse_pointer_type(ParseContext *pc, size_t *token_index, To
12111212
1212/*1213/*
1213PrefixOpExpression = PrefixOp ErrorSetExpr | SuffixOpExpression1214PrefixOpExpression = PrefixOp ErrorSetExpr | SuffixOpExpression
1214PrefixOp = "!" | "-" | "~" | ("*" option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await"1215PrefixOp = "!" | "-" | "~" | (("*" | "[*]") option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await"
1215*/1216*/
1216static AstNode *ast_parse_prefix_op_expr(ParseContext *pc, size_t *token_index, bool mandatory) {1217static AstNode *ast_parse_prefix_op_expr(ParseContext *pc, size_t *token_index, bool mandatory) {
1217 Token *token = &pc->tokens->at(*token_index);1218 Token *token = &pc->tokens->at(*token_index);
1218 if (token->id == TokenIdStar) {1219 if (token->id == TokenIdStar || token->id == TokenIdBracketStarBracket) {
1219 *token_index += 1;1220 *token_index += 1;
1220 return ast_parse_pointer_type(pc, token_index, token);1221 return ast_parse_pointer_type(pc, token_index, token);
1221 }1222 }
src/tokenizer.cpp+30-1
...@@ -219,6 +219,8 @@ enum TokenizeState {...@@ -219,6 +219,8 @@ enum TokenizeState {
219 TokenizeStateSawAtSign,219 TokenizeStateSawAtSign,
220 TokenizeStateCharCode,220 TokenizeStateCharCode,
221 TokenizeStateError,221 TokenizeStateError,
222 TokenizeStateLBracket,
223 TokenizeStateLBracketStar,
222};224};
223225
224226
...@@ -539,8 +541,8 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -539,8 +541,8 @@ void tokenize(Buf *buf, Tokenization *out) {
539 end_token(&t);541 end_token(&t);
540 break;542 break;
541 case '[':543 case '[':
544 t.state = TokenizeStateLBracket;
542 begin_token(&t, TokenIdLBracket);545 begin_token(&t, TokenIdLBracket);
543 end_token(&t);
544 break;546 break;
545 case ']':547 case ']':
546 begin_token(&t, TokenIdRBracket);548 begin_token(&t, TokenIdRBracket);
...@@ -852,6 +854,30 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -852,6 +854,30 @@ void tokenize(Buf *buf, Tokenization *out) {
852 continue;854 continue;
853 }855 }
854 break;856 break;
857 case TokenizeStateLBracket:
858 switch (c) {
859 case '*':
860 t.state = TokenizeStateLBracketStar;
861 set_token_id(&t, t.cur_tok, TokenIdBracketStarBracket);
862 break;
863 default:
864 // reinterpret as just an lbracket
865 t.pos -= 1;
866 end_token(&t);
867 t.state = TokenizeStateStart;
868 continue;
869 }
870 break;
871 case TokenizeStateLBracketStar:
872 switch (c) {
873 case ']':
874 end_token(&t);
875 t.state = TokenizeStateStart;
876 break;
877 default:
878 invalid_char_error(&t, c);
879 }
880 break;
855 case TokenizeStateSawPlusPercent:881 case TokenizeStateSawPlusPercent:
856 switch (c) {882 switch (c) {
857 case '=':883 case '=':
...@@ -1467,12 +1493,14 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1467,12 +1493,14 @@ void tokenize(Buf *buf, Tokenization *out) {
1467 case TokenizeStateLineString:1493 case TokenizeStateLineString:
1468 case TokenizeStateLineStringEnd:1494 case TokenizeStateLineStringEnd:
1469 case TokenizeStateSawBarBar:1495 case TokenizeStateSawBarBar:
1496 case TokenizeStateLBracket:
1470 end_token(&t);1497 end_token(&t);
1471 break;1498 break;
1472 case TokenizeStateSawDotDot:1499 case TokenizeStateSawDotDot:
1473 case TokenizeStateSawBackslash:1500 case TokenizeStateSawBackslash:
1474 case TokenizeStateLineStringContinue:1501 case TokenizeStateLineStringContinue:
1475 case TokenizeStateLineStringContinueC:1502 case TokenizeStateLineStringContinueC:
1503 case TokenizeStateLBracketStar:
1476 tokenize_error(&t, "unexpected EOF");1504 tokenize_error(&t, "unexpected EOF");
1477 break;1505 break;
1478 case TokenizeStateLineComment:1506 case TokenizeStateLineComment:
...@@ -1509,6 +1537,7 @@ const char * token_name(TokenId id) {...@@ -1509,6 +1537,7 @@ const char * token_name(TokenId id) {
1509 case TokenIdBitShiftRight: return ">>";1537 case TokenIdBitShiftRight: return ">>";
1510 case TokenIdBitShiftRightEq: return ">>=";1538 case TokenIdBitShiftRightEq: return ">>=";
1511 case TokenIdBitXorEq: return "^=";1539 case TokenIdBitXorEq: return "^=";
1540 case TokenIdBracketStarBracket: return "[*]";
1512 case TokenIdCharLiteral: return "CharLiteral";1541 case TokenIdCharLiteral: return "CharLiteral";
1513 case TokenIdCmpEq: return "==";1542 case TokenIdCmpEq: return "==";
1514 case TokenIdCmpGreaterOrEq: return ">=";1543 case TokenIdCmpGreaterOrEq: return ">=";
src/tokenizer.hpp+1
...@@ -28,6 +28,7 @@ enum TokenId {...@@ -28,6 +28,7 @@ enum TokenId {
28 TokenIdBitShiftRight,28 TokenIdBitShiftRight,
29 TokenIdBitShiftRightEq,29 TokenIdBitShiftRightEq,
30 TokenIdBitXorEq,30 TokenIdBitXorEq,
31 TokenIdBracketStarBracket,
31 TokenIdCharLiteral,32 TokenIdCharLiteral,
32 TokenIdCmpEq,33 TokenIdCmpEq,
33 TokenIdCmpGreaterOrEq,34 TokenIdCmpGreaterOrEq,
std/cstr.zig+4-4
...@@ -9,13 +9,13 @@ pub const line_sep = switch (builtin.os) {...@@ -9,13 +9,13 @@ pub const line_sep = switch (builtin.os) {
9 else => "\n",9 else => "\n",
10};10};
1111
12pub fn len(ptr: *const u8) usize {12pub fn len(ptr: [*]const u8) usize {
13 var count: usize = 0;13 var count: usize = 0;
14 while (ptr[count] != 0) : (count += 1) {}14 while (ptr[count] != 0) : (count += 1) {}
15 return count;15 return count;
16}16}
1717
18pub fn cmp(a: *const u8, b: *const u8) i8 {18pub fn cmp(a: [*]const u8, b: [*]const u8) i8 {
19 var index: usize = 0;19 var index: usize = 0;
20 while (a[index] == b[index] and a[index] != 0) : (index += 1) {}20 while (a[index] == b[index] and a[index] != 0) : (index += 1) {}
21 if (a[index] > b[index]) {21 if (a[index] > b[index]) {
...@@ -27,11 +27,11 @@ pub fn cmp(a: *const u8, b: *const u8) i8 {...@@ -27,11 +27,11 @@ pub fn cmp(a: *const u8, b: *const u8) i8 {
27 }27 }
28}28}
2929
30pub fn toSliceConst(str: *const u8) []const u8 {30pub fn toSliceConst(str: [*]const u8) []const u8 {
31 return str[0..len(str)];31 return str[0..len(str)];
32}32}
3333
34pub fn toSlice(str: *u8) []u8 {34pub fn toSlice(str: [*]u8) []u8 {
35 return str[0..len(str)];35 return str[0..len(str)];
36}36}
3737
std/zig/parse.zig+1-1
...@@ -3292,7 +3292,7 @@ fn tokenIdToPrefixOp(id: @TagType(Token.Id)) ?ast.Node.PrefixOp.Op {...@@ -3292,7 +3292,7 @@ fn tokenIdToPrefixOp(id: @TagType(Token.Id)) ?ast.Node.PrefixOp.Op {
3292 Token.Id.Minus => ast.Node.PrefixOp.Op{ .Negation = void{} },3292 Token.Id.Minus => ast.Node.PrefixOp.Op{ .Negation = void{} },
3293 Token.Id.MinusPercent => ast.Node.PrefixOp.Op{ .NegationWrap = void{} },3293 Token.Id.MinusPercent => ast.Node.PrefixOp.Op{ .NegationWrap = void{} },
3294 Token.Id.Ampersand => ast.Node.PrefixOp.Op{ .AddressOf = void{} },3294 Token.Id.Ampersand => ast.Node.PrefixOp.Op{ .AddressOf = void{} },
3295 Token.Id.Asterisk, Token.Id.AsteriskAsterisk => ast.Node.PrefixOp.Op{3295 Token.Id.Asterisk, Token.Id.AsteriskAsterisk, Token.Id.BracketStarBracket => ast.Node.PrefixOp.Op{
3296 .PtrType = ast.Node.PrefixOp.PtrInfo{3296 .PtrType = ast.Node.PrefixOp.PtrInfo{
3297 .align_info = null,3297 .align_info = null,
3298 .const_token = null,3298 .const_token = null,
std/zig/parser_test.zig+7
...@@ -1,3 +1,10 @@...@@ -1,3 +1,10 @@
1test "zig fmt: pointer of unknown length" {
2 try testCanonical(
3 \\fn foo(ptr: [*]u8) void {}
4 \\
5 );
6}
7
1test "zig fmt: spaces around slice operator" {8test "zig fmt: spaces around slice operator" {
2 try testCanonical(9 try testCanonical(
3 \\var a = b[c..d];10 \\var a = b[c..d];
std/zig/tokenizer.zig+39-3
...@@ -143,6 +143,7 @@ pub const Token = struct {...@@ -143,6 +143,7 @@ pub const Token = struct {
143 FloatLiteral,143 FloatLiteral,
144 LineComment,144 LineComment,
145 DocComment,145 DocComment,
146 BracketStarBracket,
146 Keyword_align,147 Keyword_align,
147 Keyword_and,148 Keyword_and,
148 Keyword_asm,149 Keyword_asm,
...@@ -263,6 +264,8 @@ pub const Tokenizer = struct {...@@ -263,6 +264,8 @@ pub const Tokenizer = struct {
263 Period,264 Period,
264 Period2,265 Period2,
265 SawAtSign,266 SawAtSign,
267 LBracket,
268 LBracketStar,
266 };269 };
267270
268 pub fn next(self: *Tokenizer) Token {271 pub fn next(self: *Tokenizer) Token {
...@@ -325,9 +328,7 @@ pub const Tokenizer = struct {...@@ -325,9 +328,7 @@ pub const Tokenizer = struct {
325 break;328 break;
326 },329 },
327 '[' => {330 '[' => {
328 result.id = Token.Id.LBracket;331 state = State.LBracket;
329 self.index += 1;
330 break;
331 },332 },
332 ']' => {333 ']' => {
333 result.id = Token.Id.RBracket;334 result.id = Token.Id.RBracket;
...@@ -429,6 +430,28 @@ pub const Tokenizer = struct {...@@ -429,6 +430,28 @@ pub const Tokenizer = struct {
429 },430 },
430 },431 },
431432
433 State.LBracket => switch (c) {
434 '*' => {
435 state = State.LBracketStar;
436 },
437 else => {
438 result.id = Token.Id.LBracket;
439 break;
440 },
441 },
442
443 State.LBracketStar => switch (c) {
444 ']' => {
445 result.id = Token.Id.BracketStarBracket;
446 self.index += 1;
447 break;
448 },
449 else => {
450 result.id = Token.Id.Invalid;
451 break;
452 },
453 },
454
432 State.Ampersand => switch (c) {455 State.Ampersand => switch (c) {
433 '=' => {456 '=' => {
434 result.id = Token.Id.AmpersandEqual;457 result.id = Token.Id.AmpersandEqual;
...@@ -1008,6 +1031,7 @@ pub const Tokenizer = struct {...@@ -1008,6 +1031,7 @@ pub const Tokenizer = struct {
1008 State.CharLiteralEscape2,1031 State.CharLiteralEscape2,
1009 State.CharLiteralEnd,1032 State.CharLiteralEnd,
1010 State.StringLiteralBackslash,1033 State.StringLiteralBackslash,
1034 State.LBracketStar,
1011 => {1035 => {
1012 result.id = Token.Id.Invalid;1036 result.id = Token.Id.Invalid;
1013 },1037 },
...@@ -1024,6 +1048,9 @@ pub const Tokenizer = struct {...@@ -1024,6 +1048,9 @@ pub const Tokenizer = struct {
1024 State.Slash => {1048 State.Slash => {
1025 result.id = Token.Id.Slash;1049 result.id = Token.Id.Slash;
1026 },1050 },
1051 State.LBracket => {
1052 result.id = Token.Id.LBracket;
1053 },
1027 State.Zero => {1054 State.Zero => {
1028 result.id = Token.Id.IntegerLiteral;1055 result.id = Token.Id.IntegerLiteral;
1029 },1056 },
...@@ -1142,6 +1169,15 @@ test "tokenizer" {...@@ -1142,6 +1169,15 @@ test "tokenizer" {
1142 testTokenize("test", []Token.Id{Token.Id.Keyword_test});1169 testTokenize("test", []Token.Id{Token.Id.Keyword_test});
1143}1170}
11441171
1172test "tokenizer - unknown length pointer" {
1173 testTokenize(
1174 \\[*]u8
1175 , []Token.Id{
1176 Token.Id.BracketStarBracket,
1177 Token.Id.Identifier,
1178 });
1179}
1180
1145test "tokenizer - char literal with hex escape" {1181test "tokenizer - char literal with hex escape" {
1146 testTokenize(1182 testTokenize(
1147 \\'\x1b'1183 \\'\x1b'