authorgravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-30 23:09:43+11:00
committergravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-30 23:47:29+11:00
log80d37a13c0e2be9aff63745e03ac4fab8f55aa98
treeb083d757da224febc9a166ac6a6ccb7622451286
parent51943ff432f3e73c2dc21015fae4bdcd8f81ae61
signature Commit is signed but in an unrecognized format.

std: use a union(enum) for std.json.Token

1 files changed, 154 insertions(+), 164 deletions(-)

lib/std/json.zig+154-164
......@@ -13,78 +13,38 @@ pub const WriteStream = @import("json/write_stream.zig").WriteStream;
1313/// A single token slice into the parent string.
1414///
1515/// Use `token.slice()` on the input at the current position to get the current slice.
16pub const Token = struct {
17 id: Id,
18
19 /// How many bytes do we skip before counting
20 offset: u1,
21
22 /// Whether string contains an escape sequence and cannot be zero-copied
23 string_has_escape: bool,
24
25 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
26 number_is_integer: bool,
27
28 /// How many bytes from the current position behind the start of this token is.
29 count: usize,
30
31 pub const Id = enum {
32 ObjectBegin,
33 ObjectEnd,
34 ArrayBegin,
35 ArrayEnd,
36 String,
37 Number,
38 True,
39 False,
40 Null,
41 };
42
43 pub fn init(id: Id, count: usize, offset: u1) Token {
44 return Token{
45 .id = id,
46 .offset = offset,
47 .string_has_escape = false,
48 .number_is_integer = true,
49 .count = count,
50 };
51 }
16pub const Token = union(enum) {
17 ObjectBegin,
18 ObjectEnd,
19 ArrayBegin,
20 ArrayEnd,
21 String: struct {
22 /// How many bytes the token is.
23 count: usize,
24
25 /// Whether string contains an escape sequence and cannot be zero-copied
26 has_escape: bool,
27
28 /// Slice into the underlying input string.
29 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
30 return input[i - self.count .. i];
31 }
32 },
33 Number: struct {
34 /// How many bytes the token is.
35 count: usize,
5236
53 pub fn initString(count: usize, has_unicode_escape: bool) Token {
54 return Token{
55 .id = Id.String,
56 .offset = 0,
57 .string_has_escape = has_unicode_escape,
58 .number_is_integer = true,
59 .count = count,
60 };
61 }
37 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
38 is_integer: bool,
6239
63 pub fn initNumber(count: usize, number_is_integer: bool) Token {
64 return Token{
65 .id = Id.Number,
66 .offset = 0,
67 .string_has_escape = false,
68 .number_is_integer = number_is_integer,
69 .count = count,
70 };
71 }
72
73 /// A marker token is a zero-length
74 pub fn initMarker(id: Id) Token {
75 return Token{
76 .id = id,
77 .offset = 0,
78 .string_has_escape = false,
79 .number_is_integer = true,
80 .count = 0,
81 };
82 }
83
84 /// Slice into the underlying input string.
85 pub fn slice(self: Token, input: []const u8, i: usize) []const u8 {
86 return input[i + self.offset - self.count .. i + self.offset];
87 }
40 /// Slice into the underlying input string.
41 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
42 return input[i - self.count .. i];
43 }
44 },
45 True,
46 False,
47 Null,
8848};
8949
9050/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
......@@ -236,7 +196,7 @@ pub const StreamingParser = struct {
236196 p.state = State.ValueBegin;
237197 p.after_string_state = State.ObjectSeparator;
238198
239 token.* = Token.initMarker(Token.Id.ObjectBegin);
199 token.* = Token.ObjectBegin;
240200 },
241201 '[' => {
242202 p.stack <<= 1;
......@@ -246,7 +206,7 @@ pub const StreamingParser = struct {
246206 p.state = State.ValueBegin;
247207 p.after_string_state = State.ValueEnd;
248208
249 token.* = Token.initMarker(Token.Id.ArrayBegin);
209 token.* = Token.ArrayBegin;
250210 },
251211 '-' => {
252212 p.number_is_integer = true;
......@@ -334,7 +294,7 @@ pub const StreamingParser = struct {
334294 },
335295 }
336296
337 token.* = Token.initMarker(Token.Id.ObjectEnd);
297 token.* = Token.ObjectEnd;
338298 },
339299 ']' => {
340300 if (p.stack & 1 != array_bit) {
......@@ -360,7 +320,7 @@ pub const StreamingParser = struct {
360320 },
361321 }
362322
363 token.* = Token.initMarker(Token.Id.ArrayEnd);
323 token.* = Token.ArrayEnd;
364324 },
365325 '{' => {
366326 if (p.stack_used == max_stack_size) {
......@@ -374,7 +334,7 @@ pub const StreamingParser = struct {
374334 p.state = State.ValueBegin;
375335 p.after_string_state = State.ObjectSeparator;
376336
377 token.* = Token.initMarker(Token.Id.ObjectBegin);
337 token.* = Token.ObjectBegin;
378338 },
379339 '[' => {
380340 if (p.stack_used == max_stack_size) {
......@@ -388,7 +348,7 @@ pub const StreamingParser = struct {
388348 p.state = State.ValueBegin;
389349 p.after_string_state = State.ValueEnd;
390350
391 token.* = Token.initMarker(Token.Id.ArrayBegin);
351 token.* = Token.ArrayBegin;
392352 },
393353 '-' => {
394354 p.number_is_integer = true;
......@@ -443,7 +403,7 @@ pub const StreamingParser = struct {
443403 p.state = State.ValueBegin;
444404 p.after_string_state = State.ObjectSeparator;
445405
446 token.* = Token.initMarker(Token.Id.ObjectBegin);
406 token.* = Token.ObjectBegin;
447407 },
448408 '[' => {
449409 if (p.stack_used == max_stack_size) {
......@@ -457,7 +417,7 @@ pub const StreamingParser = struct {
457417 p.state = State.ValueBegin;
458418 p.after_string_state = State.ValueEnd;
459419
460 token.* = Token.initMarker(Token.Id.ArrayBegin);
420 token.* = Token.ArrayBegin;
461421 },
462422 '-' => {
463423 p.number_is_integer = true;
......@@ -519,7 +479,7 @@ pub const StreamingParser = struct {
519479 p.state = State.TopLevelEnd;
520480 }
521481
522 token.* = Token.initMarker(Token.Id.ArrayEnd);
482 token.* = Token.ArrayEnd;
523483 },
524484 '}' => {
525485 if (p.stack_used == 0) {
......@@ -537,7 +497,7 @@ pub const StreamingParser = struct {
537497 p.state = State.TopLevelEnd;
538498 }
539499
540 token.* = Token.initMarker(Token.Id.ObjectEnd);
500 token.* = Token.ObjectEnd;
541501 },
542502 0x09, 0x0A, 0x0D, 0x20 => {
543503 // whitespace
......@@ -571,7 +531,12 @@ pub const StreamingParser = struct {
571531 p.complete = true;
572532 }
573533
574 token.* = Token.initString(p.count - 1, p.string_has_escape);
534 token.* = .{
535 .String = .{
536 .count = p.count - 1,
537 .has_escape = p.string_has_escape,
538 },
539 };
575540 },
576541 '\\' => {
577542 p.state = State.StringEscapeCharacter;
......@@ -686,7 +651,12 @@ pub const StreamingParser = struct {
686651 },
687652 else => {
688653 p.state = p.after_value_state;
689 token.* = Token.initNumber(p.count, p.number_is_integer);
654 token.* = .{
655 .Number = .{
656 .count = p.count,
657 .is_integer = p.number_is_integer,
658 },
659 };
690660 return true;
691661 },
692662 }
......@@ -708,7 +678,12 @@ pub const StreamingParser = struct {
708678 },
709679 else => {
710680 p.state = p.after_value_state;
711 token.* = Token.initNumber(p.count, p.number_is_integer);
681 token.* = .{
682 .Number = .{
683 .count = p.count,
684 .is_integer = p.number_is_integer,
685 },
686 };
712687 return true;
713688 },
714689 }
......@@ -738,7 +713,12 @@ pub const StreamingParser = struct {
738713 },
739714 else => {
740715 p.state = p.after_value_state;
741 token.* = Token.initNumber(p.count, p.number_is_integer);
716 token.* = .{
717 .Number = .{
718 .count = p.count,
719 .is_integer = p.number_is_integer,
720 },
721 };
742722 return true;
743723 },
744724 }
......@@ -753,7 +733,12 @@ pub const StreamingParser = struct {
753733 },
754734 else => {
755735 p.state = p.after_value_state;
756 token.* = Token.initNumber(p.count, p.number_is_integer);
736 token.* = .{
737 .Number = .{
738 .count = p.count,
739 .is_integer = p.number_is_integer,
740 },
741 };
757742 return true;
758743 },
759744 }
......@@ -791,7 +776,12 @@ pub const StreamingParser = struct {
791776 },
792777 else => {
793778 p.state = p.after_value_state;
794 token.* = Token.initNumber(p.count, p.number_is_integer);
779 token.* = .{
780 .Number = .{
781 .count = p.count,
782 .is_integer = p.number_is_integer,
783 },
784 };
795785 return true;
796786 },
797787 }
......@@ -811,7 +801,7 @@ pub const StreamingParser = struct {
811801 'e' => {
812802 p.state = p.after_value_state;
813803 p.complete = p.state == State.TopLevelEnd;
814 token.* = Token.init(Token.Id.True, p.count + 1, 1);
804 token.* = Token.True;
815805 },
816806 else => {
817807 return error.InvalidLiteral;
......@@ -837,7 +827,7 @@ pub const StreamingParser = struct {
837827 'e' => {
838828 p.state = p.after_value_state;
839829 p.complete = p.state == State.TopLevelEnd;
840 token.* = Token.init(Token.Id.False, p.count + 1, 1);
830 token.* = Token.False;
841831 },
842832 else => {
843833 return error.InvalidLiteral;
......@@ -858,7 +848,7 @@ pub const StreamingParser = struct {
858848 'l' => {
859849 p.state = p.after_value_state;
860850 p.complete = p.state == State.TopLevelEnd;
861 token.* = Token.init(Token.Id.Null, p.count + 1, 1);
851 token.* = Token.Null;
862852 },
863853 else => {
864854 return error.InvalidLiteral;
......@@ -923,9 +913,9 @@ pub const TokenStream = struct {
923913 }
924914};
925915
926fn checkNext(p: *TokenStream, id: Token.Id) void {
916fn checkNext(p: *TokenStream, id: std.meta.TagType(Token)) void {
927917 const token = (p.next() catch unreachable).?;
928 debug.assert(token.id == id);
918 debug.assert(std.meta.activeTag(token) == id);
929919}
930920
931921test "json.token" {
......@@ -948,35 +938,35 @@ test "json.token" {
948938
949939 var p = TokenStream.init(s);
950940
951 checkNext(&p, Token.Id.ObjectBegin);
952 checkNext(&p, Token.Id.String); // Image
953 checkNext(&p, Token.Id.ObjectBegin);
954 checkNext(&p, Token.Id.String); // Width
955 checkNext(&p, Token.Id.Number);
956 checkNext(&p, Token.Id.String); // Height
957 checkNext(&p, Token.Id.Number);
958 checkNext(&p, Token.Id.String); // Title
959 checkNext(&p, Token.Id.String);
960 checkNext(&p, Token.Id.String); // Thumbnail
961 checkNext(&p, Token.Id.ObjectBegin);
962 checkNext(&p, Token.Id.String); // Url
963 checkNext(&p, Token.Id.String);
964 checkNext(&p, Token.Id.String); // Height
965 checkNext(&p, Token.Id.Number);
966 checkNext(&p, Token.Id.String); // Width
967 checkNext(&p, Token.Id.Number);
968 checkNext(&p, Token.Id.ObjectEnd);
969 checkNext(&p, Token.Id.String); // Animated
970 checkNext(&p, Token.Id.False);
971 checkNext(&p, Token.Id.String); // IDs
972 checkNext(&p, Token.Id.ArrayBegin);
973 checkNext(&p, Token.Id.Number);
974 checkNext(&p, Token.Id.Number);
975 checkNext(&p, Token.Id.Number);
976 checkNext(&p, Token.Id.Number);
977 checkNext(&p, Token.Id.ArrayEnd);
978 checkNext(&p, Token.Id.ObjectEnd);
979 checkNext(&p, Token.Id.ObjectEnd);
941 checkNext(&p, .ObjectBegin);
942 checkNext(&p, .String); // Image
943 checkNext(&p, .ObjectBegin);
944 checkNext(&p, .String); // Width
945 checkNext(&p, .Number);
946 checkNext(&p, .String); // Height
947 checkNext(&p, .Number);
948 checkNext(&p, .String); // Title
949 checkNext(&p, .String);
950 checkNext(&p, .String); // Thumbnail
951 checkNext(&p, .ObjectBegin);
952 checkNext(&p, .String); // Url
953 checkNext(&p, .String);
954 checkNext(&p, .String); // Height
955 checkNext(&p, .Number);
956 checkNext(&p, .String); // Width
957 checkNext(&p, .Number);
958 checkNext(&p, .ObjectEnd);
959 checkNext(&p, .String); // Animated
960 checkNext(&p, .False);
961 checkNext(&p, .String); // IDs
962 checkNext(&p, .ArrayBegin);
963 checkNext(&p, .Number);
964 checkNext(&p, .Number);
965 checkNext(&p, .Number);
966 checkNext(&p, .Number);
967 checkNext(&p, .ArrayEnd);
968 checkNext(&p, .ObjectEnd);
969 checkNext(&p, .ObjectEnd);
980970
981971 testing.expect((try p.next()) == null);
982972}
......@@ -1122,8 +1112,8 @@ pub const Parser = struct {
11221112 // can be cleaned up on error correctly during a `parse` on call.
11231113 fn transition(p: *Parser, allocator: *Allocator, input: []const u8, i: usize, token: Token) !void {
11241114 switch (p.state) {
1125 State.ObjectKey => switch (token.id) {
1126 Token.Id.ObjectEnd => {
1115 State.ObjectKey => switch (token) {
1116 .ObjectEnd => {
11271117 if (p.stack.len == 1) {
11281118 return;
11291119 }
......@@ -1131,8 +1121,8 @@ pub const Parser = struct {
11311121 var value = p.stack.pop();
11321122 try p.pushToParent(&value);
11331123 },
1134 Token.Id.String => {
1135 try p.stack.append(try p.parseString(allocator, token, input, i));
1124 .String => |s| {
1125 try p.stack.append(try p.parseString(allocator, s, input, i));
11361126 p.state = State.ObjectValue;
11371127 },
11381128 else => {
......@@ -1146,41 +1136,41 @@ pub const Parser = struct {
11461136 var object = &p.stack.items[p.stack.len - 2].Object;
11471137 var key = p.stack.items[p.stack.len - 1].String;
11481138
1149 switch (token.id) {
1150 Token.Id.ObjectBegin => {
1139 switch (token) {
1140 .ObjectBegin => {
11511141 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
11521142 p.state = State.ObjectKey;
11531143 },
1154 Token.Id.ArrayBegin => {
1144 .ArrayBegin => {
11551145 try p.stack.append(Value{ .Array = Array.init(allocator) });
11561146 p.state = State.ArrayValue;
11571147 },
1158 Token.Id.String => {
1159 _ = try object.put(key, try p.parseString(allocator, token, input, i));
1148 .String => |s| {
1149 _ = try object.put(key, try p.parseString(allocator, s, input, i));
11601150 _ = p.stack.pop();
11611151 p.state = State.ObjectKey;
11621152 },
1163 Token.Id.Number => {
1164 _ = try object.put(key, try p.parseNumber(token, input, i));
1153 .Number => |n| {
1154 _ = try object.put(key, try p.parseNumber(n, input, i));
11651155 _ = p.stack.pop();
11661156 p.state = State.ObjectKey;
11671157 },
1168 Token.Id.True => {
1158 .True => {
11691159 _ = try object.put(key, Value{ .Bool = true });
11701160 _ = p.stack.pop();
11711161 p.state = State.ObjectKey;
11721162 },
1173 Token.Id.False => {
1163 .False => {
11741164 _ = try object.put(key, Value{ .Bool = false });
11751165 _ = p.stack.pop();
11761166 p.state = State.ObjectKey;
11771167 },
1178 Token.Id.Null => {
1168 .Null => {
11791169 _ = try object.put(key, Value.Null);
11801170 _ = p.stack.pop();
11811171 p.state = State.ObjectKey;
11821172 },
1183 Token.Id.ObjectEnd, Token.Id.ArrayEnd => {
1173 .ObjectEnd, .ArrayEnd => {
11841174 unreachable;
11851175 },
11861176 }
......@@ -1188,8 +1178,8 @@ pub const Parser = struct {
11881178 State.ArrayValue => {
11891179 var array = &p.stack.items[p.stack.len - 1].Array;
11901180
1191 switch (token.id) {
1192 Token.Id.ArrayEnd => {
1181 switch (token) {
1182 .ArrayEnd => {
11931183 if (p.stack.len == 1) {
11941184 return;
11951185 }
......@@ -1197,59 +1187,59 @@ pub const Parser = struct {
11971187 var value = p.stack.pop();
11981188 try p.pushToParent(&value);
11991189 },
1200 Token.Id.ObjectBegin => {
1190 .ObjectBegin => {
12011191 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
12021192 p.state = State.ObjectKey;
12031193 },
1204 Token.Id.ArrayBegin => {
1194 .ArrayBegin => {
12051195 try p.stack.append(Value{ .Array = Array.init(allocator) });
12061196 p.state = State.ArrayValue;
12071197 },
1208 Token.Id.String => {
1209 try array.append(try p.parseString(allocator, token, input, i));
1198 .String => |s| {
1199 try array.append(try p.parseString(allocator, s, input, i));
12101200 },
1211 Token.Id.Number => {
1212 try array.append(try p.parseNumber(token, input, i));
1201 .Number => |n| {
1202 try array.append(try p.parseNumber(n, input, i));
12131203 },
1214 Token.Id.True => {
1204 .True => {
12151205 try array.append(Value{ .Bool = true });
12161206 },
1217 Token.Id.False => {
1207 .False => {
12181208 try array.append(Value{ .Bool = false });
12191209 },
1220 Token.Id.Null => {
1210 .Null => {
12211211 try array.append(Value.Null);
12221212 },
1223 Token.Id.ObjectEnd => {
1213 .ObjectEnd => {
12241214 unreachable;
12251215 },
12261216 }
12271217 },
1228 State.Simple => switch (token.id) {
1229 Token.Id.ObjectBegin => {
1218 State.Simple => switch (token) {
1219 .ObjectBegin => {
12301220 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
12311221 p.state = State.ObjectKey;
12321222 },
1233 Token.Id.ArrayBegin => {
1223 .ArrayBegin => {
12341224 try p.stack.append(Value{ .Array = Array.init(allocator) });
12351225 p.state = State.ArrayValue;
12361226 },
1237 Token.Id.String => {
1238 try p.stack.append(try p.parseString(allocator, token, input, i));
1227 .String => |s| {
1228 try p.stack.append(try p.parseString(allocator, s, input, i));
12391229 },
1240 Token.Id.Number => {
1241 try p.stack.append(try p.parseNumber(token, input, i));
1230 .Number => |n| {
1231 try p.stack.append(try p.parseNumber(n, input, i));
12421232 },
1243 Token.Id.True => {
1233 .True => {
12441234 try p.stack.append(Value{ .Bool = true });
12451235 },
1246 Token.Id.False => {
1236 .False => {
12471237 try p.stack.append(Value{ .Bool = false });
12481238 },
1249 Token.Id.Null => {
1239 .Null => {
12501240 try p.stack.append(Value.Null);
12511241 },
1252 Token.Id.ObjectEnd, Token.Id.ArrayEnd => {
1242 .ObjectEnd, .ArrayEnd => {
12531243 unreachable;
12541244 },
12551245 },
......@@ -1277,18 +1267,18 @@ pub const Parser = struct {
12771267 }
12781268 }
12791269
1280 fn parseString(p: *Parser, allocator: *Allocator, token: Token, input: []const u8, i: usize) !Value {
1270 fn parseString(p: *Parser, allocator: *Allocator, s: std.meta.TagPayloadType(Token, Token.String), input: []const u8, i: usize) !Value {
12811271 // TODO: We don't strictly have to copy values which do not contain any escape
12821272 // characters if flagged with the option.
1283 const slice = token.slice(input, i);
1273 const slice = s.slice(input, i);
12841274 return Value{ .String = try unescapeStringAlloc(allocator, slice) };
12851275 }
12861276
1287 fn parseNumber(p: *Parser, token: Token, input: []const u8, i: usize) !Value {
1288 return if (token.number_is_integer)
1289 Value{ .Integer = try std.fmt.parseInt(i64, token.slice(input, i), 10) }
1277 fn parseNumber(p: *Parser, n: std.meta.TagPayloadType(Token, Token.Number), input: []const u8, i: usize) !Value {
1278 return if (n.is_integer)
1279 Value{ .Integer = try std.fmt.parseInt(i64, n.slice(input, i), 10) }
12901280 else
1291 Value{ .Float = try std.fmt.parseFloat(f64, token.slice(input, i)) };
1281 Value{ .Float = try std.fmt.parseFloat(f64, n.slice(input, i)) };
12921282 }
12931283};
12941284