authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-30 18:13:20-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-12-30 18:13:20-05:00
log8f8a32d297e872acb9e917559228a4a7675cd680
treebd8bb74e3414d6e78684df604472f00992aebc51
parent28a8ded95a96b1e2af5d2a73f47db2c967233476
parent42727c73f92b7b3453de643ecd13eeb35ce00b72
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4007 from daurnimator/json-cleanup

std.json: cleanups

2 files changed, 468 insertions(+), 371 deletions(-)

lib/std/json.zig+467-368
......@@ -4,83 +4,63 @@
44
55const std = @import("std.zig");
66const debug = std.debug;
7const assert = debug.assert;
78const testing = std.testing;
89const mem = std.mem;
910const maxInt = std.math.maxInt;
1011
1112pub const WriteStream = @import("json/write_stream.zig").WriteStream;
1213
14const StringEscapes = union(enum) {
15 None,
16
17 Some: struct {
18 size_diff: isize,
19 },
20};
21
1322/// A single token slice into the parent string.
1423///
1524/// Use `token.slice()` on the input at the current position to get the current slice.
16pub const Token = struct {
17 id: Id,
18 /// How many bytes do we skip before counting
19 offset: u1,
20 /// Whether string contains an escape sequence and cannot be zero-copied
21 string_has_escape: bool,
22 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
23 number_is_integer: bool,
24 /// How many bytes from the current position behind the start of this token is.
25 count: usize,
26
27 pub const Id = enum {
28 ObjectBegin,
29 ObjectEnd,
30 ArrayBegin,
31 ArrayEnd,
32 String,
33 Number,
34 True,
35 False,
36 Null,
37 };
38
39 pub fn init(id: Id, count: usize, offset: u1) Token {
40 return Token{
41 .id = id,
42 .offset = offset,
43 .string_has_escape = false,
44 .number_is_integer = true,
45 .count = count,
46 };
47 }
48
49 pub fn initString(count: usize, has_unicode_escape: bool) Token {
50 return Token{
51 .id = Id.String,
52 .offset = 0,
53 .string_has_escape = has_unicode_escape,
54 .number_is_integer = true,
55 .count = count,
56 };
57 }
25pub const Token = union(enum) {
26 ObjectBegin,
27 ObjectEnd,
28 ArrayBegin,
29 ArrayEnd,
30 String: struct {
31 /// How many bytes the token is.
32 count: usize,
33
34 /// Whether string contains an escape sequence and cannot be zero-copied
35 escapes: StringEscapes,
36
37 pub fn decodedLength(self: @This()) usize {
38 return self.count +% switch (self.escapes) {
39 .None => 0,
40 .Some => |s| @bitCast(usize, s.size_diff),
41 };
42 }
5843
59 pub fn initNumber(count: usize, number_is_integer: bool) Token {
60 return Token{
61 .id = Id.Number,
62 .offset = 0,
63 .string_has_escape = false,
64 .number_is_integer = number_is_integer,
65 .count = count,
66 };
67 }
44 /// Slice into the underlying input string.
45 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
46 return input[i - self.count .. i];
47 }
48 },
49 Number: struct {
50 /// How many bytes the token is.
51 count: usize,
6852
69 /// A marker token is a zero-length
70 pub fn initMarker(id: Id) Token {
71 return Token{
72 .id = id,
73 .offset = 0,
74 .string_has_escape = false,
75 .number_is_integer = true,
76 .count = 0,
77 };
78 }
53 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
54 is_integer: bool,
7955
80 /// Slice into the underlying input string.
81 pub fn slice(self: Token, input: []const u8, i: usize) []const u8 {
82 return input[i + self.offset - self.count .. i + self.offset];
83 }
56 /// Slice into the underlying input string.
57 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
58 return input[i - self.count .. i];
59 }
60 },
61 True,
62 False,
63 Null,
8464};
8565
8666/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
......@@ -102,7 +82,12 @@ pub const StreamingParser = struct {
10282 // If we stopped now, would the complete parsed string to now be a valid json string
10383 complete: bool,
10484 // Current token flags to pass through to the next generated, see Token.
105 string_has_escape: bool,
85 string_escapes: StringEscapes,
86 // When in .String states, was the previous character a high surrogate?
87 string_last_was_high_surrogate: bool,
88 // Used inside of StringEscapeHexUnicode* states
89 string_unicode_codepoint: u21,
90 // When in .Number states, is the number a (still) valid integer?
10691 number_is_integer: bool,
10792
10893 // Bit-stack for nested object/map literals (max 255 nestings).
......@@ -120,16 +105,18 @@ pub const StreamingParser = struct {
120105 }
121106
122107 pub fn reset(p: *StreamingParser) void {
123 p.state = State.TopLevelBegin;
108 p.state = .TopLevelBegin;
124109 p.count = 0;
125110 // Set before ever read in main transition function
126111 p.after_string_state = undefined;
127 p.after_value_state = State.ValueEnd; // handle end of values normally
112 p.after_value_state = .ValueEnd; // handle end of values normally
128113 p.stack = 0;
129114 p.stack_used = 0;
130115 p.complete = false;
131 p.string_has_escape = false;
132 p.number_is_integer = true;
116 p.string_escapes = undefined;
117 p.string_last_was_high_surrogate = undefined;
118 p.string_unicode_codepoint = undefined;
119 p.number_is_integer = undefined;
133120 }
134121
135122 pub const State = enum {
......@@ -223,66 +210,67 @@ pub const StreamingParser = struct {
223210 // Perform a single transition on the state machine and return any possible token.
224211 fn transition(p: *StreamingParser, c: u8, token: *?Token) Error!bool {
225212 switch (p.state) {
226 State.TopLevelBegin => switch (c) {
213 .TopLevelBegin => switch (c) {
227214 '{' => {
228215 p.stack <<= 1;
229216 p.stack |= object_bit;
230217 p.stack_used += 1;
231218
232 p.state = State.ValueBegin;
233 p.after_string_state = State.ObjectSeparator;
219 p.state = .ValueBegin;
220 p.after_string_state = .ObjectSeparator;
234221
235 token.* = Token.initMarker(Token.Id.ObjectBegin);
222 token.* = Token.ObjectBegin;
236223 },
237224 '[' => {
238225 p.stack <<= 1;
239226 p.stack |= array_bit;
240227 p.stack_used += 1;
241228
242 p.state = State.ValueBegin;
243 p.after_string_state = State.ValueEnd;
229 p.state = .ValueBegin;
230 p.after_string_state = .ValueEnd;
244231
245 token.* = Token.initMarker(Token.Id.ArrayBegin);
232 token.* = Token.ArrayBegin;
246233 },
247234 '-' => {
248235 p.number_is_integer = true;
249 p.state = State.Number;
250 p.after_value_state = State.TopLevelEnd;
236 p.state = .Number;
237 p.after_value_state = .TopLevelEnd;
251238 p.count = 0;
252239 },
253240 '0' => {
254241 p.number_is_integer = true;
255 p.state = State.NumberMaybeDotOrExponent;
256 p.after_value_state = State.TopLevelEnd;
242 p.state = .NumberMaybeDotOrExponent;
243 p.after_value_state = .TopLevelEnd;
257244 p.count = 0;
258245 },
259246 '1'...'9' => {
260247 p.number_is_integer = true;
261 p.state = State.NumberMaybeDigitOrDotOrExponent;
262 p.after_value_state = State.TopLevelEnd;
248 p.state = .NumberMaybeDigitOrDotOrExponent;
249 p.after_value_state = .TopLevelEnd;
263250 p.count = 0;
264251 },
265252 '"' => {
266 p.state = State.String;
267 p.after_value_state = State.TopLevelEnd;
253 p.state = .String;
254 p.after_value_state = .TopLevelEnd;
268255 // We don't actually need the following since after_value_state should override.
269 p.after_string_state = State.ValueEnd;
270 p.string_has_escape = false;
256 p.after_string_state = .ValueEnd;
257 p.string_escapes = .None;
258 p.string_last_was_high_surrogate = false;
271259 p.count = 0;
272260 },
273261 't' => {
274 p.state = State.TrueLiteral1;
275 p.after_value_state = State.TopLevelEnd;
262 p.state = .TrueLiteral1;
263 p.after_value_state = .TopLevelEnd;
276264 p.count = 0;
277265 },
278266 'f' => {
279 p.state = State.FalseLiteral1;
280 p.after_value_state = State.TopLevelEnd;
267 p.state = .FalseLiteral1;
268 p.after_value_state = .TopLevelEnd;
281269 p.count = 0;
282270 },
283271 'n' => {
284 p.state = State.NullLiteral1;
285 p.after_value_state = State.TopLevelEnd;
272 p.state = .NullLiteral1;
273 p.after_value_state = .TopLevelEnd;
286274 p.count = 0;
287275 },
288276 0x09, 0x0A, 0x0D, 0x20 => {
......@@ -293,7 +281,7 @@ pub const StreamingParser = struct {
293281 },
294282 },
295283
296 State.TopLevelEnd => switch (c) {
284 .TopLevelEnd => switch (c) {
297285 0x09, 0x0A, 0x0D, 0x20 => {
298286 // whitespace
299287 },
......@@ -302,7 +290,7 @@ pub const StreamingParser = struct {
302290 },
303291 },
304292
305 State.ValueBegin => switch (c) {
293 .ValueBegin => switch (c) {
306294 // NOTE: These are shared in ValueEnd as well, think we can reorder states to
307295 // be a bit clearer and avoid this duplication.
308296 '}' => {
......@@ -314,7 +302,7 @@ pub const StreamingParser = struct {
314302 return error.TooManyClosingItems;
315303 }
316304
317 p.state = State.ValueBegin;
305 p.state = .ValueBegin;
318306 p.after_string_state = State.fromInt(p.stack & 1);
319307
320308 p.stack >>= 1;
......@@ -323,14 +311,14 @@ pub const StreamingParser = struct {
323311 switch (p.stack_used) {
324312 0 => {
325313 p.complete = true;
326 p.state = State.TopLevelEnd;
314 p.state = .TopLevelEnd;
327315 },
328316 else => {
329 p.state = State.ValueEnd;
317 p.state = .ValueEnd;
330318 },
331319 }
332320
333 token.* = Token.initMarker(Token.Id.ObjectEnd);
321 token.* = Token.ObjectEnd;
334322 },
335323 ']' => {
336324 if (p.stack & 1 != array_bit) {
......@@ -340,7 +328,7 @@ pub const StreamingParser = struct {
340328 return error.TooManyClosingItems;
341329 }
342330
343 p.state = State.ValueBegin;
331 p.state = .ValueBegin;
344332 p.after_string_state = State.fromInt(p.stack & 1);
345333
346334 p.stack >>= 1;
......@@ -349,14 +337,14 @@ pub const StreamingParser = struct {
349337 switch (p.stack_used) {
350338 0 => {
351339 p.complete = true;
352 p.state = State.TopLevelEnd;
340 p.state = .TopLevelEnd;
353341 },
354342 else => {
355 p.state = State.ValueEnd;
343 p.state = .ValueEnd;
356344 },
357345 }
358346
359 token.* = Token.initMarker(Token.Id.ArrayEnd);
347 token.* = Token.ArrayEnd;
360348 },
361349 '{' => {
362350 if (p.stack_used == max_stack_size) {
......@@ -367,10 +355,10 @@ pub const StreamingParser = struct {
367355 p.stack |= object_bit;
368356 p.stack_used += 1;
369357
370 p.state = State.ValueBegin;
371 p.after_string_state = State.ObjectSeparator;
358 p.state = .ValueBegin;
359 p.after_string_state = .ObjectSeparator;
372360
373 token.* = Token.initMarker(Token.Id.ObjectBegin);
361 token.* = Token.ObjectBegin;
374362 },
375363 '[' => {
376364 if (p.stack_used == max_stack_size) {
......@@ -381,40 +369,42 @@ pub const StreamingParser = struct {
381369 p.stack |= array_bit;
382370 p.stack_used += 1;
383371
384 p.state = State.ValueBegin;
385 p.after_string_state = State.ValueEnd;
372 p.state = .ValueBegin;
373 p.after_string_state = .ValueEnd;
386374
387 token.* = Token.initMarker(Token.Id.ArrayBegin);
375 token.* = Token.ArrayBegin;
388376 },
389377 '-' => {
390378 p.number_is_integer = true;
391 p.state = State.Number;
379 p.state = .Number;
392380 p.count = 0;
393381 },
394382 '0' => {
395383 p.number_is_integer = true;
396 p.state = State.NumberMaybeDotOrExponent;
384 p.state = .NumberMaybeDotOrExponent;
397385 p.count = 0;
398386 },
399387 '1'...'9' => {
400388 p.number_is_integer = true;
401 p.state = State.NumberMaybeDigitOrDotOrExponent;
389 p.state = .NumberMaybeDigitOrDotOrExponent;
402390 p.count = 0;
403391 },
404392 '"' => {
405 p.state = State.String;
393 p.state = .String;
394 p.string_escapes = .None;
395 p.string_last_was_high_surrogate = false;
406396 p.count = 0;
407397 },
408398 't' => {
409 p.state = State.TrueLiteral1;
399 p.state = .TrueLiteral1;
410400 p.count = 0;
411401 },
412402 'f' => {
413 p.state = State.FalseLiteral1;
403 p.state = .FalseLiteral1;
414404 p.count = 0;
415405 },
416406 'n' => {
417 p.state = State.NullLiteral1;
407 p.state = .NullLiteral1;
418408 p.count = 0;
419409 },
420410 0x09, 0x0A, 0x0D, 0x20 => {
......@@ -426,7 +416,7 @@ pub const StreamingParser = struct {
426416 },
427417
428418 // TODO: A bit of duplication here and in the following state, redo.
429 State.ValueBeginNoClosing => switch (c) {
419 .ValueBeginNoClosing => switch (c) {
430420 '{' => {
431421 if (p.stack_used == max_stack_size) {
432422 return error.TooManyNestedItems;
......@@ -436,10 +426,10 @@ pub const StreamingParser = struct {
436426 p.stack |= object_bit;
437427 p.stack_used += 1;
438428
439 p.state = State.ValueBegin;
440 p.after_string_state = State.ObjectSeparator;
429 p.state = .ValueBegin;
430 p.after_string_state = .ObjectSeparator;
441431
442 token.* = Token.initMarker(Token.Id.ObjectBegin);
432 token.* = Token.ObjectBegin;
443433 },
444434 '[' => {
445435 if (p.stack_used == max_stack_size) {
......@@ -450,40 +440,42 @@ pub const StreamingParser = struct {
450440 p.stack |= array_bit;
451441 p.stack_used += 1;
452442
453 p.state = State.ValueBegin;
454 p.after_string_state = State.ValueEnd;
443 p.state = .ValueBegin;
444 p.after_string_state = .ValueEnd;
455445
456 token.* = Token.initMarker(Token.Id.ArrayBegin);
446 token.* = Token.ArrayBegin;
457447 },
458448 '-' => {
459449 p.number_is_integer = true;
460 p.state = State.Number;
450 p.state = .Number;
461451 p.count = 0;
462452 },
463453 '0' => {
464454 p.number_is_integer = true;
465 p.state = State.NumberMaybeDotOrExponent;
455 p.state = .NumberMaybeDotOrExponent;
466456 p.count = 0;
467457 },
468458 '1'...'9' => {
469459 p.number_is_integer = true;
470 p.state = State.NumberMaybeDigitOrDotOrExponent;
460 p.state = .NumberMaybeDigitOrDotOrExponent;
471461 p.count = 0;
472462 },
473463 '"' => {
474 p.state = State.String;
464 p.state = .String;
465 p.string_escapes = .None;
466 p.string_last_was_high_surrogate = false;
475467 p.count = 0;
476468 },
477469 't' => {
478 p.state = State.TrueLiteral1;
470 p.state = .TrueLiteral1;
479471 p.count = 0;
480472 },
481473 'f' => {
482 p.state = State.FalseLiteral1;
474 p.state = .FalseLiteral1;
483475 p.count = 0;
484476 },
485477 'n' => {
486 p.state = State.NullLiteral1;
478 p.state = .NullLiteral1;
487479 p.count = 0;
488480 },
489481 0x09, 0x0A, 0x0D, 0x20 => {
......@@ -494,17 +486,17 @@ pub const StreamingParser = struct {
494486 },
495487 },
496488
497 State.ValueEnd => switch (c) {
489 .ValueEnd => switch (c) {
498490 ',' => {
499491 p.after_string_state = State.fromInt(p.stack & 1);
500 p.state = State.ValueBeginNoClosing;
492 p.state = .ValueBeginNoClosing;
501493 },
502494 ']' => {
503495 if (p.stack_used == 0) {
504496 return error.UnbalancedBrackets;
505497 }
506498
507 p.state = State.ValueEnd;
499 p.state = .ValueEnd;
508500 p.after_string_state = State.fromInt(p.stack & 1);
509501
510502 p.stack >>= 1;
......@@ -512,17 +504,17 @@ pub const StreamingParser = struct {
512504
513505 if (p.stack_used == 0) {
514506 p.complete = true;
515 p.state = State.TopLevelEnd;
507 p.state = .TopLevelEnd;
516508 }
517509
518 token.* = Token.initMarker(Token.Id.ArrayEnd);
510 token.* = Token.ArrayEnd;
519511 },
520512 '}' => {
521513 if (p.stack_used == 0) {
522514 return error.UnbalancedBraces;
523515 }
524516
525 p.state = State.ValueEnd;
517 p.state = .ValueEnd;
526518 p.after_string_state = State.fromInt(p.stack & 1);
527519
528520 p.stack >>= 1;
......@@ -530,10 +522,10 @@ pub const StreamingParser = struct {
530522
531523 if (p.stack_used == 0) {
532524 p.complete = true;
533 p.state = State.TopLevelEnd;
525 p.state = .TopLevelEnd;
534526 }
535527
536 token.* = Token.initMarker(Token.Id.ObjectEnd);
528 token.* = Token.ObjectEnd;
537529 },
538530 0x09, 0x0A, 0x0D, 0x20 => {
539531 // whitespace
......@@ -543,10 +535,10 @@ pub const StreamingParser = struct {
543535 },
544536 },
545537
546 State.ObjectSeparator => switch (c) {
538 .ObjectSeparator => switch (c) {
547539 ':' => {
548 p.state = State.ValueBegin;
549 p.after_string_state = State.ValueEnd;
540 p.state = .ValueBegin;
541 p.after_string_state = .ValueEnd;
550542 },
551543 0x09, 0x0A, 0x0D, 0x20 => {
552544 // whitespace
......@@ -556,55 +548,72 @@ pub const StreamingParser = struct {
556548 },
557549 },
558550
559 State.String => switch (c) {
551 .String => switch (c) {
560552 0x00...0x1F => {
561553 return error.InvalidControlCharacter;
562554 },
563555 '"' => {
564556 p.state = p.after_string_state;
565 if (p.after_value_state == State.TopLevelEnd) {
566 p.state = State.TopLevelEnd;
557 if (p.after_value_state == .TopLevelEnd) {
558 p.state = .TopLevelEnd;
567559 p.complete = true;
568560 }
569561
570 token.* = Token.initString(p.count - 1, p.string_has_escape);
562 token.* = .{
563 .String = .{
564 .count = p.count - 1,
565 .escapes = p.string_escapes,
566 },
567 };
568 p.string_escapes = undefined;
569 p.string_last_was_high_surrogate = undefined;
571570 },
572571 '\\' => {
573 p.state = State.StringEscapeCharacter;
572 p.state = .StringEscapeCharacter;
573 switch (p.string_escapes) {
574 .None => {
575 p.string_escapes = .{ .Some = .{ .size_diff = 0 } };
576 },
577 .Some => {},
578 }
574579 },
575580 0x20, 0x21, 0x23...0x5B, 0x5D...0x7F => {
576581 // non-control ascii
582 p.string_last_was_high_surrogate = false;
577583 },
578584 0xC0...0xDF => {
579 p.state = State.StringUtf8Byte1;
585 p.state = .StringUtf8Byte1;
580586 },
581587 0xE0...0xEF => {
582 p.state = State.StringUtf8Byte2;
588 p.state = .StringUtf8Byte2;
583589 },
584590 0xF0...0xFF => {
585 p.state = State.StringUtf8Byte3;
591 p.state = .StringUtf8Byte3;
586592 },
587593 else => {
588594 return error.InvalidUtf8Byte;
589595 },
590596 },
591597
592 State.StringUtf8Byte3 => switch (c >> 6) {
593 0b10 => p.state = State.StringUtf8Byte2,
598 .StringUtf8Byte3 => switch (c >> 6) {
599 0b10 => p.state = .StringUtf8Byte2,
594600 else => return error.InvalidUtf8Byte,
595601 },
596602
597 State.StringUtf8Byte2 => switch (c >> 6) {
598 0b10 => p.state = State.StringUtf8Byte1,
603 .StringUtf8Byte2 => switch (c >> 6) {
604 0b10 => p.state = .StringUtf8Byte1,
599605 else => return error.InvalidUtf8Byte,
600606 },
601607
602 State.StringUtf8Byte1 => switch (c >> 6) {
603 0b10 => p.state = State.String,
608 .StringUtf8Byte1 => switch (c >> 6) {
609 0b10 => {
610 p.state = .String;
611 p.string_last_was_high_surrogate = false;
612 },
604613 else => return error.InvalidUtf8Byte,
605614 },
606615
607 State.StringEscapeCharacter => switch (c) {
616 .StringEscapeCharacter => switch (c) {
608617 // NOTE: '/' is allowed as an escaped character but it also is allowed
609618 // as unescaped according to the RFC. There is a reported errata which suggests
610619 // removing the non-escaped variant but it makes more sense to simply disallow
......@@ -614,54 +623,121 @@ pub const StreamingParser = struct {
614623 // however, so we default to the status quo where both are accepted until this
615624 // is further clarified.
616625 '"', '\\', '/', 'b', 'f', 'n', 'r', 't' => {
617 p.string_has_escape = true;
618 p.state = State.String;
626 p.string_escapes.Some.size_diff -= 1;
627 p.state = .String;
628 p.string_last_was_high_surrogate = false;
619629 },
620630 'u' => {
621 p.string_has_escape = true;
622 p.state = State.StringEscapeHexUnicode4;
631 p.state = .StringEscapeHexUnicode4;
623632 },
624633 else => {
625634 return error.InvalidEscapeCharacter;
626635 },
627636 },
628637
629 State.StringEscapeHexUnicode4 => switch (c) {
630 '0'...'9', 'A'...'F', 'a'...'f' => {
631 p.state = State.StringEscapeHexUnicode3;
632 },
633 else => return error.InvalidUnicodeHexSymbol,
638 .StringEscapeHexUnicode4 => {
639 var codepoint: u21 = undefined;
640 switch (c) {
641 else => return error.InvalidUnicodeHexSymbol,
642 '0'...'9' => {
643 codepoint = c - '0';
644 },
645 'A'...'F' => {
646 codepoint = c - 'A' + 10;
647 },
648 'a'...'f' => {
649 codepoint = c - 'a' + 10;
650 },
651 }
652 p.state = .StringEscapeHexUnicode3;
653 p.string_unicode_codepoint = codepoint << 12;
634654 },
635655
636 State.StringEscapeHexUnicode3 => switch (c) {
637 '0'...'9', 'A'...'F', 'a'...'f' => {
638 p.state = State.StringEscapeHexUnicode2;
639 },
640 else => return error.InvalidUnicodeHexSymbol,
656 .StringEscapeHexUnicode3 => {
657 var codepoint: u21 = undefined;
658 switch (c) {
659 else => return error.InvalidUnicodeHexSymbol,
660 '0'...'9' => {
661 codepoint = c - '0';
662 },
663 'A'...'F' => {
664 codepoint = c - 'A' + 10;
665 },
666 'a'...'f' => {
667 codepoint = c - 'a' + 10;
668 },
669 }
670 p.state = .StringEscapeHexUnicode2;
671 p.string_unicode_codepoint |= codepoint << 8;
641672 },
642673
643 State.StringEscapeHexUnicode2 => switch (c) {
644 '0'...'9', 'A'...'F', 'a'...'f' => {
645 p.state = State.StringEscapeHexUnicode1;
646 },
647 else => return error.InvalidUnicodeHexSymbol,
674 .StringEscapeHexUnicode2 => {
675 var codepoint: u21 = undefined;
676 switch (c) {
677 else => return error.InvalidUnicodeHexSymbol,
678 '0'...'9' => {
679 codepoint = c - '0';
680 },
681 'A'...'F' => {
682 codepoint = c - 'A' + 10;
683 },
684 'a'...'f' => {
685 codepoint = c - 'a' + 10;
686 },
687 }
688 p.state = .StringEscapeHexUnicode1;
689 p.string_unicode_codepoint |= codepoint << 4;
648690 },
649691
650 State.StringEscapeHexUnicode1 => switch (c) {
651 '0'...'9', 'A'...'F', 'a'...'f' => {
652 p.state = State.String;
653 },
654 else => return error.InvalidUnicodeHexSymbol,
692 .StringEscapeHexUnicode1 => {
693 var codepoint: u21 = undefined;
694 switch (c) {
695 else => return error.InvalidUnicodeHexSymbol,
696 '0'...'9' => {
697 codepoint = c - '0';
698 },
699 'A'...'F' => {
700 codepoint = c - 'A' + 10;
701 },
702 'a'...'f' => {
703 codepoint = c - 'a' + 10;
704 },
705 }
706 p.state = .String;
707 p.string_unicode_codepoint |= codepoint;
708 if (p.string_unicode_codepoint < 0xD800 or p.string_unicode_codepoint >= 0xE000) {
709 // not part of surrogate pair
710 p.string_escapes.Some.size_diff -= @as(isize, 6 - (std.unicode.utf8CodepointSequenceLength(p.string_unicode_codepoint) catch unreachable));
711 p.string_last_was_high_surrogate = false;
712 } else if (p.string_unicode_codepoint < 0xDC00) {
713 // 'high' surrogate
714 // takes 3 bytes to encode a half surrogate pair into wtf8
715 p.string_escapes.Some.size_diff -= 6 - 3;
716 p.string_last_was_high_surrogate = true;
717 } else {
718 // 'low' surrogate
719 p.string_escapes.Some.size_diff -= 6;
720 if (p.string_last_was_high_surrogate) {
721 // takes 4 bytes to encode a full surrogate pair into utf8
722 // 3 bytes are already reserved by high surrogate
723 p.string_escapes.Some.size_diff -= -1;
724 } else {
725 // takes 3 bytes to encode a half surrogate pair into wtf8
726 p.string_escapes.Some.size_diff -= -3;
727 }
728 p.string_last_was_high_surrogate = false;
729 }
730 p.string_unicode_codepoint = undefined;
655731 },
656732
657 State.Number => {
658 p.complete = p.after_value_state == State.TopLevelEnd;
733 .Number => {
734 p.complete = p.after_value_state == .TopLevelEnd;
659735 switch (c) {
660736 '0' => {
661 p.state = State.NumberMaybeDotOrExponent;
737 p.state = .NumberMaybeDotOrExponent;
662738 },
663739 '1'...'9' => {
664 p.state = State.NumberMaybeDigitOrDotOrExponent;
740 p.state = .NumberMaybeDigitOrDotOrExponent;
665741 },
666742 else => {
667743 return error.InvalidNumber;
......@@ -669,52 +745,63 @@ pub const StreamingParser = struct {
669745 }
670746 },
671747
672 State.NumberMaybeDotOrExponent => {
673 p.complete = p.after_value_state == State.TopLevelEnd;
748 .NumberMaybeDotOrExponent => {
749 p.complete = p.after_value_state == .TopLevelEnd;
674750 switch (c) {
675751 '.' => {
676752 p.number_is_integer = false;
677 p.state = State.NumberFractionalRequired;
753 p.state = .NumberFractionalRequired;
678754 },
679755 'e', 'E' => {
680756 p.number_is_integer = false;
681 p.state = State.NumberExponent;
757 p.state = .NumberExponent;
682758 },
683759 else => {
684760 p.state = p.after_value_state;
685 token.* = Token.initNumber(p.count, p.number_is_integer);
761 token.* = .{
762 .Number = .{
763 .count = p.count,
764 .is_integer = p.number_is_integer,
765 },
766 };
767 p.number_is_integer = undefined;
686768 return true;
687769 },
688770 }
689771 },
690772
691 State.NumberMaybeDigitOrDotOrExponent => {
692 p.complete = p.after_value_state == State.TopLevelEnd;
773 .NumberMaybeDigitOrDotOrExponent => {
774 p.complete = p.after_value_state == .TopLevelEnd;
693775 switch (c) {
694776 '.' => {
695777 p.number_is_integer = false;
696 p.state = State.NumberFractionalRequired;
778 p.state = .NumberFractionalRequired;
697779 },
698780 'e', 'E' => {
699781 p.number_is_integer = false;
700 p.state = State.NumberExponent;
782 p.state = .NumberExponent;
701783 },
702784 '0'...'9' => {
703785 // another digit
704786 },
705787 else => {
706788 p.state = p.after_value_state;
707 token.* = Token.initNumber(p.count, p.number_is_integer);
789 token.* = .{
790 .Number = .{
791 .count = p.count,
792 .is_integer = p.number_is_integer,
793 },
794 };
708795 return true;
709796 },
710797 }
711798 },
712799
713 State.NumberFractionalRequired => {
714 p.complete = p.after_value_state == State.TopLevelEnd;
800 .NumberFractionalRequired => {
801 p.complete = p.after_value_state == .TopLevelEnd;
715802 switch (c) {
716803 '0'...'9' => {
717 p.state = State.NumberFractional;
804 p.state = .NumberFractional;
718805 },
719806 else => {
720807 return error.InvalidNumber;
......@@ -722,139 +809,154 @@ pub const StreamingParser = struct {
722809 }
723810 },
724811
725 State.NumberFractional => {
726 p.complete = p.after_value_state == State.TopLevelEnd;
812 .NumberFractional => {
813 p.complete = p.after_value_state == .TopLevelEnd;
727814 switch (c) {
728815 '0'...'9' => {
729816 // another digit
730817 },
731818 'e', 'E' => {
732819 p.number_is_integer = false;
733 p.state = State.NumberExponent;
820 p.state = .NumberExponent;
734821 },
735822 else => {
736823 p.state = p.after_value_state;
737 token.* = Token.initNumber(p.count, p.number_is_integer);
824 token.* = .{
825 .Number = .{
826 .count = p.count,
827 .is_integer = p.number_is_integer,
828 },
829 };
738830 return true;
739831 },
740832 }
741833 },
742834
743 State.NumberMaybeExponent => {
744 p.complete = p.after_value_state == State.TopLevelEnd;
835 .NumberMaybeExponent => {
836 p.complete = p.after_value_state == .TopLevelEnd;
745837 switch (c) {
746838 'e', 'E' => {
747839 p.number_is_integer = false;
748 p.state = State.NumberExponent;
840 p.state = .NumberExponent;
749841 },
750842 else => {
751843 p.state = p.after_value_state;
752 token.* = Token.initNumber(p.count, p.number_is_integer);
844 token.* = .{
845 .Number = .{
846 .count = p.count,
847 .is_integer = p.number_is_integer,
848 },
849 };
753850 return true;
754851 },
755852 }
756853 },
757854
758 State.NumberExponent => switch (c) {
855 .NumberExponent => switch (c) {
759856 '-', '+' => {
760857 p.complete = false;
761 p.state = State.NumberExponentDigitsRequired;
858 p.state = .NumberExponentDigitsRequired;
762859 },
763860 '0'...'9' => {
764 p.complete = p.after_value_state == State.TopLevelEnd;
765 p.state = State.NumberExponentDigits;
861 p.complete = p.after_value_state == .TopLevelEnd;
862 p.state = .NumberExponentDigits;
766863 },
767864 else => {
768865 return error.InvalidNumber;
769866 },
770867 },
771868
772 State.NumberExponentDigitsRequired => switch (c) {
869 .NumberExponentDigitsRequired => switch (c) {
773870 '0'...'9' => {
774 p.complete = p.after_value_state == State.TopLevelEnd;
775 p.state = State.NumberExponentDigits;
871 p.complete = p.after_value_state == .TopLevelEnd;
872 p.state = .NumberExponentDigits;
776873 },
777874 else => {
778875 return error.InvalidNumber;
779876 },
780877 },
781878
782 State.NumberExponentDigits => {
783 p.complete = p.after_value_state == State.TopLevelEnd;
879 .NumberExponentDigits => {
880 p.complete = p.after_value_state == .TopLevelEnd;
784881 switch (c) {
785882 '0'...'9' => {
786883 // another digit
787884 },
788885 else => {
789886 p.state = p.after_value_state;
790 token.* = Token.initNumber(p.count, p.number_is_integer);
887 token.* = .{
888 .Number = .{
889 .count = p.count,
890 .is_integer = p.number_is_integer,
891 },
892 };
791893 return true;
792894 },
793895 }
794896 },
795897
796 State.TrueLiteral1 => switch (c) {
797 'r' => p.state = State.TrueLiteral2,
898 .TrueLiteral1 => switch (c) {
899 'r' => p.state = .TrueLiteral2,
798900 else => return error.InvalidLiteral,
799901 },
800902
801 State.TrueLiteral2 => switch (c) {
802 'u' => p.state = State.TrueLiteral3,
903 .TrueLiteral2 => switch (c) {
904 'u' => p.state = .TrueLiteral3,
803905 else => return error.InvalidLiteral,
804906 },
805907
806 State.TrueLiteral3 => switch (c) {
908 .TrueLiteral3 => switch (c) {
807909 'e' => {
808910 p.state = p.after_value_state;
809 p.complete = p.state == State.TopLevelEnd;
810 token.* = Token.init(Token.Id.True, p.count + 1, 1);
911 p.complete = p.state == .TopLevelEnd;
912 token.* = Token.True;
811913 },
812914 else => {
813915 return error.InvalidLiteral;
814916 },
815917 },
816918
817 State.FalseLiteral1 => switch (c) {
818 'a' => p.state = State.FalseLiteral2,
919 .FalseLiteral1 => switch (c) {
920 'a' => p.state = .FalseLiteral2,
819921 else => return error.InvalidLiteral,
820922 },
821923
822 State.FalseLiteral2 => switch (c) {
823 'l' => p.state = State.FalseLiteral3,
924 .FalseLiteral2 => switch (c) {
925 'l' => p.state = .FalseLiteral3,
824926 else => return error.InvalidLiteral,
825927 },
826928
827 State.FalseLiteral3 => switch (c) {
828 's' => p.state = State.FalseLiteral4,
929 .FalseLiteral3 => switch (c) {
930 's' => p.state = .FalseLiteral4,
829931 else => return error.InvalidLiteral,
830932 },
831933
832 State.FalseLiteral4 => switch (c) {
934 .FalseLiteral4 => switch (c) {
833935 'e' => {
834936 p.state = p.after_value_state;
835 p.complete = p.state == State.TopLevelEnd;
836 token.* = Token.init(Token.Id.False, p.count + 1, 1);
937 p.complete = p.state == .TopLevelEnd;
938 token.* = Token.False;
837939 },
838940 else => {
839941 return error.InvalidLiteral;
840942 },
841943 },
842944
843 State.NullLiteral1 => switch (c) {
844 'u' => p.state = State.NullLiteral2,
945 .NullLiteral1 => switch (c) {
946 'u' => p.state = .NullLiteral2,
845947 else => return error.InvalidLiteral,
846948 },
847949
848 State.NullLiteral2 => switch (c) {
849 'l' => p.state = State.NullLiteral3,
950 .NullLiteral2 => switch (c) {
951 'l' => p.state = .NullLiteral3,
850952 else => return error.InvalidLiteral,
851953 },
852954
853 State.NullLiteral3 => switch (c) {
955 .NullLiteral3 => switch (c) {
854956 'l' => {
855957 p.state = p.after_value_state;
856 p.complete = p.state == State.TopLevelEnd;
857 token.* = Token.init(Token.Id.Null, p.count + 1, 1);
958 p.complete = p.state == .TopLevelEnd;
959 token.* = Token.Null;
858960 },
859961 else => {
860962 return error.InvalidLiteral;
......@@ -905,7 +1007,7 @@ pub const TokenStream = struct {
9051007 }
9061008 }
9071009
908 // Without this a bare number fails, becasue the streaming parser doesn't know it ended
1010 // Without this a bare number fails, the streaming parser doesn't know the input ended
9091011 try self.parser.feed(' ', &t1, &t2);
9101012 self.i += 1;
9111013
......@@ -919,9 +1021,9 @@ pub const TokenStream = struct {
9191021 }
9201022};
9211023
922fn checkNext(p: *TokenStream, id: Token.Id) void {
1024fn checkNext(p: *TokenStream, id: std.meta.TagType(Token)) void {
9231025 const token = (p.next() catch unreachable).?;
924 debug.assert(token.id == id);
1026 debug.assert(std.meta.activeTag(token) == id);
9251027}
9261028
9271029test "json.token" {
......@@ -944,35 +1046,35 @@ test "json.token" {
9441046
9451047 var p = TokenStream.init(s);
9461048
947 checkNext(&p, Token.Id.ObjectBegin);
948 checkNext(&p, Token.Id.String); // Image
949 checkNext(&p, Token.Id.ObjectBegin);
950 checkNext(&p, Token.Id.String); // Width
951 checkNext(&p, Token.Id.Number);
952 checkNext(&p, Token.Id.String); // Height
953 checkNext(&p, Token.Id.Number);
954 checkNext(&p, Token.Id.String); // Title
955 checkNext(&p, Token.Id.String);
956 checkNext(&p, Token.Id.String); // Thumbnail
957 checkNext(&p, Token.Id.ObjectBegin);
958 checkNext(&p, Token.Id.String); // Url
959 checkNext(&p, Token.Id.String);
960 checkNext(&p, Token.Id.String); // Height
961 checkNext(&p, Token.Id.Number);
962 checkNext(&p, Token.Id.String); // Width
963 checkNext(&p, Token.Id.Number);
964 checkNext(&p, Token.Id.ObjectEnd);
965 checkNext(&p, Token.Id.String); // Animated
966 checkNext(&p, Token.Id.False);
967 checkNext(&p, Token.Id.String); // IDs
968 checkNext(&p, Token.Id.ArrayBegin);
969 checkNext(&p, Token.Id.Number);
970 checkNext(&p, Token.Id.Number);
971 checkNext(&p, Token.Id.Number);
972 checkNext(&p, Token.Id.Number);
973 checkNext(&p, Token.Id.ArrayEnd);
974 checkNext(&p, Token.Id.ObjectEnd);
975 checkNext(&p, Token.Id.ObjectEnd);
1049 checkNext(&p, .ObjectBegin);
1050 checkNext(&p, .String); // Image
1051 checkNext(&p, .ObjectBegin);
1052 checkNext(&p, .String); // Width
1053 checkNext(&p, .Number);
1054 checkNext(&p, .String); // Height
1055 checkNext(&p, .Number);
1056 checkNext(&p, .String); // Title
1057 checkNext(&p, .String);
1058 checkNext(&p, .String); // Thumbnail
1059 checkNext(&p, .ObjectBegin);
1060 checkNext(&p, .String); // Url
1061 checkNext(&p, .String);
1062 checkNext(&p, .String); // Height
1063 checkNext(&p, .Number);
1064 checkNext(&p, .String); // Width
1065 checkNext(&p, .Number);
1066 checkNext(&p, .ObjectEnd);
1067 checkNext(&p, .String); // Animated
1068 checkNext(&p, .False);
1069 checkNext(&p, .String); // IDs
1070 checkNext(&p, .ArrayBegin);
1071 checkNext(&p, .Number);
1072 checkNext(&p, .Number);
1073 checkNext(&p, .Number);
1074 checkNext(&p, .Number);
1075 checkNext(&p, .ArrayEnd);
1076 checkNext(&p, .ObjectEnd);
1077 checkNext(&p, .ObjectEnd);
9761078
9771079 testing.expect((try p.next()) == null);
9781080}
......@@ -1081,7 +1183,7 @@ pub const Parser = struct {
10811183 pub fn init(allocator: *Allocator, copy_strings: bool) Parser {
10821184 return Parser{
10831185 .allocator = allocator,
1084 .state = State.Simple,
1186 .state = .Simple,
10851187 .copy_strings = copy_strings,
10861188 .stack = Array.init(allocator),
10871189 };
......@@ -1092,7 +1194,7 @@ pub const Parser = struct {
10921194 }
10931195
10941196 pub fn reset(p: *Parser) void {
1095 p.state = State.Simple;
1197 p.state = .Simple;
10961198 p.stack.shrink(0);
10971199 }
10981200
......@@ -1118,8 +1220,8 @@ pub const Parser = struct {
11181220 // can be cleaned up on error correctly during a `parse` on call.
11191221 fn transition(p: *Parser, allocator: *Allocator, input: []const u8, i: usize, token: Token) !void {
11201222 switch (p.state) {
1121 State.ObjectKey => switch (token.id) {
1122 Token.Id.ObjectEnd => {
1223 .ObjectKey => switch (token) {
1224 .ObjectEnd => {
11231225 if (p.stack.len == 1) {
11241226 return;
11251227 }
......@@ -1127,9 +1229,9 @@ pub const Parser = struct {
11271229 var value = p.stack.pop();
11281230 try p.pushToParent(&value);
11291231 },
1130 Token.Id.String => {
1131 try p.stack.append(try p.parseString(allocator, token, input, i));
1132 p.state = State.ObjectValue;
1232 .String => |s| {
1233 try p.stack.append(try p.parseString(allocator, s, input, i));
1234 p.state = .ObjectValue;
11331235 },
11341236 else => {
11351237 // The streaming parser would return an error eventually.
......@@ -1138,54 +1240,54 @@ pub const Parser = struct {
11381240 return error.InvalidLiteral;
11391241 },
11401242 },
1141 State.ObjectValue => {
1243 .ObjectValue => {
11421244 var object = &p.stack.items[p.stack.len - 2].Object;
11431245 var key = p.stack.items[p.stack.len - 1].String;
11441246
1145 switch (token.id) {
1146 Token.Id.ObjectBegin => {
1247 switch (token) {
1248 .ObjectBegin => {
11471249 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1148 p.state = State.ObjectKey;
1250 p.state = .ObjectKey;
11491251 },
1150 Token.Id.ArrayBegin => {
1252 .ArrayBegin => {
11511253 try p.stack.append(Value{ .Array = Array.init(allocator) });
1152 p.state = State.ArrayValue;
1254 p.state = .ArrayValue;
11531255 },
1154 Token.Id.String => {
1155 _ = try object.put(key, try p.parseString(allocator, token, input, i));
1256 .String => |s| {
1257 _ = try object.put(key, try p.parseString(allocator, s, input, i));
11561258 _ = p.stack.pop();
1157 p.state = State.ObjectKey;
1259 p.state = .ObjectKey;
11581260 },
1159 Token.Id.Number => {
1160 _ = try object.put(key, try p.parseNumber(token, input, i));
1261 .Number => |n| {
1262 _ = try object.put(key, try p.parseNumber(n, input, i));
11611263 _ = p.stack.pop();
1162 p.state = State.ObjectKey;
1264 p.state = .ObjectKey;
11631265 },
1164 Token.Id.True => {
1266 .True => {
11651267 _ = try object.put(key, Value{ .Bool = true });
11661268 _ = p.stack.pop();
1167 p.state = State.ObjectKey;
1269 p.state = .ObjectKey;
11681270 },
1169 Token.Id.False => {
1271 .False => {
11701272 _ = try object.put(key, Value{ .Bool = false });
11711273 _ = p.stack.pop();
1172 p.state = State.ObjectKey;
1274 p.state = .ObjectKey;
11731275 },
1174 Token.Id.Null => {
1276 .Null => {
11751277 _ = try object.put(key, Value.Null);
11761278 _ = p.stack.pop();
1177 p.state = State.ObjectKey;
1279 p.state = .ObjectKey;
11781280 },
1179 Token.Id.ObjectEnd, Token.Id.ArrayEnd => {
1281 .ObjectEnd, .ArrayEnd => {
11801282 unreachable;
11811283 },
11821284 }
11831285 },
1184 State.ArrayValue => {
1286 .ArrayValue => {
11851287 var array = &p.stack.items[p.stack.len - 1].Array;
11861288
1187 switch (token.id) {
1188 Token.Id.ArrayEnd => {
1289 switch (token) {
1290 .ArrayEnd => {
11891291 if (p.stack.len == 1) {
11901292 return;
11911293 }
......@@ -1193,59 +1295,59 @@ pub const Parser = struct {
11931295 var value = p.stack.pop();
11941296 try p.pushToParent(&value);
11951297 },
1196 Token.Id.ObjectBegin => {
1298 .ObjectBegin => {
11971299 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1198 p.state = State.ObjectKey;
1300 p.state = .ObjectKey;
11991301 },
1200 Token.Id.ArrayBegin => {
1302 .ArrayBegin => {
12011303 try p.stack.append(Value{ .Array = Array.init(allocator) });
1202 p.state = State.ArrayValue;
1304 p.state = .ArrayValue;
12031305 },
1204 Token.Id.String => {
1205 try array.append(try p.parseString(allocator, token, input, i));
1306 .String => |s| {
1307 try array.append(try p.parseString(allocator, s, input, i));
12061308 },
1207 Token.Id.Number => {
1208 try array.append(try p.parseNumber(token, input, i));
1309 .Number => |n| {
1310 try array.append(try p.parseNumber(n, input, i));
12091311 },
1210 Token.Id.True => {
1312 .True => {
12111313 try array.append(Value{ .Bool = true });
12121314 },
1213 Token.Id.False => {
1315 .False => {
12141316 try array.append(Value{ .Bool = false });
12151317 },
1216 Token.Id.Null => {
1318 .Null => {
12171319 try array.append(Value.Null);
12181320 },
1219 Token.Id.ObjectEnd => {
1321 .ObjectEnd => {
12201322 unreachable;
12211323 },
12221324 }
12231325 },
1224 State.Simple => switch (token.id) {
1225 Token.Id.ObjectBegin => {
1326 .Simple => switch (token) {
1327 .ObjectBegin => {
12261328 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1227 p.state = State.ObjectKey;
1329 p.state = .ObjectKey;
12281330 },
1229 Token.Id.ArrayBegin => {
1331 .ArrayBegin => {
12301332 try p.stack.append(Value{ .Array = Array.init(allocator) });
1231 p.state = State.ArrayValue;
1333 p.state = .ArrayValue;
12321334 },
1233 Token.Id.String => {
1234 try p.stack.append(try p.parseString(allocator, token, input, i));
1335 .String => |s| {
1336 try p.stack.append(try p.parseString(allocator, s, input, i));
12351337 },
1236 Token.Id.Number => {
1237 try p.stack.append(try p.parseNumber(token, input, i));
1338 .Number => |n| {
1339 try p.stack.append(try p.parseNumber(n, input, i));
12381340 },
1239 Token.Id.True => {
1341 .True => {
12401342 try p.stack.append(Value{ .Bool = true });
12411343 },
1242 Token.Id.False => {
1344 .False => {
12431345 try p.stack.append(Value{ .Bool = false });
12441346 },
1245 Token.Id.Null => {
1347 .Null => {
12461348 try p.stack.append(Value.Null);
12471349 },
1248 Token.Id.ObjectEnd, Token.Id.ArrayEnd => {
1350 .ObjectEnd, .ArrayEnd => {
12491351 unreachable;
12501352 },
12511353 },
......@@ -1260,12 +1362,12 @@ pub const Parser = struct {
12601362
12611363 var object = &p.stack.items[p.stack.len - 1].Object;
12621364 _ = try object.put(key, value.*);
1263 p.state = State.ObjectKey;
1365 p.state = .ObjectKey;
12641366 },
12651367 // Array Parent -> [ ..., <array>, value ]
12661368 Value.Array => |*array| {
12671369 try array.append(value.*);
1268 p.state = State.ArrayValue;
1370 p.state = .ArrayValue;
12691371 },
12701372 else => {
12711373 unreachable;
......@@ -1273,80 +1375,78 @@ pub const Parser = struct {
12731375 }
12741376 }
12751377
1276 fn parseString(p: *Parser, allocator: *Allocator, token: Token, input: []const u8, i: usize) !Value {
1378 fn parseString(p: *Parser, allocator: *Allocator, s: std.meta.TagPayloadType(Token, Token.String), input: []const u8, i: usize) !Value {
12771379 // TODO: We don't strictly have to copy values which do not contain any escape
12781380 // characters if flagged with the option.
1279 const slice = token.slice(input, i);
1280 return Value{ .String = try unescapeStringAlloc(allocator, slice) };
1381 const slice = s.slice(input, i);
1382 switch (s.escapes) {
1383 .None => return Value{ .String = try mem.dupe(allocator, u8, slice) },
1384 .Some => |some_escapes| {
1385 const output = try allocator.alloc(u8, s.decodedLength());
1386 errdefer allocator.free(output);
1387 try unescapeString(output, slice);
1388 return Value{ .String = output };
1389 },
1390 }
12811391 }
12821392
1283 fn parseNumber(p: *Parser, token: Token, input: []const u8, i: usize) !Value {
1284 return if (token.number_is_integer)
1285 Value{ .Integer = try std.fmt.parseInt(i64, token.slice(input, i), 10) }
1393 fn parseNumber(p: *Parser, n: std.meta.TagPayloadType(Token, Token.Number), input: []const u8, i: usize) !Value {
1394 return if (n.is_integer)
1395 Value{ .Integer = try std.fmt.parseInt(i64, n.slice(input, i), 10) }
12861396 else
1287 Value{ .Float = try std.fmt.parseFloat(f64, token.slice(input, i)) };
1397 Value{ .Float = try std.fmt.parseFloat(f64, n.slice(input, i)) };
12881398 }
12891399};
12901400
12911401// Unescape a JSON string
12921402// Only to be used on strings already validated by the parser
12931403// (note the unreachable statements and lack of bounds checking)
1294// Optimized for arena allocators, uses Allocator.shrink
1295//
1296// Idea: count how many bytes we will need to allocate in the streaming parser and store it
1297// in the token to avoid allocating too much memory or iterating through the string again
1298// Downside: need to find how many bytes a unicode escape sequence will produce twice
1299fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1300 const output = try alloc.alloc(u8, input.len);
1301 errdefer alloc.free(output);
1302
1404fn unescapeString(output: []u8, input: []const u8) !void {
13031405 var inIndex: usize = 0;
13041406 var outIndex: usize = 0;
13051407
1306 while(inIndex < input.len) {
1307 if(input[inIndex] != '\\'){
1408 while (inIndex < input.len) {
1409 if (input[inIndex] != '\\') {
13081410 // not an escape sequence
13091411 output[outIndex] = input[inIndex];
13101412 inIndex += 1;
13111413 outIndex += 1;
1312 } else if(input[inIndex + 1] != 'u'){
1414 } else if (input[inIndex + 1] != 'u') {
13131415 // a simple escape sequence
1314 output[outIndex] = @as(u8,
1315 switch(input[inIndex + 1]){
1316 '\\' => '\\',
1317 '/' => '/',
1318 'n' => '\n',
1319 'r' => '\r',
1320 't' => '\t',
1321 'f' => 12,
1322 'b' => 8,
1323 '"' => '"',
1324 else => unreachable
1325 }
1326 );
1416 output[outIndex] = @as(u8, switch (input[inIndex + 1]) {
1417 '\\' => '\\',
1418 '/' => '/',
1419 'n' => '\n',
1420 'r' => '\r',
1421 't' => '\t',
1422 'f' => 12,
1423 'b' => 8,
1424 '"' => '"',
1425 else => unreachable,
1426 });
13271427 inIndex += 2;
13281428 outIndex += 1;
13291429 } else {
13301430 // a unicode escape sequence
1331 const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex+2 .. inIndex+6], 16) catch unreachable;
1431 const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex + 2 .. inIndex + 6], 16) catch unreachable;
13321432
13331433 // guess optimistically that it's not a surrogate pair
1334 if(std.unicode.utf8Encode(firstCodeUnit, output[outIndex..])) |byteCount| {
1434 if (std.unicode.utf8Encode(firstCodeUnit, output[outIndex..])) |byteCount| {
13351435 outIndex += byteCount;
13361436 inIndex += 6;
13371437 } else |err| {
13381438 // it might be a surrogate pair
1339 if(err != error.Utf8CannotEncodeSurrogateHalf) {
1439 if (err != error.Utf8CannotEncodeSurrogateHalf) {
13401440 return error.InvalidUnicodeHexSymbol;
13411441 }
13421442 // check if a second code unit is present
1343 if(inIndex + 7 >= input.len or input[inIndex + 6] != '\\' or input[inIndex + 7] != 'u'){
1443 if (inIndex + 7 >= input.len or input[inIndex + 6] != '\\' or input[inIndex + 7] != 'u') {
13441444 return error.InvalidUnicodeHexSymbol;
13451445 }
1346
1347 const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex+8 .. inIndex+12], 16) catch unreachable;
1348
1349 if(std.unicode.utf16leToUtf8(output[outIndex..], &[2]u16{ firstCodeUnit, secondCodeUnit })) |byteCount| {
1446
1447 const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex + 8 .. inIndex + 12], 16) catch unreachable;
1448
1449 if (std.unicode.utf16leToUtf8(output[outIndex..], &[2]u16{ firstCodeUnit, secondCodeUnit })) |byteCount| {
13501450 outIndex += byteCount;
13511451 inIndex += 12;
13521452 } else |_| {
......@@ -1355,8 +1455,7 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
13551455 }
13561456 }
13571457 }
1358
1359 return alloc.shrink(output, outIndex);
1458 assert(outIndex == output.len);
13601459}
13611460
13621461test "json.parser.dynamic" {
lib/std/meta.zig+1-3
......@@ -364,10 +364,8 @@ test "std.meta.activeTag" {
364364
365365///Given a tagged union type, and an enum, return the type of the union
366366/// field corresponding to the enum tag.
367pub fn TagPayloadType(comptime U: type, tag: var) type {
368 const Tag = @TypeOf(tag);
367pub fn TagPayloadType(comptime U: type, tag: @TagType(U)) type {
369368 testing.expect(trait.is(builtin.TypeId.Union)(U));
370 testing.expect(trait.is(builtin.TypeId.Enum)(Tag));
371369
372370 const info = @typeInfo(U).Union;
373371