1const std = @import("std");
2const Scanner = @import("Scanner.zig");
3const Token = Scanner.Token;
4const TokenType = Scanner.TokenType;
5const Diagnostics = Scanner.Diagnostics;
6const Error = Scanner.Error;
7const validate = Scanner.validate;
8const isNumberFormattedLikeAnInteger = Scanner.isNumberFormattedLikeAnInteger;
9
10const example_document_str =
11 \\{
12 \\ "Image": {
13 \\ "Width": 800,
14 \\ "Height": 600,
15 \\ "Title": "View from 15th Floor",
16 \\ "Thumbnail": {
17 \\ "Url": "http://www.example.com/image/481989943",
18 \\ "Height": 125,
19 \\ "Width": 100
20 \\ },
21 \\ "Animated" : false,
22 \\ "IDs": [116, 943, 234, 38793]
23 \\ }
24 \\}
25;
26
27fn expectNext(scanner_or_reader: anytype, expected_token: Token) !void {
28 return expectEqualTokens(expected_token, try scanner_or_reader.next());
29}
30
31fn expectPeekNext(scanner_or_reader: anytype, expected_token_type: TokenType, expected_token: Token) !void {
32 try std.testing.expectEqual(expected_token_type, try scanner_or_reader.peekNextTokenType());
33 try expectEqualTokens(expected_token, try scanner_or_reader.next());
34}
35
36test "token" {
37 var scanner = Scanner.initCompleteInput(std.testing.allocator, example_document_str);
38 defer scanner.deinit();
39
40 try expectNext(&scanner, .object_begin);
41 try expectNext(&scanner, Token{ .string = "Image" });
42 try expectNext(&scanner, .object_begin);
43 try expectNext(&scanner, Token{ .string = "Width" });
44 try expectNext(&scanner, Token{ .number = "800" });
45 try expectNext(&scanner, Token{ .string = "Height" });
46 try expectNext(&scanner, Token{ .number = "600" });
47 try expectNext(&scanner, Token{ .string = "Title" });
48 try expectNext(&scanner, Token{ .string = "View from 15th Floor" });
49 try expectNext(&scanner, Token{ .string = "Thumbnail" });
50 try expectNext(&scanner, .object_begin);
51 try expectNext(&scanner, Token{ .string = "Url" });
52 try expectNext(&scanner, Token{ .string = "http://www.example.com/image/481989943" });
53 try expectNext(&scanner, Token{ .string = "Height" });
54 try expectNext(&scanner, Token{ .number = "125" });
55 try expectNext(&scanner, Token{ .string = "Width" });
56 try expectNext(&scanner, Token{ .number = "100" });
57 try expectNext(&scanner, .object_end);
58 try expectNext(&scanner, Token{ .string = "Animated" });
59 try expectNext(&scanner, .false);
60 try expectNext(&scanner, Token{ .string = "IDs" });
61 try expectNext(&scanner, .array_begin);
62 try expectNext(&scanner, Token{ .number = "116" });
63 try expectNext(&scanner, Token{ .number = "943" });
64 try expectNext(&scanner, Token{ .number = "234" });
65 try expectNext(&scanner, Token{ .number = "38793" });
66 try expectNext(&scanner, .array_end);
67 try expectNext(&scanner, .object_end);
68 try expectNext(&scanner, .object_end);
69 try expectNext(&scanner, .end_of_document);
70}
71
72const all_types_test_case =
73 \\[
74 \\ "", "a\nb",
75 \\ 0, 0.0, -1.1e-1,
76 \\ true, false, null,
77 \\ {"a": {}},
78 \\ []
79 \\]
80;
81
82fn testAllTypes(source: anytype, large_buffer: bool) !void {
83 try expectPeekNext(source, .array_begin, .array_begin);
84 try expectPeekNext(source, .string, Token{ .string = "" });
85 try expectPeekNext(source, .string, Token{ .partial_string = "a" });
86 try expectPeekNext(source, .string, Token{ .partial_string_escaped_1 = "\n".* });
87 if (large_buffer) {
88 try expectPeekNext(source, .string, Token{ .string = "b" });
89 } else {
90 try expectPeekNext(source, .string, Token{ .partial_string = "b" });
91 try expectPeekNext(source, .string, Token{ .string = "" });
92 }
93 if (large_buffer) {
94 try expectPeekNext(source, .number, Token{ .number = "0" });
95 } else {
96 try expectPeekNext(source, .number, Token{ .partial_number = "0" });
97 try expectPeekNext(source, .number, Token{ .number = "" });
98 }
99 if (large_buffer) {
100 try expectPeekNext(source, .number, Token{ .number = "0.0" });
101 } else {
102 try expectPeekNext(source, .number, Token{ .partial_number = "0" });
103 try expectPeekNext(source, .number, Token{ .partial_number = "." });
104 try expectPeekNext(source, .number, Token{ .partial_number = "0" });
105 try expectPeekNext(source, .number, Token{ .number = "" });
106 }
107 if (large_buffer) {
108 try expectPeekNext(source, .number, Token{ .number = "-1.1e-1" });
109 } else {
110 try expectPeekNext(source, .number, Token{ .partial_number = "-" });
111 try expectPeekNext(source, .number, Token{ .partial_number = "1" });
112 try expectPeekNext(source, .number, Token{ .partial_number = "." });
113 try expectPeekNext(source, .number, Token{ .partial_number = "1" });
114 try expectPeekNext(source, .number, Token{ .partial_number = "e" });
115 try expectPeekNext(source, .number, Token{ .partial_number = "-" });
116 try expectPeekNext(source, .number, Token{ .partial_number = "1" });
117 try expectPeekNext(source, .number, Token{ .number = "" });
118 }
119 try expectPeekNext(source, .true, .true);
120 try expectPeekNext(source, .false, .false);
121 try expectPeekNext(source, .null, .null);
122 try expectPeekNext(source, .object_begin, .object_begin);
123 if (large_buffer) {
124 try expectPeekNext(source, .string, Token{ .string = "a" });
125 } else {
126 try expectPeekNext(source, .string, Token{ .partial_string = "a" });
127 try expectPeekNext(source, .string, Token{ .string = "" });
128 }
129 try expectPeekNext(source, .object_begin, .object_begin);
130 try expectPeekNext(source, .object_end, .object_end);
131 try expectPeekNext(source, .object_end, .object_end);
132 try expectPeekNext(source, .array_begin, .array_begin);
133 try expectPeekNext(source, .array_end, .array_end);
134 try expectPeekNext(source, .array_end, .array_end);
135 try expectPeekNext(source, .end_of_document, .end_of_document);
136}
137
138test "peek all types" {
139 var scanner = Scanner.initCompleteInput(std.testing.allocator, all_types_test_case);
140 defer scanner.deinit();
141 try testAllTypes(&scanner, true);
142
143 var stream: std.Io.Reader = .fixed(all_types_test_case);
144 var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream);
145 defer json_reader.deinit();
146 try testAllTypes(&json_reader, true);
147
148 var tiny_buffer: [1]u8 = undefined;
149 var tiny_stream: std.testing.Reader = .init(&tiny_buffer, &.{.{ .buffer = all_types_test_case }});
150 tiny_stream.artificial_limit = .limited(1);
151 var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream.interface);
152 defer tiny_json_reader.deinit();
153 try testAllTypes(&tiny_json_reader, false);
154}
155
156test "token mismatched close" {
157 var scanner = Scanner.initCompleteInput(std.testing.allocator, "[102, 111, 111 }");
158 defer scanner.deinit();
159 try expectNext(&scanner, .array_begin);
160 try expectNext(&scanner, Token{ .number = "102" });
161 try expectNext(&scanner, Token{ .number = "111" });
162 try expectNext(&scanner, Token{ .number = "111" });
163 try std.testing.expectError(error.SyntaxError, scanner.next());
164}
165
166test "token premature object close" {
167 var scanner = Scanner.initCompleteInput(std.testing.allocator, "{ \"key\": }");
168 defer scanner.deinit();
169 try expectNext(&scanner, .object_begin);
170 try expectNext(&scanner, Token{ .string = "key" });
171 try std.testing.expectError(error.SyntaxError, scanner.next());
172}
173
174test "Scanner basic" {
175 var scanner = Scanner.initCompleteInput(std.testing.allocator, example_document_str);
176 defer scanner.deinit();
177
178 while (true) {
179 const token = try scanner.next();
180 if (token == .end_of_document) break;
181 }
182}
183
184test "Scanner.Reader basic" {
185 var stream: std.Io.Reader = .fixed(example_document_str);
186
187 var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream);
188 defer json_reader.deinit();
189
190 while (true) {
191 const token = try json_reader.next();
192 if (token == .end_of_document) break;
193 }
194}
195
196const number_test_stems = .{
197 .{ "", "-" },
198 .{ "0", "1", "10", "9999999999999999999999999" },
199 .{ "", ".0", ".999999999999999999999999" },
200 .{ "", "e0", "E0", "e+0", "e-0", "e9999999999999999999999999999" },
201};
202const number_test_items = blk: {
203 var ret: []const []const u8 = &[_][]const u8{};
204 for (number_test_stems[0]) |s0| {
205 for (number_test_stems[1]) |s1| {
206 for (number_test_stems[2]) |s2| {
207 for (number_test_stems[3]) |s3| {
208 ret = ret ++ &[_][]const u8{s0 ++ s1 ++ s2 ++ s3};
209 }
210 }
211 }
212 }
213 break :blk ret;
214};
215
216test "numbers" {
217 for (number_test_items) |number_str| {
218 var scanner = Scanner.initCompleteInput(std.testing.allocator, number_str);
219 defer scanner.deinit();
220
221 const token = try scanner.next();
222 const value = token.number; // assert this is a number
223 try std.testing.expectEqualStrings(number_str, value);
224
225 try std.testing.expectEqual(Token.end_of_document, try scanner.next());
226 }
227}
228
229const string_test_cases = .{
230 // The left is JSON without the "quotes".
231 // The right is the expected unescaped content.
232 .{ "", "" },
233 .{ "\\\\", "\\" },
234 .{ "a\\\\b", "a\\b" },
235 .{ "a\\\"b", "a\"b" },
236 .{ "\\n", "\n" },
237 .{ "\\u000a", "\n" },
238 .{ "𝄞", "\u{1D11E}" },
239 .{ "\\uD834\\uDD1E", "\u{1D11E}" },
240 .{ "\\uD87F\\uDFFE", "\u{2FFFE}" },
241 .{ "\\uff20", "＠" },
242};
243
244test "strings" {
245 inline for (string_test_cases) |tuple| {
246 var stream: std.Io.Reader = .fixed("\"" ++ tuple[0] ++ "\"");
247 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
248 defer arena.deinit();
249 var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream);
250 defer json_reader.deinit();
251
252 const token = try json_reader.nextAlloc(arena.allocator(), .alloc_if_needed);
253 const value = switch (token) {
254 .string => |value| value,
255 .allocated_string => |value| value,
256 else => return error.ExpectedString,
257 };
258 try std.testing.expectEqualStrings(tuple[1], value);
259
260 try std.testing.expectEqual(Token.end_of_document, try json_reader.next());
261 }
262}
263
264const nesting_test_cases = cases: {
265 const open_arrays: *const [1000]u8 = &@splat('[');
266 const close_arrays: *const [1000]u8 = &@splat(']');
267
268 const open_objects_buf: [1000][4]u8 = @splat("{\"\":".*);
269 const open_objects: []const u8 = @ptrCast(&open_objects_buf);
270 const close_objects: *const [1000]u8 = &@splat('}');
271
272 break :cases .{
273 .{ null, "[]" },
274 .{ null, "{}" },
275 .{ error.SyntaxError, "[}" },
276 .{ error.SyntaxError, "{]" },
277 .{ null, open_arrays ++ close_arrays },
278 .{ null, open_objects ++ "0" ++ close_objects },
279 .{ error.SyntaxError, open_arrays ++ close_arrays[0..999] ++ "}" },
280 .{ error.SyntaxError, open_objects ++ "0" ++ close_objects[0..999] ++ "]" },
281 .{ error.SyntaxError, open_arrays ++ close_arrays ++ "]" },
282 .{ error.SyntaxError, open_objects ++ "0" ++ close_objects ++ "}" },
283 .{ error.UnexpectedEndOfInput, open_arrays ++ close_arrays[0..999] },
284 .{ error.UnexpectedEndOfInput, open_objects ++ "0" ++ close_objects[0..999] },
285 };
286};
287
288test "nesting" {
289 inline for (nesting_test_cases) |tuple| {
290 const maybe_error = tuple[0];
291 const document_str = tuple[1];
292
293 expectMaybeError(document_str, maybe_error) catch |err| {
294 std.debug.print("in json document: {s}\n", .{document_str});
295 return err;
296 };
297 }
298}
299
300fn expectMaybeError(document_str: []const u8, maybe_error: ?Error) !void {
301 var scanner = Scanner.initCompleteInput(std.testing.allocator, document_str);
302 defer scanner.deinit();
303
304 while (true) {
305 const token = scanner.next() catch |err| {
306 if (maybe_error) |expected_err| {
307 if (err == expected_err) return;
308 }
309 return err;
310 };
311 if (token == .end_of_document) break;
312 }
313 if (maybe_error != null) return error.ExpectedError;
314}
315
316fn expectEqualTokens(expected_token: Token, actual_token: Token) !void {
317 try std.testing.expectEqual(std.meta.activeTag(expected_token), std.meta.activeTag(actual_token));
318 switch (expected_token) {
319 .number => |expected_value| {
320 try std.testing.expectEqualStrings(expected_value, actual_token.number);
321 },
322 .allocated_number => |expected_value| {
323 try std.testing.expectEqualStrings(expected_value, actual_token.allocated_number);
324 },
325 .partial_number => |expected_value| {
326 try std.testing.expectEqualStrings(expected_value, actual_token.partial_number);
327 },
328
329 .string => |expected_value| {
330 try std.testing.expectEqualStrings(expected_value, actual_token.string);
331 },
332 .allocated_string => |expected_value| {
333 try std.testing.expectEqualStrings(expected_value, actual_token.allocated_string);
334 },
335 .partial_string => |expected_value| {
336 try std.testing.expectEqualStrings(expected_value, actual_token.partial_string);
337 },
338 .partial_string_escaped_1 => |expected_value| {
339 try std.testing.expectEqualStrings(&expected_value, &actual_token.partial_string_escaped_1);
340 },
341 .partial_string_escaped_2 => |expected_value| {
342 try std.testing.expectEqualStrings(&expected_value, &actual_token.partial_string_escaped_2);
343 },
344 .partial_string_escaped_3 => |expected_value| {
345 try std.testing.expectEqualStrings(&expected_value, &actual_token.partial_string_escaped_3);
346 },
347 .partial_string_escaped_4 => |expected_value| {
348 try std.testing.expectEqualStrings(&expected_value, &actual_token.partial_string_escaped_4);
349 },
350
351 .object_begin,
352 .object_end,
353 .array_begin,
354 .array_end,
355 .true,
356 .false,
357 .null,
358 .end_of_document,
359 => {},
360 }
361}
362
363fn testTinyBufferSize(document_str: []const u8) !void {
364 var tiny_stream: std.Io.Reader = .fixed(document_str);
365 var normal_stream: std.Io.Reader = .fixed(document_str);
366
367 var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream);
368 defer tiny_json_reader.deinit();
369 var normal_json_reader: Scanner.Reader = .init(std.testing.allocator, &normal_stream);
370 defer normal_json_reader.deinit();
371
372 expectEqualStreamOfTokens(&normal_json_reader, &tiny_json_reader) catch |err| {
373 std.debug.print("in json document: {s}\n", .{document_str});
374 return err;
375 };
376}
377fn expectEqualStreamOfTokens(control_json_reader: anytype, test_json_reader: anytype) !void {
378 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
379 defer arena.deinit();
380 while (true) {
381 const control_token = try control_json_reader.nextAlloc(arena.allocator(), .alloc_always);
382 const test_token = try test_json_reader.nextAlloc(arena.allocator(), .alloc_always);
383 try expectEqualTokens(control_token, test_token);
384 if (control_token == .end_of_document) break;
385 _ = arena.reset(.retain_capacity);
386 }
387}
388
389test "BufferUnderrun" {
390 try testTinyBufferSize(example_document_str);
391 for (number_test_items) |number_str| {
392 try testTinyBufferSize(number_str);
393 }
394 inline for (string_test_cases) |tuple| {
395 try testTinyBufferSize("\"" ++ tuple[0] ++ "\"");
396 }
397}
398
399test "validate" {
400 try std.testing.expectEqual(true, try validate(std.testing.allocator, "{}"));
401 try std.testing.expectEqual(true, try validate(std.testing.allocator, "[]"));
402 try std.testing.expectEqual(false, try validate(std.testing.allocator, "[{[[[[{}]]]]}]"));
403 try std.testing.expectEqual(false, try validate(std.testing.allocator, "{]"));
404 try std.testing.expectEqual(false, try validate(std.testing.allocator, "[}"));
405 try std.testing.expectEqual(false, try validate(std.testing.allocator, "{{{{[]}}}]"));
406}
407
408fn testSkipValue(s: []const u8) !void {
409 var scanner = Scanner.initCompleteInput(std.testing.allocator, s);
410 defer scanner.deinit();
411 try scanner.skipValue();
412 try expectEqualTokens(.end_of_document, try scanner.next());
413
414 var stream: std.Io.Reader = .fixed(s);
415 var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream);
416 defer json_reader.deinit();
417 try json_reader.skipValue();
418 try expectEqualTokens(.end_of_document, try json_reader.next());
419}
420
421test "skipValue" {
422 try testSkipValue("false");
423 try testSkipValue("true");
424 try testSkipValue("null");
425 try testSkipValue("42");
426 try testSkipValue("42.0");
427 try testSkipValue("\"foo\"");
428 try testSkipValue("[101, 111, 121]");
429 try testSkipValue("{}");
430 try testSkipValue("{\"foo\": \"bar\\nbaz\"}");
431
432 // An absurd number of nestings
433 const open_all: [1000]u8 = @splat('[');
434 const close_all: [1000]u8 = @splat(']');
435 try testSkipValue(&(open_all ++ close_all));
436
437 // Would a number token cause problems in a deeply-nested array?
438 try testSkipValue(&open_all ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ &close_all);
439
440 // Mismatched brace/square bracket
441 try std.testing.expectError(error.SyntaxError, testSkipValue("[102, 111, 111}"));
442}
443
444fn testDiagnosticsFromSource(expected_error: ?anyerror, line: u64, col: u64, byte_offset: u64, source: anytype) !void {
445 var diagnostics = Diagnostics{};
446 source.enableDiagnostics(&diagnostics);
447
448 if (expected_error) |expected_err| {
449 try std.testing.expectError(expected_err, source.skipValue());
450 } else {
451 try source.skipValue();
452 try std.testing.expectEqual(Token.end_of_document, try source.next());
453 }
454 try std.testing.expectEqual(line, diagnostics.getLine());
455 try std.testing.expectEqual(col, diagnostics.getColumn());
456 try std.testing.expectEqual(byte_offset, diagnostics.getByteOffset());
457}
458fn testDiagnostics(expected_error: ?anyerror, line: u64, col: u64, byte_offset: u64, s: []const u8) !void {
459 var scanner = Scanner.initCompleteInput(std.testing.allocator, s);
460 defer scanner.deinit();
461 try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &scanner);
462
463 var tiny_stream: std.Io.Reader = .fixed(s);
464 var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream);
465 defer tiny_json_reader.deinit();
466 try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &tiny_json_reader);
467
468 var medium_stream: std.Io.Reader = .fixed(s);
469 var medium_json_reader: Scanner.Reader = .init(std.testing.allocator, &medium_stream);
470 defer medium_json_reader.deinit();
471 try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &medium_json_reader);
472}
473test "enableDiagnostics" {
474 try testDiagnostics(error.UnexpectedEndOfInput, 1, 1, 0, "");
475 try testDiagnostics(null, 1, 3, 2, "[]");
476 try testDiagnostics(null, 2, 2, 3, "[\n]");
477 try testDiagnostics(null, 14, 2, example_document_str.len, example_document_str);
478
479 try testDiagnostics(error.SyntaxError, 3, 1, 25,
480 \\{
481 \\ "common": "mistake",
482 \\}
483 );
484
485 inline for ([_]comptime_int{ 5, 6, 7, 99 }) |reps| {
486 // The error happens 1 byte before the end.
487 const open_all: [reps]u8 = @splat('[');
488 const s = &open_all ++ "}";
489 try testDiagnostics(error.SyntaxError, 1, s.len, s.len - 1, s);
490 }
491}
492
493test isNumberFormattedLikeAnInteger {
494 try std.testing.expect(isNumberFormattedLikeAnInteger("0"));
495 try std.testing.expect(isNumberFormattedLikeAnInteger("1"));
496 try std.testing.expect(isNumberFormattedLikeAnInteger("123"));
497 try std.testing.expect(!isNumberFormattedLikeAnInteger("-0"));
498 try std.testing.expect(!isNumberFormattedLikeAnInteger("0.0"));
499 try std.testing.expect(!isNumberFormattedLikeAnInteger("1.0"));
500 try std.testing.expect(!isNumberFormattedLikeAnInteger("1.23"));
501 try std.testing.expect(!isNumberFormattedLikeAnInteger("1e10"));
502 try std.testing.expect(!isNumberFormattedLikeAnInteger("1E10"));
503}