authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-05-04 07:58:22-04:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-05-29 08:22:01-04:00
log925e17879b851cb44d1a419eb08a2cb31d9e5eb7
treef22abad0bbbef153c6acaba3b4bc01a3c8a61554
parente60cd8058091e6c97743cd56206072ba5d1d8fce

second attempt

2 files changed, 84 insertions(+), 38 deletions(-)

lib/std/json/scanner.zig+77-28
...@@ -32,6 +32,7 @@ const std = @import("std");...@@ -32,6 +32,7 @@ const std = @import("std");
3232
33const Allocator = std.mem.Allocator;33const Allocator = std.mem.Allocator;
34const ArrayList = std.ArrayList;34const ArrayList = std.ArrayList;
35const ArrayListUnmanaged = std.ArrayListUnmanaged;
35const assert = std.debug.assert;36const assert = std.debug.assert;
36const BitStack = std.BitStack;37const BitStack = std.BitStack;
3738
...@@ -193,15 +194,17 @@ pub const TokenType = enum {...@@ -193,15 +194,17 @@ pub const TokenType = enum {
193/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()`194/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()`
194/// to get meaningful information from this.195/// to get meaningful information from this.
195pub const Diagnostics = struct {196pub const Diagnostics = struct {
197 // continually updated by Scanner:
196 line_number: u64 = 1,198 line_number: u64 = 1,
197 line_start_cursor: usize = @as(usize, @bitCast(@as(isize, -1))), // Start just "before" the input buffer to get a 1-based column for line 1.199 line_start_cursor: usize = @bitCast(@as(isize, -1)), // Start just "before" the input buffer to get a 1-based column for line 1.
198 total_bytes_before_current_input: u64 = 0,200 total_bytes_before_current_input: u64 = 0,
199 /// While the source is operational, this is a pointer into it.201
200 /// If the source is destroyed, this becomes a literal value.202 // updated by Scanner.saveDiagnostics:
201 cursor: union(enum) {203 cursor_in_current_input: usize = undefined,
202 pointer: *const usize,204 current_input: []const u8 = undefined,
203 value: usize,205
204 } = undefined,206 // updated by recordContext().
207 context_stack: ArrayListUnmanaged([]const u8) = .{},
205208
206 /// Starts at 1.209 /// Starts at 1.
207 pub fn getLine(self: *const @This()) u64 {210 pub fn getLine(self: *const @This()) u64 {
...@@ -209,25 +212,68 @@ pub const Diagnostics = struct {...@@ -209,25 +212,68 @@ pub const Diagnostics = struct {
209 }212 }
210 /// Starts at 1.213 /// Starts at 1.
211 pub fn getColumn(self: *const @This()) u64 {214 pub fn getColumn(self: *const @This()) u64 {
212 return self.getCursor() -% self.line_start_cursor;215 return self.cursor_in_current_input -% self.line_start_cursor;
213 }216 }
214 /// Starts at 0. Measures the byte offset since the start of the input.217 /// Starts at 0. Measures the byte offset since the start of the input.
215 pub fn getByteOffset(self: *const @This()) u64 {218 pub fn getByteOffset(self: *const @This()) u64 {
216 return self.total_bytes_before_current_input + self.getCursor();219 return self.total_bytes_before_current_input + self.cursor_in_current_input;
217 }220 }
218221
219 fn getCursor(self: *const @This()) usize {222 pub fn recordContext(self: *@This(), allocator: Allocator, context: []const u8) Allocator.Error!void {
220 return switch (self.cursor) {223 return self.context_stack.append(allocator, context);
221 .pointer => |p| p.*,
222 .value => |v| v,
223 };
224 }224 }
225 fn saveCursor(self: *@This()) void {225
226 const value = self.getCursor();226 /// Pretty-print diagnostic information to the given writer, such as `std.io.getStdErr().writer()`.
227 self.cursor = .{ .value = value };227 /// file_name if non-null will be printed in a line with the line and column numbers;
228 /// it is purely aesthetic and is not touched on any actual file system.
229 pub fn dump(self: *const @This(), writer: anytype, err: anyerror, file_name: ?[]const u8) !void {
230 try writer.print("{s}:{}:{}: {s}\n", .{file_name orelse "<json>", self.getLine(), self.getColumn(), @errorName(err)});
231
232 // Show a "line" of context, or in case of very long lines, just an excerpt of the line.
233 // (Very long lines are common in minified JSON such as in an HTTP API or other machine-to-machine contexts.)
234 var start = self.cursor_in_current_input;
235 var start_elipsis: []const u8 = "";
236 while (true) {
237 if (start == 0 or self.current_input[start - 1] == '\n') break; // found start of line.
238 if (start + 40 <= self.cursor_in_current_input) {
239 // Too far into the line. Show part of the line.
240 start_elipsis = "...";
241 break;
242 }
243 start -= 1;
244 }
245 var end = start;
246 var end_elipsis: []const u8 = "";
247 while (true) {
248 if (end + 1 < self.current_input.len and self.current_input[end + 1] == '\n') break; // found end of line.
249 if (end == self.current_input.len) {
250 // found end of input.
251 // TODO: put elipsis when not is_end_of_input.
252 break;
253 }
254 if (end >= start + 70) {
255 // Line is too long. Show part of it.
256 end_elipsis = "...";
257 break;
258 }
259 end += 1;
260 }
261 try writer.print("{s}{s}{s}\n", .{start_elipsis, self.current_input[start..end], end_elipsis});
262 try writer.writeByteNTimes(' ', start_elipsis.len + self.cursor_in_current_input - start);
263 try writer.writeAll("^\n");
264
265 for (self.context_stack.items) |item| {
266 try writer.print(" in {s}\n", .{item});
267 }
228 }268 }
229};269};
230270
271pub inline fn maybeRecordDiagnosticContext(allocator: Allocator, maybe_diagnostics: ?*Diagnostics, context: []const u8) Allocator.Error!void {
272 if (maybe_diagnostics) |diag| {
273 try diag.recordContext(allocator, context);
274 }
275}
276
231/// See the documentation for `std.json.Token`.277/// See the documentation for `std.json.Token`.
232pub const AllocWhen = enum { alloc_if_needed, alloc_always };278pub const AllocWhen = enum { alloc_if_needed, alloc_always };
233279
...@@ -260,10 +306,6 @@ pub fn Reader(comptime buffer_size: usize, comptime ReaderType: type) type {...@@ -260,10 +306,6 @@ pub fn Reader(comptime buffer_size: usize, comptime ReaderType: type) type {
260 pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void {306 pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void {
261 self.scanner.enableDiagnostics(diagnostics);307 self.scanner.enableDiagnostics(diagnostics);
262 }308 }
263 /// Calls `std.json.Scanner.saveDiagnostics`.
264 pub fn saveDiagnostics(self: *const @This()) void {
265 self.scanner.saveDiagnostics();
266 }
267309
268 pub const NextError = ReaderType.Error || Error || Allocator.Error;310 pub const NextError = ReaderType.Error || Error || Allocator.Error;
269 pub const SkipError = NextError;311 pub const SkipError = NextError;
...@@ -466,18 +508,18 @@ pub const Scanner = struct {...@@ -466,18 +508,18 @@ pub const Scanner = struct {
466 self.* = undefined;508 self.* = undefined;
467 }509 }
468510
469 /// See also `saveDiagnostics()`.
470 pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void {511 pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void {
471 diagnostics.cursor = .{ .pointer = &self.cursor };
472 std.log.warn("cursor(enableDiagnostics): {}", .{diagnostics.getCursor()});
473 self.diagnostics = diagnostics;512 self.diagnostics = diagnostics;
474 }513 }
475 /// Call this just before `deinit()` to make the diagnostics available after the `deinit()`.514 /// For performance reasons, the diagnostics (see `enableDiagnostics`) are not kept up to date continually.
515 /// Call this method to update the diagnostics with the latest information.
516 /// Because diagnostics are usually consulted in case of an error, it is common to call this in an errdefer.
517 /// It is safe to call this regardless of whether diagnostics have been enabled.
518 /// This is already called in an errdefer block in every relevant public method of this class.
476 pub fn saveDiagnostics(self: *const @This()) void {519 pub fn saveDiagnostics(self: *const @This()) void {
477 if (self.diagnostics) |diag| {520 if (self.diagnostics) |diag| {
478 std.log.warn("cursor(deinit presave): {}", .{diag.getCursor()});521 diag.cursor_in_current_input = self.cursor;
479 diag.saveCursor();522 diag.current_input = self.input;
480 std.log.warn("cursor(deinit postsave): {}", .{diag.getCursor()});
481 }523 }
482 }524 }
483525
...@@ -520,6 +562,7 @@ pub const Scanner = struct {...@@ -520,6 +562,7 @@ pub const Scanner = struct {
520 /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.562 /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
521 pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token {563 pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token {
522 assert(self.is_end_of_input); // This function is not available in streaming mode.564 assert(self.is_end_of_input); // This function is not available in streaming mode.
565 errdefer self.saveDiagnostics();
523 const token_type = self.peekNextTokenType() catch |e| switch (e) {566 const token_type = self.peekNextTokenType() catch |e| switch (e) {
524 error.BufferUnderrun => unreachable,567 error.BufferUnderrun => unreachable,
525 else => |err| return err,568 else => |err| return err,
...@@ -577,6 +620,7 @@ pub const Scanner = struct {...@@ -577,6 +620,7 @@ pub const Scanner = struct {
577 /// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type;620 /// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type;
578 /// the caller of this method is expected to know which type of token is being processed.621 /// the caller of this method is expected to know which type of token is being processed.
579 pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 {622 pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 {
623 errdefer self.saveDiagnostics();
580 while (true) {624 while (true) {
581 const token = try self.next();625 const token = try self.next();
582 switch (token) {626 switch (token) {
...@@ -642,6 +686,7 @@ pub const Scanner = struct {...@@ -642,6 +686,7 @@ pub const Scanner = struct {
642 /// see `peekNextTokenType()`.686 /// see `peekNextTokenType()`.
643 pub fn skipValue(self: *@This()) SkipError!void {687 pub fn skipValue(self: *@This()) SkipError!void {
644 assert(self.is_end_of_input); // This function is not available in streaming mode.688 assert(self.is_end_of_input); // This function is not available in streaming mode.
689 errdefer self.saveDiagnostics();
645 switch (self.peekNextTokenType() catch |e| switch (e) {690 switch (self.peekNextTokenType() catch |e| switch (e) {
646 error.BufferUnderrun => unreachable,691 error.BufferUnderrun => unreachable,
647 else => |err| return err,692 else => |err| return err,
...@@ -686,6 +731,7 @@ pub const Scanner = struct {...@@ -686,6 +731,7 @@ pub const Scanner = struct {
686 /// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height.731 /// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height.
687 /// Unlike `skipValue()`, this function is available in streaming mode.732 /// Unlike `skipValue()`, this function is available in streaming mode.
688 pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void {733 pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void {
734 errdefer self.saveDiagnostics();
689 while (true) {735 while (true) {
690 switch (try self.next()) {736 switch (try self.next()) {
691 .object_end, .array_end => {737 .object_end, .array_end => {
...@@ -705,11 +751,13 @@ pub const Scanner = struct {...@@ -705,11 +751,13 @@ pub const Scanner = struct {
705 /// Pre allocate memory to hold the given number of nesting levels.751 /// Pre allocate memory to hold the given number of nesting levels.
706 /// `stackHeight()` up to the given number will not cause allocations.752 /// `stackHeight()` up to the given number will not cause allocations.
707 pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void {753 pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void {
754 errdefer self.saveDiagnostics();
708 try self.stack.ensureTotalCapacity(height);755 try self.stack.ensureTotalCapacity(height);
709 }756 }
710757
711 /// See `std.json.Token` for documentation of this function.758 /// See `std.json.Token` for documentation of this function.
712 pub fn next(self: *@This()) NextError!Token {759 pub fn next(self: *@This()) NextError!Token {
760 errdefer self.saveDiagnostics();
713 state_loop: while (true) {761 state_loop: while (true) {
714 switch (self.state) {762 switch (self.state) {
715 .value => {763 .value => {
...@@ -1463,6 +1511,7 @@ pub const Scanner = struct {...@@ -1463,6 +1511,7 @@ pub const Scanner = struct {
1463 /// determines which type of token will be returned from the next `next*()` call.1511 /// determines which type of token will be returned from the next `next*()` call.
1464 /// This function is idempotent, only advancing past commas, colons, and inter-token whitespace.1512 /// This function is idempotent, only advancing past commas, colons, and inter-token whitespace.
1465 pub fn peekNextTokenType(self: *@This()) PeekError!TokenType {1513 pub fn peekNextTokenType(self: *@This()) PeekError!TokenType {
1514 errdefer self.saveDiagnostics();
1466 state_loop: while (true) {1515 state_loop: while (true) {
1467 switch (self.state) {1516 switch (self.state) {
1468 .value => {1517 .value => {
lib/std/json/static.zig+7-10
...@@ -10,6 +10,7 @@ const AllocWhen = @import("./scanner.zig").AllocWhen;...@@ -10,6 +10,7 @@ const AllocWhen = @import("./scanner.zig").AllocWhen;
10const Diagnostics = @import("./scanner.zig").Diagnostics;10const Diagnostics = @import("./scanner.zig").Diagnostics;
11const default_max_value_len = @import("./scanner.zig").default_max_value_len;11const default_max_value_len = @import("./scanner.zig").default_max_value_len;
12const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger;12const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger;
13const maybeRecordDiagnosticContext = @import("./scanner.zig").maybeRecordDiagnosticContext;
1314
14const Value = @import("./dynamic.zig").Value;15const Value = @import("./dynamic.zig").Value;
15const Array = @import("./dynamic.zig").Array;16const Array = @import("./dynamic.zig").Array;
...@@ -144,11 +145,6 @@ pub fn parseFromTokenSourceLeaky(...@@ -144,11 +145,6 @@ pub fn parseFromTokenSourceLeaky(
144 if (resolved_options.diagnostics) |diag| {145 if (resolved_options.diagnostics) |diag| {
145 scanner_or_reader.enableDiagnostics(diag);146 scanner_or_reader.enableDiagnostics(diag);
146 }147 }
147 defer {
148 if (resolved_options.diagnostics) |_| {
149 scanner_or_reader.saveDiagnostics();
150 }
151 }
152148
153 const value = try innerParse(T, allocator, scanner_or_reader, resolved_options);149 const value = try innerParse(T, allocator, scanner_or_reader, resolved_options);
154150
...@@ -222,6 +218,8 @@ pub fn innerParse(...@@ -222,6 +218,8 @@ pub fn innerParse(
222 source: anytype,218 source: anytype,
223 options: ParseOptions,219 options: ParseOptions,
224) ParseError(@TypeOf(source.*))!T {220) ParseError(@TypeOf(source.*))!T {
221 errdefer source.saveDiagnostics();
222 errdefer maybeRecordDiagnosticContext(allocator, options.diagnostics, @typeName(T)) catch {};
225 switch (@typeInfo(T)) {223 switch (@typeInfo(T)) {
226 .Bool => {224 .Bool => {
227 return switch (try source.next()) {225 return switch (try source.next()) {
...@@ -299,7 +297,7 @@ pub fn innerParse(...@@ -299,7 +297,7 @@ pub fn innerParse(
299 if (u_field.type == void) {297 if (u_field.type == void) {
300 // void isn't really a json type, but we can support void payload union tags with {} as a value.298 // void isn't really a json type, but we can support void payload union tags with {} as a value.
301 if (.object_begin != try source.next()) return error.UnexpectedToken;299 if (.object_begin != try source.next()) return error.UnexpectedToken;
302 if (.object_end != try source.next()) return error.UnexpectedToken;300 if (.object_end != try source.next()) return error.UnknownField;
303 result = @unionInit(T, u_field.name, {});301 result = @unionInit(T, u_field.name, {});
304 } else {302 } else {
305 // Recurse.303 // Recurse.
...@@ -347,9 +345,7 @@ pub fn innerParse(...@@ -347,9 +345,7 @@ pub fn innerParse(
347 .object_end => { // No more fields.345 .object_end => { // No more fields.
348 break;346 break;
349 },347 },
350 else => {348 else => unreachable, // Not possible while in an object.
351 return error.UnexpectedToken;
352 },
353 };349 };
354350
355 inline for (structInfo.fields, 0..) |field, i| {351 inline for (structInfo.fields, 0..) |field, i| {
...@@ -358,6 +354,7 @@ pub fn innerParse(...@@ -358,6 +354,7 @@ pub fn innerParse(
358 // Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.354 // Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
359 // (Recursing into innerParse() might trigger more allocations.)355 // (Recursing into innerParse() might trigger more allocations.)
360 freeAllocated(allocator, name_token.?);356 freeAllocated(allocator, name_token.?);
357 errdefer maybeRecordDiagnosticContext(allocator, options.diagnostics, @typeName(T) ++ "." ++ field.name) catch {};
361 name_token = null;358 name_token = null;
362 if (fields_seen[i]) {359 if (fields_seen[i]) {
363 switch (options.duplicate_field_behavior) {360 switch (options.duplicate_field_behavior) {
...@@ -624,7 +621,7 @@ pub fn innerParseFromValue(...@@ -624,7 +621,7 @@ pub fn innerParseFromValue(
624 if (u_field.type == void) {621 if (u_field.type == void) {
625 // void isn't really a json type, but we can support void payload union tags with {} as a value.622 // void isn't really a json type, but we can support void payload union tags with {} as a value.
626 if (kv.value_ptr.* != .object) return error.UnexpectedToken;623 if (kv.value_ptr.* != .object) return error.UnexpectedToken;
627 if (kv.value_ptr.*.object.count() != 0) return error.UnexpectedToken;624 if (kv.value_ptr.*.object.count() != 0) return error.UnknownField;
628 return @unionInit(T, u_field.name, {});625 return @unionInit(T, u_field.name, {});
629 }626 }
630 // Recurse.627 // Recurse.