authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-09-21 21:34:51+02:00
committergravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-11-20 08:36:10+01:00
log6b39167fdc2aa4c4c57d0459f3de792e001562aa
treebbee4c49f6dc61d2f72cd02417a396e2cb0d9552
parent73be59433f36f5a8f5c7fa7bbffcb8eb17b1735d

std: Rewrite the fmt parser

Turn the FSM parser into a linear one so that's easier to implement new features and/or more error checking without adding more and more states. Functionally-speaking the two parsers are at feature parity.

1 files changed, 178 insertions(+), 190 deletions(-)

lib/std/fmt.zig+178-190
......@@ -27,18 +27,6 @@ pub const FormatOptions = struct {
2727 fill: u8 = ' ',
2828};
2929
30fn peekIsAlign(comptime fmt: []const u8) bool {
31 // Should only be called during a state transition to the format segment.
32 comptime assert(fmt[0] == ':');
33
34 inline for (([_]u8{ 1, 2 })[0..]) |i| {
35 if (fmt.len > i and (fmt[i] == '<' or fmt[i] == '^' or fmt[i] == '>')) {
36 return true;
37 }
38 }
39 return false;
40}
41
4230/// Renders fmt string with args, calling output with slices of bytes.
4331/// If `output` returns an error, the error is returned from `format` and
4432/// `output` is not called again.
......@@ -103,22 +91,6 @@ pub fn format(
10391 @compileError("32 arguments max are supported per format call");
10492 }
10593
106 const State = enum {
107 Start,
108 Positional,
109 CloseBrace,
110 Specifier,
111 FormatFillAndAlign,
112 FormatWidth,
113 FormatPrecision,
114 };
115
116 comptime var start_index = 0;
117 comptime var state = State.Start;
118 comptime var maybe_pos_arg: ?comptime_int = null;
119 comptime var specifier_start = 0;
120 comptime var specifier_end = 0;
121 comptime var options = FormatOptions{};
12294 comptime var arg_state: struct {
12395 next_arg: usize = 0,
12496 used_args: ArgSetType = 0,
......@@ -128,7 +100,7 @@ pub fn format(
128100 return (@popCount(ArgSetType, self.used_args) != self.args_len);
129101 }
130102
131 fn nextArg(comptime self: *@This(), comptime pos_arg: ?comptime_int) comptime_int {
103 fn nextArg(comptime self: *@This(), comptime pos_arg: ?usize) comptime_int {
132104 const next_idx = pos_arg orelse blk: {
133105 const arg = self.next_arg;
134106 self.next_arg += 1;
......@@ -146,182 +118,193 @@ pub fn format(
146118 }
147119 } = .{};
148120
149 inline for (fmt) |c, i| {
150 switch (state) {
151 .Start => switch (c) {
152 '{' => {
153 if (start_index < i) {
154 try writer.writeAll(fmt[start_index..i]);
155 }
121 comptime var parser: struct {
122 buf: []const u8 = undefined,
123 pos: usize = 0,
124
125 // Returns a decimal number or null if the current character is not a
126 // digit
127 fn number(comptime self: *@This()) ?usize {
128 var r: ?usize = null;
129
130 while (self.pos < self.buf.len) : (self.pos += 1) {
131 switch (self.buf[self.pos]) {
132 '0'...'9' => {
133 if (r == null) r = 0;
134 r.? *= 10;
135 r.? += self.buf[self.pos] - '0';
136 },
137 else => break,
138 }
139 }
156140
157 start_index = i;
158 specifier_start = i + 1;
159 specifier_end = i + 1;
160 maybe_pos_arg = null;
161 state = .Positional;
162 options = FormatOptions{};
163 },
164 '}' => {
165 if (start_index < i) {
166 try writer.writeAll(fmt[start_index..i]);
167 }
168 state = .CloseBrace;
169 },
170 else => {},
171 },
172 .Positional => switch (c) {
173 '{' => {
174 state = .Start;
175 start_index = i;
176 },
177 ':' => {
178 state = if (comptime peekIsAlign(fmt[i..])) State.FormatFillAndAlign else State.FormatWidth;
179 specifier_end = i;
180 },
181 '0'...'9' => {
182 if (maybe_pos_arg == null) {
183 maybe_pos_arg = 0;
184 }
141 return r;
142 }
185143
186 maybe_pos_arg.? *= 10;
187 maybe_pos_arg.? += c - '0';
188 specifier_start = i + 1;
144 // Returns a substring of the input starting from the current position
145 // and ending where `ch` is found or until the end if not found
146 fn until(comptime self: *@This(), comptime ch: u8) []const u8 {
147 const start = self.pos;
189148
190 if (maybe_pos_arg.? >= args.len) {
191 @compileError("Positional value refers to non-existent argument");
192 }
193 },
194 '}' => {
195 const arg_to_print = comptime arg_state.nextArg(maybe_pos_arg);
196
197 try formatType(
198 args[arg_to_print],
199 fmt[0..0],
200 options,
201 writer,
202 default_max_depth,
203 );
204
205 state = .Start;
206 start_index = i + 1;
207 },
208 else => {
209 state = .Specifier;
210 specifier_start = i;
211 },
212 },
213 .CloseBrace => switch (c) {
214 '}' => {
215 state = .Start;
216 start_index = i;
217 },
218 else => @compileError("Single '}' encountered in format string"),
219 },
220 .Specifier => switch (c) {
221 ':' => {
222 specifier_end = i;
223 state = if (comptime peekIsAlign(fmt[i..])) State.FormatFillAndAlign else State.FormatWidth;
224 },
225 '}' => {
226 const arg_to_print = comptime arg_state.nextArg(maybe_pos_arg);
227
228 try formatType(
229 args[arg_to_print],
230 fmt[specifier_start..i],
231 options,
232 writer,
233 default_max_depth,
234 );
235 state = .Start;
236 start_index = i + 1;
237 },
149 if (start >= self.buf.len)
150 return &[_]u8{};
151
152 while (self.pos < self.buf.len) : (self.pos += 1) {
153 if (self.buf[self.pos] == ch) break;
154 }
155 return self.buf[start..self.pos];
156 }
157
158 // Returns one character, if available
159 fn char(comptime self: *@This()) ?u8 {
160 if (self.pos < self.buf.len) {
161 const ch = self.buf[self.pos];
162 self.pos += 1;
163 return ch;
164 }
165 return null;
166 }
167
168 // Returns the n-th next character or null if that's past the end
169 fn peek(comptime self: *@This(), comptime n: usize) ?u8 {
170 return if (self.pos + n < self.buf.len) self.buf[self.pos + n] else null;
171 }
172 } = .{};
173
174 comptime var options: FormatOptions = .{};
175
176 @setEvalBranchQuota(2000000);
177
178 comptime var i = 0;
179 inline while (i < fmt.len) {
180 comptime const start_index = i;
181
182 inline while (i < fmt.len) : (i += 1) {
183 switch (fmt[i]) {
184 '{', '}' => break,
238185 else => {},
239 },
240 // Only entered if the format string contains a fill/align segment.
241 .FormatFillAndAlign => switch (c) {
186 }
187 }
188
189 comptime var end_index = i;
190 comptime var unescape_brace = false;
191
192 // Handle {{ and }}, those are un-escaped as single braces
193 if (i + 1 < fmt.len and fmt[i + 1] == fmt[i]) {
194 unescape_brace = true;
195 // Make the first brace part of the literal...
196 end_index += 1;
197 // ...and skip both
198 i += 2;
199 }
200
201 // Write out the literal
202 if (start_index != end_index) {
203 try writer.writeAll(fmt[start_index..end_index]);
204 }
205
206 // We've already skipped the other brace, restart the loop
207 if (unescape_brace) continue;
208
209 if (i >= fmt.len) break;
210
211 if (fmt[i] == '}') {
212 @compileError("missing opening {");
213 }
214
215 // Get past the {
216 comptime assert(fmt[i] == '{');
217 i += 1;
218
219 comptime const fmt_begin = i;
220 // Find the closing brace
221 inline while (i < fmt.len and fmt[i] != '}') : (i += 1) {}
222 comptime const fmt_end = i;
223
224 if (i >= fmt.len) {
225 @compileError("missing closing }");
226 }
227
228 // Get past the }
229 comptime assert(fmt[i] == '}');
230 i += 1;
231
232 options = .{};
233
234 // Parse the format fragment between braces
235 parser.buf = fmt[fmt_begin..fmt_end];
236 parser.pos = 0;
237
238 // Parse the positional argument number
239 comptime var opt_pos_arg = comptime parser.number();
240
241 // Parse the format specifier
242 comptime var specifier_arg = comptime parser.until(':');
243
244 // Skip the colon, if present
245 if (comptime parser.char()) |ch| {
246 if (ch != ':')
247 @compileError("expected : or }, found '" ++ [1]u8{ch} ++ "'");
248 }
249
250 // Parse the fill character
251 // The fill parameter requires the alignment parameter to be specified
252 // too
253 if (comptime parser.peek(1)) |ch| {
254 if (comptime mem.indexOfScalar(u8, "<^>", ch) != null) {
255 options.fill = comptime parser.char().?;
256 }
257 }
258
259 // Parse the alignment parameter
260 if (comptime parser.peek(0)) |ch| {
261 switch (ch) {
242262 '<' => {
243 options.alignment = Alignment.Left;
244 state = .FormatWidth;
263 options.alignment = .Left;
264 _ = comptime parser.char();
245265 },
246266 '^' => {
247 options.alignment = Alignment.Center;
248 state = .FormatWidth;
267 options.alignment = .Center;
268 _ = comptime parser.char();
249269 },
250270 '>' => {
251 options.alignment = Alignment.Right;
252 state = .FormatWidth;
271 options.alignment = .Right;
272 _ = comptime parser.char();
253273 },
254 else => {
255 options.fill = c;
256 },
257 },
258 .FormatWidth => switch (c) {
259 '0'...'9' => {
260 if (options.width == null) {
261 options.width = 0;
262 }
274 else => {},
275 }
276 }
263277
264 options.width.? *= 10;
265 options.width.? += c - '0';
266 },
267 '.' => {
268 state = .FormatPrecision;
269 },
270 '}' => {
271 const arg_to_print = comptime arg_state.nextArg(maybe_pos_arg);
272
273 try formatType(
274 args[arg_to_print],
275 fmt[specifier_start..specifier_end],
276 options,
277 writer,
278 default_max_depth,
279 );
280 state = .Start;
281 start_index = i + 1;
282 },
283 else => {
284 @compileError("Unexpected character in width value: " ++ [_]u8{c});
285 },
286 },
287 .FormatPrecision => switch (c) {
288 '0'...'9' => {
289 if (options.precision == null) {
290 options.precision = 0;
291 }
278 // Parse the width parameter
279 comptime var opt_width_arg = comptime parser.number();
280 options.width = opt_width_arg;
292281
293 options.precision.? *= 10;
294 options.precision.? += c - '0';
295 },
296 '}' => {
297 const arg_to_print = comptime arg_state.nextArg(maybe_pos_arg);
298
299 try formatType(
300 args[arg_to_print],
301 fmt[specifier_start..specifier_end],
302 options,
303 writer,
304 default_max_depth,
305 );
306 state = .Start;
307 start_index = i + 1;
308 },
309 else => {
310 @compileError("Unexpected character in precision value: " ++ [_]u8{c});
311 },
312 },
313 }
314 }
315 comptime {
316 if (comptime arg_state.hasUnusedArgs()) {
317 @compileError("Unused arguments");
282 // Skip the dot, if present
283 if (comptime parser.char()) |ch| {
284 if (ch != '.')
285 @compileError("expected . or }, found '" ++ [1]u8{ch} ++ "'");
318286 }
319 if (state != State.Start) {
320 @compileError("Incomplete format string: " ++ fmt);
287
288 // Parse the precision parameter
289 comptime var opt_precision_arg = comptime parser.number();
290 options.precision = opt_precision_arg;
291
292 if (comptime parser.char()) |ch| {
293 @compileError("extraneous trailing character '" ++ [1]u8{ch} ++ "'");
321294 }
295
296 const arg_to_print = comptime arg_state.nextArg(opt_pos_arg);
297 try formatType(
298 args[arg_to_print],
299 specifier_arg,
300 options,
301 writer,
302 default_max_depth,
303 );
322304 }
323 if (start_index < fmt.len) {
324 try writer.writeAll(fmt[start_index..]);
305
306 if (comptime arg_state.hasUnusedArgs()) {
307 @compileError("Unused arguments");
325308 }
326309}
327310
......@@ -1371,6 +1354,11 @@ test "parse unsigned comptime" {
13711354 }
13721355}
13731356
1357test "escaped braces" {
1358 try testFmt("escaped: {{foo}}\n", "escaped: {{{{foo}}}}\n", .{});
1359 try testFmt("escaped: {foo}\n", "escaped: {{foo}}\n", .{});
1360}
1361
13741362test "optional" {
13751363 {
13761364 const value: ?i32 = 1234;