authorgravatar for xq@random-projects.netFelix "xq" Queißner <xq@random-projects.net> 2023-01-05 11:51:26+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-06 17:56:05-07:00
log08496aa2aaca0771bbd16bccf640da7ccc050158
treeab4bc46ffa58f5daaf1d615d947df2411231b86a
parent3806091a106dca684fdcce2cb5c5dfb4941e825f

Ports zig-uri to stdlib.

1 files changed, 495 insertions(+), 78 deletions(-)

lib/std/Url.zig+495-78
......@@ -1,98 +1,515 @@
1scheme: []const u8,
2host: []const u8,
3path: []const u8,
1//! Implements URI parsing roughly adhering to <https://tools.ietf.org/html/rfc3986>.
2//! Does not do perfect grammar and character class checking, but should be robust against URIs in the wild.
3
4const Url = @This();
5const std = @import("std.zig");
6const testing = std.testing;
7
8scheme: ?[]const u8,
9user: ?[]const u8,
10password: ?[]const u8,
11host: ?[]const u8,
412port: ?u16,
13path: []const u8,
14query: ?[]const u8,
15fragment: ?[]const u8,
16
17/// Applies URI encoding and replaces all reserved characters with their respective %XX code.
18pub fn escapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
19 var outsize: usize = 0;
20 for (input) |c| {
21 outsize += if (isUnreserved(c)) @as(usize, 1) else 3;
22 }
23 var output = try allocator.alloc(u8, outsize);
24 var outptr: usize = 0;
25
26 for (input) |c| {
27 if (isUnreserved(c)) {
28 output[outptr] = c;
29 outptr += 1;
30 } else {
31 var buf: [2]u8 = undefined;
32 _ = std.fmt.bufPrint(&buf, "{X:0>2}", .{c}) catch unreachable;
33
34 output[outptr + 0] = '%';
35 output[outptr + 1] = buf[0];
36 output[outptr + 2] = buf[1];
37 outptr += 3;
38 }
39 }
40 return output;
41}
42
43/// Parses a URI string and unescapes all %XX where XX is a valid hex number. Otherwise, verbatim copies
44/// them to the output.
45pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
46 var outsize: usize = 0;
47 var inptr: usize = 0;
48 while (inptr < input.len) {
49 if (input[inptr] == '%') {
50 inptr += 1;
51 if (inptr + 2 <= input.len) {
52 _ = std.fmt.parseInt(u8, input[inptr..][0..2], 16) catch {
53 outsize += 3;
54 inptr += 2;
55 continue;
56 };
57 inptr += 2;
58 outsize += 1;
59 }
60 } else {
61 inptr += 1;
62 outsize += 1;
63 }
64 }
65
66 var output = try allocator.alloc(u8, outsize);
67 var outptr: usize = 0;
68 inptr = 0;
69 while (inptr < input.len) {
70 if (input[inptr] == '%') {
71 inptr += 1;
72 if (inptr + 2 <= input.len) {
73 const value = std.fmt.parseInt(u8, input[inptr..][0..2], 16) catch {
74 output[outptr + 0] = input[inptr + 0];
75 output[outptr + 1] = input[inptr + 1];
76 inptr += 2;
77 outptr += 2;
78 continue;
79 };
80
81 output[outptr] = value;
582
6/// TODO: redo this implementation according to RFC 1738. This code is only a
7/// placeholder for now.
8pub fn parse(s: []const u8) !Url {
9 var scheme_end: usize = 0;
10 var host_start: usize = 0;
11 var host_end: usize = 0;
12 var path_start: usize = 0;
13 var port_start: usize = 0;
14 var port_end: usize = 0;
15 var state: enum {
16 scheme,
17 scheme_slash1,
18 scheme_slash2,
19 host,
20 port,
21 path,
22 } = .scheme;
23
24 for (s) |b, i| switch (state) {
25 .scheme => switch (b) {
26 ':' => {
27 state = .scheme_slash1;
28 scheme_end = i;
29 },
30 else => {},
31 },
32 .scheme_slash1 => switch (b) {
33 '/' => {
34 state = .scheme_slash2;
35 },
36 else => return error.InvalidUrl,
37 },
38 .scheme_slash2 => switch (b) {
39 '/' => {
40 state = .host;
41 host_start = i + 1;
42 },
43 else => return error.InvalidUrl,
44 },
45 .host => switch (b) {
46 ':' => {
47 state = .port;
48 host_end = i;
49 port_start = i + 1;
50 },
51 '/' => {
52 state = .path;
53 host_end = i;
54 path_start = i;
55 },
56 else => {},
57 },
58 .port => switch (b) {
59 '/' => {
60 port_end = i;
61 state = .path;
62 path_start = i;
63 },
64 else => {},
65 },
66 .path => {},
83 inptr += 2;
84 outptr += 1;
85 }
86 } else {
87 output[outptr] = input[inptr];
88 inptr += 1;
89 outptr += 1;
90 }
91 }
92 return output;
93}
94
95pub const ParseError = error{ UnexpectedCharacter, InvalidFormat, InvalidPort };
96
97/// Parses the URI or returns an error.
98/// The return value will contain unescaped strings pointing into the
99/// original `text`. Each component that is provided, will be non-`null`.
100pub fn parse(text: []const u8) ParseError!Url {
101 var uri = Url{
102 .scheme = null,
103 .user = null,
104 .password = null,
105 .host = null,
106 .port = null,
107 .path = "", // path is always set, but empty by default.
108 .query = null,
109 .fragment = null,
67110 };
68111
69 const port_slice = s[port_start..port_end];
70 const port = if (port_slice.len == 0) null else try std.fmt.parseInt(u16, port_slice, 10);
112 var reader = SliceReader{ .slice = text };
113
114 uri.scheme = reader.readWhile(isSchemeChar);
115
116 // after the scheme, a ':' must appear
117 if (reader.get()) |c| {
118 if (c != ':')
119 return error.UnexpectedCharacter;
120 } else {
121 return error.InvalidFormat;
122 }
123
124 if (reader.peekPrefix("//")) { // authority part
125 std.debug.assert(reader.get().? == '/');
126 std.debug.assert(reader.get().? == '/');
127
128 const authority = reader.readUntil(isAuthoritySeparator);
129 if (authority.len == 0)
130 return error.InvalidFormat;
131
132 var start_of_host: usize = 0;
133 if (std.mem.indexOf(u8, authority, "@")) |index| {
134 start_of_host = index + 1;
135 const user_info = authority[0..index];
136
137 if (std.mem.indexOf(u8, user_info, ":")) |idx| {
138 uri.user = user_info[0..idx];
139 if (idx < user_info.len - 1) { // empty password is also "no password"
140 uri.password = user_info[idx + 1 ..];
141 }
142 } else {
143 uri.user = user_info;
144 uri.password = null;
145 }
146 }
147
148 var end_of_host: usize = authority.len;
149
150 if (authority[start_of_host] == '[') { // IPv6
151 end_of_host = std.mem.lastIndexOf(u8, authority, "]") orelse return error.InvalidFormat;
152 end_of_host += 1;
153
154 if (std.mem.lastIndexOf(u8, authority, ":")) |index| {
155 if (index >= end_of_host) { // if not part of the V6 address field
156 end_of_host = std.math.min(end_of_host, index);
157 uri.port = std.fmt.parseInt(u16, authority[index + 1 ..], 10) catch return error.InvalidPort;
158 }
159 }
160 } else if (std.mem.lastIndexOf(u8, authority, ":")) |index| {
161 if (index >= start_of_host) { // if not part of the userinfo field
162 end_of_host = std.math.min(end_of_host, index);
163 uri.port = std.fmt.parseInt(u16, authority[index + 1 ..], 10) catch return error.InvalidPort;
164 }
165 }
166
167 uri.host = authority[start_of_host..end_of_host];
168 }
71169
72 return .{
73 .scheme = s[0..scheme_end],
74 .host = s[host_start..host_end],
75 .path = s[path_start..],
76 .port = port,
170 uri.path = reader.readUntil(isPathSeparator);
171
172 if ((reader.peek() orelse 0) == '?') { // query part
173 std.debug.assert(reader.get().? == '?');
174 uri.query = reader.readUntil(isQuerySeparator);
175 }
176
177 if ((reader.peek() orelse 0) == '#') { // fragment part
178 std.debug.assert(reader.get().? == '#');
179 uri.fragment = reader.readUntilEof();
180 }
181
182 return uri;
183}
184
185const SliceReader = struct {
186 const Self = @This();
187
188 slice: []const u8,
189 offset: usize = 0,
190
191 fn get(self: *Self) ?u8 {
192 if (self.offset >= self.slice.len)
193 return null;
194 const c = self.slice[self.offset];
195 self.offset += 1;
196 return c;
197 }
198
199 fn peek(self: Self) ?u8 {
200 if (self.offset >= self.slice.len)
201 return null;
202 return self.slice[self.offset];
203 }
204
205 fn readWhile(self: *Self, comptime predicate: fn (u8) bool) []const u8 {
206 const start = self.offset;
207 var end = start;
208 while (end < self.slice.len and predicate(self.slice[end])) {
209 end += 1;
210 }
211 self.offset = end;
212 return self.slice[start..end];
213 }
214
215 fn readUntil(self: *Self, comptime predicate: fn (u8) bool) []const u8 {
216 const start = self.offset;
217 var end = start;
218 while (end < self.slice.len and !predicate(self.slice[end])) {
219 end += 1;
220 }
221 self.offset = end;
222 return self.slice[start..end];
223 }
224
225 fn readUntilEof(self: *Self) []const u8 {
226 const start = self.offset;
227 self.offset = self.slice.len;
228 return self.slice[start..];
229 }
230
231 fn peekPrefix(self: Self, prefix: []const u8) bool {
232 if (self.offset + prefix.len > self.slice.len)
233 return false;
234 return std.mem.eql(u8, self.slice[self.offset..][0..prefix.len], prefix);
235 }
236};
237
238/// scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
239fn isSchemeChar(c: u8) bool {
240 return switch (c) {
241 'A'...'Z', 'a'...'z', '0'...'9', '+', '-', '.' => true,
242 else => false,
77243 };
78244}
79245
80const Url = @This();
81const std = @import("std.zig");
82const testing = std.testing;
246fn isAuthoritySeparator(c: u8) bool {
247 return switch (c) {
248 '/', '?', '#' => true,
249 else => false,
250 };
251}
252
253/// reserved = gen-delims / sub-delims
254fn isReserved(c: u8) bool {
255 return isGenLimit(c) or isSubLimit(c);
256}
257
258/// gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
259fn isGenLimit(c: u8) bool {
260 return switch (c) {
261 ':', ',', '?', '#', '[', ']', '@' => true,
262 else => false,
263 };
264}
265
266/// sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
267/// / "*" / "+" / "," / ";" / "="
268fn isSubLimit(c: u8) bool {
269 return switch (c) {
270 '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=' => true,
271 else => false,
272 };
273}
274
275/// unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
276fn isUnreserved(c: u8) bool {
277 return switch (c) {
278 'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_', '~' => true,
279 else => false,
280 };
281}
282
283fn isPathSeparator(c: u8) bool {
284 return switch (c) {
285 '?', '#' => true,
286 else => false,
287 };
288}
289
290fn isQuerySeparator(c: u8) bool {
291 return switch (c) {
292 '#' => true,
293 else => false,
294 };
295}
83296
84297test "basic" {
85298 const parsed = try parse("https://ziglang.org/download");
86 try testing.expectEqualStrings("https", parsed.scheme);
87 try testing.expectEqualStrings("ziglang.org", parsed.host);
299 try testing.expectEqualStrings("https", parsed.scheme orelse return error.UnexpectedNull);
300 try testing.expectEqualStrings("ziglang.org", parsed.host orelse return error.UnexpectedNull);
88301 try testing.expectEqualStrings("/download", parsed.path);
89302 try testing.expectEqual(@as(?u16, null), parsed.port);
90303}
91304
92305test "with port" {
93306 const parsed = try parse("http://example:1337/");
94 try testing.expectEqualStrings("http", parsed.scheme);
95 try testing.expectEqualStrings("example", parsed.host);
307 try testing.expectEqualStrings("http", parsed.scheme orelse return error.UnexpectedNull);
308 try testing.expectEqualStrings("example", parsed.host orelse return error.UnexpectedNull);
96309 try testing.expectEqualStrings("/", parsed.path);
97310 try testing.expectEqual(@as(?u16, 1337), parsed.port);
98311}
312
313test "should fail gracefully" {
314 try std.testing.expectEqual(@as(ParseError!Url, error.InvalidFormat), parse("foobar://"));
315}
316
317test "scheme" {
318 try std.testing.expectEqualSlices(u8, "http", (try parse("http:_")).scheme.?);
319 try std.testing.expectEqualSlices(u8, "scheme-mee", (try parse("scheme-mee:_")).scheme.?);
320 try std.testing.expectEqualSlices(u8, "a.b.c", (try parse("a.b.c:_")).scheme.?);
321 try std.testing.expectEqualSlices(u8, "ab+", (try parse("ab+:_")).scheme.?);
322 try std.testing.expectEqualSlices(u8, "X+++", (try parse("X+++:_")).scheme.?);
323 try std.testing.expectEqualSlices(u8, "Y+-.", (try parse("Y+-.:_")).scheme.?);
324}
325
326test "authority" {
327 try std.testing.expectEqualSlices(u8, "hostname", (try parse("scheme://hostname")).host.?);
328
329 try std.testing.expectEqualSlices(u8, "hostname", (try parse("scheme://userinfo@hostname")).host.?);
330 try std.testing.expectEqualSlices(u8, "userinfo", (try parse("scheme://userinfo@hostname")).user.?);
331 try std.testing.expectEqual(@as(?[]const u8, null), (try parse("scheme://userinfo@hostname")).password);
332
333 try std.testing.expectEqualSlices(u8, "hostname", (try parse("scheme://user:password@hostname")).host.?);
334 try std.testing.expectEqualSlices(u8, "user", (try parse("scheme://user:password@hostname")).user.?);
335 try std.testing.expectEqualSlices(u8, "password", (try parse("scheme://user:password@hostname")).password.?);
336
337 try std.testing.expectEqualSlices(u8, "hostname", (try parse("scheme://hostname:0")).host.?);
338 try std.testing.expectEqual(@as(u16, 1234), (try parse("scheme://hostname:1234")).port.?);
339
340 try std.testing.expectEqualSlices(u8, "hostname", (try parse("scheme://userinfo@hostname:1234")).host.?);
341 try std.testing.expectEqual(@as(u16, 1234), (try parse("scheme://userinfo@hostname:1234")).port.?);
342 try std.testing.expectEqualSlices(u8, "userinfo", (try parse("scheme://userinfo@hostname:1234")).user.?);
343 try std.testing.expectEqual(@as(?[]const u8, null), (try parse("scheme://userinfo@hostname:1234")).password);
344
345 try std.testing.expectEqualSlices(u8, "hostname", (try parse("scheme://user:password@hostname:1234")).host.?);
346 try std.testing.expectEqual(@as(u16, 1234), (try parse("scheme://user:password@hostname:1234")).port.?);
347 try std.testing.expectEqualSlices(u8, "user", (try parse("scheme://user:password@hostname:1234")).user.?);
348 try std.testing.expectEqualSlices(u8, "password", (try parse("scheme://user:password@hostname:1234")).password.?);
349}
350
351test "authority.password" {
352 try std.testing.expectEqualSlices(u8, "username", (try parse("scheme://username@a")).user.?);
353 try std.testing.expectEqual(@as(?[]const u8, null), (try parse("scheme://username@a")).password);
354
355 try std.testing.expectEqualSlices(u8, "username", (try parse("scheme://username:@a")).user.?);
356 try std.testing.expectEqual(@as(?[]const u8, null), (try parse("scheme://username:@a")).password);
357
358 try std.testing.expectEqualSlices(u8, "username", (try parse("scheme://username:password@a")).user.?);
359 try std.testing.expectEqualSlices(u8, "password", (try parse("scheme://username:password@a")).password.?);
360
361 try std.testing.expectEqualSlices(u8, "username", (try parse("scheme://username::@a")).user.?);
362 try std.testing.expectEqualSlices(u8, ":", (try parse("scheme://username::@a")).password.?);
363}
364
365fn testAuthorityHost(comptime hostlist: anytype) !void {
366 inline for (hostlist) |hostname| {
367 try std.testing.expectEqualSlices(u8, hostname, (try parse("scheme://" ++ hostname)).host.?);
368 }
369}
370
371test "authority.dns-names" {
372 try testAuthorityHost(.{
373 "a",
374 "a.b",
375 "example.com",
376 "www.example.com",
377 "example.org.",
378 "www.example.org.",
379 "xn--nw2a.xn--j6w193g", // internationalized URI: 見.香港
380 "fe80--1ff-fe23-4567-890as3.ipv6-literal.net",
381 });
382}
383
384test "authority.IPv4" {
385 try testAuthorityHost(.{
386 "127.0.0.1",
387 "255.255.255.255",
388 "0.0.0.0",
389 "8.8.8.8",
390 "1.2.3.4",
391 "192.168.0.1",
392 "10.42.0.0",
393 });
394}
395
396test "authority.IPv6" {
397 try testAuthorityHost(.{
398 "[2001:db8:0:0:0:0:2:1]",
399 "[2001:db8::2:1]",
400 "[2001:db8:0000:1:1:1:1:1]",
401 "[2001:db8:0:1:1:1:1:1]",
402 "[0:0:0:0:0:0:0:0]",
403 "[0:0:0:0:0:0:0:1]",
404 "[::1]",
405 "[::]",
406 "[2001:db8:85a3:8d3:1319:8a2e:370:7348]",
407 "[fe80::1ff:fe23:4567:890a%25eth2]",
408 "[fe80::1ff:fe23:4567:890a]",
409 "[fe80::1ff:fe23:4567:890a%253]",
410 "[fe80:3::1ff:fe23:4567:890a]",
411 });
412}
413
414test "RFC example 1" {
415 const uri = "foo://example.com:8042/over/there?name=ferret#nose";
416 try std.testing.expectEqual(Url{
417 .scheme = uri[0..3],
418 .user = null,
419 .password = null,
420 .host = uri[6..17],
421 .port = 8042,
422 .path = uri[22..33],
423 .query = uri[34..45],
424 .fragment = uri[46..50],
425 }, try parse(uri));
426}
427
428test "RFX example 2" {
429 const uri = "urn:example:animal:ferret:nose";
430 try std.testing.expectEqual(Url{
431 .scheme = uri[0..3],
432 .user = null,
433 .password = null,
434 .host = null,
435 .port = null,
436 .path = uri[4..],
437 .query = null,
438 .fragment = null,
439 }, try parse(uri));
440}
441
442// source:
443// https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#Examples
444test "Examples from wikipedia" {
445 const list = [_][]const u8{
446 "https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top",
447 "ldap://[2001:db8::7]/c=GB?objectClass?one",
448 "mailto:John.Doe@example.com",
449 "news:comp.infosystems.www.servers.unix",
450 "tel:+1-816-555-1212",
451 "telnet://192.0.2.16:80/",
452 "urn:oasis:names:specification:docbook:dtd:xml:4.1.2",
453 "http://a/b/c/d;p?q",
454 };
455 for (list) |uri| {
456 _ = try parse(uri);
457 }
458}
459
460// source:
461// https://tools.ietf.org/html/rfc3986#section-5.4.1
462test "Examples from RFC3986" {
463 const list = [_][]const u8{
464 "http://a/b/c/g",
465 "http://a/b/c/g",
466 "http://a/b/c/g/",
467 "http://a/g",
468 "http://g",
469 "http://a/b/c/d;p?y",
470 "http://a/b/c/g?y",
471 "http://a/b/c/d;p?q#s",
472 "http://a/b/c/g#s",
473 "http://a/b/c/g?y#s",
474 "http://a/b/c/;x",
475 "http://a/b/c/g;x",
476 "http://a/b/c/g;x?y#s",
477 "http://a/b/c/d;p?q",
478 "http://a/b/c/",
479 "http://a/b/c/",
480 "http://a/b/",
481 "http://a/b/",
482 "http://a/b/g",
483 "http://a/",
484 "http://a/",
485 "http://a/g",
486 };
487 for (list) |uri| {
488 _ = try parse(uri);
489 }
490}
491
492test "Special test" {
493 // This is for all of you code readers ♥
494 _ = try parse("https://www.youtube.com/watch?v=dQw4w9WgXcQ&feature=youtu.be&t=0");
495}
496
497test "URI escaping" {
498 const input = "\\ö/ äöß ~~.adas-https://canvas:123/#ads&&sad";
499 const expected = "%5C%C3%B6%2F%20%C3%A4%C3%B6%C3%9F%20~~.adas-https%3A%2F%2Fcanvas%3A123%2F%23ads%26%26sad";
500
501 const actual = try escapeString(std.testing.allocator, input);
502 defer std.testing.allocator.free(actual);
503
504 try std.testing.expectEqualSlices(u8, expected, actual);
505}
506
507test "URI unescaping" {
508 const input = "%5C%C3%B6%2F%20%C3%A4%C3%B6%C3%9F%20~~.adas-https%3A%2F%2Fcanvas%3A123%2F%23ads%26%26sad";
509 const expected = "\\ö/ äöß ~~.adas-https://canvas:123/#ads&&sad";
510
511 const actual = try unescapeString(std.testing.allocator, input);
512 defer std.testing.allocator.free(actual);
513
514 try std.testing.expectEqualSlices(u8, expected, actual);
515}