authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-17 02:21:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log651aa5e8e497a12dc14d31f383777f99b92a2739
tree458ebb9f05ec821747a2f3267b4def1fab085db0
parent107992d50e6a07d98b325f8768882f17e454b4a4

std.http.Client: eliminate arena allocator usage

Before, this code constructed an arena allocator and then used it when handling redirects. You know what's better than having threads fight over an allocator? Avoiding dynamic memory allocation in the first place. This commit reuses the http headers static buffer for handling redirects. The new location is copied to the beginning of the static header buffer and then the subsequent request uses a subslice of that buffer.

2 files changed, 156 insertions(+), 120 deletions(-)

lib/std/Uri.zig+127-93
......@@ -342,7 +342,7 @@ pub fn format(
342342/// The return value will contain unescaped strings pointing into the
343343/// original `text`. Each component that is provided, will be non-`null`.
344344pub fn parse(text: []const u8) ParseError!Uri {
345 var reader = SliceReader{ .slice = text };
345 var reader: SliceReader = .{ .slice = text };
346346 const scheme = reader.readWhile(isSchemeChar);
347347
348348 // after the scheme, a ':' must appear
......@@ -359,111 +359,145 @@ pub fn parse(text: []const u8) ParseError!Uri {
359359 return uri;
360360}
361361
362/// Implementation of RFC 3986, Section 5.2.4. Removes dot segments from a URI path.
363///
364/// `std.fs.path.resolvePosix` is not sufficient here because it may return relative paths and does not preserve trailing slashes.
365fn removeDotSegments(allocator: Allocator, paths: []const []const u8) Allocator.Error![]const u8 {
366 var result = std.ArrayList(u8).init(allocator);
367 defer result.deinit();
368
369 for (paths) |p| {
370 var it = std.mem.tokenizeScalar(u8, p, '/');
371 while (it.next()) |component| {
372 if (std.mem.eql(u8, component, ".")) {
373 continue;
374 } else if (std.mem.eql(u8, component, "..")) {
375 if (result.items.len == 0)
376 continue;
362pub const ResolveInplaceError = ParseError || error{OutOfMemory};
377363
378 while (true) {
379 const ends_with_slash = result.items[result.items.len - 1] == '/';
380 result.items.len -= 1;
381 if (ends_with_slash or result.items.len == 0) break;
382 }
383 } else {
384 try result.ensureUnusedCapacity(1 + component.len);
385 result.appendAssumeCapacity('/');
386 result.appendSliceAssumeCapacity(component);
387 }
388 }
389 }
364/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.
365/// Copies `new` to the beginning of `aux_buf`, allowing the slices to overlap,
366/// then parses `new` as a URI, and then resolves the path in place.
367/// If a merge needs to take place, the newly constructed path will be stored
368/// in `aux_buf` just after the copied `new`.
369pub fn resolve_inplace(base: Uri, new: []const u8, aux_buf: []u8) ResolveInplaceError!Uri {
370 std.mem.copyBackwards(u8, aux_buf, new);
371 // At this point, new is an invalid pointer.
372 const new_mut = aux_buf[0..new.len];
373
374 const new_parsed, const has_scheme = p: {
375 break :p .{
376 parse(new_mut) catch |first_err| {
377 break :p .{
378 parseWithoutScheme(new_mut) catch return first_err,
379 false,
380 };
381 },
382 true,
383 };
384 };
390385
391 // ensure a trailing slash is kept
392 const last_path = paths[paths.len - 1];
393 if (last_path.len > 0 and last_path[last_path.len - 1] == '/') {
394 try result.append('/');
395 }
386 // As you can see above, `new_mut` is not a const pointer.
387 const new_path: []u8 = @constCast(new_parsed.path);
388
389 if (has_scheme) return .{
390 .scheme = new_parsed.scheme,
391 .user = new_parsed.user,
392 .host = new_parsed.host,
393 .port = new_parsed.port,
394 .path = remove_dot_segments(new_path),
395 .query = new_parsed.query,
396 .fragment = new_parsed.fragment,
397 };
396398
397 return result.toOwnedSlice();
398}
399 if (new_parsed.host) |host| return .{
400 .scheme = base.scheme,
401 .user = new_parsed.user,
402 .host = host,
403 .port = new_parsed.port,
404 .path = remove_dot_segments(new_path),
405 .query = new_parsed.query,
406 .fragment = new_parsed.fragment,
407 };
399408
400/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.
401///
402/// Assumes `arena` owns all memory in `base` and `ref`. `arena` will own all memory in the returned URI.
403pub fn resolve(base: Uri, ref: Uri, strict: bool, arena: Allocator) Allocator.Error!Uri {
404 var target: Uri = Uri{
405 .scheme = "",
406 .user = null,
407 .password = null,
408 .host = null,
409 .port = null,
410 .path = "",
411 .query = null,
412 .fragment = null,
409 const path, const query = b: {
410 if (new_path.len == 0)
411 break :b .{
412 base.path,
413 new_parsed.query orelse base.query,
414 };
415
416 if (new_path[0] == '/')
417 break :b .{
418 remove_dot_segments(new_path),
419 new_parsed.query,
420 };
421
422 break :b .{
423 try merge_paths(base.path, new_path, aux_buf[new_mut.len..]),
424 new_parsed.query,
425 };
413426 };
414427
415 if (ref.scheme.len > 0 and (strict or !std.mem.eql(u8, ref.scheme, base.scheme))) {
416 target.scheme = ref.scheme;
417 target.user = ref.user;
418 target.host = ref.host;
419 target.port = ref.port;
420 target.path = try removeDotSegments(arena, &.{ref.path});
421 target.query = ref.query;
422 } else {
423 target.scheme = base.scheme;
424 if (ref.host) |host| {
425 target.user = ref.user;
426 target.host = host;
427 target.port = ref.port;
428 target.path = ref.path;
429 target.path = try removeDotSegments(arena, &.{ref.path});
430 target.query = ref.query;
428 return .{
429 .scheme = base.scheme,
430 .user = base.user,
431 .host = base.host,
432 .port = base.port,
433 .path = path,
434 .query = query,
435 .fragment = new_parsed.fragment,
436 };
437}
438
439/// In-place implementation of RFC 3986, Section 5.2.4.
440fn remove_dot_segments(path: []u8) []u8 {
441 var in_i: usize = 0;
442 var out_i: usize = 0;
443 while (in_i < path.len) {
444 if (std.mem.startsWith(u8, path[in_i..], "./")) {
445 in_i += 2;
446 } else if (std.mem.startsWith(u8, path[in_i..], "../")) {
447 in_i += 3;
448 } else if (std.mem.startsWith(u8, path[in_i..], "/./")) {
449 in_i += 2;
450 } else if (std.mem.eql(u8, path[in_i..], "/.")) {
451 in_i += 1;
452 path[in_i] = '/';
453 } else if (std.mem.startsWith(u8, path[in_i..], "/../")) {
454 in_i += 3;
455 while (out_i > 0) {
456 out_i -= 1;
457 if (path[out_i] == '/') break;
458 }
459 } else if (std.mem.eql(u8, path[in_i..], "/..")) {
460 in_i += 2;
461 path[in_i] = '/';
462 while (out_i > 0) {
463 out_i -= 1;
464 if (path[out_i] == '/') break;
465 }
466 } else if (std.mem.eql(u8, path[in_i..], ".")) {
467 in_i += 1;
468 } else if (std.mem.eql(u8, path[in_i..], "..")) {
469 in_i += 2;
431470 } else {
432 if (ref.path.len == 0) {
433 target.path = base.path;
434 target.query = ref.query orelse base.query;
435 } else {
436 if (ref.path[0] == '/') {
437 target.path = try removeDotSegments(arena, &.{ref.path});
438 } else {
439 target.path = try removeDotSegments(arena, &.{ std.fs.path.dirnamePosix(base.path) orelse "", ref.path });
440 }
441 target.query = ref.query;
471 while (true) {
472 path[out_i] = path[in_i];
473 out_i += 1;
474 in_i += 1;
475 if (in_i >= path.len or path[in_i] == '/') break;
442476 }
443
444 target.user = base.user;
445 target.host = base.host;
446 target.port = base.port;
447477 }
448478 }
449
450 target.fragment = ref.fragment;
451
452 return target;
479 return path[0..out_i];
453480}
454481
455test resolve {
456 const base = try parse("http://a/b/c/d;p?q");
457
458 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
459 defer arena.deinit();
482test remove_dot_segments {
483 {
484 var buffer = "/a/b/c/./../../g".*;
485 try std.testing.expectEqualStrings("/a/g", remove_dot_segments(&buffer));
486 }
487}
460488
461 try std.testing.expectEqualDeep(try parse("http://a/b/c/blog/"), try base.resolve(try parseWithoutScheme("blog/"), true, arena.allocator()));
462 try std.testing.expectEqualDeep(try parse("http://a/b/c/blog/?k"), try base.resolve(try parseWithoutScheme("blog/?k"), true, arena.allocator()));
463 try std.testing.expectEqualDeep(try parse("http://a/b/blog/"), try base.resolve(try parseWithoutScheme("../blog/"), true, arena.allocator()));
464 try std.testing.expectEqualDeep(try parse("http://a/b/blog"), try base.resolve(try parseWithoutScheme("../blog"), true, arena.allocator()));
465 try std.testing.expectEqualDeep(try parse("http://e"), try base.resolve(try parseWithoutScheme("//e"), true, arena.allocator()));
466 try std.testing.expectEqualDeep(try parse("https://a:1/"), try base.resolve(try parse("https://a:1/"), true, arena.allocator()));
489/// 5.2.3. Merge Paths
490fn merge_paths(base: []const u8, new: []u8, aux: []u8) error{OutOfMemory}![]u8 {
491 if (aux.len < base.len + 1 + new.len) return error.OutOfMemory;
492 if (base.len == 0) {
493 aux[0] = '/';
494 @memcpy(aux[1..][0..new.len], new);
495 return remove_dot_segments(aux[0 .. new.len + 1]);
496 }
497 const pos = std.mem.lastIndexOfScalar(u8, base, '/') orelse return remove_dot_segments(new);
498 @memcpy(aux[0 .. pos + 1], base[0 .. pos + 1]);
499 @memcpy(aux[pos + 1 ..][0..new.len], new);
500 return remove_dot_segments(aux[0 .. pos + 1 + new.len]);
467501}
468502
469503const SliceReader = struct {
lib/std/http/Client.zig+29-27
......@@ -597,9 +597,6 @@ pub const Request = struct {
597597 /// This field is undefined until `wait` is called.
598598 response: Response,
599599
600 /// Used as a allocator for resolving redirects locations.
601 arena: std.heap.ArenaAllocator,
602
603600 /// Standard headers that have default, but overridable, behavior.
604601 headers: Headers,
605602
......@@ -661,8 +658,6 @@ pub const Request = struct {
661658 }
662659 req.client.connection_pool.release(req.client.allocator, connection);
663660 }
664
665 req.arena.deinit();
666661 req.* = undefined;
667662 }
668663
......@@ -842,11 +837,12 @@ pub const Request = struct {
842837 }
843838
844839 pub const WaitError = RequestError || SendError || TransferReadError ||
845 proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError ||
840 proto.HeadersParser.CheckCompleteHeadError || Response.ParseError ||
846841 error{ // TODO: file zig fmt issue for this bad indentation
847842 TooManyHttpRedirects,
848843 RedirectRequiresResend,
849 HttpRedirectMissingLocation,
844 HttpRedirectLocationMissing,
845 HttpRedirectLocationInvalid,
850846 CompressionInitializationFailed,
851847 CompressionUnsupported,
852848 };
......@@ -927,31 +923,40 @@ pub const Request = struct {
927923 }
928924
929925 if (req.response.status.class() == .redirect and req.redirect_behavior != .unhandled) {
930 req.response.skip = true;
931
932926 // skip the body of the redirect response, this will at least
933927 // leave the connection in a known good state.
928 req.response.skip = true;
934929 assert(try req.transferRead(&.{}) == 0); // we're skipping, no buffer is necessary
935930
936931 if (req.redirect_behavior == .not_allowed) return error.TooManyHttpRedirects;
937932
938933 const location = req.response.location orelse
939 return error.HttpRedirectMissingLocation;
940
941 const arena = req.arena.allocator();
942
943 const location_duped = try arena.dupe(u8, location);
944
945 const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);
946 const resolved_url = try req.uri.resolve(new_url, false, arena);
934 return error.HttpRedirectLocationMissing;
935
936 // This mutates the beginning of header_buffer and uses that
937 // for the backing memory of the returned new_uri.
938 const header_buffer = req.response.parser.header_bytes_buffer;
939 const new_uri = req.uri.resolve_inplace(location, header_buffer) catch
940 return error.HttpRedirectLocationInvalid;
941
942 // The new URI references the beginning of header_bytes_buffer memory.
943 // That memory will be kept, but everything after it will be
944 // reused by the subsequent request. In other words,
945 // header_bytes_buffer must be large enough to store all
946 // redirect locations as well as the final request header.
947 const path_end = new_uri.path.ptr + new_uri.path.len;
948 // https://github.com/ziglang/zig/issues/1738
949 const path_offset = @intFromPtr(path_end) - @intFromPtr(header_buffer.ptr);
950 const end_offset = @max(path_offset, location.len);
951 req.response.parser.header_bytes_buffer = header_buffer[end_offset..];
947952
948953 const is_same_domain_or_subdomain =
949 std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and
950 (resolved_url.host.?.len == req.uri.host.?.len or
951 resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.');
954 std.ascii.endsWithIgnoreCase(new_uri.host.?, req.uri.host.?) and
955 (new_uri.host.?.len == req.uri.host.?.len or
956 new_uri.host.?[new_uri.host.?.len - req.uri.host.?.len - 1] == '.');
952957
953 if (resolved_url.host == null or !is_same_domain_or_subdomain or
954 !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme))
958 if (new_uri.host == null or !is_same_domain_or_subdomain or
959 !std.ascii.eqlIgnoreCase(new_uri.scheme, req.uri.scheme))
955960 {
956961 // When redirecting to a different domain, strip privileged headers.
957962 req.privileged_headers = &.{};
......@@ -975,7 +980,7 @@ pub const Request = struct {
975980 return error.RedirectRequiresResend;
976981 }
977982
978 try req.redirect(resolved_url);
983 try req.redirect(new_uri);
979984 try req.send(.{});
980985 } else {
981986 req.response.skip = false;
......@@ -1341,7 +1346,7 @@ pub fn connectTunnel(
13411346 client.connection_pool.release(client.allocator, conn);
13421347 }
13431348
1344 const uri = Uri{
1349 const uri: Uri = .{
13451350 .scheme = "http",
13461351 .user = null,
13471352 .password = null,
......@@ -1548,15 +1553,12 @@ pub fn open(
15481553 .version = undefined,
15491554 .parser = proto.HeadersParser.init(options.server_header_buffer),
15501555 },
1551 .arena = undefined,
15521556 .headers = options.headers,
15531557 .extra_headers = options.extra_headers,
15541558 .privileged_headers = options.privileged_headers,
15551559 };
15561560 errdefer req.deinit();
15571561
1558 req.arena = std.heap.ArenaAllocator.init(client.allocator);
1559
15601562 return req;
15611563}
15621564