authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-17 02:21:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log651aa5e8e497a12dc14d31f383777f99b92a2739
tree458ebb9f05ec821747a2f3267b4def1fab085db0
parent107992d50e6a07d98b325f8768882f17e454b4a4

std.http.Client: eliminate arena allocator usage

Before, this code constructed an arena allocator and then used it when handling redirects. You know what's better than having threads fight over an allocator? Avoiding dynamic memory allocation in the first place. This commit reuses the http headers static buffer for handling redirects. The new location is copied to the beginning of the static header buffer and then the subsequent request uses a subslice of that buffer.

2 files changed, 156 insertions(+), 120 deletions(-)

lib/std/Uri.zig+127-93
...@@ -342,7 +342,7 @@ pub fn format(...@@ -342,7 +342,7 @@ pub fn format(
342/// The return value will contain unescaped strings pointing into the342/// The return value will contain unescaped strings pointing into the
343/// original `text`. Each component that is provided, will be non-`null`.343/// original `text`. Each component that is provided, will be non-`null`.
344pub fn parse(text: []const u8) ParseError!Uri {344pub fn parse(text: []const u8) ParseError!Uri {
345 var reader = SliceReader{ .slice = text };345 var reader: SliceReader = .{ .slice = text };
346 const scheme = reader.readWhile(isSchemeChar);346 const scheme = reader.readWhile(isSchemeChar);
347347
348 // after the scheme, a ':' must appear348 // after the scheme, a ':' must appear
...@@ -359,111 +359,145 @@ pub fn parse(text: []const u8) ParseError!Uri {...@@ -359,111 +359,145 @@ pub fn parse(text: []const u8) ParseError!Uri {
359 return uri;359 return uri;
360}360}
361361
362/// Implementation of RFC 3986, Section 5.2.4. Removes dot segments from a URI path.362pub const ResolveInplaceError = ParseError || error{OutOfMemory};
363///
364/// `std.fs.path.resolvePosix` is not sufficient here because it may return relative paths and does not preserve trailing slashes.
365fn removeDotSegments(allocator: Allocator, paths: []const []const u8) Allocator.Error![]const u8 {
366 var result = std.ArrayList(u8).init(allocator);
367 defer result.deinit();
368
369 for (paths) |p| {
370 var it = std.mem.tokenizeScalar(u8, p, '/');
371 while (it.next()) |component| {
372 if (std.mem.eql(u8, component, ".")) {
373 continue;
374 } else if (std.mem.eql(u8, component, "..")) {
375 if (result.items.len == 0)
376 continue;
377363
378 while (true) {364/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.
379 const ends_with_slash = result.items[result.items.len - 1] == '/';365/// Copies `new` to the beginning of `aux_buf`, allowing the slices to overlap,
380 result.items.len -= 1;366/// then parses `new` as a URI, and then resolves the path in place.
381 if (ends_with_slash or result.items.len == 0) break;367/// If a merge needs to take place, the newly constructed path will be stored
382 }368/// in `aux_buf` just after the copied `new`.
383 } else {369pub fn resolve_inplace(base: Uri, new: []const u8, aux_buf: []u8) ResolveInplaceError!Uri {
384 try result.ensureUnusedCapacity(1 + component.len);370 std.mem.copyBackwards(u8, aux_buf, new);
385 result.appendAssumeCapacity('/');371 // At this point, new is an invalid pointer.
386 result.appendSliceAssumeCapacity(component);372 const new_mut = aux_buf[0..new.len];
387 }373
388 }374 const new_parsed, const has_scheme = p: {
389 }375 break :p .{
376 parse(new_mut) catch |first_err| {
377 break :p .{
378 parseWithoutScheme(new_mut) catch return first_err,
379 false,
380 };
381 },
382 true,
383 };
384 };
390385
391 // ensure a trailing slash is kept386 // As you can see above, `new_mut` is not a const pointer.
392 const last_path = paths[paths.len - 1];387 const new_path: []u8 = @constCast(new_parsed.path);
393 if (last_path.len > 0 and last_path[last_path.len - 1] == '/') {388
394 try result.append('/');389 if (has_scheme) return .{
395 }390 .scheme = new_parsed.scheme,
391 .user = new_parsed.user,
392 .host = new_parsed.host,
393 .port = new_parsed.port,
394 .path = remove_dot_segments(new_path),
395 .query = new_parsed.query,
396 .fragment = new_parsed.fragment,
397 };
396398
397 return result.toOwnedSlice();399 if (new_parsed.host) |host| return .{
398}400 .scheme = base.scheme,
401 .user = new_parsed.user,
402 .host = host,
403 .port = new_parsed.port,
404 .path = remove_dot_segments(new_path),
405 .query = new_parsed.query,
406 .fragment = new_parsed.fragment,
407 };
399408
400/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.409 const path, const query = b: {
401///410 if (new_path.len == 0)
402/// Assumes `arena` owns all memory in `base` and `ref`. `arena` will own all memory in the returned URI.411 break :b .{
403pub fn resolve(base: Uri, ref: Uri, strict: bool, arena: Allocator) Allocator.Error!Uri {412 base.path,
404 var target: Uri = Uri{413 new_parsed.query orelse base.query,
405 .scheme = "",414 };
406 .user = null,415
407 .password = null,416 if (new_path[0] == '/')
408 .host = null,417 break :b .{
409 .port = null,418 remove_dot_segments(new_path),
410 .path = "",419 new_parsed.query,
411 .query = null,420 };
412 .fragment = null,421
422 break :b .{
423 try merge_paths(base.path, new_path, aux_buf[new_mut.len..]),
424 new_parsed.query,
425 };
413 };426 };
414427
415 if (ref.scheme.len > 0 and (strict or !std.mem.eql(u8, ref.scheme, base.scheme))) {428 return .{
416 target.scheme = ref.scheme;429 .scheme = base.scheme,
417 target.user = ref.user;430 .user = base.user,
418 target.host = ref.host;431 .host = base.host,
419 target.port = ref.port;432 .port = base.port,
420 target.path = try removeDotSegments(arena, &.{ref.path});433 .path = path,
421 target.query = ref.query;434 .query = query,
422 } else {435 .fragment = new_parsed.fragment,
423 target.scheme = base.scheme;436 };
424 if (ref.host) |host| {437}
425 target.user = ref.user;438
426 target.host = host;439/// In-place implementation of RFC 3986, Section 5.2.4.
427 target.port = ref.port;440fn remove_dot_segments(path: []u8) []u8 {
428 target.path = ref.path;441 var in_i: usize = 0;
429 target.path = try removeDotSegments(arena, &.{ref.path});442 var out_i: usize = 0;
430 target.query = ref.query;443 while (in_i < path.len) {
444 if (std.mem.startsWith(u8, path[in_i..], "./")) {
445 in_i += 2;
446 } else if (std.mem.startsWith(u8, path[in_i..], "../")) {
447 in_i += 3;
448 } else if (std.mem.startsWith(u8, path[in_i..], "/./")) {
449 in_i += 2;
450 } else if (std.mem.eql(u8, path[in_i..], "/.")) {
451 in_i += 1;
452 path[in_i] = '/';
453 } else if (std.mem.startsWith(u8, path[in_i..], "/../")) {
454 in_i += 3;
455 while (out_i > 0) {
456 out_i -= 1;
457 if (path[out_i] == '/') break;
458 }
459 } else if (std.mem.eql(u8, path[in_i..], "/..")) {
460 in_i += 2;
461 path[in_i] = '/';
462 while (out_i > 0) {
463 out_i -= 1;
464 if (path[out_i] == '/') break;
465 }
466 } else if (std.mem.eql(u8, path[in_i..], ".")) {
467 in_i += 1;
468 } else if (std.mem.eql(u8, path[in_i..], "..")) {
469 in_i += 2;
431 } else {470 } else {
432 if (ref.path.len == 0) {471 while (true) {
433 target.path = base.path;472 path[out_i] = path[in_i];
434 target.query = ref.query orelse base.query;473 out_i += 1;
435 } else {474 in_i += 1;
436 if (ref.path[0] == '/') {475 if (in_i >= path.len or path[in_i] == '/') break;
437 target.path = try removeDotSegments(arena, &.{ref.path});
438 } else {
439 target.path = try removeDotSegments(arena, &.{ std.fs.path.dirnamePosix(base.path) orelse "", ref.path });
440 }
441 target.query = ref.query;
442 }476 }
443
444 target.user = base.user;
445 target.host = base.host;
446 target.port = base.port;
447 }477 }
448 }478 }
449479 return path[0..out_i];
450 target.fragment = ref.fragment;
451
452 return target;
453}480}
454481
455test resolve {482test remove_dot_segments {
456 const base = try parse("http://a/b/c/d;p?q");483 {
457484 var buffer = "/a/b/c/./../../g".*;
458 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);485 try std.testing.expectEqualStrings("/a/g", remove_dot_segments(&buffer));
459 defer arena.deinit();486 }
487}
460488
461 try std.testing.expectEqualDeep(try parse("http://a/b/c/blog/"), try base.resolve(try parseWithoutScheme("blog/"), true, arena.allocator()));489/// 5.2.3. Merge Paths
462 try std.testing.expectEqualDeep(try parse("http://a/b/c/blog/?k"), try base.resolve(try parseWithoutScheme("blog/?k"), true, arena.allocator()));490fn merge_paths(base: []const u8, new: []u8, aux: []u8) error{OutOfMemory}![]u8 {
463 try std.testing.expectEqualDeep(try parse("http://a/b/blog/"), try base.resolve(try parseWithoutScheme("../blog/"), true, arena.allocator()));491 if (aux.len < base.len + 1 + new.len) return error.OutOfMemory;
464 try std.testing.expectEqualDeep(try parse("http://a/b/blog"), try base.resolve(try parseWithoutScheme("../blog"), true, arena.allocator()));492 if (base.len == 0) {
465 try std.testing.expectEqualDeep(try parse("http://e"), try base.resolve(try parseWithoutScheme("//e"), true, arena.allocator()));493 aux[0] = '/';
466 try std.testing.expectEqualDeep(try parse("https://a:1/"), try base.resolve(try parse("https://a:1/"), true, arena.allocator()));494 @memcpy(aux[1..][0..new.len], new);
495 return remove_dot_segments(aux[0 .. new.len + 1]);
496 }
497 const pos = std.mem.lastIndexOfScalar(u8, base, '/') orelse return remove_dot_segments(new);
498 @memcpy(aux[0 .. pos + 1], base[0 .. pos + 1]);
499 @memcpy(aux[pos + 1 ..][0..new.len], new);
500 return remove_dot_segments(aux[0 .. pos + 1 + new.len]);
467}501}
468502
469const SliceReader = struct {503const SliceReader = struct {
lib/std/http/Client.zig+29-27
...@@ -597,9 +597,6 @@ pub const Request = struct {...@@ -597,9 +597,6 @@ pub const Request = struct {
597 /// This field is undefined until `wait` is called.597 /// This field is undefined until `wait` is called.
598 response: Response,598 response: Response,
599599
600 /// Used as a allocator for resolving redirects locations.
601 arena: std.heap.ArenaAllocator,
602
603 /// Standard headers that have default, but overridable, behavior.600 /// Standard headers that have default, but overridable, behavior.
604 headers: Headers,601 headers: Headers,
605602
...@@ -661,8 +658,6 @@ pub const Request = struct {...@@ -661,8 +658,6 @@ pub const Request = struct {
661 }658 }
662 req.client.connection_pool.release(req.client.allocator, connection);659 req.client.connection_pool.release(req.client.allocator, connection);
663 }660 }
664
665 req.arena.deinit();
666 req.* = undefined;661 req.* = undefined;
667 }662 }
668663
...@@ -842,11 +837,12 @@ pub const Request = struct {...@@ -842,11 +837,12 @@ pub const Request = struct {
842 }837 }
843838
844 pub const WaitError = RequestError || SendError || TransferReadError ||839 pub const WaitError = RequestError || SendError || TransferReadError ||
845 proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError ||840 proto.HeadersParser.CheckCompleteHeadError || Response.ParseError ||
846 error{ // TODO: file zig fmt issue for this bad indentation841 error{ // TODO: file zig fmt issue for this bad indentation
847 TooManyHttpRedirects,842 TooManyHttpRedirects,
848 RedirectRequiresResend,843 RedirectRequiresResend,
849 HttpRedirectMissingLocation,844 HttpRedirectLocationMissing,
845 HttpRedirectLocationInvalid,
850 CompressionInitializationFailed,846 CompressionInitializationFailed,
851 CompressionUnsupported,847 CompressionUnsupported,
852 };848 };
...@@ -927,31 +923,40 @@ pub const Request = struct {...@@ -927,31 +923,40 @@ pub const Request = struct {
927 }923 }
928924
929 if (req.response.status.class() == .redirect and req.redirect_behavior != .unhandled) {925 if (req.response.status.class() == .redirect and req.redirect_behavior != .unhandled) {
930 req.response.skip = true;
931
932 // skip the body of the redirect response, this will at least926 // skip the body of the redirect response, this will at least
933 // leave the connection in a known good state.927 // leave the connection in a known good state.
928 req.response.skip = true;
934 assert(try req.transferRead(&.{}) == 0); // we're skipping, no buffer is necessary929 assert(try req.transferRead(&.{}) == 0); // we're skipping, no buffer is necessary
935930
936 if (req.redirect_behavior == .not_allowed) return error.TooManyHttpRedirects;931 if (req.redirect_behavior == .not_allowed) return error.TooManyHttpRedirects;
937932
938 const location = req.response.location orelse933 const location = req.response.location orelse
939 return error.HttpRedirectMissingLocation;934 return error.HttpRedirectLocationMissing;
940935
941 const arena = req.arena.allocator();936 // This mutates the beginning of header_buffer and uses that
942937 // for the backing memory of the returned new_uri.
943 const location_duped = try arena.dupe(u8, location);938 const header_buffer = req.response.parser.header_bytes_buffer;
944939 const new_uri = req.uri.resolve_inplace(location, header_buffer) catch
945 const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);940 return error.HttpRedirectLocationInvalid;
946 const resolved_url = try req.uri.resolve(new_url, false, arena);941
942 // The new URI references the beginning of header_bytes_buffer memory.
943 // That memory will be kept, but everything after it will be
944 // reused by the subsequent request. In other words,
945 // header_bytes_buffer must be large enough to store all
946 // redirect locations as well as the final request header.
947 const path_end = new_uri.path.ptr + new_uri.path.len;
948 // https://github.com/ziglang/zig/issues/1738
949 const path_offset = @intFromPtr(path_end) - @intFromPtr(header_buffer.ptr);
950 const end_offset = @max(path_offset, location.len);
951 req.response.parser.header_bytes_buffer = header_buffer[end_offset..];
947952
948 const is_same_domain_or_subdomain =953 const is_same_domain_or_subdomain =
949 std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and954 std.ascii.endsWithIgnoreCase(new_uri.host.?, req.uri.host.?) and
950 (resolved_url.host.?.len == req.uri.host.?.len or955 (new_uri.host.?.len == req.uri.host.?.len or
951 resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.');956 new_uri.host.?[new_uri.host.?.len - req.uri.host.?.len - 1] == '.');
952957
953 if (resolved_url.host == null or !is_same_domain_or_subdomain or958 if (new_uri.host == null or !is_same_domain_or_subdomain or
954 !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme))959 !std.ascii.eqlIgnoreCase(new_uri.scheme, req.uri.scheme))
955 {960 {
956 // When redirecting to a different domain, strip privileged headers.961 // When redirecting to a different domain, strip privileged headers.
957 req.privileged_headers = &.{};962 req.privileged_headers = &.{};
...@@ -975,7 +980,7 @@ pub const Request = struct {...@@ -975,7 +980,7 @@ pub const Request = struct {
975 return error.RedirectRequiresResend;980 return error.RedirectRequiresResend;
976 }981 }
977982
978 try req.redirect(resolved_url);983 try req.redirect(new_uri);
979 try req.send(.{});984 try req.send(.{});
980 } else {985 } else {
981 req.response.skip = false;986 req.response.skip = false;
...@@ -1341,7 +1346,7 @@ pub fn connectTunnel(...@@ -1341,7 +1346,7 @@ pub fn connectTunnel(
1341 client.connection_pool.release(client.allocator, conn);1346 client.connection_pool.release(client.allocator, conn);
1342 }1347 }
13431348
1344 const uri = Uri{1349 const uri: Uri = .{
1345 .scheme = "http",1350 .scheme = "http",
1346 .user = null,1351 .user = null,
1347 .password = null,1352 .password = null,
...@@ -1548,15 +1553,12 @@ pub fn open(...@@ -1548,15 +1553,12 @@ pub fn open(
1548 .version = undefined,1553 .version = undefined,
1549 .parser = proto.HeadersParser.init(options.server_header_buffer),1554 .parser = proto.HeadersParser.init(options.server_header_buffer),
1550 },1555 },
1551 .arena = undefined,
1552 .headers = options.headers,1556 .headers = options.headers,
1553 .extra_headers = options.extra_headers,1557 .extra_headers = options.extra_headers,
1554 .privileged_headers = options.privileged_headers,1558 .privileged_headers = options.privileged_headers,
1555 };1559 };
1556 errdefer req.deinit();1560 errdefer req.deinit();
15571561
1558 req.arena = std.heap.ArenaAllocator.init(client.allocator);
1559
1560 return req;1562 return req;
1561}1563}
15621564