authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 17:41:38-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-02-23 17:41:38-08:00
logcfce81f7d5f11ab93b2d5fd26df41edf967f333b
tree11e52ad0a44620f4a4519683abd945146c11b312
parent7230b68b350b16c637e84f3ff224be24d23214ce
parent653d4158cdcb20be82ff525e122277064e6acb92
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #18955 from ziglang/std.http.Server

take std.http in a different direction

26 files changed, 3624 insertions(+), 3558 deletions(-)

lib/std/Uri.zig+133-98
...@@ -4,6 +4,7 @@...@@ -4,6 +4,7 @@
4const Uri = @This();4const Uri = @This();
5const std = @import("std.zig");5const std = @import("std.zig");
6const testing = std.testing;6const testing = std.testing;
7const Allocator = std.mem.Allocator;
78
8scheme: []const u8,9scheme: []const u8,
9user: ?[]const u8 = null,10user: ?[]const u8 = null,
...@@ -15,15 +16,15 @@ query: ?[]const u8 = null,...@@ -15,15 +16,15 @@ query: ?[]const u8 = null,
15fragment: ?[]const u8 = null,16fragment: ?[]const u8 = null,
1617
17/// Applies URI encoding and replaces all reserved characters with their respective %XX code.18/// Applies URI encoding and replaces all reserved characters with their respective %XX code.
18pub fn escapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]u8 {19pub fn escapeString(allocator: Allocator, input: []const u8) error{OutOfMemory}![]u8 {
19 return escapeStringWithFn(allocator, input, isUnreserved);20 return escapeStringWithFn(allocator, input, isUnreserved);
20}21}
2122
22pub fn escapePath(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]u8 {23pub fn escapePath(allocator: Allocator, input: []const u8) error{OutOfMemory}![]u8 {
23 return escapeStringWithFn(allocator, input, isPathChar);24 return escapeStringWithFn(allocator, input, isPathChar);
24}25}
2526
26pub fn escapeQuery(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]u8 {27pub fn escapeQuery(allocator: Allocator, input: []const u8) error{OutOfMemory}![]u8 {
27 return escapeStringWithFn(allocator, input, isQueryChar);28 return escapeStringWithFn(allocator, input, isQueryChar);
28}29}
2930
...@@ -39,7 +40,7 @@ pub fn writeEscapedQuery(writer: anytype, input: []const u8) !void {...@@ -39,7 +40,7 @@ pub fn writeEscapedQuery(writer: anytype, input: []const u8) !void {
39 return writeEscapedStringWithFn(writer, input, isQueryChar);40 return writeEscapedStringWithFn(writer, input, isQueryChar);
40}41}
4142
42pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) std.mem.Allocator.Error![]u8 {43pub fn escapeStringWithFn(allocator: Allocator, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) Allocator.Error![]u8 {
43 var outsize: usize = 0;44 var outsize: usize = 0;
44 for (input) |c| {45 for (input) |c| {
45 outsize += if (keepUnescaped(c)) @as(usize, 1) else 3;46 outsize += if (keepUnescaped(c)) @as(usize, 1) else 3;
...@@ -76,7 +77,7 @@ pub fn writeEscapedStringWithFn(writer: anytype, input: []const u8, comptime kee...@@ -76,7 +77,7 @@ pub fn writeEscapedStringWithFn(writer: anytype, input: []const u8, comptime kee
7677
77/// Parses a URI string and unescapes all %XX where XX is a valid hex number. Otherwise, verbatim copies78/// Parses a URI string and unescapes all %XX where XX is a valid hex number. Otherwise, verbatim copies
78/// them to the output.79/// them to the output.
79pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]u8 {80pub fn unescapeString(allocator: Allocator, input: []const u8) error{OutOfMemory}![]u8 {
80 var outsize: usize = 0;81 var outsize: usize = 0;
81 var inptr: usize = 0;82 var inptr: usize = 0;
82 while (inptr < input.len) {83 while (inptr < input.len) {
...@@ -341,7 +342,7 @@ pub fn format(...@@ -341,7 +342,7 @@ pub fn format(
341/// The return value will contain unescaped strings pointing into the342/// The return value will contain unescaped strings pointing into the
342/// original `text`. Each component that is provided, will be non-`null`.343/// original `text`. Each component that is provided, will be non-`null`.
343pub fn parse(text: []const u8) ParseError!Uri {344pub fn parse(text: []const u8) ParseError!Uri {
344 var reader = SliceReader{ .slice = text };345 var reader: SliceReader = .{ .slice = text };
345 const scheme = reader.readWhile(isSchemeChar);346 const scheme = reader.readWhile(isSchemeChar);
346347
347 // after the scheme, a ':' must appear348 // after the scheme, a ':' must appear
...@@ -358,111 +359,145 @@ pub fn parse(text: []const u8) ParseError!Uri {...@@ -358,111 +359,145 @@ pub fn parse(text: []const u8) ParseError!Uri {
358 return uri;359 return uri;
359}360}
360361
361/// Implementation of RFC 3986, Section 5.2.4. Removes dot segments from a URI path.362pub const ResolveInplaceError = ParseError || error{OutOfMemory};
362///
363/// `std.fs.path.resolvePosix` is not sufficient here because it may return relative paths and does not preserve trailing slashes.
364fn removeDotSegments(allocator: std.mem.Allocator, paths: []const []const u8) std.mem.Allocator.Error![]const u8 {
365 var result = std.ArrayList(u8).init(allocator);
366 defer result.deinit();
367
368 for (paths) |p| {
369 var it = std.mem.tokenizeScalar(u8, p, '/');
370 while (it.next()) |component| {
371 if (std.mem.eql(u8, component, ".")) {
372 continue;
373 } else if (std.mem.eql(u8, component, "..")) {
374 if (result.items.len == 0)
375 continue;
376363
377 while (true) {364/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.
378 const ends_with_slash = result.items[result.items.len - 1] == '/';365/// Copies `new` to the beginning of `aux_buf`, allowing the slices to overlap,
379 result.items.len -= 1;366/// then parses `new` as a URI, and then resolves the path in place.
380 if (ends_with_slash or result.items.len == 0) break;367/// If a merge needs to take place, the newly constructed path will be stored
381 }368/// in `aux_buf` just after the copied `new`.
382 } else {369pub fn resolve_inplace(base: Uri, new: []const u8, aux_buf: []u8) ResolveInplaceError!Uri {
383 try result.ensureUnusedCapacity(1 + component.len);370 std.mem.copyBackwards(u8, aux_buf, new);
384 result.appendAssumeCapacity('/');371 // At this point, new is an invalid pointer.
385 result.appendSliceAssumeCapacity(component);372 const new_mut = aux_buf[0..new.len];
386 }373
387 }374 const new_parsed, const has_scheme = p: {
388 }375 break :p .{
376 parse(new_mut) catch |first_err| {
377 break :p .{
378 parseWithoutScheme(new_mut) catch return first_err,
379 false,
380 };
381 },
382 true,
383 };
384 };
389385
390 // ensure a trailing slash is kept386 // As you can see above, `new_mut` is not a const pointer.
391 const last_path = paths[paths.len - 1];387 const new_path: []u8 = @constCast(new_parsed.path);
392 if (last_path.len > 0 and last_path[last_path.len - 1] == '/') {388
393 try result.append('/');389 if (has_scheme) return .{
394 }390 .scheme = new_parsed.scheme,
391 .user = new_parsed.user,
392 .host = new_parsed.host,
393 .port = new_parsed.port,
394 .path = remove_dot_segments(new_path),
395 .query = new_parsed.query,
396 .fragment = new_parsed.fragment,
397 };
395398
396 return result.toOwnedSlice();399 if (new_parsed.host) |host| return .{
397}400 .scheme = base.scheme,
401 .user = new_parsed.user,
402 .host = host,
403 .port = new_parsed.port,
404 .path = remove_dot_segments(new_path),
405 .query = new_parsed.query,
406 .fragment = new_parsed.fragment,
407 };
398408
399/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.409 const path, const query = b: {
400///410 if (new_path.len == 0)
401/// Assumes `arena` owns all memory in `base` and `ref`. `arena` will own all memory in the returned URI.411 break :b .{
402pub fn resolve(base: Uri, ref: Uri, strict: bool, arena: std.mem.Allocator) std.mem.Allocator.Error!Uri {412 base.path,
403 var target: Uri = Uri{413 new_parsed.query orelse base.query,
404 .scheme = "",414 };
405 .user = null,415
406 .password = null,416 if (new_path[0] == '/')
407 .host = null,417 break :b .{
408 .port = null,418 remove_dot_segments(new_path),
409 .path = "",419 new_parsed.query,
410 .query = null,420 };
411 .fragment = null,421
422 break :b .{
423 try merge_paths(base.path, new_path, aux_buf[new_mut.len..]),
424 new_parsed.query,
425 };
412 };426 };
413427
414 if (ref.scheme.len > 0 and (strict or !std.mem.eql(u8, ref.scheme, base.scheme))) {428 return .{
415 target.scheme = ref.scheme;429 .scheme = base.scheme,
416 target.user = ref.user;430 .user = base.user,
417 target.host = ref.host;431 .host = base.host,
418 target.port = ref.port;432 .port = base.port,
419 target.path = try removeDotSegments(arena, &.{ref.path});433 .path = path,
420 target.query = ref.query;434 .query = query,
421 } else {435 .fragment = new_parsed.fragment,
422 target.scheme = base.scheme;436 };
423 if (ref.host) |host| {437}
424 target.user = ref.user;438
425 target.host = host;439/// In-place implementation of RFC 3986, Section 5.2.4.
426 target.port = ref.port;440fn remove_dot_segments(path: []u8) []u8 {
427 target.path = ref.path;441 var in_i: usize = 0;
428 target.path = try removeDotSegments(arena, &.{ref.path});442 var out_i: usize = 0;
429 target.query = ref.query;443 while (in_i < path.len) {
444 if (std.mem.startsWith(u8, path[in_i..], "./")) {
445 in_i += 2;
446 } else if (std.mem.startsWith(u8, path[in_i..], "../")) {
447 in_i += 3;
448 } else if (std.mem.startsWith(u8, path[in_i..], "/./")) {
449 in_i += 2;
450 } else if (std.mem.eql(u8, path[in_i..], "/.")) {
451 in_i += 1;
452 path[in_i] = '/';
453 } else if (std.mem.startsWith(u8, path[in_i..], "/../")) {
454 in_i += 3;
455 while (out_i > 0) {
456 out_i -= 1;
457 if (path[out_i] == '/') break;
458 }
459 } else if (std.mem.eql(u8, path[in_i..], "/..")) {
460 in_i += 2;
461 path[in_i] = '/';
462 while (out_i > 0) {
463 out_i -= 1;
464 if (path[out_i] == '/') break;
465 }
466 } else if (std.mem.eql(u8, path[in_i..], ".")) {
467 in_i += 1;
468 } else if (std.mem.eql(u8, path[in_i..], "..")) {
469 in_i += 2;
430 } else {470 } else {
431 if (ref.path.len == 0) {471 while (true) {
432 target.path = base.path;472 path[out_i] = path[in_i];
433 target.query = ref.query orelse base.query;473 out_i += 1;
434 } else {474 in_i += 1;
435 if (ref.path[0] == '/') {475 if (in_i >= path.len or path[in_i] == '/') break;
436 target.path = try removeDotSegments(arena, &.{ref.path});
437 } else {
438 target.path = try removeDotSegments(arena, &.{ std.fs.path.dirnamePosix(base.path) orelse "", ref.path });
439 }
440 target.query = ref.query;
441 }476 }
442
443 target.user = base.user;
444 target.host = base.host;
445 target.port = base.port;
446 }477 }
447 }478 }
448479 return path[0..out_i];
449 target.fragment = ref.fragment;
450
451 return target;
452}480}
453481
454test resolve {482test remove_dot_segments {
455 const base = try parse("http://a/b/c/d;p?q");483 {
456484 var buffer = "/a/b/c/./../../g".*;
457 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);485 try std.testing.expectEqualStrings("/a/g", remove_dot_segments(&buffer));
458 defer arena.deinit();486 }
487}
459488
460 try std.testing.expectEqualDeep(try parse("http://a/b/c/blog/"), try base.resolve(try parseWithoutScheme("blog/"), true, arena.allocator()));489/// 5.2.3. Merge Paths
461 try std.testing.expectEqualDeep(try parse("http://a/b/c/blog/?k"), try base.resolve(try parseWithoutScheme("blog/?k"), true, arena.allocator()));490fn merge_paths(base: []const u8, new: []u8, aux: []u8) error{OutOfMemory}![]u8 {
462 try std.testing.expectEqualDeep(try parse("http://a/b/blog/"), try base.resolve(try parseWithoutScheme("../blog/"), true, arena.allocator()));491 if (aux.len < base.len + 1 + new.len) return error.OutOfMemory;
463 try std.testing.expectEqualDeep(try parse("http://a/b/blog"), try base.resolve(try parseWithoutScheme("../blog"), true, arena.allocator()));492 if (base.len == 0) {
464 try std.testing.expectEqualDeep(try parse("http://e"), try base.resolve(try parseWithoutScheme("//e"), true, arena.allocator()));493 aux[0] = '/';
465 try std.testing.expectEqualDeep(try parse("https://a:1/"), try base.resolve(try parse("https://a:1/"), true, arena.allocator()));494 @memcpy(aux[1..][0..new.len], new);
495 return remove_dot_segments(aux[0 .. new.len + 1]);
496 }
497 const pos = std.mem.lastIndexOfScalar(u8, base, '/') orelse return remove_dot_segments(new);
498 @memcpy(aux[0 .. pos + 1], base[0 .. pos + 1]);
499 @memcpy(aux[pos + 1 ..][0..new.len], new);
500 return remove_dot_segments(aux[0 .. pos + 1 + new.len]);
466}501}
467502
468const SliceReader = struct {503const SliceReader = struct {
lib/std/array_list.zig+21-2
...@@ -937,14 +937,33 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -937,14 +937,33 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
937 return .{ .context = .{ .self = self, .allocator = allocator } };937 return .{ .context = .{ .self = self, .allocator = allocator } };
938 }938 }
939939
940 /// Same as `append` except it returns the number of bytes written, which is always the same940 /// Same as `append` except it returns the number of bytes written,
941 /// as `m.len`. The purpose of this function existing is to match `std.io.Writer` API.941 /// which is always the same as `m.len`. The purpose of this function
942 /// existing is to match `std.io.Writer` API.
942 /// Invalidates element pointers if additional memory is needed.943 /// Invalidates element pointers if additional memory is needed.
943 fn appendWrite(context: WriterContext, m: []const u8) Allocator.Error!usize {944 fn appendWrite(context: WriterContext, m: []const u8) Allocator.Error!usize {
944 try context.self.appendSlice(context.allocator, m);945 try context.self.appendSlice(context.allocator, m);
945 return m.len;946 return m.len;
946 }947 }
947948
949 pub const FixedWriter = std.io.Writer(*Self, Allocator.Error, appendWriteFixed);
950
951 /// Initializes a Writer which will append to the list but will return
952 /// `error.OutOfMemory` rather than increasing capacity.
953 pub fn fixedWriter(self: *Self) FixedWriter {
954 return .{ .context = self };
955 }
956
957 /// The purpose of this function existing is to match `std.io.Writer` API.
958 fn appendWriteFixed(self: *Self, m: []const u8) error{OutOfMemory}!usize {
959 const available_capacity = self.capacity - self.items.len;
960 if (m.len > available_capacity)
961 return error.OutOfMemory;
962
963 self.appendSliceAssumeCapacity(m);
964 return m.len;
965 }
966
948 /// Append a value to the list `n` times.967 /// Append a value to the list `n` times.
949 /// Allocates more memory as necessary.968 /// Allocates more memory as necessary.
950 /// Invalidates element pointers if additional memory is needed.969 /// Invalidates element pointers if additional memory is needed.
lib/std/compress/zstandard.zig+103-114
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const Allocator = std.mem.Allocator;
3const RingBuffer = std.RingBuffer;2const RingBuffer = std.RingBuffer;
43
5const types = @import("zstandard/types.zig");4const types = @import("zstandard/types.zig");
...@@ -8,32 +7,41 @@ pub const compressed_block = types.compressed_block;...@@ -8,32 +7,41 @@ pub const compressed_block = types.compressed_block;
87
9pub const decompress = @import("zstandard/decompress.zig");8pub const decompress = @import("zstandard/decompress.zig");
109
11pub const DecompressStreamOptions = struct {10pub const DecompressorOptions = struct {
12 verify_checksum: bool = true,11 verify_checksum: bool = true,
13 window_size_max: usize = 1 << 23, // 8MiB default maximum window size12 window_buffer: []u8,
13
14 /// Recommended amount by the standard. Lower than this may result
15 /// in inability to decompress common streams.
16 pub const default_window_buffer_len = 8 * 1024 * 1024;
14};17};
1518
16pub fn DecompressStream(19pub fn Decompressor(comptime ReaderType: type) type {
17 comptime ReaderType: type,
18 comptime options: DecompressStreamOptions,
19) type {
20 return struct {20 return struct {
21 const Self = @This();21 const Self = @This();
2222
23 allocator: Allocator,23 const table_size_max = types.compressed_block.table_size_max;
24
24 source: std.io.CountingReader(ReaderType),25 source: std.io.CountingReader(ReaderType),
25 state: enum { NewFrame, InFrame, LastBlock },26 state: enum { NewFrame, InFrame, LastBlock },
26 decode_state: decompress.block.DecodeState,27 decode_state: decompress.block.DecodeState,
27 frame_context: decompress.FrameContext,28 frame_context: decompress.FrameContext,
28 buffer: RingBuffer,29 buffer: WindowBuffer,
29 literal_fse_buffer: []types.compressed_block.Table.Fse,30 literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
30 match_fse_buffer: []types.compressed_block.Table.Fse,31 match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
31 offset_fse_buffer: []types.compressed_block.Table.Fse,32 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
32 literals_buffer: []u8,33 literals_buffer: [types.block_size_max]u8,
33 sequence_buffer: []u8,34 sequence_buffer: [types.block_size_max]u8,
34 checksum: if (options.verify_checksum) ?u32 else void,35 verify_checksum: bool,
36 checksum: ?u32,
35 current_frame_decompressed_size: usize,37 current_frame_decompressed_size: usize,
3638
39 const WindowBuffer = struct {
40 data: []u8 = undefined,
41 read_index: usize = 0,
42 write_index: usize = 0,
43 };
44
37 pub const Error = ReaderType.Error || error{45 pub const Error = ReaderType.Error || error{
38 ChecksumFailure,46 ChecksumFailure,
39 DictionaryIdFlagUnsupported,47 DictionaryIdFlagUnsupported,
...@@ -44,19 +52,19 @@ pub fn DecompressStream(...@@ -44,19 +52,19 @@ pub fn DecompressStream(
4452
45 pub const Reader = std.io.Reader(*Self, Error, read);53 pub const Reader = std.io.Reader(*Self, Error, read);
4654
47 pub fn init(allocator: Allocator, source: ReaderType) Self {55 pub fn init(source: ReaderType, options: DecompressorOptions) Self {
48 return Self{56 return .{
49 .allocator = allocator,
50 .source = std.io.countingReader(source),57 .source = std.io.countingReader(source),
51 .state = .NewFrame,58 .state = .NewFrame,
52 .decode_state = undefined,59 .decode_state = undefined,
53 .frame_context = undefined,60 .frame_context = undefined,
54 .buffer = undefined,61 .buffer = .{ .data = options.window_buffer },
55 .literal_fse_buffer = undefined,62 .literal_fse_buffer = undefined,
56 .match_fse_buffer = undefined,63 .match_fse_buffer = undefined,
57 .offset_fse_buffer = undefined,64 .offset_fse_buffer = undefined,
58 .literals_buffer = undefined,65 .literals_buffer = undefined,
59 .sequence_buffer = undefined,66 .sequence_buffer = undefined,
67 .verify_checksum = options.verify_checksum,
60 .checksum = undefined,68 .checksum = undefined,
61 .current_frame_decompressed_size = undefined,69 .current_frame_decompressed_size = undefined,
62 };70 };
...@@ -72,53 +80,20 @@ pub fn DecompressStream(...@@ -72,53 +80,20 @@ pub fn DecompressStream(
72 .zstandard => |header| {80 .zstandard => |header| {
73 const frame_context = try decompress.FrameContext.init(81 const frame_context = try decompress.FrameContext.init(
74 header,82 header,
75 options.window_size_max,83 self.buffer.data.len,
76 options.verify_checksum,84 self.verify_checksum,
77 );
78
79 const literal_fse_buffer = try self.allocator.alloc(
80 types.compressed_block.Table.Fse,
81 types.compressed_block.table_size_max.literal,
82 );85 );
83 errdefer self.allocator.free(literal_fse_buffer);
84
85 const match_fse_buffer = try self.allocator.alloc(
86 types.compressed_block.Table.Fse,
87 types.compressed_block.table_size_max.match,
88 );
89 errdefer self.allocator.free(match_fse_buffer);
90
91 const offset_fse_buffer = try self.allocator.alloc(
92 types.compressed_block.Table.Fse,
93 types.compressed_block.table_size_max.offset,
94 );
95 errdefer self.allocator.free(offset_fse_buffer);
9686
97 const decode_state = decompress.block.DecodeState.init(87 const decode_state = decompress.block.DecodeState.init(
98 literal_fse_buffer,88 &self.literal_fse_buffer,
99 match_fse_buffer,89 &self.match_fse_buffer,
100 offset_fse_buffer,90 &self.offset_fse_buffer,
101 );91 );
102 const buffer = try RingBuffer.init(self.allocator, frame_context.window_size);
103
104 const literals_data = try self.allocator.alloc(u8, options.window_size_max);
105 errdefer self.allocator.free(literals_data);
106
107 const sequence_data = try self.allocator.alloc(u8, options.window_size_max);
108 errdefer self.allocator.free(sequence_data);
109
110 self.literal_fse_buffer = literal_fse_buffer;
111 self.match_fse_buffer = match_fse_buffer;
112 self.offset_fse_buffer = offset_fse_buffer;
113 self.literals_buffer = literals_data;
114 self.sequence_buffer = sequence_data;
115
116 self.buffer = buffer;
11792
118 self.decode_state = decode_state;93 self.decode_state = decode_state;
119 self.frame_context = frame_context;94 self.frame_context = frame_context;
12095
121 self.checksum = if (options.verify_checksum) null else {};96 self.checksum = null;
122 self.current_frame_decompressed_size = 0;97 self.current_frame_decompressed_size = 0;
12398
124 self.state = .InFrame;99 self.state = .InFrame;
...@@ -126,16 +101,6 @@ pub fn DecompressStream(...@@ -126,16 +101,6 @@ pub fn DecompressStream(
126 }101 }
127 }102 }
128103
129 pub fn deinit(self: *Self) void {
130 if (self.state == .NewFrame) return;
131 self.allocator.free(self.decode_state.literal_fse_buffer);
132 self.allocator.free(self.decode_state.match_fse_buffer);
133 self.allocator.free(self.decode_state.offset_fse_buffer);
134 self.allocator.free(self.literals_buffer);
135 self.allocator.free(self.sequence_buffer);
136 self.buffer.deinit(self.allocator);
137 }
138
139 pub fn reader(self: *Self) Reader {104 pub fn reader(self: *Self) Reader {
140 return .{ .context = self };105 return .{ .context = self };
141 }106 }
...@@ -153,7 +118,6 @@ pub fn DecompressStream(...@@ -153,7 +118,6 @@ pub fn DecompressStream(
153 0118 0
154 else119 else
155 error.MalformedFrame,120 error.MalformedFrame,
156 error.OutOfMemory => return error.OutOfMemory,
157 else => return error.MalformedFrame,121 else => return error.MalformedFrame,
158 };122 };
159 }123 }
...@@ -165,20 +129,30 @@ pub fn DecompressStream(...@@ -165,20 +129,30 @@ pub fn DecompressStream(
165 fn readInner(self: *Self, buffer: []u8) Error!usize {129 fn readInner(self: *Self, buffer: []u8) Error!usize {
166 std.debug.assert(self.state != .NewFrame);130 std.debug.assert(self.state != .NewFrame);
167131
132 var ring_buffer = RingBuffer{
133 .data = self.buffer.data,
134 .read_index = self.buffer.read_index,
135 .write_index = self.buffer.write_index,
136 };
137 defer {
138 self.buffer.read_index = ring_buffer.read_index;
139 self.buffer.write_index = ring_buffer.write_index;
140 }
141
168 const source_reader = self.source.reader();142 const source_reader = self.source.reader();
169 while (self.buffer.isEmpty() and self.state != .LastBlock) {143 while (ring_buffer.isEmpty() and self.state != .LastBlock) {
170 const header_bytes = source_reader.readBytesNoEof(3) catch144 const header_bytes = source_reader.readBytesNoEof(3) catch
171 return error.MalformedFrame;145 return error.MalformedFrame;
172 const block_header = decompress.block.decodeBlockHeader(&header_bytes);146 const block_header = decompress.block.decodeBlockHeader(&header_bytes);
173147
174 decompress.block.decodeBlockReader(148 decompress.block.decodeBlockReader(
175 &self.buffer,149 &ring_buffer,
176 source_reader,150 source_reader,
177 block_header,151 block_header,
178 &self.decode_state,152 &self.decode_state,
179 self.frame_context.block_size_max,153 self.frame_context.block_size_max,
180 self.literals_buffer,154 &self.literals_buffer,
181 self.sequence_buffer,155 &self.sequence_buffer,
182 ) catch156 ) catch
183 return error.MalformedBlock;157 return error.MalformedBlock;
184158
...@@ -186,12 +160,12 @@ pub fn DecompressStream(...@@ -186,12 +160,12 @@ pub fn DecompressStream(
186 if (self.current_frame_decompressed_size > size) return error.MalformedFrame;160 if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
187 }161 }
188162
189 const size = self.buffer.len();163 const size = ring_buffer.len();
190 self.current_frame_decompressed_size += size;164 self.current_frame_decompressed_size += size;
191165
192 if (self.frame_context.hasher_opt) |*hasher| {166 if (self.frame_context.hasher_opt) |*hasher| {
193 if (size > 0) {167 if (size > 0) {
194 const written_slice = self.buffer.sliceLast(size);168 const written_slice = ring_buffer.sliceLast(size);
195 hasher.update(written_slice.first);169 hasher.update(written_slice.first);
196 hasher.update(written_slice.second);170 hasher.update(written_slice.second);
197 }171 }
...@@ -201,7 +175,7 @@ pub fn DecompressStream(...@@ -201,7 +175,7 @@ pub fn DecompressStream(
201 if (self.frame_context.has_checksum) {175 if (self.frame_context.has_checksum) {
202 const checksum = source_reader.readInt(u32, .little) catch176 const checksum = source_reader.readInt(u32, .little) catch
203 return error.MalformedFrame;177 return error.MalformedFrame;
204 if (comptime options.verify_checksum) {178 if (self.verify_checksum) {
205 if (self.frame_context.hasher_opt) |*hasher| {179 if (self.frame_context.hasher_opt) |*hasher| {
206 if (checksum != decompress.computeChecksum(hasher))180 if (checksum != decompress.computeChecksum(hasher))
207 return error.ChecksumFailure;181 return error.ChecksumFailure;
...@@ -216,43 +190,28 @@ pub fn DecompressStream(...@@ -216,43 +190,28 @@ pub fn DecompressStream(
216 }190 }
217 }191 }
218192
219 const size = @min(self.buffer.len(), buffer.len);193 const size = @min(ring_buffer.len(), buffer.len);
220 if (size > 0) {194 if (size > 0) {
221 self.buffer.readFirstAssumeLength(buffer, size);195 ring_buffer.readFirstAssumeLength(buffer, size);
222 }196 }
223 if (self.state == .LastBlock and self.buffer.len() == 0) {197 if (self.state == .LastBlock and ring_buffer.len() == 0) {
224 self.state = .NewFrame;198 self.state = .NewFrame;
225 self.allocator.free(self.literal_fse_buffer);
226 self.allocator.free(self.match_fse_buffer);
227 self.allocator.free(self.offset_fse_buffer);
228 self.allocator.free(self.literals_buffer);
229 self.allocator.free(self.sequence_buffer);
230 self.buffer.deinit(self.allocator);
231 }199 }
232 return size;200 return size;
233 }201 }
234 };202 };
235}203}
236204
237pub fn decompressStreamOptions(205pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {
238 allocator: Allocator,206 return Decompressor(@TypeOf(reader)).init(reader, options);
239 reader: anytype,
240 comptime options: DecompressStreamOptions,
241) DecompressStream(@TypeOf(reader, options)) {
242 return DecompressStream(@TypeOf(reader), options).init(allocator, reader);
243}
244
245pub fn decompressStream(
246 allocator: Allocator,
247 reader: anytype,
248) DecompressStream(@TypeOf(reader), .{}) {
249 return DecompressStream(@TypeOf(reader), .{}).init(allocator, reader);
250}207}
251208
252fn testDecompress(data: []const u8) ![]u8 {209fn testDecompress(data: []const u8) ![]u8 {
210 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
211 defer std.testing.allocator.free(window_buffer);
212
253 var in_stream = std.io.fixedBufferStream(data);213 var in_stream = std.io.fixedBufferStream(data);
254 var zstd_stream = decompressStream(std.testing.allocator, in_stream.reader());214 var zstd_stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
255 defer zstd_stream.deinit();
256 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));215 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
257 return result;216 return result;
258}217}
...@@ -278,38 +237,48 @@ test "zstandard decompression" {...@@ -278,38 +237,48 @@ test "zstandard decompression" {
278 const res19 = try decompress.decode(buffer, compressed19, true);237 const res19 = try decompress.decode(buffer, compressed19, true);
279 try std.testing.expectEqual(uncompressed.len, res19);238 try std.testing.expectEqual(uncompressed.len, res19);
280 try std.testing.expectEqualSlices(u8, uncompressed, buffer);239 try std.testing.expectEqualSlices(u8, uncompressed, buffer);
240}
241
242test "zstandard streaming decompression" {
243 // default stack size for wasm32 is too low for Decompressor - slightly
244 // over 1MiB stack space is needed via the --stack CLI flag
245 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
246
247 const uncompressed = @embedFile("testdata/rfc8478.txt");
248 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
249 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
281250
282 try testReader(compressed3, uncompressed);251 try testReader(compressed3, uncompressed);
283 try testReader(compressed19, uncompressed);252 try testReader(compressed19, uncompressed);
284}253}
285254
286fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {255fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {
287 const allocator = std.testing.allocator;
288
289 {256 {
290 const result = try decompress.decodeAlloc(allocator, input, false, 1 << 23);257 const result = try decompress.decodeAlloc(std.testing.allocator, input, false, 1 << 23);
291 defer allocator.free(result);258 defer std.testing.allocator.free(result);
292 try std.testing.expectEqualStrings(expected, result);259 try std.testing.expectEqualStrings(expected, result);
293 }260 }
294261
295 {262 {
296 var buffer = try allocator.alloc(u8, 2 * expected.len);263 var buffer = try std.testing.allocator.alloc(u8, 2 * expected.len);
297 defer allocator.free(buffer);264 defer std.testing.allocator.free(buffer);
298265
299 const size = try decompress.decode(buffer, input, false);266 const size = try decompress.decode(buffer, input, false);
300 try std.testing.expectEqualStrings(expected, buffer[0..size]);267 try std.testing.expectEqualStrings(expected, buffer[0..size]);
301 }268 }
269}
302270
303 {271fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
304 var in_stream = std.io.fixedBufferStream(input);272 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
305 var stream = decompressStream(allocator, in_stream.reader());273 defer std.testing.allocator.free(window_buffer);
306 defer stream.deinit();
307274
308 const result = try stream.reader().readAllAlloc(allocator, std.math.maxInt(usize));275 var in_stream = std.io.fixedBufferStream(input);
309 defer allocator.free(result);276 var stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
310277
311 try std.testing.expectEqualStrings(expected, result);278 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
312 }279 defer std.testing.allocator.free(result);
280
281 try std.testing.expectEqualStrings(expected, result);
313}282}
314283
315test "zero sized block" {284test "zero sized block" {
...@@ -327,3 +296,23 @@ test "zero sized block" {...@@ -327,3 +296,23 @@ test "zero sized block" {
327 try expectEqualDecoded("", input_raw);296 try expectEqualDecoded("", input_raw);
328 try expectEqualDecoded("", input_rle);297 try expectEqualDecoded("", input_rle);
329}298}
299
300test "zero sized block streaming" {
301 // default stack size for wasm32 is too low for Decompressor - slightly
302 // over 1MiB stack space is needed via the --stack CLI flag
303 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
304
305 const input_raw =
306 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
307 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
308 "\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
309
310 const input_rle =
311 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
312 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
313 "\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
314 "\xaa"; // block_content
315
316 try expectEqualDecodedStreaming("", input_raw);
317 try expectEqualDecodedStreaming("", input_rle);
318}
lib/std/compress/zstandard/decompress.zig+1-1
...@@ -409,7 +409,7 @@ pub const FrameContext = struct {...@@ -409,7 +409,7 @@ pub const FrameContext = struct {
409 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,409 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
410 .window_size = window_size,410 .window_size = window_size,
411 .has_checksum = frame_header.descriptor.content_checksum_flag,411 .has_checksum = frame_header.descriptor.content_checksum_flag,
412 .block_size_max = @min(1 << 17, window_size),412 .block_size_max = @min(types.block_size_max, window_size),
413 .content_size = content_size,413 .content_size = content_size,
414 };414 };
415 }415 }
lib/std/compress/zstandard/types.zig+3-1
...@@ -1,3 +1,5 @@...@@ -1,3 +1,5 @@
1pub const block_size_max = 1 << 17;
2
1pub const frame = struct {3pub const frame = struct {
2 pub const Kind = enum { zstandard, skippable };4 pub const Kind = enum { zstandard, skippable };
35
...@@ -391,7 +393,7 @@ pub const compressed_block = struct {...@@ -391,7 +393,7 @@ pub const compressed_block = struct {
391 pub const table_size_max = struct {393 pub const table_size_max = struct {
392 pub const literal = 1 << table_accuracy_log_max.literal;394 pub const literal = 1 << table_accuracy_log_max.literal;
393 pub const match = 1 << table_accuracy_log_max.match;395 pub const match = 1 << table_accuracy_log_max.match;
394 pub const offset = 1 << table_accuracy_log_max.match;396 pub const offset = 1 << table_accuracy_log_max.offset;
395 };397 };
396};398};
397399
lib/std/http.zig+17-11
...@@ -1,12 +1,9 @@...@@ -1,12 +1,9 @@
1const std = @import("std.zig");
2
3pub const Client = @import("http/Client.zig");1pub const Client = @import("http/Client.zig");
4pub const Server = @import("http/Server.zig");2pub const Server = @import("http/Server.zig");
5pub const protocol = @import("http/protocol.zig");3pub const protocol = @import("http/protocol.zig");
6const headers = @import("http/Headers.zig");4pub const HeadParser = @import("http/HeadParser.zig");
75pub const ChunkParser = @import("http/ChunkParser.zig");
8pub const Headers = headers.Headers;6pub const HeaderIterator = @import("http/HeaderIterator.zig");
9pub const Field = headers.Field;
107
11pub const Version = enum {8pub const Version = enum {
12 @"HTTP/1.0",9 @"HTTP/1.0",
...@@ -18,7 +15,7 @@ pub const Version = enum {...@@ -18,7 +15,7 @@ pub const Version = enum {
18/// https://datatracker.ietf.org/doc/html/rfc7231#section-4 Initial definition15/// https://datatracker.ietf.org/doc/html/rfc7231#section-4 Initial definition
19///16///
20/// https://datatracker.ietf.org/doc/html/rfc5789#section-2 PATCH17/// https://datatracker.ietf.org/doc/html/rfc5789#section-2 PATCH
21pub const Method = enum(u64) { // TODO: should be u192 or u256, but neither is supported by the C backend, and therefore cannot pass CI18pub const Method = enum(u64) {
22 GET = parse("GET"),19 GET = parse("GET"),
23 HEAD = parse("HEAD"),20 HEAD = parse("HEAD"),
24 POST = parse("POST"),21 POST = parse("POST"),
...@@ -46,10 +43,6 @@ pub const Method = enum(u64) { // TODO: should be u192 or u256, but neither is s...@@ -46,10 +43,6 @@ pub const Method = enum(u64) { // TODO: should be u192 or u256, but neither is s
46 try w.writeAll(str);43 try w.writeAll(str);
47 }44 }
4845
49 pub fn format(value: Method, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) @TypeOf(writer).Error!void {
50 return try value.write(writer);
51 }
52
53 /// Returns true if a request of this method is allowed to have a body46 /// Returns true if a request of this method is allowed to have a body
54 /// Actual behavior from servers may vary and should still be checked47 /// Actual behavior from servers may vary and should still be checked
55 pub fn requestHasBody(self: Method) bool {48 pub fn requestHasBody(self: Method) bool {
...@@ -309,9 +302,22 @@ pub const Connection = enum {...@@ -309,9 +302,22 @@ pub const Connection = enum {
309 close,302 close,
310};303};
311304
305pub const Header = struct {
306 name: []const u8,
307 value: []const u8,
308};
309
310const builtin = @import("builtin");
311const std = @import("std.zig");
312
312test {313test {
313 _ = Client;314 _ = Client;
314 _ = Method;315 _ = Method;
315 _ = Server;316 _ = Server;
316 _ = Status;317 _ = Status;
318 _ = HeadParser;
319 _ = ChunkParser;
320 if (builtin.os.tag != .wasi) {
321 _ = @import("http/test.zig");
322 }
317}323}
lib/std/http/ChunkParser.zig created+131
...@@ -0,0 +1,131 @@
1//! Parser for transfer-encoding: chunked.
2
3state: State,
4chunk_len: u64,
5
6pub const init: ChunkParser = .{
7 .state = .head_size,
8 .chunk_len = 0,
9};
10
11pub const State = enum {
12 head_size,
13 head_ext,
14 head_r,
15 data,
16 data_suffix,
17 data_suffix_r,
18 invalid,
19};
20
21/// Returns the number of bytes consumed by the chunk size. This is always
22/// less than or equal to `bytes.len`.
23///
24/// After this function returns, `chunk_len` will contain the parsed chunk size
25/// in bytes when `state` is `data`. Alternately, `state` may become `invalid`,
26/// indicating a syntax error in the input stream.
27///
28/// If the amount returned is less than `bytes.len`, the parser is in the
29/// `chunk_data` state and the first byte of the chunk is at `bytes[result]`.
30///
31/// Asserts `state` is neither `data` nor `invalid`.
32pub fn feed(p: *ChunkParser, bytes: []const u8) usize {
33 for (bytes, 0..) |c, i| switch (p.state) {
34 .data_suffix => switch (c) {
35 '\r' => p.state = .data_suffix_r,
36 '\n' => p.state = .head_size,
37 else => {
38 p.state = .invalid;
39 return i;
40 },
41 },
42 .data_suffix_r => switch (c) {
43 '\n' => p.state = .head_size,
44 else => {
45 p.state = .invalid;
46 return i;
47 },
48 },
49 .head_size => {
50 const digit = switch (c) {
51 '0'...'9' => |b| b - '0',
52 'A'...'Z' => |b| b - 'A' + 10,
53 'a'...'z' => |b| b - 'a' + 10,
54 '\r' => {
55 p.state = .head_r;
56 continue;
57 },
58 '\n' => {
59 p.state = .data;
60 return i + 1;
61 },
62 else => {
63 p.state = .head_ext;
64 continue;
65 },
66 };
67
68 const new_len = p.chunk_len *% 16 +% digit;
69 if (new_len <= p.chunk_len and p.chunk_len != 0) {
70 p.state = .invalid;
71 return i;
72 }
73
74 p.chunk_len = new_len;
75 },
76 .head_ext => switch (c) {
77 '\r' => p.state = .head_r,
78 '\n' => {
79 p.state = .data;
80 return i + 1;
81 },
82 else => continue,
83 },
84 .head_r => switch (c) {
85 '\n' => {
86 p.state = .data;
87 return i + 1;
88 },
89 else => {
90 p.state = .invalid;
91 return i;
92 },
93 },
94 .data => unreachable,
95 .invalid => unreachable,
96 };
97 return bytes.len;
98}
99
100const ChunkParser = @This();
101const std = @import("std");
102
103test feed {
104 const testing = std.testing;
105
106 const data = "Ff\r\nf0f000 ; ext\n0\r\nffffffffffffffffffffffffffffffffffffffff\r\n";
107
108 var p = init;
109 const first = p.feed(data[0..]);
110 try testing.expectEqual(@as(u32, 4), first);
111 try testing.expectEqual(@as(u64, 0xff), p.chunk_len);
112 try testing.expectEqual(.data, p.state);
113
114 p = init;
115 const second = p.feed(data[first..]);
116 try testing.expectEqual(@as(u32, 13), second);
117 try testing.expectEqual(@as(u64, 0xf0f000), p.chunk_len);
118 try testing.expectEqual(.data, p.state);
119
120 p = init;
121 const third = p.feed(data[first + second ..]);
122 try testing.expectEqual(@as(u32, 3), third);
123 try testing.expectEqual(@as(u64, 0), p.chunk_len);
124 try testing.expectEqual(.data, p.state);
125
126 p = init;
127 const fourth = p.feed(data[first + second + third ..]);
128 try testing.expectEqual(@as(u32, 16), fourth);
129 try testing.expectEqual(@as(u64, 0xffffffffffffffff), p.chunk_len);
130 try testing.expectEqual(.invalid, p.state);
131}
lib/std/http/Client.zig+545-555
...@@ -20,9 +20,7 @@ const proto = @import("protocol.zig");...@@ -20,9 +20,7 @@ const proto = @import("protocol.zig");
2020
21pub const disable_tls = std.options.http_disable_tls;21pub const disable_tls = std.options.http_disable_tls;
2222
23/// Allocator used for all allocations made by the client.23/// Used for all client allocations. Must be thread-safe.
24///
25/// This allocator must be thread-safe.
26allocator: Allocator,24allocator: Allocator,
2725
28ca_bundle: if (disable_tls) void else std.crypto.Certificate.Bundle = if (disable_tls) {} else .{},26ca_bundle: if (disable_tls) void else std.crypto.Certificate.Bundle = if (disable_tls) {} else .{},
...@@ -35,14 +33,25 @@ next_https_rescan_certs: bool = true,...@@ -35,14 +33,25 @@ next_https_rescan_certs: bool = true,
35/// The pool of connections that can be reused (and currently in use).33/// The pool of connections that can be reused (and currently in use).
36connection_pool: ConnectionPool = .{},34connection_pool: ConnectionPool = .{},
3735
38/// This is the proxy that will handle http:// connections. It *must not* be modified when the client has any active connections.36/// If populated, all http traffic travels through this third party.
39http_proxy: ?Proxy = null,37/// This field cannot be modified while the client has active connections.
4038/// Pointer to externally-owned memory.
41/// This is the proxy that will handle https:// connections. It *must not* be modified when the client has any active connections.39http_proxy: ?*Proxy = null,
42https_proxy: ?Proxy = null,40/// If populated, all https traffic travels through this third party.
41/// This field cannot be modified while the client has active connections.
42/// Pointer to externally-owned memory.
43https_proxy: ?*Proxy = null,
4344
44/// A set of linked lists of connections that can be reused.45/// A set of linked lists of connections that can be reused.
45pub const ConnectionPool = struct {46pub const ConnectionPool = struct {
47 mutex: std.Thread.Mutex = .{},
48 /// Open connections that are currently in use.
49 used: Queue = .{},
50 /// Open connections that are not currently in use.
51 free: Queue = .{},
52 free_len: usize = 0,
53 free_size: usize = 32,
54
46 /// The criteria for a connection to be considered a match.55 /// The criteria for a connection to be considered a match.
47 pub const Criteria = struct {56 pub const Criteria = struct {
48 host: []const u8,57 host: []const u8,
...@@ -53,14 +62,6 @@ pub const ConnectionPool = struct {...@@ -53,14 +62,6 @@ pub const ConnectionPool = struct {
53 const Queue = std.DoublyLinkedList(Connection);62 const Queue = std.DoublyLinkedList(Connection);
54 pub const Node = Queue.Node;63 pub const Node = Queue.Node;
5564
56 mutex: std.Thread.Mutex = .{},
57 /// Open connections that are currently in use.
58 used: Queue = .{},
59 /// Open connections that are not currently in use.
60 free: Queue = .{},
61 free_len: usize = 0,
62 free_size: usize = 32,
63
64 /// Finds and acquires a connection from the connection pool matching the criteria. This function is threadsafe.65 /// Finds and acquires a connection from the connection pool matching the criteria. This function is threadsafe.
65 /// If no connection is found, null is returned.66 /// If no connection is found, null is returned.
66 pub fn findConnection(pool: *ConnectionPool, criteria: Criteria) ?*Connection {67 pub fn findConnection(pool: *ConnectionPool, criteria: Criteria) ?*Connection {
...@@ -189,11 +190,6 @@ pub const ConnectionPool = struct {...@@ -189,11 +190,6 @@ pub const ConnectionPool = struct {
189190
190/// An interface to either a plain or TLS connection.191/// An interface to either a plain or TLS connection.
191pub const Connection = struct {192pub const Connection = struct {
192 pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
193 const BufferSize = std.math.IntFittingRange(0, buffer_size);
194
195 pub const Protocol = enum { plain, tls };
196
197 stream: net.Stream,193 stream: net.Stream,
198 /// undefined unless protocol is tls.194 /// undefined unless protocol is tls.
199 tls_client: if (!disable_tls) *std.crypto.tls.Client else void,195 tls_client: if (!disable_tls) *std.crypto.tls.Client else void,
...@@ -219,6 +215,11 @@ pub const Connection = struct {...@@ -219,6 +215,11 @@ pub const Connection = struct {
219 read_buf: [buffer_size]u8 = undefined,215 read_buf: [buffer_size]u8 = undefined,
220 write_buf: [buffer_size]u8 = undefined,216 write_buf: [buffer_size]u8 = undefined,
221217
218 pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
219 const BufferSize = std.math.IntFittingRange(0, buffer_size);
220
221 pub const Protocol = enum { plain, tls };
222
222 pub fn readvDirectTls(conn: *Connection, buffers: []std.os.iovec) ReadError!usize {223 pub fn readvDirectTls(conn: *Connection, buffers: []std.os.iovec) ReadError!usize {
223 return conn.tls_client.readv(conn.stream, buffers) catch |err| {224 return conn.tls_client.readv(conn.stream, buffers) catch |err| {
224 // https://github.com/ziglang/zig/issues/2473225 // https://github.com/ziglang/zig/issues/2473
...@@ -406,31 +407,63 @@ pub const RequestTransfer = union(enum) {...@@ -406,31 +407,63 @@ pub const RequestTransfer = union(enum) {
406pub const Compression = union(enum) {407pub const Compression = union(enum) {
407 pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader);408 pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader);
408 pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader);409 pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader);
409 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});410 // https://github.com/ziglang/zig/issues/18937
411 //pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
410412
411 deflate: DeflateDecompressor,413 deflate: DeflateDecompressor,
412 gzip: GzipDecompressor,414 gzip: GzipDecompressor,
413 zstd: ZstdDecompressor,415 // https://github.com/ziglang/zig/issues/18937
416 //zstd: ZstdDecompressor,
414 none: void,417 none: void,
415};418};
416419
417/// A HTTP response originating from a server.420/// A HTTP response originating from a server.
418pub const Response = struct {421pub const Response = struct {
419 pub const ParseError = Allocator.Error || error{422 version: http.Version,
423 status: http.Status,
424 reason: []const u8,
425
426 /// Points into the user-provided `server_header_buffer`.
427 location: ?[]const u8 = null,
428 /// Points into the user-provided `server_header_buffer`.
429 content_type: ?[]const u8 = null,
430 /// Points into the user-provided `server_header_buffer`.
431 content_disposition: ?[]const u8 = null,
432
433 keep_alive: bool = false,
434
435 /// If present, the number of bytes in the response body.
436 content_length: ?u64 = null,
437
438 /// If present, the transfer encoding of the response body, otherwise none.
439 transfer_encoding: http.TransferEncoding = .none,
440
441 /// If present, the compression of the response body, otherwise identity (no compression).
442 transfer_compression: http.ContentEncoding = .identity,
443
444 parser: proto.HeadersParser,
445 compression: Compression = .none,
446
447 /// Whether the response body should be skipped. Any data read from the
448 /// response body will be discarded.
449 skip: bool = false,
450
451 pub const ParseError = error{
420 HttpHeadersInvalid,452 HttpHeadersInvalid,
421 HttpHeaderContinuationsUnsupported,453 HttpHeaderContinuationsUnsupported,
422 HttpTransferEncodingUnsupported,454 HttpTransferEncodingUnsupported,
423 HttpConnectionHeaderUnsupported,455 HttpConnectionHeaderUnsupported,
424 InvalidContentLength,456 InvalidContentLength,
425 CompressionNotSupported,457 CompressionUnsupported,
426 };458 };
427459
428 pub fn parse(res: *Response, bytes: []const u8, trailing: bool) ParseError!void {460 pub fn parse(res: *Response, bytes: []const u8) ParseError!void {
429 var it = mem.tokenizeAny(u8, bytes, "\r\n");461 var it = mem.splitSequence(u8, bytes, "\r\n");
430462
431 const first_line = it.next() orelse return error.HttpHeadersInvalid;463 const first_line = it.next().?;
432 if (first_line.len < 12)464 if (first_line.len < 12) {
433 return error.HttpHeadersInvalid;465 return error.HttpHeadersInvalid;
466 }
434467
435 const version: http.Version = switch (int64(first_line[0..8])) {468 const version: http.Version = switch (int64(first_line[0..8])) {
436 int64("HTTP/1.0") => .@"HTTP/1.0",469 int64("HTTP/1.0") => .@"HTTP/1.0",
...@@ -445,24 +478,27 @@ pub const Response = struct {...@@ -445,24 +478,27 @@ pub const Response = struct {
445 res.status = status;478 res.status = status;
446 res.reason = reason;479 res.reason = reason;
447480
448 res.headers.clearRetainingCapacity();
449
450 while (it.next()) |line| {481 while (it.next()) |line| {
451 if (line.len == 0) return error.HttpHeadersInvalid;482 if (line.len == 0) return;
452 switch (line[0]) {483 switch (line[0]) {
453 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,484 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
454 else => {},485 else => {},
455 }486 }
456487
457 var line_it = mem.tokenizeAny(u8, line, ": ");488 var line_it = mem.splitSequence(u8, line, ": ");
458 const header_name = line_it.next() orelse return error.HttpHeadersInvalid;489 const header_name = line_it.next().?;
459 const header_value = line_it.rest();490 const header_value = line_it.rest();
460491 if (header_value.len == 0) return error.HttpHeadersInvalid;
461 try res.headers.append(header_name, header_value);492
462493 if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
463 if (trailing) continue;494 res.keep_alive = !std.ascii.eqlIgnoreCase(header_value, "close");
464495 } else if (std.ascii.eqlIgnoreCase(header_name, "content-type")) {
465 if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {496 res.content_type = header_value;
497 } else if (std.ascii.eqlIgnoreCase(header_name, "location")) {
498 res.location = header_value;
499 } else if (std.ascii.eqlIgnoreCase(header_name, "content-disposition")) {
500 res.content_disposition = header_value;
501 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
466 // Transfer-Encoding: second, first502 // Transfer-Encoding: second, first
467 // Transfer-Encoding: deflate, chunked503 // Transfer-Encoding: deflate, chunked
468 var iter = mem.splitBackwardsScalar(u8, header_value, ',');504 var iter = mem.splitBackwardsScalar(u8, header_value, ',');
...@@ -508,6 +544,7 @@ pub const Response = struct {...@@ -508,6 +544,7 @@ pub const Response = struct {
508 }544 }
509 }545 }
510 }546 }
547 return error.HttpHeadersInvalid; // missing empty line
511 }548 }
512549
513 inline fn int64(array: *const [8]u8) u64 {550 inline fn int64(array: *const [8]u8) u64 {
...@@ -531,60 +568,25 @@ pub const Response = struct {...@@ -531,60 +568,25 @@ pub const Response = struct {
531 try expectEqual(@as(u10, 999), parseInt3("999"));568 try expectEqual(@as(u10, 999), parseInt3("999"));
532 }569 }
533570
534 /// The HTTP version this response is using.571 pub fn iterateHeaders(r: Response) http.HeaderIterator {
535 version: http.Version,572 return http.HeaderIterator.init(r.parser.get());
536573 }
537 /// The status code of the response.
538 status: http.Status,
539
540 /// The reason phrase of the response.
541 reason: []const u8,
542
543 /// If present, the number of bytes in the response body.
544 content_length: ?u64 = null,
545
546 /// If present, the transfer encoding of the response body, otherwise none.
547 transfer_encoding: http.TransferEncoding = .none,
548
549 /// If present, the compression of the response body, otherwise identity (no compression).
550 transfer_compression: http.ContentEncoding = .identity,
551
552 /// The headers received from the server.
553 headers: http.Headers,
554 parser: proto.HeadersParser,
555 compression: Compression = .none,
556
557 /// Whether the response body should be skipped. Any data read from the response body will be discarded.
558 skip: bool = false,
559};574};
560575
561/// A HTTP request that has been sent.576/// A HTTP request that has been sent.
562///577///
563/// Order of operations: open -> send[ -> write -> finish] -> wait -> read578/// Order of operations: open -> send[ -> write -> finish] -> wait -> read
564pub const Request = struct {579pub const Request = struct {
565 /// The uri that this request is being sent to.
566 uri: Uri,580 uri: Uri,
567
568 /// The client that this request was created from.
569 client: *Client,581 client: *Client,
570582 /// This is null when the connection is released.
571 /// Underlying connection to the server. This is null when the connection is released.
572 connection: ?*Connection,583 connection: ?*Connection,
584 keep_alive: bool,
573585
574 method: http.Method,586 method: http.Method,
575 version: http.Version = .@"HTTP/1.1",587 version: http.Version = .@"HTTP/1.1",
576588 transfer_encoding: RequestTransfer,
577 /// The list of HTTP request headers.589 redirect_behavior: RedirectBehavior,
578 headers: http.Headers,
579
580 /// The transfer encoding of the request body.
581 transfer_encoding: RequestTransfer = .none,
582
583 /// The redirect quota left for this request.
584 redirects_left: u32,
585
586 /// Whether the request should follow redirects.
587 handle_redirects: bool,
588590
589 /// Whether the request should handle a 100-continue response before sending the request body.591 /// Whether the request should handle a 100-continue response before sending the request body.
590 handle_continue: bool,592 handle_continue: bool,
...@@ -594,25 +596,60 @@ pub const Request = struct {...@@ -594,25 +596,60 @@ pub const Request = struct {
594 /// This field is undefined until `wait` is called.596 /// This field is undefined until `wait` is called.
595 response: Response,597 response: Response,
596598
597 /// Used as a allocator for resolving redirects locations.599 /// Standard headers that have default, but overridable, behavior.
598 arena: std.heap.ArenaAllocator,600 headers: Headers,
601
602 /// These headers are kept including when following a redirect to a
603 /// different domain.
604 /// Externally-owned; must outlive the Request.
605 extra_headers: []const http.Header,
606
607 /// These headers are stripped when following a redirect to a different
608 /// domain.
609 /// Externally-owned; must outlive the Request.
610 privileged_headers: []const http.Header,
611
612 pub const Headers = struct {
613 host: Value = .default,
614 authorization: Value = .default,
615 user_agent: Value = .default,
616 connection: Value = .default,
617 accept_encoding: Value = .default,
618 content_type: Value = .default,
619
620 pub const Value = union(enum) {
621 default,
622 omit,
623 override: []const u8,
624 };
625 };
599626
600 /// Frees all resources associated with the request.627 /// Any value other than `not_allowed` or `unhandled` means that integer represents
601 pub fn deinit(req: *Request) void {628 /// how many remaining redirects are allowed.
602 switch (req.response.compression) {629 pub const RedirectBehavior = enum(u16) {
603 .none => {},630 /// The next redirect will cause an error.
604 .deflate => {},631 not_allowed = 0,
605 .gzip => {},632 /// Redirects are passed to the client to analyze the redirect response
606 .zstd => |*zstd| zstd.deinit(),633 /// directly.
634 unhandled = std.math.maxInt(u16),
635 _,
636
637 pub fn subtractOne(rb: *RedirectBehavior) void {
638 switch (rb.*) {
639 .not_allowed => unreachable,
640 .unhandled => unreachable,
641 _ => rb.* = @enumFromInt(@intFromEnum(rb.*) - 1),
642 }
607 }643 }
608644
609 req.headers.deinit();645 pub fn remaining(rb: RedirectBehavior) u16 {
610 req.response.headers.deinit();646 assert(rb != .unhandled);
611647 return @intFromEnum(rb);
612 if (req.response.parser.header_bytes_owned) {
613 req.response.parser.header_bytes.deinit(req.client.allocator);
614 }648 }
649 };
615650
651 /// Frees all resources associated with the request.
652 pub fn deinit(req: *Request) void {
616 if (req.connection) |connection| {653 if (req.connection) |connection| {
617 if (!req.response.parser.done) {654 if (!req.response.parser.done) {
618 // If the response wasn't fully read, then we need to close the connection.655 // If the response wasn't fully read, then we need to close the connection.
...@@ -620,23 +657,15 @@ pub const Request = struct {...@@ -620,23 +657,15 @@ pub const Request = struct {
620 }657 }
621 req.client.connection_pool.release(req.client.allocator, connection);658 req.client.connection_pool.release(req.client.allocator, connection);
622 }659 }
623
624 req.arena.deinit();
625 req.* = undefined;660 req.* = undefined;
626 }661 }
627662
628 // This function must deallocate all resources associated with the request, or keep those which will be used663 // This function must deallocate all resources associated with the request,
629 // This needs to be kept in sync with deinit and request664 // or keep those which will be used.
665 // This needs to be kept in sync with deinit and request.
630 fn redirect(req: *Request, uri: Uri) !void {666 fn redirect(req: *Request, uri: Uri) !void {
631 assert(req.response.parser.done);667 assert(req.response.parser.done);
632668
633 switch (req.response.compression) {
634 .none => {},
635 .deflate => {},
636 .gzip => {},
637 .zstd => |*zstd| zstd.deinit(),
638 }
639
640 req.client.connection_pool.release(req.client.allocator, req.connection.?);669 req.client.connection_pool.release(req.client.allocator, req.connection.?);
641 req.connection = null;670 req.connection = null;
642671
...@@ -651,15 +680,13 @@ pub const Request = struct {...@@ -651,15 +680,13 @@ pub const Request = struct {
651680
652 req.uri = uri;681 req.uri = uri;
653 req.connection = try req.client.connect(host, port, protocol);682 req.connection = try req.client.connect(host, port, protocol);
654 req.redirects_left -= 1;683 req.redirect_behavior.subtractOne();
655 req.response.headers.clearRetainingCapacity();
656 req.response.parser.reset();684 req.response.parser.reset();
657685
658 req.response = .{686 req.response = .{
659 .status = undefined,687 .status = undefined,
660 .reason = undefined,688 .reason = undefined,
661 .version = undefined,689 .version = undefined,
662 .headers = req.response.headers,
663 .parser = req.response.parser,690 .parser = req.response.parser,
664 };691 };
665 }692 }
...@@ -667,15 +694,17 @@ pub const Request = struct {...@@ -667,15 +694,17 @@ pub const Request = struct {
667 pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };694 pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };
668695
669 pub const SendOptions = struct {696 pub const SendOptions = struct {
670 /// Specifies that the uri should be used as is. You guarantee that the uri is already escaped.697 /// Specifies that the uri is already escaped.
671 raw_uri: bool = false,698 raw_uri: bool = false,
672 };699 };
673700
674 /// Send the HTTP request headers to the server.701 /// Send the HTTP request headers to the server.
675 pub fn send(req: *Request, options: SendOptions) SendError!void {702 pub fn send(req: *Request, options: SendOptions) SendError!void {
676 if (!req.method.requestHasBody() and req.transfer_encoding != .none) return error.UnsupportedTransferEncoding;703 if (!req.method.requestHasBody() and req.transfer_encoding != .none)
704 return error.UnsupportedTransferEncoding;
677705
678 const w = req.connection.?.writer();706 const connection = req.connection.?;
707 const w = connection.writer();
679708
680 try req.method.write(w);709 try req.method.write(w);
681 try w.writeByte(' ');710 try w.writeByte(' ');
...@@ -684,9 +713,9 @@ pub const Request = struct {...@@ -684,9 +713,9 @@ pub const Request = struct {
684 try req.uri.writeToStream(.{ .authority = true }, w);713 try req.uri.writeToStream(.{ .authority = true }, w);
685 } else {714 } else {
686 try req.uri.writeToStream(.{715 try req.uri.writeToStream(.{
687 .scheme = req.connection.?.proxied,716 .scheme = connection.proxied,
688 .authentication = req.connection.?.proxied,717 .authentication = connection.proxied,
689 .authority = req.connection.?.proxied,718 .authority = connection.proxied,
690 .path = true,719 .path = true,
691 .query = true,720 .query = true,
692 .raw = options.raw_uri,721 .raw = options.raw_uri,
...@@ -696,97 +725,93 @@ pub const Request = struct {...@@ -696,97 +725,93 @@ pub const Request = struct {
696 try w.writeAll(@tagName(req.version));725 try w.writeAll(@tagName(req.version));
697 try w.writeAll("\r\n");726 try w.writeAll("\r\n");
698727
699 if (!req.headers.contains("host")) {728 if (try emitOverridableHeader("host: ", req.headers.host, w)) {
700 try w.writeAll("Host: ");729 try w.writeAll("host: ");
701 try req.uri.writeToStream(.{ .authority = true }, w);730 try req.uri.writeToStream(.{ .authority = true }, w);
702 try w.writeAll("\r\n");731 try w.writeAll("\r\n");
703 }732 }
704733
705 if ((req.uri.user != null or req.uri.password != null) and734 if (try emitOverridableHeader("authorization: ", req.headers.authorization, w)) {
706 !req.headers.contains("authorization"))735 if (req.uri.user != null or req.uri.password != null) {
707 {736 try w.writeAll("authorization: ");
708 try w.writeAll("Authorization: ");737 const authorization = try connection.allocWriteBuffer(
709 const authorization = try req.connection.?.allocWriteBuffer(738 @intCast(basic_authorization.valueLengthFromUri(req.uri)),
710 @intCast(basic_authorization.valueLengthFromUri(req.uri)),739 );
711 );740 assert(basic_authorization.value(req.uri, authorization).len == authorization.len);
712 std.debug.assert(basic_authorization.value(req.uri, authorization).len == authorization.len);741 try w.writeAll("\r\n");
713 try w.writeAll("\r\n");742 }
714 }743 }
715744
716 if (!req.headers.contains("user-agent")) {745 if (try emitOverridableHeader("user-agent: ", req.headers.user_agent, w)) {
717 try w.writeAll("User-Agent: zig/");746 try w.writeAll("user-agent: zig/");
718 try w.writeAll(builtin.zig_version_string);747 try w.writeAll(builtin.zig_version_string);
719 try w.writeAll(" (std.http)\r\n");748 try w.writeAll(" (std.http)\r\n");
720 }749 }
721750
722 if (!req.headers.contains("connection")) {751 if (try emitOverridableHeader("connection: ", req.headers.connection, w)) {
723 try w.writeAll("Connection: keep-alive\r\n");752 if (req.keep_alive) {
753 try w.writeAll("connection: keep-alive\r\n");
754 } else {
755 try w.writeAll("connection: close\r\n");
756 }
724 }757 }
725758
726 if (!req.headers.contains("accept-encoding")) {759 if (try emitOverridableHeader("accept-encoding: ", req.headers.accept_encoding, w)) {
727 try w.writeAll("Accept-Encoding: gzip, deflate, zstd\r\n");760 // https://github.com/ziglang/zig/issues/18937
761 //try w.writeAll("accept-encoding: gzip, deflate, zstd\r\n");
762 try w.writeAll("accept-encoding: gzip, deflate\r\n");
728 }763 }
729764
730 if (!req.headers.contains("te")) {765 switch (req.transfer_encoding) {
731 try w.writeAll("TE: gzip, deflate, trailers\r\n");766 .chunked => try w.writeAll("transfer-encoding: chunked\r\n"),
767 .content_length => |len| try w.print("content-length: {d}\r\n", .{len}),
768 .none => {},
732 }769 }
733770
734 const has_transfer_encoding = req.headers.contains("transfer-encoding");771 if (try emitOverridableHeader("content-type: ", req.headers.content_type, w)) {
735 const has_content_length = req.headers.contains("content-length");772 // The default is to omit content-type if not provided because
736773 // "application/octet-stream" is redundant.
737 if (!has_transfer_encoding and !has_content_length) {
738 switch (req.transfer_encoding) {
739 .chunked => try w.writeAll("Transfer-Encoding: chunked\r\n"),
740 .content_length => |content_length| try w.print("Content-Length: {d}\r\n", .{content_length}),
741 .none => {},
742 }
743 } else {
744 if (has_transfer_encoding) {
745 const transfer_encoding = req.headers.getFirstValue("transfer-encoding").?;
746 if (std.mem.eql(u8, transfer_encoding, "chunked")) {
747 req.transfer_encoding = .chunked;
748 } else {
749 return error.UnsupportedTransferEncoding;
750 }
751 } else if (has_content_length) {
752 const content_length = std.fmt.parseInt(u64, req.headers.getFirstValue("content-length").?, 10) catch return error.InvalidContentLength;
753
754 req.transfer_encoding = .{ .content_length = content_length };
755 } else {
756 req.transfer_encoding = .none;
757 }
758 }774 }
759775
760 for (req.headers.list.items) |entry| {776 for (req.extra_headers) |header| {
761 if (entry.value.len == 0) continue;777 assert(header.value.len != 0);
762778
763 try w.writeAll(entry.name);779 try w.writeAll(header.name);
764 try w.writeAll(": ");780 try w.writeAll(": ");
765 try w.writeAll(entry.value);781 try w.writeAll(header.value);
766 try w.writeAll("\r\n");782 try w.writeAll("\r\n");
767 }783 }
768784
769 if (req.connection.?.proxied) {785 if (connection.proxied) proxy: {
770 const proxy_headers: ?http.Headers = switch (req.connection.?.protocol) {786 const proxy = switch (connection.protocol) {
771 .plain => if (req.client.http_proxy) |proxy| proxy.headers else null,787 .plain => req.client.http_proxy,
772 .tls => if (req.client.https_proxy) |proxy| proxy.headers else null,788 .tls => req.client.https_proxy,
773 };789 } orelse break :proxy;
774
775 if (proxy_headers) |headers| {
776 for (headers.list.items) |entry| {
777 if (entry.value.len == 0) continue;
778790
779 try w.writeAll(entry.name);791 const authorization = proxy.authorization orelse break :proxy;
780 try w.writeAll(": ");792 try w.writeAll("proxy-authorization: ");
781 try w.writeAll(entry.value);793 try w.writeAll(authorization);
782 try w.writeAll("\r\n");794 try w.writeAll("\r\n");
783 }
784 }
785 }795 }
786796
787 try w.writeAll("\r\n");797 try w.writeAll("\r\n");
788798
789 try req.connection.?.flush();799 try connection.flush();
800 }
801
802 /// Returns true if the default behavior is required, otherwise handles
803 /// writing (or not writing) the header.
804 fn emitOverridableHeader(prefix: []const u8, v: Headers.Value, w: anytype) !bool {
805 switch (v) {
806 .default => return true,
807 .omit => return false,
808 .override => |x| {
809 try w.writeAll(prefix);
810 try w.writeAll(x);
811 try w.writeAll("\r\n");
812 return false;
813 },
814 }
790 }815 }
791816
792 const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;817 const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;
...@@ -810,145 +835,169 @@ pub const Request = struct {...@@ -810,145 +835,169 @@ pub const Request = struct {
810 return index;835 return index;
811 }836 }
812837
813 pub const WaitError = RequestError || SendError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, RedirectRequiresResend, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };838 pub const WaitError = RequestError || SendError || TransferReadError ||
839 proto.HeadersParser.CheckCompleteHeadError || Response.ParseError ||
840 error{ // TODO: file zig fmt issue for this bad indentation
841 TooManyHttpRedirects,
842 RedirectRequiresResend,
843 HttpRedirectLocationMissing,
844 HttpRedirectLocationInvalid,
845 CompressionInitializationFailed,
846 CompressionUnsupported,
847 };
814848
815 /// Waits for a response from the server and parses any headers that are sent.849 /// Waits for a response from the server and parses any headers that are sent.
816 /// This function will block until the final response is received.850 /// This function will block until the final response is received.
817 ///851 ///
818 /// If `handle_redirects` is true and the request has no payload, then this function will automatically follow852 /// If handling redirects and the request has no payload, then this
819 /// redirects. If a request payload is present, then this function will error with error.RedirectRequiresResend.853 /// function will automatically follow redirects. If a request payload is
854 /// present, then this function will error with
855 /// error.RedirectRequiresResend.
820 ///856 ///
821 /// Must be called after `send` and, if any data was written to the request body, then also after `finish`.857 /// Must be called after `send` and, if any data was written to the request
858 /// body, then also after `finish`.
822 pub fn wait(req: *Request) WaitError!void {859 pub fn wait(req: *Request) WaitError!void {
860 const connection = req.connection.?;
861
823 while (true) { // handle redirects862 while (true) { // handle redirects
824 while (true) { // read headers863 while (true) { // read headers
825 try req.connection.?.fill();864 try connection.fill();
826865
827 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.?.peek());866 const nchecked = try req.response.parser.checkCompleteHead(connection.peek());
828 req.connection.?.drop(@intCast(nchecked));867 connection.drop(@intCast(nchecked));
829868
830 if (req.response.parser.state.isContent()) break;869 if (req.response.parser.state.isContent()) break;
831 }870 }
832871
833 try req.response.parse(req.response.parser.header_bytes.items, false);872 try req.response.parse(req.response.parser.get());
834873
835 if (req.response.status == .@"continue") {874 if (req.response.status == .@"continue") {
836 req.response.parser.done = true; // we're done parsing the continue response, reset to prepare for the real response875 // We're done parsing the continue response; reset to prepare
876 // for the real response.
877 req.response.parser.done = true;
837 req.response.parser.reset();878 req.response.parser.reset();
838879
839 if (req.handle_continue)880 if (req.handle_continue)
840 continue;881 continue;
841882
842 return; // we're not handling the 100-continue, return to the caller883 return; // we're not handling the 100-continue
843 }884 }
844885
845 // we're switching protocols, so this connection is no longer doing http886 // we're switching protocols, so this connection is no longer doing http
846 if (req.method == .CONNECT and req.response.status.class() == .success) {887 if (req.method == .CONNECT and req.response.status.class() == .success) {
847 req.connection.?.closing = false;888 connection.closing = false;
848 req.response.parser.done = true;889 req.response.parser.done = true;
849890 return; // the connection is not HTTP past this point
850 return; // the connection is not HTTP past this point, return to the caller
851 }891 }
852892
853 // we default to using keep-alive if not provided in the client if the server asks for it893 connection.closing = !req.response.keep_alive or !req.keep_alive;
854 const req_connection = req.headers.getFirstValue("connection");
855 const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
856
857 const res_connection = req.response.headers.getFirstValue("connection");
858 const res_keepalive = res_connection != null and !std.ascii.eqlIgnoreCase("close", res_connection.?);
859 if (res_keepalive and (req_keepalive or req_connection == null)) {
860 req.connection.?.closing = false;
861 } else {
862 req.connection.?.closing = true;
863 }
864894
865 // Any response to a HEAD request and any response with a 1xx (Informational), 204 (No Content), or 304 (Not Modified)895 // Any response to a HEAD request and any response with a 1xx
866 // status code is always terminated by the first empty line after the header fields, regardless of the header fields896 // (Informational), 204 (No Content), or 304 (Not Modified) status
867 // present in the message897 // code is always terminated by the first empty line after the
868 if (req.method == .HEAD or req.response.status.class() == .informational or req.response.status == .no_content or req.response.status == .not_modified) {898 // header fields, regardless of the header fields present in the
899 // message.
900 if (req.method == .HEAD or req.response.status.class() == .informational or
901 req.response.status == .no_content or req.response.status == .not_modified)
902 {
869 req.response.parser.done = true;903 req.response.parser.done = true;
870904 return; // The response is empty; no further setup or redirection is necessary.
871 return; // the response is empty, no further setup or redirection is necessary
872 }905 }
873906
874 if (req.response.transfer_encoding != .none) {907 switch (req.response.transfer_encoding) {
875 switch (req.response.transfer_encoding) {908 .none => {
876 .none => unreachable,909 if (req.response.content_length) |cl| {
877 .chunked => {910 req.response.parser.next_chunk_length = cl;
878 req.response.parser.next_chunk_length = 0;
879 req.response.parser.state = .chunk_head_size;
880 },
881 }
882 } else if (req.response.content_length) |cl| {
883 req.response.parser.next_chunk_length = cl;
884911
885 if (cl == 0) req.response.parser.done = true;912 if (cl == 0) req.response.parser.done = true;
886 } else {913 } else {
887 // read until the connection is closed914 // read until the connection is closed
888 req.response.parser.next_chunk_length = std.math.maxInt(u64);915 req.response.parser.next_chunk_length = std.math.maxInt(u64);
916 }
917 },
918 .chunked => {
919 req.response.parser.next_chunk_length = 0;
920 req.response.parser.state = .chunk_head_size;
921 },
889 }922 }
890923
891 if (req.response.status.class() == .redirect and req.handle_redirects) {924 if (req.response.status.class() == .redirect and req.redirect_behavior != .unhandled) {
925 // skip the body of the redirect response, this will at least
926 // leave the connection in a known good state.
892 req.response.skip = true;927 req.response.skip = true;
893928 assert(try req.transferRead(&.{}) == 0); // we're skipping, no buffer is necessary
894 // skip the body of the redirect response, this will at least leave the connection in a known good state.929
895 const empty = @as([*]u8, undefined)[0..0];930 if (req.redirect_behavior == .not_allowed) return error.TooManyHttpRedirects;
896 assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary931
897932 const location = req.response.location orelse
898 if (req.redirects_left == 0) return error.TooManyHttpRedirects;933 return error.HttpRedirectLocationMissing;
899934
900 const location = req.response.headers.getFirstValue("location") orelse935 // This mutates the beginning of header_buffer and uses that
901 return error.HttpRedirectMissingLocation;936 // for the backing memory of the returned new_uri.
902937 const header_buffer = req.response.parser.header_bytes_buffer;
903 const arena = req.arena.allocator();938 const new_uri = req.uri.resolve_inplace(location, header_buffer) catch
904939 return error.HttpRedirectLocationInvalid;
905 const location_duped = try arena.dupe(u8, location);940
906941 // The new URI references the beginning of header_bytes_buffer memory.
907 const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);942 // That memory will be kept, but everything after it will be
908 const resolved_url = try req.uri.resolve(new_url, false, arena);943 // reused by the subsequent request. In other words,
909944 // header_bytes_buffer must be large enough to store all
910 // is the redirect location on the same domain, or a subdomain of the original request?945 // redirect locations as well as the final request header.
911 const is_same_domain_or_subdomain = std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and (resolved_url.host.?.len == req.uri.host.?.len or resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.');946 const path_end = new_uri.path.ptr + new_uri.path.len;
912947 // https://github.com/ziglang/zig/issues/1738
913 if (resolved_url.host == null or !is_same_domain_or_subdomain or !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme)) {948 const path_offset = @intFromPtr(path_end) - @intFromPtr(header_buffer.ptr);
914 // we're redirecting to a different domain, strip privileged headers like cookies949 const end_offset = @max(path_offset, location.len);
915 _ = req.headers.delete("authorization");950 req.response.parser.header_bytes_buffer = header_buffer[end_offset..];
916 _ = req.headers.delete("www-authenticate");951
917 _ = req.headers.delete("cookie");952 const is_same_domain_or_subdomain =
918 _ = req.headers.delete("cookie2");953 std.ascii.endsWithIgnoreCase(new_uri.host.?, req.uri.host.?) and
954 (new_uri.host.?.len == req.uri.host.?.len or
955 new_uri.host.?[new_uri.host.?.len - req.uri.host.?.len - 1] == '.');
956
957 if (new_uri.host == null or !is_same_domain_or_subdomain or
958 !std.ascii.eqlIgnoreCase(new_uri.scheme, req.uri.scheme))
959 {
960 // When redirecting to a different domain, strip privileged headers.
961 req.privileged_headers = &.{};
919 }962 }
920963
921 if (req.response.status == .see_other or ((req.response.status == .moved_permanently or req.response.status == .found) and req.method == .POST)) {964 if (switch (req.response.status) {
922 // we're redirecting to a GET, so we need to change the method and remove the body965 .see_other => true,
966 .moved_permanently, .found => req.method == .POST,
967 else => false,
968 }) {
969 // A redirect to a GET must change the method and remove the body.
923 req.method = .GET;970 req.method = .GET;
924 req.transfer_encoding = .none;971 req.transfer_encoding = .none;
925 _ = req.headers.delete("transfer-encoding");972 req.headers.content_type = .omit;
926 _ = req.headers.delete("content-length");
927 _ = req.headers.delete("content-type");
928 }973 }
929974
930 if (req.transfer_encoding != .none) {975 if (req.transfer_encoding != .none) {
931 return error.RedirectRequiresResend; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually.976 // The request body has already been sent. The request is
977 // still in a valid state, but the redirect must be handled
978 // manually.
979 return error.RedirectRequiresResend;
932 }980 }
933981
934 try req.redirect(resolved_url);982 try req.redirect(new_uri);
935
936 try req.send(.{});983 try req.send(.{});
937 } else {984 } else {
938 req.response.skip = false;985 req.response.skip = false;
939 if (!req.response.parser.done) {986 if (!req.response.parser.done) {
940 switch (req.response.transfer_compression) {987 switch (req.response.transfer_compression) {
941 .identity => req.response.compression = .none,988 .identity => req.response.compression = .none,
942 .compress, .@"x-compress" => return error.CompressionNotSupported,989 .compress, .@"x-compress" => return error.CompressionUnsupported,
943 .deflate => req.response.compression = .{990 .deflate => req.response.compression = .{
944 .deflate = std.compress.zlib.decompressor(req.transferReader()),991 .deflate = std.compress.zlib.decompressor(req.transferReader()),
945 },992 },
946 .gzip, .@"x-gzip" => req.response.compression = .{993 .gzip, .@"x-gzip" => req.response.compression = .{
947 .gzip = std.compress.gzip.decompressor(req.transferReader()),994 .gzip = std.compress.gzip.decompressor(req.transferReader()),
948 },995 },
949 .zstd => req.response.compression = .{996 // https://github.com/ziglang/zig/issues/18937
950 .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),997 //.zstd => req.response.compression = .{
951 },998 // .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),
999 //},
1000 .zstd => return error.CompressionUnsupported,
952 }1001 }
953 }1002 }
9541003
...@@ -957,7 +1006,8 @@ pub const Request = struct {...@@ -957,7 +1006,8 @@ pub const Request = struct {
957 }1006 }
958 }1007 }
9591008
960 pub const ReadError = TransferReadError || proto.HeadersParser.CheckCompleteHeadError || error{ DecompressionFailure, InvalidTrailers };1009 pub const ReadError = TransferReadError || proto.HeadersParser.CheckCompleteHeadError ||
1010 error{ DecompressionFailure, InvalidTrailers };
9611011
962 pub const Reader = std.io.Reader(*Request, ReadError, read);1012 pub const Reader = std.io.Reader(*Request, ReadError, read);
9631013
...@@ -970,28 +1020,20 @@ pub const Request = struct {...@@ -970,28 +1020,20 @@ pub const Request = struct {
970 const out_index = switch (req.response.compression) {1020 const out_index = switch (req.response.compression) {
971 .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure,1021 .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure,
972 .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure,1022 .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure,
973 .zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,1023 // https://github.com/ziglang/zig/issues/18937
1024 //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,
974 else => try req.transferRead(buffer),1025 else => try req.transferRead(buffer),
975 };1026 };
1027 if (out_index > 0) return out_index;
9761028
977 if (out_index == 0) {1029 while (!req.response.parser.state.isContent()) { // read trailing headers
978 const has_trail = !req.response.parser.state.isContent();1030 try req.connection.?.fill();
979
980 while (!req.response.parser.state.isContent()) { // read trailing headers
981 try req.connection.?.fill();
982
983 const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.?.peek());
984 req.connection.?.drop(@intCast(nchecked));
985 }
9861031
987 if (has_trail) {1032 const nchecked = try req.response.parser.checkCompleteHead(req.connection.?.peek());
988 // The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error.1033 req.connection.?.drop(@intCast(nchecked));
989 // This will *only* fail for a malformed trailer.
990 req.response.parse(req.response.parser.header_bytes.items, true) catch return error.InvalidTrailers;
991 }
992 }1034 }
9931035
994 return out_index;1036 return 0;
995 }1037 }
9961038
997 /// Reads data from the response body. Must be called after `wait`.1039 /// Reads data from the response body. Must be called after `wait`.
...@@ -1061,16 +1103,12 @@ pub const Request = struct {...@@ -1061,16 +1103,12 @@ pub const Request = struct {
1061 }1103 }
1062};1104};
10631105
1064/// A HTTP proxy server.
1065pub const Proxy = struct {1106pub const Proxy = struct {
1066 allocator: Allocator,
1067 headers: http.Headers,
1068
1069 protocol: Connection.Protocol,1107 protocol: Connection.Protocol,
1070 host: []const u8,1108 host: []const u8,
1109 authorization: ?[]const u8,
1071 port: u16,1110 port: u16,
10721111 supports_connect: bool,
1073 supports_connect: bool = true,
1074};1112};
10751113
1076/// Release all associated resources with the client.1114/// Release all associated resources with the client.
...@@ -1082,116 +1120,71 @@ pub fn deinit(client: *Client) void {...@@ -1082,116 +1120,71 @@ pub fn deinit(client: *Client) void {
10821120
1083 client.connection_pool.deinit(client.allocator);1121 client.connection_pool.deinit(client.allocator);
10841122
1085 if (client.http_proxy) |*proxy| {
1086 proxy.allocator.free(proxy.host);
1087 proxy.headers.deinit();
1088 }
1089
1090 if (client.https_proxy) |*proxy| {
1091 proxy.allocator.free(proxy.host);
1092 proxy.headers.deinit();
1093 }
1094
1095 if (!disable_tls)1123 if (!disable_tls)
1096 client.ca_bundle.deinit(client.allocator);1124 client.ca_bundle.deinit(client.allocator);
10971125
1098 client.* = undefined;1126 client.* = undefined;
1099}1127}
11001128
1101/// Uses the *_proxy environment variable to set any unset proxies for the client.1129/// Populates `http_proxy` and `http_proxy` via standard proxy environment variables.
1102/// This function *must not* be called when the client has any active connections.1130/// Asserts the client has no active connections.
1103pub fn loadDefaultProxies(client: *Client) !void {1131/// Uses `arena` for a few small allocations that must outlive the client, or
1132/// at least until those fields are set to different values.
1133pub fn initDefaultProxies(client: *Client, arena: Allocator) !void {
1104 // Prevent any new connections from being created.1134 // Prevent any new connections from being created.
1105 client.connection_pool.mutex.lock();1135 client.connection_pool.mutex.lock();
1106 defer client.connection_pool.mutex.unlock();1136 defer client.connection_pool.mutex.unlock();
11071137
1108 assert(client.connection_pool.used.first == null); // There are still active requests.1138 assert(client.connection_pool.used.first == null); // There are active requests.
11091139
1110 if (client.http_proxy == null) http: {1140 if (client.http_proxy == null) {
1111 const content: []const u8 = if (std.process.hasEnvVarConstant("http_proxy"))1141 client.http_proxy = try createProxyFromEnvVar(arena, &.{
1112 try std.process.getEnvVarOwned(client.allocator, "http_proxy")1142 "http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY",
1113 else if (std.process.hasEnvVarConstant("HTTP_PROXY"))1143 });
1114 try std.process.getEnvVarOwned(client.allocator, "HTTP_PROXY")1144 }
1115 else if (std.process.hasEnvVarConstant("all_proxy"))
1116 try std.process.getEnvVarOwned(client.allocator, "all_proxy")
1117 else if (std.process.hasEnvVarConstant("ALL_PROXY"))
1118 try std.process.getEnvVarOwned(client.allocator, "ALL_PROXY")
1119 else
1120 break :http;
1121 defer client.allocator.free(content);
1122
1123 const uri = Uri.parse(content) catch
1124 Uri.parseWithoutScheme(content) catch
1125 break :http;
1126
1127 const protocol = if (uri.scheme.len == 0)
1128 .plain // No scheme, assume http://
1129 else
1130 protocol_map.get(uri.scheme) orelse break :http; // Unknown scheme, ignore
1131
1132 const host = if (uri.host) |host| try client.allocator.dupe(u8, host) else break :http; // Missing host, ignore
1133 client.http_proxy = .{
1134 .allocator = client.allocator,
1135 .headers = .{ .allocator = client.allocator },
1136
1137 .protocol = protocol,
1138 .host = host,
1139 .port = uri.port orelse switch (protocol) {
1140 .plain => 80,
1141 .tls => 443,
1142 },
1143 };
11441145
1145 if (uri.user != null or uri.password != null) {1146 if (client.https_proxy == null) {
1146 const authorization = try client.allocator.alloc(u8, basic_authorization.valueLengthFromUri(uri));1147 client.https_proxy = try createProxyFromEnvVar(arena, &.{
1147 errdefer client.allocator.free(authorization);1148 "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY",
1148 std.debug.assert(basic_authorization.value(uri, authorization).len == authorization.len);1149 });
1149 try client.http_proxy.?.headers.appendOwned(.{ .unowned = "proxy-authorization" }, .{ .owned = authorization });
1150 }
1151 }1150 }
1151}
11521152
1153 if (client.https_proxy == null) https: {1153fn createProxyFromEnvVar(arena: Allocator, env_var_names: []const []const u8) !?*Proxy {
1154 const content: []const u8 = if (std.process.hasEnvVarConstant("https_proxy"))1154 const content = for (env_var_names) |name| {
1155 try std.process.getEnvVarOwned(client.allocator, "https_proxy")1155 break std.process.getEnvVarOwned(arena, name) catch |err| switch (err) {
1156 else if (std.process.hasEnvVarConstant("HTTPS_PROXY"))1156 error.EnvironmentVariableNotFound => continue,
1157 try std.process.getEnvVarOwned(client.allocator, "HTTPS_PROXY")1157 else => |e| return e,
1158 else if (std.process.hasEnvVarConstant("all_proxy"))
1159 try std.process.getEnvVarOwned(client.allocator, "all_proxy")
1160 else if (std.process.hasEnvVarConstant("ALL_PROXY"))
1161 try std.process.getEnvVarOwned(client.allocator, "ALL_PROXY")
1162 else
1163 break :https;
1164 defer client.allocator.free(content);
1165
1166 const uri = Uri.parse(content) catch
1167 Uri.parseWithoutScheme(content) catch
1168 break :https;
1169
1170 const protocol = if (uri.scheme.len == 0)
1171 .plain // No scheme, assume http://
1172 else
1173 protocol_map.get(uri.scheme) orelse break :https; // Unknown scheme, ignore
1174
1175 const host = if (uri.host) |host| try client.allocator.dupe(u8, host) else break :https; // Missing host, ignore
1176 client.https_proxy = .{
1177 .allocator = client.allocator,
1178 .headers = .{ .allocator = client.allocator },
1179
1180 .protocol = protocol,
1181 .host = host,
1182 .port = uri.port orelse switch (protocol) {
1183 .plain => 80,
1184 .tls => 443,
1185 },
1186 };1158 };
1159 } else return null;
11871160
1188 if (uri.user != null or uri.password != null) {1161 const uri = Uri.parse(content) catch try Uri.parseWithoutScheme(content);
1189 const authorization = try client.allocator.alloc(u8, basic_authorization.valueLengthFromUri(uri));1162
1190 errdefer client.allocator.free(authorization);1163 const protocol = if (uri.scheme.len == 0)
1191 std.debug.assert(basic_authorization.value(uri, authorization).len == authorization.len);1164 .plain // No scheme, assume http://
1192 try client.https_proxy.?.headers.appendOwned(.{ .unowned = "proxy-authorization" }, .{ .owned = authorization });1165 else
1193 }1166 protocol_map.get(uri.scheme) orelse return null; // Unknown scheme, ignore
1194 }1167
1168 const host = uri.host orelse return error.HttpProxyMissingHost;
1169
1170 const authorization: ?[]const u8 = if (uri.user != null or uri.password != null) a: {
1171 const authorization = try arena.alloc(u8, basic_authorization.valueLengthFromUri(uri));
1172 assert(basic_authorization.value(uri, authorization).len == authorization.len);
1173 break :a authorization;
1174 } else null;
1175
1176 const proxy = try arena.create(Proxy);
1177 proxy.* = .{
1178 .protocol = protocol,
1179 .host = host,
1180 .authorization = authorization,
1181 .port = uri.port orelse switch (protocol) {
1182 .plain => 80,
1183 .tls => 443,
1184 },
1185 .supports_connect = true,
1186 };
1187 return proxy;
1195}1188}
11961189
1197pub const basic_authorization = struct {1190pub const basic_authorization = struct {
...@@ -1213,8 +1206,8 @@ pub const basic_authorization = struct {...@@ -1213,8 +1206,8 @@ pub const basic_authorization = struct {
1213 }1206 }
12141207
1215 pub fn value(uri: Uri, out: []u8) []u8 {1208 pub fn value(uri: Uri, out: []u8) []u8 {
1216 std.debug.assert(uri.user == null or uri.user.?.len <= max_user_len);1209 assert(uri.user == null or uri.user.?.len <= max_user_len);
1217 std.debug.assert(uri.password == null or uri.password.?.len <= max_password_len);1210 assert(uri.password == null or uri.password.?.len <= max_password_len);
12181211
1219 @memcpy(out[0..prefix.len], prefix);1212 @memcpy(out[0..prefix.len], prefix);
12201213
...@@ -1288,14 +1281,12 @@ pub fn connectTcp(client: *Client, host: []const u8, port: u16, protocol: Connec...@@ -1288,14 +1281,12 @@ pub fn connectTcp(client: *Client, host: []const u8, port: u16, protocol: Connec
1288 return &conn.data;1281 return &conn.data;
1289}1282}
12901283
1291pub const ConnectUnixError = Allocator.Error || std.os.SocketError || error{ NameTooLong, Unsupported } || std.os.ConnectError;1284pub const ConnectUnixError = Allocator.Error || std.os.SocketError || error{NameTooLong} || std.os.ConnectError;
12921285
1293/// Connect to `path` as a unix domain socket. This will reuse a connection if one is already open.1286/// Connect to `path` as a unix domain socket. This will reuse a connection if one is already open.
1294///1287///
1295/// This function is threadsafe.1288/// This function is threadsafe.
1296pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connection {1289pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connection {
1297 if (!net.has_unix_sockets) return error.Unsupported;
1298
1299 if (client.connection_pool.findConnection(.{1290 if (client.connection_pool.findConnection(.{
1300 .host = path,1291 .host = path,
1301 .port = 0,1292 .port = 0,
...@@ -1325,7 +1316,8 @@ pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connecti...@@ -1325,7 +1316,8 @@ pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connecti
1325 return &conn.data;1316 return &conn.data;
1326}1317}
13271318
1328/// Connect to `tunnel_host:tunnel_port` using the specified proxy with HTTP CONNECT. This will reuse a connection if one is already open.1319/// Connect to `tunnel_host:tunnel_port` using the specified proxy with HTTP
1320/// CONNECT. This will reuse a connection if one is already open.
1329///1321///
1330/// This function is threadsafe.1322/// This function is threadsafe.
1331pub fn connectTunnel(1323pub fn connectTunnel(
...@@ -1351,7 +1343,7 @@ pub fn connectTunnel(...@@ -1351,7 +1343,7 @@ pub fn connectTunnel(
1351 client.connection_pool.release(client.allocator, conn);1343 client.connection_pool.release(client.allocator, conn);
1352 }1344 }
13531345
1354 const uri = Uri{1346 const uri: Uri = .{
1355 .scheme = "http",1347 .scheme = "http",
1356 .user = null,1348 .user = null,
1357 .password = null,1349 .password = null,
...@@ -1362,13 +1354,11 @@ pub fn connectTunnel(...@@ -1362,13 +1354,11 @@ pub fn connectTunnel(
1362 .fragment = null,1354 .fragment = null,
1363 };1355 };
13641356
1365 // we can use a small buffer here because a CONNECT response should be very small
1366 var buffer: [8096]u8 = undefined;1357 var buffer: [8096]u8 = undefined;
13671358 var req = client.open(.CONNECT, uri, .{
1368 var req = client.open(.CONNECT, uri, proxy.headers, .{1359 .redirect_behavior = .unhandled,
1369 .handle_redirects = false,
1370 .connection = conn,1360 .connection = conn,
1371 .header_strategy = .{ .static = &buffer },1361 .server_header_buffer = &buffer,
1372 }) catch |err| {1362 }) catch |err| {
1373 std.log.debug("err {}", .{err});1363 std.log.debug("err {}", .{err});
1374 break :tunnel err;1364 break :tunnel err;
...@@ -1407,45 +1397,51 @@ pub fn connectTunnel(...@@ -1407,45 +1397,51 @@ pub fn connectTunnel(
1407const ConnectErrorPartial = ConnectTcpError || error{ UnsupportedUrlScheme, ConnectionRefused };1397const ConnectErrorPartial = ConnectTcpError || error{ UnsupportedUrlScheme, ConnectionRefused };
1408pub const ConnectError = ConnectErrorPartial || RequestError;1398pub const ConnectError = ConnectErrorPartial || RequestError;
14091399
1410/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open.1400/// Connect to `host:port` using the specified protocol. This will reuse a
1411/// If a proxy is configured for the client, then the proxy will be used to connect to the host.1401/// connection if one is already open.
1402/// If a proxy is configured for the client, then the proxy will be used to
1403/// connect to the host.
1412///1404///
1413/// This function is threadsafe.1405/// This function is threadsafe.
1414pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*Connection {1406pub fn connect(
1415 // pointer required so that `supports_connect` can be updated if a CONNECT fails1407 client: *Client,
1416 const potential_proxy: ?*Proxy = switch (protocol) {1408 host: []const u8,
1417 .plain => if (client.http_proxy) |*proxy_info| proxy_info else null,1409 port: u16,
1418 .tls => if (client.https_proxy) |*proxy_info| proxy_info else null,1410 protocol: Connection.Protocol,
1419 };1411) ConnectError!*Connection {
14201412 const proxy = switch (protocol) {
1421 if (potential_proxy) |proxy| {1413 .plain => client.http_proxy,
1422 // don't attempt to proxy the proxy thru itself.1414 .tls => client.https_proxy,
1423 if (std.mem.eql(u8, proxy.host, host) and proxy.port == port and proxy.protocol == protocol) {1415 } orelse return client.connectTcp(host, port, protocol);
1424 return client.connectTcp(host, port, protocol);1416
1425 }1417 // Prevent proxying through itself.
14261418 if (std.ascii.eqlIgnoreCase(proxy.host, host) and
1427 if (proxy.supports_connect) tunnel: {1419 proxy.port == port and proxy.protocol == protocol)
1428 return connectTunnel(client, proxy, host, port) catch |err| switch (err) {1420 {
1429 error.TunnelNotSupported => break :tunnel,1421 return client.connectTcp(host, port, protocol);
1430 else => |e| return e,1422 }
1431 };
1432 }
14331423
1434 // fall back to using the proxy as a normal http proxy1424 if (proxy.supports_connect) tunnel: {
1435 const conn = try client.connectTcp(proxy.host, proxy.port, proxy.protocol);1425 return connectTunnel(client, proxy, host, port) catch |err| switch (err) {
1436 errdefer {1426 error.TunnelNotSupported => break :tunnel,
1437 conn.closing = true;1427 else => |e| return e,
1438 client.connection_pool.release(conn);1428 };
1439 }1429 }
14401430
1441 conn.proxied = true;1431 // fall back to using the proxy as a normal http proxy
1442 return conn;1432 const conn = try client.connectTcp(proxy.host, proxy.port, proxy.protocol);
1433 errdefer {
1434 conn.closing = true;
1435 client.connection_pool.release(conn);
1443 }1436 }
14441437
1445 return client.connectTcp(host, port, protocol);1438 conn.proxied = true;
1439 return conn;
1446}1440}
14471441
1448pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.SendError || std.fmt.ParseIntError || Connection.WriteError || error{1442pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.SendError ||
1443 std.fmt.ParseIntError || Connection.WriteError ||
1444 error{ // TODO: file a zig fmt issue for this bad indentation
1449 UnsupportedUrlScheme,1445 UnsupportedUrlScheme,
1450 UriMissingHost,1446 UriMissingHost,
14511447
...@@ -1456,36 +1452,44 @@ pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.SendE...@@ -1456,36 +1452,44 @@ pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.SendE
1456pub const RequestOptions = struct {1452pub const RequestOptions = struct {
1457 version: http.Version = .@"HTTP/1.1",1453 version: http.Version = .@"HTTP/1.1",
14581454
1459 /// Automatically ignore 100 Continue responses. This assumes you don't care, and will have sent the body before you1455 /// Automatically ignore 100 Continue responses. This assumes you don't
1460 /// wait for the response.1456 /// care, and will have sent the body before you wait for the response.
1461 ///1457 ///
1462 /// If this is not the case AND you know the server will send a 100 Continue, set this to false and wait for a1458 /// If this is not the case AND you know the server will send a 100
1463 /// response before sending the body. If you wait AND the server does not send a 100 Continue before you finish the1459 /// Continue, set this to false and wait for a response before sending the
1464 /// request, then the request *will* deadlock.1460 /// body. If you wait AND the server does not send a 100 Continue before
1461 /// you finish the request, then the request *will* deadlock.
1465 handle_continue: bool = true,1462 handle_continue: bool = true,
14661463
1467 /// Automatically follow redirects. This will only follow redirects for repeatable requests (ie. with no payload or the server has acknowledged the payload)1464 /// If false, close the connection after the one request. If true,
1468 handle_redirects: bool = true,1465 /// participate in the client connection pool.
1466 keep_alive: bool = true,
1467
1468 /// This field specifies whether to automatically follow redirects, and if
1469 /// so, how many redirects to follow before returning an error.
1470 ///
1471 /// This will only follow redirects for repeatable requests (ie. with no
1472 /// payload or the server has acknowledged the payload).
1473 redirect_behavior: Request.RedirectBehavior = @enumFromInt(3),
14691474
1470 /// How many redirects to follow before returning an error.1475 /// Externally-owned memory used to store the server's entire HTTP header.
1471 max_redirects: u32 = 3,1476 /// `error.HttpHeadersOversize` is returned from read() when a
1472 header_strategy: StorageStrategy = .{ .dynamic = 16 * 1024 },1477 /// client sends too many bytes of HTTP headers.
1478 server_header_buffer: []u8,
14731479
1474 /// Must be an already acquired connection.1480 /// Must be an already acquired connection.
1475 connection: ?*Connection = null,1481 connection: ?*Connection = null,
14761482
1477 pub const StorageStrategy = union(enum) {1483 /// Standard headers that have default, but overridable, behavior.
1478 /// In this case, the client's Allocator will be used to store the1484 headers: Request.Headers = .{},
1479 /// entire HTTP header. This value is the maximum total size of1485 /// These headers are kept including when following a redirect to a
1480 /// HTTP headers allowed, otherwise1486 /// different domain.
1481 /// error.HttpHeadersExceededSizeLimit is returned from read().1487 /// Externally-owned; must outlive the Request.
1482 dynamic: usize,1488 extra_headers: []const http.Header = &.{},
1483 /// This is used to store the entire HTTP header. If the HTTP1489 /// These headers are stripped when following a redirect to a different
1484 /// header is too big to fit, `error.HttpHeadersExceededSizeLimit`1490 /// domain.
1485 /// is returned from read(). When this is used, `error.OutOfMemory`1491 /// Externally-owned; must outlive the Request.
1486 /// cannot be returned from `read()`.1492 privileged_headers: []const http.Header = &.{},
1487 static: []u8,
1488 };
1489};1493};
14901494
1491pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{1495pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{
...@@ -1498,11 +1502,29 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{...@@ -1498,11 +1502,29 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{
1498/// Open a connection to the host specified by `uri` and prepare to send a HTTP request.1502/// Open a connection to the host specified by `uri` and prepare to send a HTTP request.
1499///1503///
1500/// `uri` must remain alive during the entire request.1504/// `uri` must remain alive during the entire request.
1501/// `headers` is cloned and may be freed after this function returns.
1502///1505///
1503/// The caller is responsible for calling `deinit()` on the `Request`.1506/// The caller is responsible for calling `deinit()` on the `Request`.
1504/// This function is threadsafe.1507/// This function is threadsafe.
1505pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: RequestOptions) RequestError!Request {1508///
1509/// Asserts that "\r\n" does not occur in any header name or value.
1510pub fn open(
1511 client: *Client,
1512 method: http.Method,
1513 uri: Uri,
1514 options: RequestOptions,
1515) RequestError!Request {
1516 if (std.debug.runtime_safety) {
1517 for (options.extra_headers) |header| {
1518 assert(std.mem.indexOfScalar(u8, header.name, ':') == null);
1519 assert(std.mem.indexOfPosLinear(u8, header.name, 0, "\r\n") == null);
1520 assert(std.mem.indexOfPosLinear(u8, header.value, 0, "\r\n") == null);
1521 }
1522 for (options.privileged_headers) |header| {
1523 assert(std.mem.indexOfPosLinear(u8, header.name, 0, "\r\n") == null);
1524 assert(std.mem.indexOfPosLinear(u8, header.value, 0, "\r\n") == null);
1525 }
1526 }
1527
1506 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;1528 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
15071529
1508 const port: u16 = uri.port orelse switch (protocol) {1530 const port: u16 = uri.port orelse switch (protocol) {
...@@ -1530,163 +1552,131 @@ pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Header...@@ -1530,163 +1552,131 @@ pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Header
1530 .uri = uri,1552 .uri = uri,
1531 .client = client,1553 .client = client,
1532 .connection = conn,1554 .connection = conn,
1533 .headers = try headers.clone(client.allocator), // Headers must be cloned to properly handle header transformations in redirects.1555 .keep_alive = options.keep_alive,
1534 .method = method,1556 .method = method,
1535 .version = options.version,1557 .version = options.version,
1536 .redirects_left = options.max_redirects,1558 .transfer_encoding = .none,
1537 .handle_redirects = options.handle_redirects,1559 .redirect_behavior = options.redirect_behavior,
1538 .handle_continue = options.handle_continue,1560 .handle_continue = options.handle_continue,
1539 .response = .{1561 .response = .{
1540 .status = undefined,1562 .status = undefined,
1541 .reason = undefined,1563 .reason = undefined,
1542 .version = undefined,1564 .version = undefined,
1543 .headers = http.Headers{ .allocator = client.allocator, .owned = false },1565 .parser = proto.HeadersParser.init(options.server_header_buffer),
1544 .parser = switch (options.header_strategy) {
1545 .dynamic => |max| proto.HeadersParser.initDynamic(max),
1546 .static => |buf| proto.HeadersParser.initStatic(buf),
1547 },
1548 },1566 },
1549 .arena = undefined,1567 .headers = options.headers,
1568 .extra_headers = options.extra_headers,
1569 .privileged_headers = options.privileged_headers,
1550 };1570 };
1551 errdefer req.deinit();1571 errdefer req.deinit();
15521572
1553 req.arena = std.heap.ArenaAllocator.init(client.allocator);
1554
1555 return req;1573 return req;
1556}1574}
15571575
1558pub const FetchOptions = struct {1576pub const FetchOptions = struct {
1577 server_header_buffer: ?[]u8 = null,
1578 redirect_behavior: ?Request.RedirectBehavior = null,
1579
1580 /// If the server sends a body, it will be appended to this ArrayList.
1581 /// `max_append_size` provides an upper limit for how much they can grow.
1582 response_storage: ResponseStorage = .ignore,
1583 max_append_size: ?usize = null,
1584
1585 location: Location,
1586 method: ?http.Method = null,
1587 payload: ?[]const u8 = null,
1588 raw_uri: bool = false,
1589 keep_alive: bool = true,
1590
1591 /// Standard headers that have default, but overridable, behavior.
1592 headers: Request.Headers = .{},
1593 /// These headers are kept including when following a redirect to a
1594 /// different domain.
1595 /// Externally-owned; must outlive the Request.
1596 extra_headers: []const http.Header = &.{},
1597 /// These headers are stripped when following a redirect to a different
1598 /// domain.
1599 /// Externally-owned; must outlive the Request.
1600 privileged_headers: []const http.Header = &.{},
1601
1559 pub const Location = union(enum) {1602 pub const Location = union(enum) {
1560 url: []const u8,1603 url: []const u8,
1561 uri: Uri,1604 uri: Uri,
1562 };1605 };
15631606
1564 pub const Payload = union(enum) {1607 pub const ResponseStorage = union(enum) {
1565 string: []const u8,1608 ignore,
1566 file: std.fs.File,1609 /// Only the existing capacity will be used.
1567 none,1610 static: *std.ArrayListUnmanaged(u8),
1611 dynamic: *std.ArrayList(u8),
1568 };1612 };
1569
1570 pub const ResponseStrategy = union(enum) {
1571 storage: RequestOptions.StorageStrategy,
1572 file: std.fs.File,
1573 none,
1574 };
1575
1576 header_strategy: RequestOptions.StorageStrategy = .{ .dynamic = 16 * 1024 },
1577 response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } },
1578
1579 location: Location,
1580 method: http.Method = .GET,
1581 headers: http.Headers = http.Headers{ .allocator = std.heap.page_allocator, .owned = false },
1582 payload: Payload = .none,
1583 raw_uri: bool = false,
1584};1613};
15851614
1586pub const FetchResult = struct {1615pub const FetchResult = struct {
1587 status: http.Status,1616 status: http.Status,
1588 body: ?[]const u8 = null,
1589 headers: http.Headers,
1590
1591 allocator: Allocator,
1592 options: FetchOptions,
1593
1594 pub fn deinit(res: *FetchResult) void {
1595 if (res.options.response_strategy == .storage and res.options.response_strategy.storage == .dynamic) {
1596 if (res.body) |body| res.allocator.free(body);
1597 }
1598
1599 res.headers.deinit();
1600 }
1601};1617};
16021618
1603/// Perform a one-shot HTTP request with the provided options.1619/// Perform a one-shot HTTP request with the provided options.
1604///1620///
1605/// This function is threadsafe.1621/// This function is threadsafe.
1606pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !FetchResult {1622pub fn fetch(client: *Client, options: FetchOptions) !FetchResult {
1607 const has_transfer_encoding = options.headers.contains("transfer-encoding");
1608 const has_content_length = options.headers.contains("content-length");
1609
1610 if (has_content_length or has_transfer_encoding) return error.UnsupportedHeader;
1611
1612 const uri = switch (options.location) {1623 const uri = switch (options.location) {
1613 .url => |u| try Uri.parse(u),1624 .url => |u| try Uri.parse(u),
1614 .uri => |u| u,1625 .uri => |u| u,
1615 };1626 };
16161627 var server_header_buffer: [16 * 1024]u8 = undefined;
1617 var req = try open(client, options.method, uri, options.headers, .{1628
1618 .header_strategy = options.header_strategy,1629 const method: http.Method = options.method orelse
1619 .handle_redirects = options.payload == .none,1630 if (options.payload != null) .POST else .GET;
1631
1632 var req = try open(client, method, uri, .{
1633 .server_header_buffer = options.server_header_buffer orelse &server_header_buffer,
1634 .redirect_behavior = options.redirect_behavior orelse
1635 if (options.payload == null) @enumFromInt(3) else .unhandled,
1636 .headers = options.headers,
1637 .extra_headers = options.extra_headers,
1638 .privileged_headers = options.privileged_headers,
1639 .keep_alive = options.keep_alive,
1620 });1640 });
1621 defer req.deinit();1641 defer req.deinit();
16221642
1623 { // Block to maintain lock of file to attempt to prevent a race condition where another process modifies the file while we are reading it.1643 if (options.payload) |payload| req.transfer_encoding = .{ .content_length = payload.len };
1624 // This relies on other processes actually obeying the advisory lock, which is not guaranteed.
1625 if (options.payload == .file) try options.payload.file.lock(.shared);
1626 defer if (options.payload == .file) options.payload.file.unlock();
16271644
1628 switch (options.payload) {1645 try req.send(.{ .raw_uri = options.raw_uri });
1629 .string => |str| req.transfer_encoding = .{ .content_length = str.len },
1630 .file => |file| req.transfer_encoding = .{ .content_length = (try file.stat()).size },
1631 .none => {},
1632 }
1633
1634 try req.send(.{ .raw_uri = options.raw_uri });
16351646
1636 switch (options.payload) {1647 if (options.payload) |payload| try req.writeAll(payload);
1637 .string => |str| try req.writeAll(str),
1638 .file => |file| {
1639 try file.seekTo(0);
1640 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
1641 try fifo.pump(file.reader(), req.writer());
1642 },
1643 .none => {},
1644 }
1645
1646 try req.finish();
1647 }
16481648
1649 try req.finish();
1649 try req.wait();1650 try req.wait();
16501651
1651 var res = FetchResult{1652 switch (options.response_storage) {
1652 .status = req.response.status,1653 .ignore => {
1653 .headers = try req.response.headers.clone(allocator),1654 // Take advantage of request internals to discard the response body
16541655 // and make the connection available for another request.
1655 .allocator = allocator,1656 req.response.skip = true;
1656 .options = options,1657 assert(try req.transferRead(&.{}) == 0); // No buffer is necessary when skipping.
1657 };
1658
1659 switch (options.response_strategy) {
1660 .storage => |storage| switch (storage) {
1661 .dynamic => |max| res.body = try req.reader().readAllAlloc(allocator, max),
1662 .static => |buf| res.body = buf[0..try req.reader().readAll(buf)],
1663 },1658 },
1664 .file => |file| {1659 .dynamic => |list| {
1665 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();1660 const max_append_size = options.max_append_size orelse 2 * 1024 * 1024;
1666 try fifo.pump(req.reader(), file.writer());1661 try req.reader().readAllArrayList(list, max_append_size);
1667 },1662 },
1668 .none => { // Take advantage of request internals to discard the response body and make the connection available for another request.1663 .static => |list| {
1669 req.response.skip = true;1664 const buf = b: {
16701665 const buf = list.unusedCapacitySlice();
1671 const empty = @as([*]u8, undefined)[0..0];1666 if (options.max_append_size) |len| {
1672 assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary1667 if (len < buf.len) break :b buf[0..len];
1668 }
1669 break :b buf;
1670 };
1671 list.items.len += try req.reader().readAll(buf);
1673 },1672 },
1674 }1673 }
16751674
1676 return res;1675 return .{
1676 .status = req.response.status,
1677 };
1677}1678}
16781679
1679test {1680test {
1680 const native_endian = comptime builtin.cpu.arch.endian();1681 _ = &initDefaultProxies;
1681 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
1682 // https://github.com/ziglang/zig/issues/13782
1683 return error.SkipZigTest;
1684 }
1685
1686 if (builtin.os.tag == .wasi) return error.SkipZigTest;
1687
1688 if (builtin.zig_backend == .stage2_x86_64 and
1689 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) return error.SkipZigTest;
1690
1691 std.testing.refAllDecls(@This());
1692}1682}
lib/std/http/HeadParser.zig created+371
...@@ -0,0 +1,371 @@
1//! Finds the end of an HTTP head in a stream.
2
3state: State = .start,
4
5pub const State = enum {
6 start,
7 seen_n,
8 seen_r,
9 seen_rn,
10 seen_rnr,
11 finished,
12};
13
14/// Returns the number of bytes consumed by headers. This is always less
15/// than or equal to `bytes.len`.
16///
17/// If the amount returned is less than `bytes.len`, the parser is in a
18/// content state and the first byte of content is located at
19/// `bytes[result]`.
20pub fn feed(p: *HeadParser, bytes: []const u8) usize {
21 const vector_len: comptime_int = @max(std.simd.suggestVectorLength(u8) orelse 1, 8);
22 var index: usize = 0;
23
24 while (true) {
25 switch (p.state) {
26 .finished => return index,
27 .start => switch (bytes.len - index) {
28 0 => return index,
29 1 => {
30 switch (bytes[index]) {
31 '\r' => p.state = .seen_r,
32 '\n' => p.state = .seen_n,
33 else => {},
34 }
35
36 return index + 1;
37 },
38 2 => {
39 const b16 = int16(bytes[index..][0..2]);
40 const b8 = intShift(u8, b16);
41
42 switch (b8) {
43 '\r' => p.state = .seen_r,
44 '\n' => p.state = .seen_n,
45 else => {},
46 }
47
48 switch (b16) {
49 int16("\r\n") => p.state = .seen_rn,
50 int16("\n\n") => p.state = .finished,
51 else => {},
52 }
53
54 return index + 2;
55 },
56 3 => {
57 const b24 = int24(bytes[index..][0..3]);
58 const b16 = intShift(u16, b24);
59 const b8 = intShift(u8, b24);
60
61 switch (b8) {
62 '\r' => p.state = .seen_r,
63 '\n' => p.state = .seen_n,
64 else => {},
65 }
66
67 switch (b16) {
68 int16("\r\n") => p.state = .seen_rn,
69 int16("\n\n") => p.state = .finished,
70 else => {},
71 }
72
73 switch (b24) {
74 int24("\r\n\r") => p.state = .seen_rnr,
75 else => {},
76 }
77
78 return index + 3;
79 },
80 4...vector_len - 1 => {
81 const b32 = int32(bytes[index..][0..4]);
82 const b24 = intShift(u24, b32);
83 const b16 = intShift(u16, b32);
84 const b8 = intShift(u8, b32);
85
86 switch (b8) {
87 '\r' => p.state = .seen_r,
88 '\n' => p.state = .seen_n,
89 else => {},
90 }
91
92 switch (b16) {
93 int16("\r\n") => p.state = .seen_rn,
94 int16("\n\n") => p.state = .finished,
95 else => {},
96 }
97
98 switch (b24) {
99 int24("\r\n\r") => p.state = .seen_rnr,
100 else => {},
101 }
102
103 switch (b32) {
104 int32("\r\n\r\n") => p.state = .finished,
105 else => {},
106 }
107
108 index += 4;
109 continue;
110 },
111 else => {
112 const chunk = bytes[index..][0..vector_len];
113 const matches = if (use_vectors) matches: {
114 const Vector = @Vector(vector_len, u8);
115 // const BoolVector = @Vector(vector_len, bool);
116 const BitVector = @Vector(vector_len, u1);
117 const SizeVector = @Vector(vector_len, u8);
118
119 const v: Vector = chunk.*;
120 const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
121 const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
122 const matches_or: SizeVector = matches_r | matches_n;
123
124 break :matches @reduce(.Add, matches_or);
125 } else matches: {
126 var matches: u8 = 0;
127 for (chunk) |byte| switch (byte) {
128 '\r', '\n' => matches += 1,
129 else => {},
130 };
131 break :matches matches;
132 };
133 switch (matches) {
134 0 => {},
135 1 => switch (chunk[vector_len - 1]) {
136 '\r' => p.state = .seen_r,
137 '\n' => p.state = .seen_n,
138 else => {},
139 },
140 2 => {
141 const b16 = int16(chunk[vector_len - 2 ..][0..2]);
142 const b8 = intShift(u8, b16);
143
144 switch (b8) {
145 '\r' => p.state = .seen_r,
146 '\n' => p.state = .seen_n,
147 else => {},
148 }
149
150 switch (b16) {
151 int16("\r\n") => p.state = .seen_rn,
152 int16("\n\n") => p.state = .finished,
153 else => {},
154 }
155 },
156 3 => {
157 const b24 = int24(chunk[vector_len - 3 ..][0..3]);
158 const b16 = intShift(u16, b24);
159 const b8 = intShift(u8, b24);
160
161 switch (b8) {
162 '\r' => p.state = .seen_r,
163 '\n' => p.state = .seen_n,
164 else => {},
165 }
166
167 switch (b16) {
168 int16("\r\n") => p.state = .seen_rn,
169 int16("\n\n") => p.state = .finished,
170 else => {},
171 }
172
173 switch (b24) {
174 int24("\r\n\r") => p.state = .seen_rnr,
175 else => {},
176 }
177 },
178 4...vector_len => {
179 inline for (0..vector_len - 3) |i_usize| {
180 const i = @as(u32, @truncate(i_usize));
181
182 const b32 = int32(chunk[i..][0..4]);
183 const b16 = intShift(u16, b32);
184
185 if (b32 == int32("\r\n\r\n")) {
186 p.state = .finished;
187 return index + i + 4;
188 } else if (b16 == int16("\n\n")) {
189 p.state = .finished;
190 return index + i + 2;
191 }
192 }
193
194 const b24 = int24(chunk[vector_len - 3 ..][0..3]);
195 const b16 = intShift(u16, b24);
196 const b8 = intShift(u8, b24);
197
198 switch (b8) {
199 '\r' => p.state = .seen_r,
200 '\n' => p.state = .seen_n,
201 else => {},
202 }
203
204 switch (b16) {
205 int16("\r\n") => p.state = .seen_rn,
206 int16("\n\n") => p.state = .finished,
207 else => {},
208 }
209
210 switch (b24) {
211 int24("\r\n\r") => p.state = .seen_rnr,
212 else => {},
213 }
214 },
215 else => unreachable,
216 }
217
218 index += vector_len;
219 continue;
220 },
221 },
222 .seen_n => switch (bytes.len - index) {
223 0 => return index,
224 else => {
225 switch (bytes[index]) {
226 '\n' => p.state = .finished,
227 else => p.state = .start,
228 }
229
230 index += 1;
231 continue;
232 },
233 },
234 .seen_r => switch (bytes.len - index) {
235 0 => return index,
236 1 => {
237 switch (bytes[index]) {
238 '\n' => p.state = .seen_rn,
239 '\r' => p.state = .seen_r,
240 else => p.state = .start,
241 }
242
243 return index + 1;
244 },
245 2 => {
246 const b16 = int16(bytes[index..][0..2]);
247 const b8 = intShift(u8, b16);
248
249 switch (b8) {
250 '\r' => p.state = .seen_r,
251 '\n' => p.state = .seen_rn,
252 else => p.state = .start,
253 }
254
255 switch (b16) {
256 int16("\r\n") => p.state = .seen_rn,
257 int16("\n\r") => p.state = .seen_rnr,
258 int16("\n\n") => p.state = .finished,
259 else => {},
260 }
261
262 return index + 2;
263 },
264 else => {
265 const b24 = int24(bytes[index..][0..3]);
266 const b16 = intShift(u16, b24);
267 const b8 = intShift(u8, b24);
268
269 switch (b8) {
270 '\r' => p.state = .seen_r,
271 '\n' => p.state = .seen_n,
272 else => p.state = .start,
273 }
274
275 switch (b16) {
276 int16("\r\n") => p.state = .seen_rn,
277 int16("\n\n") => p.state = .finished,
278 else => {},
279 }
280
281 switch (b24) {
282 int24("\n\r\n") => p.state = .finished,
283 else => {},
284 }
285
286 index += 3;
287 continue;
288 },
289 },
290 .seen_rn => switch (bytes.len - index) {
291 0 => return index,
292 1 => {
293 switch (bytes[index]) {
294 '\r' => p.state = .seen_rnr,
295 '\n' => p.state = .seen_n,
296 else => p.state = .start,
297 }
298
299 return index + 1;
300 },
301 else => {
302 const b16 = int16(bytes[index..][0..2]);
303 const b8 = intShift(u8, b16);
304
305 switch (b8) {
306 '\r' => p.state = .seen_rnr,
307 '\n' => p.state = .seen_n,
308 else => p.state = .start,
309 }
310
311 switch (b16) {
312 int16("\r\n") => p.state = .finished,
313 int16("\n\n") => p.state = .finished,
314 else => {},
315 }
316
317 index += 2;
318 continue;
319 },
320 },
321 .seen_rnr => switch (bytes.len - index) {
322 0 => return index,
323 else => {
324 switch (bytes[index]) {
325 '\n' => p.state = .finished,
326 else => p.state = .start,
327 }
328
329 index += 1;
330 continue;
331 },
332 },
333 }
334
335 return index;
336 }
337}
338
339inline fn int16(array: *const [2]u8) u16 {
340 return @bitCast(array.*);
341}
342
343inline fn int24(array: *const [3]u8) u24 {
344 return @bitCast(array.*);
345}
346
347inline fn int32(array: *const [4]u8) u32 {
348 return @bitCast(array.*);
349}
350
351inline fn intShift(comptime T: type, x: anytype) T {
352 switch (@import("builtin").cpu.arch.endian()) {
353 .little => return @truncate(x >> (@bitSizeOf(@TypeOf(x)) - @bitSizeOf(T))),
354 .big => return @truncate(x),
355 }
356}
357
358const HeadParser = @This();
359const std = @import("std");
360const use_vectors = builtin.zig_backend != .stage2_x86_64;
361const builtin = @import("builtin");
362
363test feed {
364 const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\nHello";
365
366 for (0..36) |i| {
367 var p: HeadParser = .{};
368 try std.testing.expectEqual(i, p.feed(data[0..i]));
369 try std.testing.expectEqual(35 - i, p.feed(data[i..]));
370 }
371}
lib/std/http/HeaderIterator.zig created+62
...@@ -0,0 +1,62 @@
1bytes: []const u8,
2index: usize,
3is_trailer: bool,
4
5pub fn init(bytes: []const u8) HeaderIterator {
6 return .{
7 .bytes = bytes,
8 .index = std.mem.indexOfPosLinear(u8, bytes, 0, "\r\n").? + 2,
9 .is_trailer = false,
10 };
11}
12
13pub fn next(it: *HeaderIterator) ?std.http.Header {
14 const end = std.mem.indexOfPosLinear(u8, it.bytes, it.index, "\r\n").?;
15 var kv_it = std.mem.splitSequence(u8, it.bytes[it.index..end], ": ");
16 const name = kv_it.next().?;
17 const value = kv_it.rest();
18 if (value.len == 0) {
19 if (it.is_trailer) return null;
20 const next_end = std.mem.indexOfPosLinear(u8, it.bytes, end + 2, "\r\n") orelse
21 return null;
22 it.is_trailer = true;
23 it.index = next_end + 2;
24 kv_it = std.mem.splitSequence(u8, it.bytes[end + 2 .. next_end], ": ");
25 return .{
26 .name = kv_it.next().?,
27 .value = kv_it.rest(),
28 };
29 }
30 it.index = end + 2;
31 return .{
32 .name = name,
33 .value = value,
34 };
35}
36
37test next {
38 var it = HeaderIterator.init("200 OK\r\na: b\r\nc: d\r\n\r\ne: f\r\n\r\n");
39 try std.testing.expect(!it.is_trailer);
40 {
41 const header = it.next().?;
42 try std.testing.expect(!it.is_trailer);
43 try std.testing.expectEqualStrings("a", header.name);
44 try std.testing.expectEqualStrings("b", header.value);
45 }
46 {
47 const header = it.next().?;
48 try std.testing.expect(!it.is_trailer);
49 try std.testing.expectEqualStrings("c", header.name);
50 try std.testing.expectEqualStrings("d", header.value);
51 }
52 {
53 const header = it.next().?;
54 try std.testing.expect(it.is_trailer);
55 try std.testing.expectEqualStrings("e", header.name);
56 try std.testing.expectEqualStrings("f", header.value);
57 }
58 try std.testing.expectEqual(null, it.next());
59}
60
61const HeaderIterator = @This();
62const std = @import("../std.zig");
lib/std/http/Headers.zig deleted-527
...@@ -1,527 +0,0 @@
1const std = @import("../std.zig");
2
3const Allocator = std.mem.Allocator;
4
5const testing = std.testing;
6const ascii = std.ascii;
7const assert = std.debug.assert;
8
9pub const HeaderList = std.ArrayListUnmanaged(Field);
10pub const HeaderIndexList = std.ArrayListUnmanaged(usize);
11pub const HeaderIndex = std.HashMapUnmanaged([]const u8, HeaderIndexList, CaseInsensitiveStringContext, std.hash_map.default_max_load_percentage);
12
13pub const CaseInsensitiveStringContext = struct {
14 pub fn hash(self: @This(), s: []const u8) u64 {
15 _ = self;
16 var buf: [64]u8 = undefined;
17 var i: usize = 0;
18
19 var h = std.hash.Wyhash.init(0);
20 while (i + 64 < s.len) : (i += 64) {
21 const ret = ascii.lowerString(buf[0..], s[i..][0..64]);
22 h.update(ret);
23 }
24
25 const left = @min(64, s.len - i);
26 const ret = ascii.lowerString(buf[0..], s[i..][0..left]);
27 h.update(ret);
28
29 return h.final();
30 }
31
32 pub fn eql(self: @This(), a: []const u8, b: []const u8) bool {
33 _ = self;
34 return ascii.eqlIgnoreCase(a, b);
35 }
36};
37
38/// A single HTTP header field.
39pub const Field = struct {
40 name: []const u8,
41 value: []const u8,
42
43 fn lessThan(ctx: void, a: Field, b: Field) bool {
44 _ = ctx;
45 if (a.name.ptr == b.name.ptr) return false;
46
47 return ascii.lessThanIgnoreCase(a.name, b.name);
48 }
49};
50
51/// A list of HTTP header fields.
52pub const Headers = struct {
53 allocator: Allocator,
54 list: HeaderList = .{},
55 index: HeaderIndex = .{},
56
57 /// When this is false, names and values will not be duplicated.
58 /// Use with caution.
59 owned: bool = true,
60
61 /// Initialize an empty list of headers.
62 pub fn init(allocator: Allocator) Headers {
63 return .{ .allocator = allocator };
64 }
65
66 /// Initialize a pre-populated list of headers from a list of fields.
67 pub fn initList(allocator: Allocator, list: []const Field) !Headers {
68 var new = Headers.init(allocator);
69
70 try new.list.ensureTotalCapacity(allocator, list.len);
71 try new.index.ensureTotalCapacity(allocator, @intCast(list.len));
72 for (list) |field| {
73 try new.append(field.name, field.value);
74 }
75
76 return new;
77 }
78
79 /// Deallocate all memory associated with the headers.
80 ///
81 /// If the `owned` field is false, this will not free the names and values of the headers.
82 pub fn deinit(headers: *Headers) void {
83 headers.deallocateIndexListsAndFields();
84 headers.index.deinit(headers.allocator);
85 headers.list.deinit(headers.allocator);
86
87 headers.* = undefined;
88 }
89
90 /// Appends a header to the list.
91 ///
92 /// If the `owned` field is true, both name and value will be copied.
93 pub fn append(headers: *Headers, name: []const u8, value: []const u8) !void {
94 try headers.appendOwned(.{ .unowned = name }, .{ .unowned = value });
95 }
96
97 pub const OwnedString = union(enum) {
98 /// A string allocated by the `allocator` field.
99 owned: []u8,
100 /// A string to be copied by the `allocator` field.
101 unowned: []const u8,
102 };
103
104 /// Appends a header to the list.
105 ///
106 /// If the `owned` field is true, `name` and `value` will be copied if unowned.
107 pub fn appendOwned(headers: *Headers, name: OwnedString, value: OwnedString) !void {
108 const n = headers.list.items.len;
109 try headers.list.ensureUnusedCapacity(headers.allocator, 1);
110
111 const owned_value = switch (value) {
112 .owned => |owned| owned,
113 .unowned => |unowned| if (headers.owned)
114 try headers.allocator.dupe(u8, unowned)
115 else
116 unowned,
117 };
118 errdefer if (value == .unowned and headers.owned) headers.allocator.free(owned_value);
119
120 var entry = Field{ .name = undefined, .value = owned_value };
121
122 if (headers.index.getEntry(switch (name) {
123 inline else => |string| string,
124 })) |kv| {
125 defer switch (name) {
126 .owned => |owned| headers.allocator.free(owned),
127 .unowned => {},
128 };
129
130 entry.name = kv.key_ptr.*;
131 try kv.value_ptr.append(headers.allocator, n);
132 } else {
133 const owned_name = switch (name) {
134 .owned => |owned| owned,
135 .unowned => |unowned| if (headers.owned)
136 try std.ascii.allocLowerString(headers.allocator, unowned)
137 else
138 unowned,
139 };
140 errdefer if (name == .unowned and headers.owned) headers.allocator.free(owned_name);
141
142 entry.name = owned_name;
143
144 var new_index = try HeaderIndexList.initCapacity(headers.allocator, 1);
145 errdefer new_index.deinit(headers.allocator);
146
147 new_index.appendAssumeCapacity(n);
148 try headers.index.put(headers.allocator, owned_name, new_index);
149 }
150
151 headers.list.appendAssumeCapacity(entry);
152 }
153
154 /// Returns true if this list of headers contains the given name.
155 pub fn contains(headers: Headers, name: []const u8) bool {
156 return headers.index.contains(name);
157 }
158
159 /// Removes all headers with the given name.
160 pub fn delete(headers: *Headers, name: []const u8) bool {
161 if (headers.index.fetchRemove(name)) |kv| {
162 var index = kv.value;
163
164 // iterate backwards
165 var i = index.items.len;
166 while (i > 0) {
167 i -= 1;
168 const data_index = index.items[i];
169 const removed = headers.list.orderedRemove(data_index);
170
171 assert(ascii.eqlIgnoreCase(removed.name, name)); // ensure the index hasn't been corrupted
172 if (headers.owned) headers.allocator.free(removed.value);
173 }
174
175 if (headers.owned) headers.allocator.free(kv.key);
176 index.deinit(headers.allocator);
177 headers.rebuildIndex();
178
179 return true;
180 } else {
181 return false;
182 }
183 }
184
185 /// Returns the index of the first occurrence of a header with the given name.
186 pub fn firstIndexOf(headers: Headers, name: []const u8) ?usize {
187 const index = headers.index.get(name) orelse return null;
188
189 return index.items[0];
190 }
191
192 /// Returns a list of indices containing headers with the given name.
193 pub fn getIndices(headers: Headers, name: []const u8) ?[]const usize {
194 const index = headers.index.get(name) orelse return null;
195
196 return index.items;
197 }
198
199 /// Returns the entry of the first occurrence of a header with the given name.
200 pub fn getFirstEntry(headers: Headers, name: []const u8) ?Field {
201 const first_index = headers.firstIndexOf(name) orelse return null;
202
203 return headers.list.items[first_index];
204 }
205
206 /// Returns a slice containing each header with the given name.
207 /// The caller owns the returned slice, but NOT the values in the slice.
208 pub fn getEntries(headers: Headers, allocator: Allocator, name: []const u8) !?[]const Field {
209 const indices = headers.getIndices(name) orelse return null;
210
211 const buf = try allocator.alloc(Field, indices.len);
212 for (indices, 0..) |idx, n| {
213 buf[n] = headers.list.items[idx];
214 }
215
216 return buf;
217 }
218
219 /// Returns the value in the entry of the first occurrence of a header with the given name.
220 pub fn getFirstValue(headers: Headers, name: []const u8) ?[]const u8 {
221 const first_index = headers.firstIndexOf(name) orelse return null;
222
223 return headers.list.items[first_index].value;
224 }
225
226 /// Returns a slice containing the value of each header with the given name.
227 /// The caller owns the returned slice, but NOT the values in the slice.
228 pub fn getValues(headers: Headers, allocator: Allocator, name: []const u8) !?[]const []const u8 {
229 const indices = headers.getIndices(name) orelse return null;
230
231 const buf = try allocator.alloc([]const u8, indices.len);
232 for (indices, 0..) |idx, n| {
233 buf[n] = headers.list.items[idx].value;
234 }
235
236 return buf;
237 }
238
239 fn rebuildIndex(headers: *Headers) void {
240 // clear out the indexes
241 var it = headers.index.iterator();
242 while (it.next()) |entry| {
243 entry.value_ptr.shrinkRetainingCapacity(0);
244 }
245
246 // fill up indexes again; we know capacity is fine from before
247 for (headers.list.items, 0..) |entry, i| {
248 headers.index.getEntry(entry.name).?.value_ptr.appendAssumeCapacity(i);
249 }
250 }
251
252 /// Sorts the headers in lexicographical order.
253 pub fn sort(headers: *Headers) void {
254 std.mem.sort(Field, headers.list.items, {}, Field.lessThan);
255 headers.rebuildIndex();
256 }
257
258 /// Writes the headers to the given stream.
259 pub fn format(
260 headers: Headers,
261 comptime fmt: []const u8,
262 options: std.fmt.FormatOptions,
263 out_stream: anytype,
264 ) !void {
265 _ = fmt;
266 _ = options;
267
268 for (headers.list.items) |entry| {
269 if (entry.value.len == 0) continue;
270
271 try out_stream.writeAll(entry.name);
272 try out_stream.writeAll(": ");
273 try out_stream.writeAll(entry.value);
274 try out_stream.writeAll("\r\n");
275 }
276 }
277
278 /// Writes all of the headers with the given name to the given stream, separated by commas.
279 ///
280 /// This is useful for headers like `Set-Cookie` which can have multiple values. RFC 9110, Section 5.2
281 pub fn formatCommaSeparated(
282 headers: Headers,
283 name: []const u8,
284 out_stream: anytype,
285 ) !void {
286 const indices = headers.getIndices(name) orelse return;
287
288 try out_stream.writeAll(name);
289 try out_stream.writeAll(": ");
290
291 for (indices, 0..) |idx, n| {
292 if (n != 0) try out_stream.writeAll(", ");
293 try out_stream.writeAll(headers.list.items[idx].value);
294 }
295
296 try out_stream.writeAll("\r\n");
297 }
298
299 /// Frees all `HeaderIndexList`s within `index`.
300 /// Frees names and values of all fields if they are owned.
301 fn deallocateIndexListsAndFields(headers: *Headers) void {
302 var it = headers.index.iterator();
303 while (it.next()) |entry| {
304 entry.value_ptr.deinit(headers.allocator);
305
306 if (headers.owned) headers.allocator.free(entry.key_ptr.*);
307 }
308
309 if (headers.owned) {
310 for (headers.list.items) |entry| {
311 headers.allocator.free(entry.value);
312 }
313 }
314 }
315
316 /// Clears and frees the underlying data structures.
317 /// Frees names and values if they are owned.
318 pub fn clearAndFree(headers: *Headers) void {
319 headers.deallocateIndexListsAndFields();
320 headers.index.clearAndFree(headers.allocator);
321 headers.list.clearAndFree(headers.allocator);
322 }
323
324 /// Clears the underlying data structures while retaining their capacities.
325 /// Frees names and values if they are owned.
326 pub fn clearRetainingCapacity(headers: *Headers) void {
327 headers.deallocateIndexListsAndFields();
328 headers.index.clearRetainingCapacity();
329 headers.list.clearRetainingCapacity();
330 }
331
332 /// Creates a copy of the headers using the provided allocator.
333 pub fn clone(headers: Headers, allocator: Allocator) !Headers {
334 var new = Headers.init(allocator);
335
336 try new.list.ensureTotalCapacity(allocator, headers.list.capacity);
337 try new.index.ensureTotalCapacity(allocator, headers.index.capacity());
338 for (headers.list.items) |field| {
339 try new.append(field.name, field.value);
340 }
341
342 return new;
343 }
344};
345
346test "Headers.append" {
347 var h = Headers{ .allocator = std.testing.allocator };
348 defer h.deinit();
349
350 try h.append("foo", "bar");
351 try h.append("hello", "world");
352
353 try testing.expect(h.contains("Foo"));
354 try testing.expect(!h.contains("Bar"));
355}
356
357test "Headers.delete" {
358 var h = Headers{ .allocator = std.testing.allocator };
359 defer h.deinit();
360
361 try h.append("foo", "bar");
362 try h.append("hello", "world");
363
364 try testing.expect(h.contains("Foo"));
365
366 _ = h.delete("Foo");
367
368 try testing.expect(!h.contains("foo"));
369}
370
371test "Headers consistency" {
372 var h = Headers{ .allocator = std.testing.allocator };
373 defer h.deinit();
374
375 try h.append("foo", "bar");
376 try h.append("hello", "world");
377 _ = h.delete("Foo");
378
379 try h.append("foo", "bar");
380 try h.append("bar", "world");
381 try h.append("foo", "baz");
382 try h.append("baz", "hello");
383
384 try testing.expectEqual(@as(?usize, 0), h.firstIndexOf("hello"));
385 try testing.expectEqual(@as(?usize, 1), h.firstIndexOf("foo"));
386 try testing.expectEqual(@as(?usize, 2), h.firstIndexOf("bar"));
387 try testing.expectEqual(@as(?usize, 4), h.firstIndexOf("baz"));
388 try testing.expectEqual(@as(?usize, null), h.firstIndexOf("pog"));
389
390 try testing.expectEqualSlices(usize, &[_]usize{0}, h.getIndices("hello").?);
391 try testing.expectEqualSlices(usize, &[_]usize{ 1, 3 }, h.getIndices("foo").?);
392 try testing.expectEqualSlices(usize, &[_]usize{2}, h.getIndices("bar").?);
393 try testing.expectEqualSlices(usize, &[_]usize{4}, h.getIndices("baz").?);
394 try testing.expectEqual(@as(?[]const usize, null), h.getIndices("pog"));
395
396 try testing.expectEqualStrings("world", h.getFirstEntry("hello").?.value);
397 try testing.expectEqualStrings("bar", h.getFirstEntry("foo").?.value);
398 try testing.expectEqualStrings("world", h.getFirstEntry("bar").?.value);
399 try testing.expectEqualStrings("hello", h.getFirstEntry("baz").?.value);
400
401 const hello_entries = (try h.getEntries(testing.allocator, "hello")).?;
402 defer testing.allocator.free(hello_entries);
403 try testing.expectEqualDeep(@as([]const Field, &[_]Field{
404 .{ .name = "hello", .value = "world" },
405 }), hello_entries);
406
407 const foo_entries = (try h.getEntries(testing.allocator, "foo")).?;
408 defer testing.allocator.free(foo_entries);
409 try testing.expectEqualDeep(@as([]const Field, &[_]Field{
410 .{ .name = "foo", .value = "bar" },
411 .{ .name = "foo", .value = "baz" },
412 }), foo_entries);
413
414 const bar_entries = (try h.getEntries(testing.allocator, "bar")).?;
415 defer testing.allocator.free(bar_entries);
416 try testing.expectEqualDeep(@as([]const Field, &[_]Field{
417 .{ .name = "bar", .value = "world" },
418 }), bar_entries);
419
420 const baz_entries = (try h.getEntries(testing.allocator, "baz")).?;
421 defer testing.allocator.free(baz_entries);
422 try testing.expectEqualDeep(@as([]const Field, &[_]Field{
423 .{ .name = "baz", .value = "hello" },
424 }), baz_entries);
425
426 const pog_entries = (try h.getEntries(testing.allocator, "pog"));
427 try testing.expectEqual(@as(?[]const Field, null), pog_entries);
428
429 try testing.expectEqualStrings("world", h.getFirstValue("hello").?);
430 try testing.expectEqualStrings("bar", h.getFirstValue("foo").?);
431 try testing.expectEqualStrings("world", h.getFirstValue("bar").?);
432 try testing.expectEqualStrings("hello", h.getFirstValue("baz").?);
433 try testing.expectEqual(@as(?[]const u8, null), h.getFirstValue("pog"));
434
435 const hello_values = (try h.getValues(testing.allocator, "hello")).?;
436 defer testing.allocator.free(hello_values);
437 try testing.expectEqualDeep(@as([]const []const u8, &[_][]const u8{"world"}), hello_values);
438
439 const foo_values = (try h.getValues(testing.allocator, "foo")).?;
440 defer testing.allocator.free(foo_values);
441 try testing.expectEqualDeep(@as([]const []const u8, &[_][]const u8{ "bar", "baz" }), foo_values);
442
443 const bar_values = (try h.getValues(testing.allocator, "bar")).?;
444 defer testing.allocator.free(bar_values);
445 try testing.expectEqualDeep(@as([]const []const u8, &[_][]const u8{"world"}), bar_values);
446
447 const baz_values = (try h.getValues(testing.allocator, "baz")).?;
448 defer testing.allocator.free(baz_values);
449 try testing.expectEqualDeep(@as([]const []const u8, &[_][]const u8{"hello"}), baz_values);
450
451 const pog_values = (try h.getValues(testing.allocator, "pog"));
452 try testing.expectEqual(@as(?[]const []const u8, null), pog_values);
453
454 h.sort();
455
456 try testing.expectEqualSlices(usize, &[_]usize{0}, h.getIndices("bar").?);
457 try testing.expectEqualSlices(usize, &[_]usize{1}, h.getIndices("baz").?);
458 try testing.expectEqualSlices(usize, &[_]usize{ 2, 3 }, h.getIndices("foo").?);
459 try testing.expectEqualSlices(usize, &[_]usize{4}, h.getIndices("hello").?);
460
461 const formatted_values = try std.fmt.allocPrint(testing.allocator, "{}", .{h});
462 defer testing.allocator.free(formatted_values);
463
464 try testing.expectEqualStrings("bar: world\r\nbaz: hello\r\nfoo: bar\r\nfoo: baz\r\nhello: world\r\n", formatted_values);
465
466 var buf: [128]u8 = undefined;
467 var fbs = std.io.fixedBufferStream(&buf);
468 const writer = fbs.writer();
469
470 try h.formatCommaSeparated("foo", writer);
471 try testing.expectEqualStrings("foo: bar, baz\r\n", fbs.getWritten());
472}
473
474test "Headers.clearRetainingCapacity and clearAndFree" {
475 var h = Headers.init(std.testing.allocator);
476 defer h.deinit();
477
478 h.clearRetainingCapacity();
479
480 try h.append("foo", "bar");
481 try h.append("bar", "world");
482 try h.append("foo", "baz");
483 try h.append("baz", "hello");
484 try testing.expectEqual(@as(usize, 4), h.list.items.len);
485 try testing.expectEqual(@as(usize, 3), h.index.count());
486 const list_capacity = h.list.capacity;
487 const index_capacity = h.index.capacity();
488
489 h.clearRetainingCapacity();
490 try testing.expectEqual(@as(usize, 0), h.list.items.len);
491 try testing.expectEqual(@as(usize, 0), h.index.count());
492 try testing.expectEqual(list_capacity, h.list.capacity);
493 try testing.expectEqual(index_capacity, h.index.capacity());
494
495 try h.append("foo", "bar");
496 try h.append("bar", "world");
497 try h.append("foo", "baz");
498 try h.append("baz", "hello");
499 try testing.expectEqual(@as(usize, 4), h.list.items.len);
500 try testing.expectEqual(@as(usize, 3), h.index.count());
501 // Capacity should still be the same since we shouldn't have needed to grow
502 // when adding back the same fields
503 try testing.expectEqual(list_capacity, h.list.capacity);
504 try testing.expectEqual(index_capacity, h.index.capacity());
505
506 h.clearAndFree();
507 try testing.expectEqual(@as(usize, 0), h.list.items.len);
508 try testing.expectEqual(@as(usize, 0), h.index.count());
509 try testing.expectEqual(@as(usize, 0), h.list.capacity);
510 try testing.expectEqual(@as(usize, 0), h.index.capacity());
511}
512
513test "Headers.initList" {
514 var h = try Headers.initList(std.testing.allocator, &.{
515 .{ .name = "Accept-Encoding", .value = "gzip" },
516 .{ .name = "Authorization", .value = "it's over 9000!" },
517 });
518 defer h.deinit();
519
520 const encoding_values = (try h.getValues(testing.allocator, "Accept-Encoding")).?;
521 defer testing.allocator.free(encoding_values);
522 try testing.expectEqualDeep(@as([]const []const u8, &[_][]const u8{"gzip"}), encoding_values);
523
524 const authorization_values = (try h.getValues(testing.allocator, "Authorization")).?;
525 defer testing.allocator.free(authorization_values);
526 try testing.expectEqualDeep(@as([]const []const u8, &[_][]const u8{"it's over 9000!"}), authorization_values);
527}
lib/std/http/Server.zig+921-750
...@@ -1,873 +1,1044 @@...@@ -1,873 +1,1044 @@
1//! HTTP Server implementation.1//! Blocking HTTP server implementation.
2//!2//! Handles a single connection's lifecycle.
3//! This server assumes *all* clients are well behaved and standard compliant; it can and will deadlock if a client holds a connection open without sending a request.3
4//!4connection: net.Server.Connection,
5//! Example usage:5/// Keeps track of whether the Server is ready to accept a new request on the
6//!6/// same connection, and makes invalid API usage cause assertion failures
7//! ```zig7/// rather than HTTP protocol violations.
8//! var server = Server.init(.{ .reuse_address = true });8state: State,
9//! defer server.deinit();9/// User-provided buffer that must outlive this Server.
10//!10/// Used to store the client's entire HTTP header.
11//! try server.listen(bind_addr);11read_buffer: []u8,
12//!12/// Amount of available data inside read_buffer.
13//! while (true) {13read_buffer_len: usize,
14//! var res = try server.accept(.{ .allocator = gpa });14/// Index into `read_buffer` of the first byte of the next HTTP request.
15//! defer res.deinit();15next_request_start: usize,
16//!16
17//! while (res.reset() != .closing) {17pub const State = enum {
18//! res.wait() catch |err| switch (err) {18 /// The connection is available to be used for the first time, or reused.
19//! error.HttpHeadersInvalid => break,19 ready,
20//! error.HttpHeadersExceededSizeLimit => {20 /// An error occurred in `receiveHead`.
21//! res.status = .request_header_fields_too_large;21 receiving_head,
22//! res.send() catch break;22 /// A Request object has been obtained and from there a Response can be
23//! break;23 /// opened.
24//! },24 received_head,
25//! else => {25 /// The client is uploading something to this Server.
26//! res.status = .bad_request;26 receiving_body,
27//! res.send() catch break;27 /// The connection is eligible for another HTTP request, however the client
28//! break;28 /// and server did not negotiate connection: keep-alive.
29//! },29 closing,
30//! }30};
31//!
32//! res.status = .ok;
33//! res.transfer_encoding = .chunked;
34//!
35//! try res.send();
36//! try res.writeAll("Hello, World!\n");
37//! try res.finish();
38//! }
39//! }
40//! ```
41
42const std = @import("../std.zig");
43const testing = std.testing;
44const http = std.http;
45const mem = std.mem;
46const net = std.net;
47const Uri = std.Uri;
48const Allocator = mem.Allocator;
49const assert = std.debug.assert;
50
51const Server = @This();
52const proto = @import("protocol.zig");
53
54/// The underlying server socket.
55socket: net.StreamServer,
56
57/// An interface to a plain connection.
58pub const Connection = struct {
59 pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
60 pub const Protocol = enum { plain };
6131
62 stream: net.Stream,32/// Initialize an HTTP server that can respond to multiple requests on the same
63 protocol: Protocol,33/// connection.
6434/// The returned `Server` is ready for `receiveHead` to be called.
65 closing: bool = true,35pub fn init(connection: net.Server.Connection, read_buffer: []u8) Server {
6636 return .{
67 read_buf: [buffer_size]u8 = undefined,37 .connection = connection,
68 read_start: u16 = 0,38 .state = .ready,
69 read_end: u16 = 0,39 .read_buffer = read_buffer,
7040 .read_buffer_len = 0,
71 pub fn rawReadAtLeast(conn: *Connection, buffer: []u8, len: usize) ReadError!usize {41 .next_request_start = 0,
72 return switch (conn.protocol) {42 };
73 .plain => conn.stream.readAtLeast(buffer, len),43}
74 // .tls => conn.tls_client.readAtLeast(conn.stream, buffer, len),
75 } catch |err| {
76 switch (err) {
77 error.ConnectionResetByPeer, error.BrokenPipe => return error.ConnectionResetByPeer,
78 else => return error.UnexpectedReadFailure,
79 }
80 };
81 }
8244
83 pub fn fill(conn: *Connection) ReadError!void {45pub const ReceiveHeadError = error{
84 if (conn.read_end != conn.read_start) return;46 /// Client sent too many bytes of HTTP headers.
47 /// The HTTP specification suggests to respond with a 431 status code
48 /// before closing the connection.
49 HttpHeadersOversize,
50 /// Client sent headers that did not conform to the HTTP protocol.
51 HttpHeadersInvalid,
52 /// A low level I/O error occurred trying to read the headers.
53 HttpHeadersUnreadable,
54 /// Partial HTTP request was received but the connection was closed before
55 /// fully receiving the headers.
56 HttpRequestTruncated,
57 /// The client sent 0 bytes of headers before closing the stream.
58 /// In other words, a keep-alive connection was finally closed.
59 HttpConnectionClosing,
60};
8561
86 const nread = try conn.rawReadAtLeast(conn.read_buf[0..], 1);62/// The header bytes reference the read buffer that Server was initialized with
87 if (nread == 0) return error.EndOfStream;63/// and remain alive until the next call to receiveHead.
88 conn.read_start = 0;64pub fn receiveHead(s: *Server) ReceiveHeadError!Request {
89 conn.read_end = @as(u16, @intCast(nread));65 assert(s.state == .ready);
66 s.state = .received_head;
67 errdefer s.state = .receiving_head;
68
69 // In case of a reused connection, move the next request's bytes to the
70 // beginning of the buffer.
71 if (s.next_request_start > 0) {
72 if (s.read_buffer_len > s.next_request_start) {
73 rebase(s, 0);
74 } else {
75 s.read_buffer_len = 0;
76 }
90 }77 }
9178
92 pub fn peek(conn: *Connection) []const u8 {79 var hp: http.HeadParser = .{};
93 return conn.read_buf[conn.read_start..conn.read_end];
94 }
9580
96 pub fn drop(conn: *Connection, num: u16) void {81 if (s.read_buffer_len > 0) {
97 conn.read_start += num;82 const bytes = s.read_buffer[0..s.read_buffer_len];
83 const end = hp.feed(bytes);
84 if (hp.state == .finished)
85 return finishReceivingHead(s, end);
98 }86 }
9987
100 pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) ReadError!usize {88 while (true) {
101 assert(len <= buffer.len);89 const buf = s.read_buffer[s.read_buffer_len..];
10290 if (buf.len == 0)
103 var out_index: u16 = 0;91 return error.HttpHeadersOversize;
104 while (out_index < len) {92 const read_n = s.connection.stream.read(buf) catch
105 const available_read = conn.read_end - conn.read_start;93 return error.HttpHeadersUnreadable;
106 const available_buffer = buffer.len - out_index;94 if (read_n == 0) {
10795 if (s.read_buffer_len > 0) {
108 if (available_read > available_buffer) { // partially read buffered data96 return error.HttpRequestTruncated;
109 @memcpy(buffer[out_index..], conn.read_buf[conn.read_start..conn.read_end][0..available_buffer]);97 } else {
110 out_index += @as(u16, @intCast(available_buffer));98 return error.HttpConnectionClosing;
111 conn.read_start += @as(u16, @intCast(available_buffer));
112
113 break;
114 } else if (available_read > 0) { // fully read buffered data
115 @memcpy(buffer[out_index..][0..available_read], conn.read_buf[conn.read_start..conn.read_end]);
116 out_index += available_read;
117 conn.read_start += available_read;
118
119 if (out_index >= len) break;
120 }
121
122 const leftover_buffer = available_buffer - available_read;
123 const leftover_len = len - out_index;
124
125 if (leftover_buffer > conn.read_buf.len) {
126 // skip the buffer if the output is large enough
127 return conn.rawReadAtLeast(buffer[out_index..], leftover_len);
128 }99 }
129
130 try conn.fill();
131 }100 }
132101 s.read_buffer_len += read_n;
133 return out_index;102 const bytes = buf[0..read_n];
134 }103 const end = hp.feed(bytes);
135104 if (hp.state == .finished)
136 pub fn read(conn: *Connection, buffer: []u8) ReadError!usize {105 return finishReceivingHead(s, s.read_buffer_len - bytes.len + end);
137 return conn.readAtLeast(buffer, 1);
138 }
139
140 pub const ReadError = error{
141 ConnectionTimedOut,
142 ConnectionResetByPeer,
143 UnexpectedReadFailure,
144 EndOfStream,
145 };
146
147 pub const Reader = std.io.Reader(*Connection, ReadError, read);
148
149 pub fn reader(conn: *Connection) Reader {
150 return Reader{ .context = conn };
151 }
152
153 pub fn writeAll(conn: *Connection, buffer: []const u8) WriteError!void {
154 return switch (conn.protocol) {
155 .plain => conn.stream.writeAll(buffer),
156 // .tls => return conn.tls_client.writeAll(conn.stream, buffer),
157 } catch |err| switch (err) {
158 error.BrokenPipe, error.ConnectionResetByPeer => return error.ConnectionResetByPeer,
159 else => return error.UnexpectedWriteFailure,
160 };
161 }
162
163 pub fn write(conn: *Connection, buffer: []const u8) WriteError!usize {
164 return switch (conn.protocol) {
165 .plain => conn.stream.write(buffer),
166 // .tls => return conn.tls_client.write(conn.stream, buffer),
167 } catch |err| switch (err) {
168 error.BrokenPipe, error.ConnectionResetByPeer => return error.ConnectionResetByPeer,
169 else => return error.UnexpectedWriteFailure,
170 };
171 }106 }
107}
172108
173 pub const WriteError = error{109fn finishReceivingHead(s: *Server, head_end: usize) ReceiveHeadError!Request {
174 ConnectionResetByPeer,110 return .{
175 UnexpectedWriteFailure,111 .server = s,
112 .head_end = head_end,
113 .head = Request.Head.parse(s.read_buffer[0..head_end]) catch
114 return error.HttpHeadersInvalid,
115 .reader_state = undefined,
176 };116 };
117}
177118
178 pub const Writer = std.io.Writer(*Connection, WriteError, write);
179
180 pub fn writer(conn: *Connection) Writer {
181 return Writer{ .context = conn };
182 }
183
184 pub fn close(conn: *Connection) void {
185 conn.stream.close();
186 }
187};
188
189/// The mode of transport for responses.
190pub const ResponseTransfer = union(enum) {
191 content_length: u64,
192 chunked: void,
193 none: void,
194};
195
196/// The decompressor for request messages.
197pub const Compression = union(enum) {
198 pub const DeflateDecompressor = std.compress.zlib.Decompressor(Response.TransferReader);
199 pub const GzipDecompressor = std.compress.gzip.Decompressor(Response.TransferReader);
200 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});
201
202 deflate: DeflateDecompressor,
203 gzip: GzipDecompressor,
204 zstd: ZstdDecompressor,
205 none: void,
206};
207
208/// A HTTP request originating from a client.
209pub const Request = struct {119pub const Request = struct {
210 pub const ParseError = Allocator.Error || error{120 server: *Server,
211 UnknownHttpMethod,121 /// Index into Server's read_buffer.
212 HttpHeadersInvalid,122 head_end: usize,
213 HttpHeaderContinuationsUnsupported,123 head: Head,
214 HttpTransferEncodingUnsupported,124 reader_state: union {
215 HttpConnectionHeaderUnsupported,125 remaining_content_length: u64,
216 InvalidContentLength,126 chunk_parser: http.ChunkParser,
217 CompressionNotSupported,127 },
128
129 pub const Compression = union(enum) {
130 pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);
131 pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);
132 pub const ZstdDecompressor = std.compress.zstd.Decompressor(std.io.AnyReader);
133
134 deflate: DeflateDecompressor,
135 gzip: GzipDecompressor,
136 zstd: ZstdDecompressor,
137 none: void,
218 };138 };
219139
220 pub fn parse(req: *Request, bytes: []const u8) ParseError!void {140 pub const Head = struct {
221 var it = mem.tokenizeAny(u8, bytes, "\r\n");141 method: http.Method,
222142 target: []const u8,
223 const first_line = it.next() orelse return error.HttpHeadersInvalid;143 version: http.Version,
224 if (first_line.len < 10)144 expect: ?[]const u8,
225 return error.HttpHeadersInvalid;145 content_type: ?[]const u8,
226146 content_length: ?u64,
227 const method_end = mem.indexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;147 transfer_encoding: http.TransferEncoding,
228 if (method_end > 24) return error.HttpHeadersInvalid;148 transfer_compression: http.ContentEncoding,
229149 keep_alive: bool,
230 const method_str = first_line[0..method_end];150 compression: Compression,
231 const method: http.Method = @enumFromInt(http.Method.parse(method_str));151
232152 pub const ParseError = error{
233 const version_start = mem.lastIndexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;153 UnknownHttpMethod,
234 if (version_start == method_end) return error.HttpHeadersInvalid;154 HttpHeadersInvalid,
235155 HttpHeaderContinuationsUnsupported,
236 const version_str = first_line[version_start + 1 ..];156 HttpTransferEncodingUnsupported,
237 if (version_str.len != 8) return error.HttpHeadersInvalid;157 HttpConnectionHeaderUnsupported,
238 const version: http.Version = switch (int64(version_str[0..8])) {158 InvalidContentLength,
239 int64("HTTP/1.0") => .@"HTTP/1.0",159 CompressionUnsupported,
240 int64("HTTP/1.1") => .@"HTTP/1.1",160 MissingFinalNewline,
241 else => return error.HttpHeadersInvalid,
242 };161 };
243162
244 const target = first_line[method_end + 1 .. version_start];163 pub fn parse(bytes: []const u8) ParseError!Head {
245164 var it = mem.splitSequence(u8, bytes, "\r\n");
246 req.method = method;165
247 req.target = target;166 const first_line = it.next().?;
248 req.version = version;167 if (first_line.len < 10)
249168 return error.HttpHeadersInvalid;
250 while (it.next()) |line| {169
251 if (line.len == 0) return error.HttpHeadersInvalid;170 const method_end = mem.indexOfScalar(u8, first_line, ' ') orelse
252 switch (line[0]) {171 return error.HttpHeadersInvalid;
253 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,172 if (method_end > 24) return error.HttpHeadersInvalid;
254 else => {},173
255 }174 const method_str = first_line[0..method_end];
256175 const method: http.Method = @enumFromInt(http.Method.parse(method_str));
257 var line_it = mem.tokenizeAny(u8, line, ": ");176
258 const header_name = line_it.next() orelse return error.HttpHeadersInvalid;177 const version_start = mem.lastIndexOfScalar(u8, first_line, ' ') orelse
259 const header_value = line_it.rest();178 return error.HttpHeadersInvalid;
260179 if (version_start == method_end) return error.HttpHeadersInvalid;
261 try req.headers.append(header_name, header_value);180
262181 const version_str = first_line[version_start + 1 ..];
263 if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {182 if (version_str.len != 8) return error.HttpHeadersInvalid;
264 if (req.content_length != null) return error.HttpHeadersInvalid;183 const version: http.Version = switch (int64(version_str[0..8])) {
265 req.content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;184 int64("HTTP/1.0") => .@"HTTP/1.0",
266 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {185 int64("HTTP/1.1") => .@"HTTP/1.1",
267 // Transfer-Encoding: second, first186 else => return error.HttpHeadersInvalid,
268 // Transfer-Encoding: deflate, chunked187 };
269 var iter = mem.splitBackwardsScalar(u8, header_value, ',');188
270189 const target = first_line[method_end + 1 .. version_start];
271 const first = iter.first();190
272 const trimmed_first = mem.trim(u8, first, " ");191 var head: Head = .{
273192 .method = method,
274 var next: ?[]const u8 = first;193 .target = target,
275 if (std.meta.stringToEnum(http.TransferEncoding, trimmed_first)) |transfer| {194 .version = version,
276 if (req.transfer_encoding != .none) return error.HttpHeadersInvalid; // we already have a transfer encoding195 .expect = null,
277 req.transfer_encoding = transfer;196 .content_type = null,
278197 .content_length = null,
279 next = iter.next();198 .transfer_encoding = .none,
199 .transfer_compression = .identity,
200 .keep_alive = false,
201 .compression = .none,
202 };
203
204 while (it.next()) |line| {
205 if (line.len == 0) return head;
206 switch (line[0]) {
207 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
208 else => {},
280 }209 }
281210
282 if (next) |second| {211 var line_it = mem.splitSequence(u8, line, ": ");
283 const trimmed_second = mem.trim(u8, second, " ");212 const header_name = line_it.next().?;
284213 const header_value = line_it.rest();
285 if (std.meta.stringToEnum(http.ContentEncoding, trimmed_second)) |transfer| {214 if (header_value.len == 0) return error.HttpHeadersInvalid;
286 if (req.transfer_compression != .identity) return error.HttpHeadersInvalid; // double compression is not supported215
287 req.transfer_compression = transfer;216 if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
217 head.keep_alive = !std.ascii.eqlIgnoreCase(header_value, "close");
218 } else if (std.ascii.eqlIgnoreCase(header_name, "expect")) {
219 head.expect = header_value;
220 } else if (std.ascii.eqlIgnoreCase(header_name, "content-type")) {
221 head.content_type = header_value;
222 } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
223 if (head.content_length != null) return error.HttpHeadersInvalid;
224 head.content_length = std.fmt.parseInt(u64, header_value, 10) catch
225 return error.InvalidContentLength;
226 } else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
227 if (head.transfer_compression != .identity) return error.HttpHeadersInvalid;
228
229 const trimmed = mem.trim(u8, header_value, " ");
230
231 if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| {
232 head.transfer_compression = ce;
288 } else {233 } else {
289 return error.HttpTransferEncodingUnsupported;234 return error.HttpTransferEncodingUnsupported;
290 }235 }
291 }236 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
237 // Transfer-Encoding: second, first
238 // Transfer-Encoding: deflate, chunked
239 var iter = mem.splitBackwardsScalar(u8, header_value, ',');
292240
293 if (iter.next()) |_| return error.HttpTransferEncodingUnsupported;241 const first = iter.first();
294 } else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {242 const trimmed_first = mem.trim(u8, first, " ");
295 if (req.transfer_compression != .identity) return error.HttpHeadersInvalid;
296243
297 const trimmed = mem.trim(u8, header_value, " ");244 var next: ?[]const u8 = first;
245 if (std.meta.stringToEnum(http.TransferEncoding, trimmed_first)) |transfer| {
246 if (head.transfer_encoding != .none)
247 return error.HttpHeadersInvalid; // we already have a transfer encoding
248 head.transfer_encoding = transfer;
298249
299 if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| {250 next = iter.next();
300 req.transfer_compression = ce;251 }
301 } else {
302 return error.HttpTransferEncodingUnsupported;
303 }
304 }
305 }
306 }
307
308 inline fn int64(array: *const [8]u8) u64 {
309 return @as(u64, @bitCast(array.*));
310 }
311
312 /// The HTTP request method.
313 method: http.Method,
314
315 /// The HTTP request target.
316 target: []const u8,
317
318 /// The HTTP version of this request.
319 version: http.Version,
320
321 /// The length of the request body, if known.
322 content_length: ?u64 = null,
323
324 /// The transfer encoding of the request body, or .none if not present.
325 transfer_encoding: http.TransferEncoding = .none,
326
327 /// The compression of the request body, or .identity (no compression) if not present.
328 transfer_compression: http.ContentEncoding = .identity,
329
330 /// The list of HTTP request headers
331 headers: http.Headers,
332
333 parser: proto.HeadersParser,
334 compression: Compression = .none,
335};
336
337/// A HTTP response waiting to be sent.
338///
339/// Order of operations:
340/// ```
341/// [/ <--------------------------------------- \]
342/// accept -> wait -> send [ -> write -> finish][ -> reset /]
343/// \ -> read /
344/// ```
345pub const Response = struct {
346 version: http.Version = .@"HTTP/1.1",
347 status: http.Status = .ok,
348 reason: ?[]const u8 = null,
349
350 transfer_encoding: ResponseTransfer = .none,
351
352 /// The allocator responsible for allocating memory for this response.
353 allocator: Allocator,
354
355 /// The peer's address
356 address: net.Address,
357
358 /// The underlying connection for this response.
359 connection: Connection,
360252
361 /// The HTTP response headers253 if (next) |second| {
362 headers: http.Headers,254 const trimmed_second = mem.trim(u8, second, " ");
363255
364 /// The HTTP request that this response is responding to.256 if (std.meta.stringToEnum(http.ContentEncoding, trimmed_second)) |transfer| {
365 ///257 if (head.transfer_compression != .identity)
366 /// This field is only valid after calling `wait`.258 return error.HttpHeadersInvalid; // double compression is not supported
367 request: Request,259 head.transfer_compression = transfer;
260 } else {
261 return error.HttpTransferEncodingUnsupported;
262 }
263 }
368264
369 state: State = .first,265 if (iter.next()) |_| return error.HttpTransferEncodingUnsupported;
266 }
267 }
268 return error.MissingFinalNewline;
269 }
370270
371 const State = enum {271 inline fn int64(array: *const [8]u8) u64 {
372 first,272 return @bitCast(array.*);
373 start,273 }
374 waited,
375 responded,
376 finished,
377 };274 };
378275
379 /// Free all resources associated with this response.276 pub fn iterateHeaders(r: *Request) http.HeaderIterator {
380 pub fn deinit(res: *Response) void {277 return http.HeaderIterator.init(r.server.read_buffer[0..r.head_end]);
381 res.connection.close();
382
383 res.headers.deinit();
384 res.request.headers.deinit();
385
386 if (res.request.parser.header_bytes_owned) {
387 res.request.parser.header_bytes.deinit(res.allocator);
388 }
389 }278 }
390279
391 pub const ResetState = enum { reset, closing };280 pub const RespondOptions = struct {
281 version: http.Version = .@"HTTP/1.1",
282 status: http.Status = .ok,
283 reason: ?[]const u8 = null,
284 keep_alive: bool = true,
285 extra_headers: []const http.Header = &.{},
286 transfer_encoding: ?http.TransferEncoding = null,
287 };
392288
393 /// Reset this response to its initial state. This must be called before handling a second request on the same connection.289 /// Send an entire HTTP response to the client, including headers and body.
394 pub fn reset(res: *Response) ResetState {290 ///
395 if (res.state == .first) {291 /// Automatically handles HEAD requests by omitting the body.
396 res.state = .start;292 ///
397 return .reset;293 /// Unless `transfer_encoding` is specified, uses the "content-length"
294 /// header.
295 ///
296 /// If the request contains a body and the connection is to be reused,
297 /// discards the request body, leaving the Server in the `ready` state. If
298 /// this discarding fails, the connection is marked as not to be reused and
299 /// no error is surfaced.
300 ///
301 /// Asserts status is not `continue`.
302 /// Asserts there are at most 25 extra_headers.
303 /// Asserts that "\r\n" does not occur in any header name or value.
304 pub fn respond(
305 request: *Request,
306 content: []const u8,
307 options: RespondOptions,
308 ) Response.WriteError!void {
309 const max_extra_headers = 25;
310 assert(options.status != .@"continue");
311 assert(options.extra_headers.len <= max_extra_headers);
312 if (std.debug.runtime_safety) {
313 for (options.extra_headers) |header| {
314 assert(std.mem.indexOfScalar(u8, header.name, ':') == null);
315 assert(std.mem.indexOfPosLinear(u8, header.name, 0, "\r\n") == null);
316 assert(std.mem.indexOfPosLinear(u8, header.value, 0, "\r\n") == null);
317 }
398 }318 }
399319
400 if (!res.request.parser.done) {320 const transfer_encoding_none = (options.transfer_encoding orelse .chunked) == .none;
401 // If the response wasn't fully read, then we need to close the connection.321 const server_keep_alive = !transfer_encoding_none and options.keep_alive;
402 res.connection.closing = true;322 const keep_alive = request.discardBody(server_keep_alive);
403 return .closing;323
324 const phrase = options.reason orelse options.status.phrase() orelse "";
325
326 var first_buffer: [500]u8 = undefined;
327 var h = std.ArrayListUnmanaged(u8).initBuffer(&first_buffer);
328 if (request.head.expect != null) {
329 // reader() and hence discardBody() above sets expect to null if it
330 // is handled. So the fact that it is not null here means unhandled.
331 h.appendSliceAssumeCapacity("HTTP/1.1 417 Expectation Failed\r\n");
332 if (keep_alive) h.appendSliceAssumeCapacity("connection: keep-alive\r\n");
333 h.appendSliceAssumeCapacity("content-length: 0\r\n\r\n");
334 try request.server.connection.stream.writeAll(h.items);
335 return;
404 }336 }
337 h.fixedWriter().print("{s} {d} {s}\r\n", .{
338 @tagName(options.version), @intFromEnum(options.status), phrase,
339 }) catch unreachable;
405340
406 // A connection is only keep-alive if the Connection header is present and it's value is not "close".341 if (keep_alive) h.appendSliceAssumeCapacity("connection: keep-alive\r\n");
407 // The server and client must both agree
408 //
409 // send() defaults to using keep-alive if the client requests it.
410 const res_connection = res.headers.getFirstValue("connection");
411 const res_keepalive = res_connection != null and !std.ascii.eqlIgnoreCase("close", res_connection.?);
412
413 const req_connection = res.request.headers.getFirstValue("connection");
414 const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
415 if (req_keepalive and (res_keepalive or res_connection == null)) {
416 res.connection.closing = false;
417 } else {
418 res.connection.closing = true;
419 }
420342
421 switch (res.request.compression) {343 if (options.transfer_encoding) |transfer_encoding| switch (transfer_encoding) {
422 .none => {},344 .none => {},
423 .deflate => {},345 .chunked => h.appendSliceAssumeCapacity("transfer-encoding: chunked\r\n"),
424 .gzip => {},
425 .zstd => |*zstd| zstd.deinit(),
426 }
427
428 res.state = .start;
429 res.version = .@"HTTP/1.1";
430 res.status = .ok;
431 res.reason = null;
432
433 res.transfer_encoding = .none;
434
435 res.headers.clearRetainingCapacity();
436
437 res.request.headers.clearAndFree(); // FIXME: figure out why `clearRetainingCapacity` causes a leak in hash_map here
438 res.request.parser.reset();
439
440 res.request = Request{
441 .version = undefined,
442 .method = undefined,
443 .target = undefined,
444 .headers = res.request.headers,
445 .parser = res.request.parser,
446 };
447
448 if (res.connection.closing) {
449 return .closing;
450 } else {346 } else {
451 return .reset;347 h.fixedWriter().print("content-length: {d}\r\n", .{content.len}) catch unreachable;
452 }348 }
453 }
454349
455 pub const SendError = Connection.WriteError || error{ UnsupportedTransferEncoding, InvalidContentLength };350 var chunk_header_buffer: [18]u8 = undefined;
351 var iovecs: [max_extra_headers * 4 + 3]std.posix.iovec_const = undefined;
352 var iovecs_len: usize = 0;
456353
457 /// Send the HTTP response headers to the client.354 iovecs[iovecs_len] = .{
458 pub fn send(res: *Response) SendError!void {355 .iov_base = h.items.ptr,
459 switch (res.state) {356 .iov_len = h.items.len,
460 .waited => res.state = .responded,357 };
461 .first, .start, .responded, .finished => unreachable,358 iovecs_len += 1;
359
360 for (options.extra_headers) |header| {
361 iovecs[iovecs_len] = .{
362 .iov_base = header.name.ptr,
363 .iov_len = header.name.len,
364 };
365 iovecs_len += 1;
366
367 iovecs[iovecs_len] = .{
368 .iov_base = ": ",
369 .iov_len = 2,
370 };
371 iovecs_len += 1;
372
373 iovecs[iovecs_len] = .{
374 .iov_base = header.value.ptr,
375 .iov_len = header.value.len,
376 };
377 iovecs_len += 1;
378
379 iovecs[iovecs_len] = .{
380 .iov_base = "\r\n",
381 .iov_len = 2,
382 };
383 iovecs_len += 1;
462 }384 }
463385
464 var buffered = std.io.bufferedWriter(res.connection.writer());386 iovecs[iovecs_len] = .{
465 const w = buffered.writer();387 .iov_base = "\r\n",
466388 .iov_len = 2,
467 try w.writeAll(@tagName(res.version));389 };
468 try w.writeByte(' ');390 iovecs_len += 1;
469 try w.print("{d}", .{@intFromEnum(res.status)});391
470 try w.writeByte(' ');392 if (request.head.method != .HEAD) {
471 if (res.reason) |reason| {393 const is_chunked = (options.transfer_encoding orelse .none) == .chunked;
472 try w.writeAll(reason);394 if (is_chunked) {
473 } else if (res.status.phrase()) |phrase| {395 if (content.len > 0) {
474 try w.writeAll(phrase);396 const chunk_header = std.fmt.bufPrint(
475 }397 &chunk_header_buffer,
476 try w.writeAll("\r\n");398 "{x}\r\n",
399 .{content.len},
400 ) catch unreachable;
401
402 iovecs[iovecs_len] = .{
403 .iov_base = chunk_header.ptr,
404 .iov_len = chunk_header.len,
405 };
406 iovecs_len += 1;
407
408 iovecs[iovecs_len] = .{
409 .iov_base = content.ptr,
410 .iov_len = content.len,
411 };
412 iovecs_len += 1;
413
414 iovecs[iovecs_len] = .{
415 .iov_base = "\r\n",
416 .iov_len = 2,
417 };
418 iovecs_len += 1;
419 }
477420
478 if (res.status == .@"continue") {421 iovecs[iovecs_len] = .{
479 res.state = .waited; // we still need to send another request after this422 .iov_base = "0\r\n\r\n",
480 } else {423 .iov_len = 5,
481 if (!res.headers.contains("server")) {424 };
482 try w.writeAll("Server: zig (std.http)\r\n");425 iovecs_len += 1;
426 } else if (content.len > 0) {
427 iovecs[iovecs_len] = .{
428 .iov_base = content.ptr,
429 .iov_len = content.len,
430 };
431 iovecs_len += 1;
483 }432 }
433 }
484434
485 if (!res.headers.contains("connection")) {435 try request.server.connection.stream.writevAll(iovecs[0..iovecs_len]);
486 const req_connection = res.request.headers.getFirstValue("connection");436 }
487 const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
488
489 if (req_keepalive) {
490 try w.writeAll("Connection: keep-alive\r\n");
491 } else {
492 try w.writeAll("Connection: close\r\n");
493 }
494 }
495437
496 const has_transfer_encoding = res.headers.contains("transfer-encoding");438 pub const RespondStreamingOptions = struct {
497 const has_content_length = res.headers.contains("content-length");439 /// An externally managed slice of memory used to batch bytes before
440 /// sending. `respondStreaming` asserts this is large enough to store
441 /// the full HTTP response head.
442 ///
443 /// Must outlive the returned Response.
444 send_buffer: []u8,
445 /// If provided, the response will use the content-length header;
446 /// otherwise it will use transfer-encoding: chunked.
447 content_length: ?u64 = null,
448 /// Options that are shared with the `respond` method.
449 respond_options: RespondOptions = .{},
450 };
498451
499 if (!has_transfer_encoding and !has_content_length) {452 /// The header is buffered but not sent until Response.flush is called.
500 switch (res.transfer_encoding) {453 ///
501 .chunked => try w.writeAll("Transfer-Encoding: chunked\r\n"),454 /// If the request contains a body and the connection is to be reused,
502 .content_length => |content_length| try w.print("Content-Length: {d}\r\n", .{content_length}),455 /// discards the request body, leaving the Server in the `ready` state. If
503 .none => {},456 /// this discarding fails, the connection is marked as not to be reused and
504 }457 /// no error is surfaced.
458 ///
459 /// HEAD requests are handled transparently by setting a flag on the
460 /// returned Response to omit the body. However it may be worth noticing
461 /// that flag and skipping any expensive work that would otherwise need to
462 /// be done to satisfy the request.
463 ///
464 /// Asserts `send_buffer` is large enough to store the entire response header.
465 /// Asserts status is not `continue`.
466 pub fn respondStreaming(request: *Request, options: RespondStreamingOptions) Response {
467 const o = options.respond_options;
468 assert(o.status != .@"continue");
469 const transfer_encoding_none = (o.transfer_encoding orelse .chunked) == .none;
470 const server_keep_alive = !transfer_encoding_none and o.keep_alive;
471 const keep_alive = request.discardBody(server_keep_alive);
472 const phrase = o.reason orelse o.status.phrase() orelse "";
473
474 var h = std.ArrayListUnmanaged(u8).initBuffer(options.send_buffer);
475
476 const elide_body = if (request.head.expect != null) eb: {
477 // reader() and hence discardBody() above sets expect to null if it
478 // is handled. So the fact that it is not null here means unhandled.
479 h.appendSliceAssumeCapacity("HTTP/1.1 417 Expectation Failed\r\n");
480 if (keep_alive) h.appendSliceAssumeCapacity("connection: keep-alive\r\n");
481 h.appendSliceAssumeCapacity("content-length: 0\r\n\r\n");
482 break :eb true;
483 } else eb: {
484 h.fixedWriter().print("{s} {d} {s}\r\n", .{
485 @tagName(o.version), @intFromEnum(o.status), phrase,
486 }) catch unreachable;
487 if (keep_alive) h.appendSliceAssumeCapacity("connection: keep-alive\r\n");
488
489 if (o.transfer_encoding) |transfer_encoding| switch (transfer_encoding) {
490 .chunked => h.appendSliceAssumeCapacity("transfer-encoding: chunked\r\n"),
491 .none => {},
492 } else if (options.content_length) |len| {
493 h.fixedWriter().print("content-length: {d}\r\n", .{len}) catch unreachable;
505 } else {494 } else {
506 if (has_content_length) {495 h.appendSliceAssumeCapacity("transfer-encoding: chunked\r\n");
507 const content_length = std.fmt.parseInt(u64, res.headers.getFirstValue("content-length").?, 10) catch return error.InvalidContentLength;
508
509 res.transfer_encoding = .{ .content_length = content_length };
510 } else if (has_transfer_encoding) {
511 const transfer_encoding = res.headers.getFirstValue("transfer-encoding").?;
512 if (std.mem.eql(u8, transfer_encoding, "chunked")) {
513 res.transfer_encoding = .chunked;
514 } else {
515 return error.UnsupportedTransferEncoding;
516 }
517 } else {
518 res.transfer_encoding = .none;
519 }
520 }496 }
521497
522 try w.print("{}", .{res.headers});498 for (o.extra_headers) |header| {
523 }499 h.appendSliceAssumeCapacity(header.name);
524500 h.appendSliceAssumeCapacity(": ");
525 if (res.request.method == .HEAD) {501 h.appendSliceAssumeCapacity(header.value);
526 res.transfer_encoding = .none;502 h.appendSliceAssumeCapacity("\r\n");
527 }503 }
528504
529 try w.writeAll("\r\n");505 h.appendSliceAssumeCapacity("\r\n");
506 break :eb request.head.method == .HEAD;
507 };
530508
531 try buffered.flush();509 return .{
510 .stream = request.server.connection.stream,
511 .send_buffer = options.send_buffer,
512 .send_buffer_start = 0,
513 .send_buffer_end = h.items.len,
514 .transfer_encoding = if (o.transfer_encoding) |te| switch (te) {
515 .chunked => .chunked,
516 .none => .none,
517 } else if (options.content_length) |len| .{
518 .content_length = len,
519 } else .chunked,
520 .elide_body = elide_body,
521 .chunk_len = 0,
522 };
532 }523 }
533524
534 const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError;525 pub const ReadError = net.Stream.ReadError || error{
526 HttpChunkInvalid,
527 HttpHeadersOversize,
528 };
535529
536 const TransferReader = std.io.Reader(*Response, TransferReadError, transferRead);530 fn read_cl(context: *const anyopaque, buffer: []u8) ReadError!usize {
531 const request: *Request = @constCast(@alignCast(@ptrCast(context)));
532 const s = request.server;
537533
538 fn transferReader(res: *Response) TransferReader {534 const remaining_content_length = &request.reader_state.remaining_content_length;
539 return .{ .context = res };535 if (remaining_content_length.* == 0) {
536 s.state = .ready;
537 return 0;
538 }
539 assert(s.state == .receiving_body);
540 const available = try fill(s, request.head_end);
541 const len = @min(remaining_content_length.*, available.len, buffer.len);
542 @memcpy(buffer[0..len], available[0..len]);
543 remaining_content_length.* -= len;
544 s.next_request_start += len;
545 if (remaining_content_length.* == 0)
546 s.state = .ready;
547 return len;
540 }548 }
541549
542 fn transferRead(res: *Response, buf: []u8) TransferReadError!usize {550 fn fill(s: *Server, head_end: usize) ReadError![]u8 {
543 if (res.request.parser.done) return 0;551 const available = s.read_buffer[s.next_request_start..s.read_buffer_len];
552 if (available.len > 0) return available;
553 s.next_request_start = head_end;
554 s.read_buffer_len = head_end + try s.connection.stream.read(s.read_buffer[head_end..]);
555 return s.read_buffer[head_end..s.read_buffer_len];
556 }
544557
545 var index: usize = 0;558 fn read_chunked(context: *const anyopaque, buffer: []u8) ReadError!usize {
546 while (index == 0) {559 const request: *Request = @constCast(@alignCast(@ptrCast(context)));
547 const amt = try res.request.parser.read(&res.connection, buf[index..], false);560 const s = request.server;
548 if (amt == 0 and res.request.parser.done) break;561
549 index += amt;562 const cp = &request.reader_state.chunk_parser;
563 const head_end = request.head_end;
564
565 // Protect against returning 0 before the end of stream.
566 var out_end: usize = 0;
567 while (out_end == 0) {
568 switch (cp.state) {
569 .invalid => return 0,
570 .data => {
571 assert(s.state == .receiving_body);
572 const available = try fill(s, head_end);
573 const len = @min(cp.chunk_len, available.len, buffer.len);
574 @memcpy(buffer[0..len], available[0..len]);
575 cp.chunk_len -= len;
576 if (cp.chunk_len == 0)
577 cp.state = .data_suffix;
578 out_end += len;
579 s.next_request_start += len;
580 continue;
581 },
582 else => {
583 assert(s.state == .receiving_body);
584 const available = try fill(s, head_end);
585 const n = cp.feed(available);
586 switch (cp.state) {
587 .invalid => return error.HttpChunkInvalid,
588 .data => {
589 if (cp.chunk_len == 0) {
590 // The next bytes in the stream are trailers,
591 // or \r\n to indicate end of chunked body.
592 //
593 // This function must append the trailers at
594 // head_end so that headers and trailers are
595 // together.
596 //
597 // Since returning 0 would indicate end of
598 // stream, this function must read all the
599 // trailers before returning.
600 if (s.next_request_start > head_end) rebase(s, head_end);
601 var hp: http.HeadParser = .{};
602 {
603 const bytes = s.read_buffer[head_end..s.read_buffer_len];
604 const end = hp.feed(bytes);
605 if (hp.state == .finished) {
606 cp.state = .invalid;
607 s.state = .ready;
608 s.next_request_start = s.read_buffer_len - bytes.len + end;
609 return out_end;
610 }
611 }
612 while (true) {
613 const buf = s.read_buffer[s.read_buffer_len..];
614 if (buf.len == 0)
615 return error.HttpHeadersOversize;
616 const read_n = try s.connection.stream.read(buf);
617 s.read_buffer_len += read_n;
618 const bytes = buf[0..read_n];
619 const end = hp.feed(bytes);
620 if (hp.state == .finished) {
621 cp.state = .invalid;
622 s.state = .ready;
623 s.next_request_start = s.read_buffer_len - bytes.len + end;
624 return out_end;
625 }
626 }
627 }
628 const data = available[n..];
629 const len = @min(cp.chunk_len, data.len, buffer.len);
630 @memcpy(buffer[0..len], data[0..len]);
631 cp.chunk_len -= len;
632 if (cp.chunk_len == 0)
633 cp.state = .data_suffix;
634 out_end += len;
635 s.next_request_start += n + len;
636 continue;
637 },
638 else => continue,
639 }
640 },
641 }
550 }642 }
551643 return out_end;
552 return index;
553 }644 }
554645
555 pub const WaitError = Connection.ReadError || proto.HeadersParser.CheckCompleteHeadError || Request.ParseError || error{ CompressionInitializationFailed, CompressionNotSupported };646 pub const ReaderError = Response.WriteError || error{
647 /// The client sent an expect HTTP header value other than
648 /// "100-continue".
649 HttpExpectationFailed,
650 };
556651
557 /// Wait for the client to send a complete request head.652 /// In the case that the request contains "expect: 100-continue", this
653 /// function writes the continuation header, which means it can fail with a
654 /// write error. After sending the continuation header, it sets the
655 /// request's expect field to `null`.
558 ///656 ///
559 /// For correct behavior, the following rules must be followed:657 /// Asserts that this function is only called once.
560 ///658 pub fn reader(request: *Request) ReaderError!std.io.AnyReader {
561 /// * If this returns any error in `Connection.ReadError`, you MUST immediately close the connection by calling `deinit`.659 const s = request.server;
562 /// * If this returns `error.HttpHeadersInvalid`, you MAY immediately close the connection by calling `deinit`.660 assert(s.state == .received_head);
563 /// * If this returns `error.HttpHeadersExceededSizeLimit`, you MUST respond with a 431 status code and then call `deinit`.661 s.state = .receiving_body;
564 /// * If this returns any error in `Request.ParseError`, you MUST respond with a 400 status code and then call `deinit`.662 s.next_request_start = request.head_end;
565 /// * If this returns any other error, you MUST respond with a 400 status code and then call `deinit`.663
566 /// * If the request has an Expect header containing 100-continue, you MUST either:664 if (request.head.expect) |expect| {
567 /// * Respond with a 100 status code, then call `wait` again.665 if (mem.eql(u8, expect, "100-continue")) {
568 /// * Respond with a 417 status code.666 try request.server.connection.stream.writeAll("HTTP/1.1 100 Continue\r\n\r\n");
569 pub fn wait(res: *Response) WaitError!void {667 request.head.expect = null;
570 switch (res.state) {668 } else {
571 .first, .start => res.state = .waited,669 return error.HttpExpectationFailed;
572 .waited, .responded, .finished => unreachable,670 }
573 }671 }
574672
575 while (true) {673 switch (request.head.transfer_encoding) {
576 try res.connection.fill();674 .chunked => {
577675 request.reader_state = .{ .chunk_parser = http.ChunkParser.init };
578 const nchecked = try res.request.parser.checkCompleteHead(res.allocator, res.connection.peek());676 return .{
579 res.connection.drop(@as(u16, @intCast(nchecked)));677 .readFn = read_chunked,
580678 .context = request,
581 if (res.request.parser.state.isContent()) break;679 };
680 },
681 .none => {
682 request.reader_state = .{
683 .remaining_content_length = request.head.content_length orelse 0,
684 };
685 return .{
686 .readFn = read_cl,
687 .context = request,
688 };
689 },
582 }690 }
691 }
583692
584 res.request.headers = .{ .allocator = res.allocator, .owned = true };693 /// Returns whether the connection: keep-alive header should be sent to the client.
585 try res.request.parse(res.request.parser.header_bytes.items);694 /// If it would fail, it instead sets the Server state to `receiving_body`
586695 /// and returns false.
587 if (res.request.transfer_encoding != .none) {696 fn discardBody(request: *Request, keep_alive: bool) bool {
588 switch (res.request.transfer_encoding) {697 // Prepare to receive another request on the same connection.
589 .none => unreachable,698 // There are two factors to consider:
590 .chunked => {699 // * Any body the client sent must be discarded.
591 res.request.parser.next_chunk_length = 0;700 // * The Server's read_buffer may already have some bytes in it from
592 res.request.parser.state = .chunk_head_size;701 // whatever came after the head, which may be the next HTTP request
593 },702 // or the request body.
594 }703 // If the connection won't be kept alive, then none of this matters
595 } else if (res.request.content_length) |cl| {704 // because the connection will be severed after the response is sent.
596 res.request.parser.next_chunk_length = cl;705 const s = request.server;
597706 if (keep_alive and request.head.keep_alive) switch (s.state) {
598 if (cl == 0) res.request.parser.done = true;707 .received_head => {
708 const r = request.reader() catch return false;
709 _ = r.discard() catch return false;
710 assert(s.state == .ready);
711 return true;
712 },
713 .receiving_body, .ready => return true,
714 else => unreachable,
599 } else {715 } else {
600 res.request.parser.done = true;716 s.state = .closing;
717 return false;
601 }718 }
719 }
720};
602721
603 if (!res.request.parser.done) {722pub const Response = struct {
604 switch (res.request.transfer_compression) {723 stream: net.Stream,
605 .identity => res.request.compression = .none,724 send_buffer: []u8,
606 .compress, .@"x-compress" => return error.CompressionNotSupported,725 /// Index of the first byte in `send_buffer`.
607 .deflate => res.request.compression = .{726 /// This is 0 unless a short write happens in `write`.
608 .deflate = std.compress.zlib.decompressor(res.transferReader()),727 send_buffer_start: usize,
609 },728 /// Index of the last byte + 1 in `send_buffer`.
610 .gzip, .@"x-gzip" => res.request.compression = .{729 send_buffer_end: usize,
611 .gzip = std.compress.gzip.decompressor(res.transferReader()),730 /// `null` means transfer-encoding: chunked.
612 },731 /// As a debugging utility, counts down to zero as bytes are written.
613 .zstd => res.request.compression = .{732 transfer_encoding: TransferEncoding,
614 .zstd = std.compress.zstd.decompressStream(res.allocator, res.transferReader()),733 elide_body: bool,
615 },734 /// Indicates how much of the end of the `send_buffer` corresponds to a
616 }735 /// chunk. This amount of data will be wrapped by an HTTP chunk header.
736 chunk_len: usize,
737
738 pub const TransferEncoding = union(enum) {
739 /// End of connection signals the end of the stream.
740 none,
741 /// As a debugging utility, counts down to zero as bytes are written.
742 content_length: u64,
743 /// Each chunk is wrapped in a header and trailer.
744 chunked,
745 };
746
747 pub const WriteError = net.Stream.WriteError;
748
749 /// When using content-length, asserts that the amount of data sent matches
750 /// the value sent in the header, then calls `flush`.
751 /// Otherwise, transfer-encoding: chunked is being used, and it writes the
752 /// end-of-stream message, then flushes the stream to the system.
753 /// Respects the value of `elide_body` to omit all data after the headers.
754 pub fn end(r: *Response) WriteError!void {
755 switch (r.transfer_encoding) {
756 .content_length => |len| {
757 assert(len == 0); // Trips when end() called before all bytes written.
758 try flush_cl(r);
759 },
760 .none => {
761 try flush_cl(r);
762 },
763 .chunked => {
764 try flush_chunked(r, &.{});
765 },
617 }766 }
767 r.* = undefined;
618 }768 }
619769
620 pub const ReadError = TransferReadError || proto.HeadersParser.CheckCompleteHeadError || error{ DecompressionFailure, InvalidTrailers };770 pub const EndChunkedOptions = struct {
621771 trailers: []const http.Header = &.{},
622 pub const Reader = std.io.Reader(*Response, ReadError, read);772 };
623773
624 pub fn reader(res: *Response) Reader {774 /// Asserts that the Response is using transfer-encoding: chunked.
625 return .{ .context = res };775 /// Writes the end-of-stream message and any optional trailers, then
776 /// flushes the stream to the system.
777 /// Respects the value of `elide_body` to omit all data after the headers.
778 /// Asserts there are at most 25 trailers.
779 pub fn endChunked(r: *Response, options: EndChunkedOptions) WriteError!void {
780 assert(r.transfer_encoding == .chunked);
781 try flush_chunked(r, options.trailers);
782 r.* = undefined;
626 }783 }
627784
628 /// Reads data from the response body. Must be called after `wait`.785 /// If using content-length, asserts that writing these bytes to the client
629 pub fn read(res: *Response, buffer: []u8) ReadError!usize {786 /// would not exceed the content-length value sent in the HTTP header.
630 switch (res.state) {787 /// May return 0, which does not indicate end of stream. The caller decides
631 .waited, .responded, .finished => {},788 /// when the end of stream occurs by calling `end`.
632 .first, .start => unreachable,789 pub fn write(r: *Response, bytes: []const u8) WriteError!usize {
790 switch (r.transfer_encoding) {
791 .content_length, .none => return write_cl(r, bytes),
792 .chunked => return write_chunked(r, bytes),
633 }793 }
794 }
634795
635 const out_index = switch (res.request.compression) {796 fn write_cl(context: *const anyopaque, bytes: []const u8) WriteError!usize {
636 .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure,797 const r: *Response = @constCast(@alignCast(@ptrCast(context)));
637 .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure,
638 .zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,
639 else => try res.transferRead(buffer),
640 };
641798
642 if (out_index == 0) {799 var trash: u64 = std.math.maxInt(u64);
643 const has_trail = !res.request.parser.state.isContent();800 const len = switch (r.transfer_encoding) {
801 .content_length => |*len| len,
802 else => &trash,
803 };
644804
645 while (!res.request.parser.state.isContent()) { // read trailing headers805 if (r.elide_body) {
646 try res.connection.fill();806 len.* -= bytes.len;
807 return bytes.len;
808 }
647809
648 const nchecked = try res.request.parser.checkCompleteHead(res.allocator, res.connection.peek());810 if (bytes.len + r.send_buffer_end > r.send_buffer.len) {
649 res.connection.drop(@as(u16, @intCast(nchecked)));811 const send_buffer_len = r.send_buffer_end - r.send_buffer_start;
812 var iovecs: [2]std.posix.iovec_const = .{
813 .{
814 .iov_base = r.send_buffer.ptr + r.send_buffer_start,
815 .iov_len = send_buffer_len,
816 },
817 .{
818 .iov_base = bytes.ptr,
819 .iov_len = bytes.len,
820 },
821 };
822 const n = try r.stream.writev(&iovecs);
823
824 if (n >= send_buffer_len) {
825 // It was enough to reset the buffer.
826 r.send_buffer_start = 0;
827 r.send_buffer_end = 0;
828 const bytes_n = n - send_buffer_len;
829 len.* -= bytes_n;
830 return bytes_n;
650 }831 }
651832
652 if (has_trail) {833 // It didn't even make it through the existing buffer, let
653 res.request.headers = http.Headers{ .allocator = res.allocator, .owned = false };834 // alone the new bytes provided.
654835 r.send_buffer_start += n;
655 // The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error.836 return 0;
656 // This will *only* fail for a malformed trailer.
657 res.request.parse(res.request.parser.header_bytes.items) catch return error.InvalidTrailers;
658 }
659 }837 }
660838
661 return out_index;839 // All bytes can be stored in the remaining space of the buffer.
662 }840 @memcpy(r.send_buffer[r.send_buffer_end..][0..bytes.len], bytes);
663841 r.send_buffer_end += bytes.len;
664 /// Reads data from the response body. Must be called after `wait`.842 len.* -= bytes.len;
665 pub fn readAll(res: *Response, buffer: []u8) !usize {843 return bytes.len;
666 var index: usize = 0;
667 while (index < buffer.len) {
668 const amt = try read(res, buffer[index..]);
669 if (amt == 0) break;
670 index += amt;
671 }
672 return index;
673 }844 }
674845
675 pub const WriteError = Connection.WriteError || error{ NotWriteable, MessageTooLong };846 fn write_chunked(context: *const anyopaque, bytes: []const u8) WriteError!usize {
847 const r: *Response = @constCast(@alignCast(@ptrCast(context)));
848 assert(r.transfer_encoding == .chunked);
676849
677 pub const Writer = std.io.Writer(*Response, WriteError, write);850 if (r.elide_body)
851 return bytes.len;
678852
679 pub fn writer(res: *Response) Writer {853 if (bytes.len + r.send_buffer_end > r.send_buffer.len) {
680 return .{ .context = res };854 const send_buffer_len = r.send_buffer_end - r.send_buffer_start;
681 }855 const chunk_len = r.chunk_len + bytes.len;
856 var header_buf: [18]u8 = undefined;
857 const chunk_header = std.fmt.bufPrint(&header_buf, "{x}\r\n", .{chunk_len}) catch unreachable;
682858
683 /// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.859 var iovecs: [5]std.posix.iovec_const = .{
684 /// Must be called after `send` and before `finish`.860 .{
685 pub fn write(res: *Response, bytes: []const u8) WriteError!usize {861 .iov_base = r.send_buffer.ptr + r.send_buffer_start,
686 switch (res.state) {862 .iov_len = send_buffer_len - r.chunk_len,
687 .responded => {},863 },
688 .first, .waited, .start, .finished => unreachable,864 .{
865 .iov_base = chunk_header.ptr,
866 .iov_len = chunk_header.len,
867 },
868 .{
869 .iov_base = r.send_buffer.ptr + r.send_buffer_end - r.chunk_len,
870 .iov_len = r.chunk_len,
871 },
872 .{
873 .iov_base = bytes.ptr,
874 .iov_len = bytes.len,
875 },
876 .{
877 .iov_base = "\r\n",
878 .iov_len = 2,
879 },
880 };
881 // TODO make this writev instead of writevAll, which involves
882 // complicating the logic of this function.
883 try r.stream.writevAll(&iovecs);
884 r.send_buffer_start = 0;
885 r.send_buffer_end = 0;
886 r.chunk_len = 0;
887 return bytes.len;
689 }888 }
690889
691 switch (res.transfer_encoding) {890 // All bytes can be stored in the remaining space of the buffer.
692 .chunked => {891 @memcpy(r.send_buffer[r.send_buffer_end..][0..bytes.len], bytes);
693 try res.connection.writer().print("{x}\r\n", .{bytes.len});892 r.send_buffer_end += bytes.len;
694 try res.connection.writeAll(bytes);893 r.chunk_len += bytes.len;
695 try res.connection.writeAll("\r\n");894 return bytes.len;
696
697 return bytes.len;
698 },
699 .content_length => |*len| {
700 if (len.* < bytes.len) return error.MessageTooLong;
701
702 const amt = try res.connection.write(bytes);
703 len.* -= amt;
704 return amt;
705 },
706 .none => return error.NotWriteable,
707 }
708 }895 }
709896
710 /// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.897 /// If using content-length, asserts that writing these bytes to the client
711 /// Must be called after `send` and before `finish`.898 /// would not exceed the content-length value sent in the HTTP header.
712 pub fn writeAll(req: *Response, bytes: []const u8) WriteError!void {899 pub fn writeAll(r: *Response, bytes: []const u8) WriteError!void {
713 var index: usize = 0;900 var index: usize = 0;
714 while (index < bytes.len) {901 while (index < bytes.len) {
715 index += try write(req, bytes[index..]);902 index += try write(r, bytes[index..]);
716 }903 }
717 }904 }
718905
719 pub const FinishError = WriteError || error{MessageNotCompleted};906 /// Sends all buffered data to the client.
720907 /// This is redundant after calling `end`.
721 /// Finish the body of a request. This notifies the server that you have no more data to send.908 /// Respects the value of `elide_body` to omit all data after the headers.
722 /// Must be called after `send`.909 pub fn flush(r: *Response) WriteError!void {
723 pub fn finish(res: *Response) FinishError!void {910 switch (r.transfer_encoding) {
724 switch (res.state) {911 .none, .content_length => return flush_cl(r),
725 .responded => res.state = .finished,912 .chunked => return flush_chunked(r, null),
726 .first, .waited, .start, .finished => unreachable,
727 }913 }
914 }
728915
729 switch (res.transfer_encoding) {916 fn flush_cl(r: *Response) WriteError!void {
730 .chunked => try res.connection.writeAll("0\r\n\r\n"),917 try r.stream.writeAll(r.send_buffer[r.send_buffer_start..r.send_buffer_end]);
731 .content_length => |len| if (len != 0) return error.MessageNotCompleted,918 r.send_buffer_start = 0;
732 .none => {},919 r.send_buffer_end = 0;
733 }
734 }920 }
735};
736921
737/// Create a new HTTP server.922 fn flush_chunked(r: *Response, end_trailers: ?[]const http.Header) WriteError!void {
738pub fn init(options: net.StreamServer.Options) Server {923 const max_trailers = 25;
739 return .{924 if (end_trailers) |trailers| assert(trailers.len <= max_trailers);
740 .socket = net.StreamServer.init(options),925 assert(r.transfer_encoding == .chunked);
741 };
742}
743926
744/// Free all resources associated with this server.927 const http_headers = r.send_buffer[r.send_buffer_start .. r.send_buffer_end - r.chunk_len];
745pub fn deinit(server: *Server) void {
746 server.socket.deinit();
747}
748928
749pub const ListenError = std.os.SocketError || std.os.BindError || std.os.ListenError || std.os.SetSockOptError || std.os.GetSockNameError;929 if (r.elide_body) {
930 try r.stream.writeAll(http_headers);
931 r.send_buffer_start = 0;
932 r.send_buffer_end = 0;
933 r.chunk_len = 0;
934 return;
935 }
750936
751/// Start the HTTP server listening on the given address.937 var header_buf: [18]u8 = undefined;
752pub fn listen(server: *Server, address: net.Address) ListenError!void {938 const chunk_header = std.fmt.bufPrint(&header_buf, "{x}\r\n", .{r.chunk_len}) catch unreachable;
753 try server.socket.listen(address);
754}
755939
756pub const AcceptError = net.StreamServer.AcceptError || Allocator.Error;940 var iovecs: [max_trailers * 4 + 5]std.posix.iovec_const = undefined;
757941 var iovecs_len: usize = 0;
758pub const HeaderStrategy = union(enum) {
759 /// In this case, the client's Allocator will be used to store the
760 /// entire HTTP header. This value is the maximum total size of
761 /// HTTP headers allowed, otherwise
762 /// error.HttpHeadersExceededSizeLimit is returned from read().
763 dynamic: usize,
764 /// This is used to store the entire HTTP header. If the HTTP
765 /// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
766 /// is returned from read(). When this is used, `error.OutOfMemory`
767 /// cannot be returned from `read()`.
768 static: []u8,
769};
770942
771pub const AcceptOptions = struct {943 iovecs[iovecs_len] = .{
772 allocator: Allocator,944 .iov_base = http_headers.ptr,
773 header_strategy: HeaderStrategy = .{ .dynamic = 8192 },945 .iov_len = http_headers.len,
774};946 };
947 iovecs_len += 1;
948
949 if (r.chunk_len > 0) {
950 iovecs[iovecs_len] = .{
951 .iov_base = chunk_header.ptr,
952 .iov_len = chunk_header.len,
953 };
954 iovecs_len += 1;
955
956 iovecs[iovecs_len] = .{
957 .iov_base = r.send_buffer.ptr + r.send_buffer_end - r.chunk_len,
958 .iov_len = r.chunk_len,
959 };
960 iovecs_len += 1;
961
962 iovecs[iovecs_len] = .{
963 .iov_base = "\r\n",
964 .iov_len = 2,
965 };
966 iovecs_len += 1;
967 }
775968
776/// Accept a new connection.969 if (end_trailers) |trailers| {
777pub fn accept(server: *Server, options: AcceptOptions) AcceptError!Response {970 iovecs[iovecs_len] = .{
778 const in = try server.socket.accept();971 .iov_base = "0\r\n",
779972 .iov_len = 3,
780 return Response{973 };
781 .allocator = options.allocator,974 iovecs_len += 1;
782 .address = in.address,975
783 .connection = .{976 for (trailers) |trailer| {
784 .stream = in.stream,977 iovecs[iovecs_len] = .{
785 .protocol = .plain,978 .iov_base = trailer.name.ptr,
786 },979 .iov_len = trailer.name.len,
787 .headers = .{ .allocator = options.allocator },980 };
788 .request = .{981 iovecs_len += 1;
789 .version = undefined,982
790 .method = undefined,983 iovecs[iovecs_len] = .{
791 .target = undefined,984 .iov_base = ": ",
792 .headers = .{ .allocator = options.allocator, .owned = false },985 .iov_len = 2,
793 .parser = switch (options.header_strategy) {986 };
794 .dynamic => |max| proto.HeadersParser.initDynamic(max),987 iovecs_len += 1;
795 .static => |buf| proto.HeadersParser.initStatic(buf),988
796 },989 iovecs[iovecs_len] = .{
797 },990 .iov_base = trailer.value.ptr,
798 };991 .iov_len = trailer.value.len,
799}992 };
993 iovecs_len += 1;
994
995 iovecs[iovecs_len] = .{
996 .iov_base = "\r\n",
997 .iov_len = 2,
998 };
999 iovecs_len += 1;
1000 }
8001001
801test "HTTP server handles a chunked transfer coding request" {1002 iovecs[iovecs_len] = .{
802 const builtin = @import("builtin");1003 .iov_base = "\r\n",
1004 .iov_len = 2,
1005 };
1006 iovecs_len += 1;
1007 }
8031008
804 // This test requires spawning threads.1009 try r.stream.writevAll(iovecs[0..iovecs_len]);
805 if (builtin.single_threaded) {1010 r.send_buffer_start = 0;
806 return error.SkipZigTest;1011 r.send_buffer_end = 0;
1012 r.chunk_len = 0;
807 }1013 }
8081014
809 const native_endian = comptime builtin.cpu.arch.endian();1015 pub fn writer(r: *Response) std.io.AnyWriter {
810 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {1016 return .{
811 // https://github.com/ziglang/zig/issues/137821017 .writeFn = switch (r.transfer_encoding) {
812 return error.SkipZigTest;1018 .none, .content_length => write_cl,
1019 .chunked => write_chunked,
1020 },
1021 .context = r,
1022 };
813 }1023 }
1024};
8141025
815 if (builtin.os.tag == .wasi) return error.SkipZigTest;1026fn rebase(s: *Server, index: usize) void {
8161027 const leftover = s.read_buffer[s.next_request_start..s.read_buffer_len];
817 const allocator = std.testing.allocator;1028 const dest = s.read_buffer[index..][0..leftover.len];
818 const expect = std.testing.expect;1029 if (leftover.len <= s.next_request_start - index) {
8191030 @memcpy(dest, leftover);
820 const max_header_size = 8192;1031 } else {
821 var server = std.http.Server.init(.{ .reuse_address = true });1032 mem.copyBackwards(u8, dest, leftover);
822 defer server.deinit();1033 }
8231034 s.read_buffer_len = index + leftover.len;
824 const address = try std.net.Address.parseIp("127.0.0.1", 0);
825 try server.listen(address);
826 const server_port = server.socket.listen_address.in.getPort();
827
828 const server_thread = try std.Thread.spawn(.{}, (struct {
829 fn apply(s: *std.http.Server) !void {
830 var res = try s.accept(.{
831 .allocator = allocator,
832 .header_strategy = .{ .dynamic = max_header_size },
833 });
834 defer res.deinit();
835 defer _ = res.reset();
836 try res.wait();
837
838 try expect(res.request.transfer_encoding == .chunked);
839
840 const server_body: []const u8 = "message from server!\n";
841 res.transfer_encoding = .{ .content_length = server_body.len };
842 try res.headers.append("content-type", "text/plain");
843 try res.headers.append("connection", "close");
844 try res.send();
845
846 var buf: [128]u8 = undefined;
847 const n = try res.readAll(&buf);
848 try expect(std.mem.eql(u8, buf[0..n], "ABCD"));
849 _ = try res.writer().writeAll(server_body);
850 try res.finish();
851 }
852 }).apply, .{&server});
853
854 const request_bytes =
855 "POST / HTTP/1.1\r\n" ++
856 "Content-Type: text/plain\r\n" ++
857 "Transfer-Encoding: chunked\r\n" ++
858 "\r\n" ++
859 "1\r\n" ++
860 "A\r\n" ++
861 "1\r\n" ++
862 "B\r\n" ++
863 "2\r\n" ++
864 "CD\r\n" ++
865 "0\r\n" ++
866 "\r\n";
867
868 const stream = try std.net.tcpConnectToHost(allocator, "127.0.0.1", server_port);
869 defer stream.close();
870 _ = try stream.writeAll(request_bytes[0..]);
871
872 server_thread.join();
873}1035}
1036
1037const std = @import("../std.zig");
1038const http = std.http;
1039const mem = std.mem;
1040const net = std.net;
1041const Uri = std.Uri;
1042const assert = std.debug.assert;
1043
1044const Server = @This();
lib/std/http/protocol.zig+111-521
...@@ -7,15 +7,19 @@ const assert = std.debug.assert;...@@ -7,15 +7,19 @@ const assert = std.debug.assert;
7const use_vectors = builtin.zig_backend != .stage2_x86_64;7const use_vectors = builtin.zig_backend != .stage2_x86_64;
88
9pub const State = enum {9pub const State = enum {
10 /// Begin header parsing states.
11 invalid,10 invalid,
11
12 // Begin header and trailer parsing states.
13
12 start,14 start,
13 seen_n,15 seen_n,
14 seen_r,16 seen_r,
15 seen_rn,17 seen_rn,
16 seen_rnr,18 seen_rnr,
17 finished,19 finished,
18 /// Begin transfer-encoding: chunked parsing states.20
21 // Begin transfer-encoding: chunked parsing states.
22
19 chunk_head_size,23 chunk_head_size,
20 chunk_head_ext,24 chunk_head_ext,
21 chunk_head_r,25 chunk_head_r,
...@@ -34,484 +38,114 @@ pub const State = enum {...@@ -34,484 +38,114 @@ pub const State = enum {
3438
35pub const HeadersParser = struct {39pub const HeadersParser = struct {
36 state: State = .start,40 state: State = .start,
37 /// Whether or not `header_bytes` is allocated or was provided as a fixed buffer.41 /// A fixed buffer of len `max_header_bytes`.
38 header_bytes_owned: bool,
39 /// Either a fixed buffer of len `max_header_bytes` or a dynamic buffer that can grow up to `max_header_bytes`.
40 /// Pointers into this buffer are not stable until after a message is complete.42 /// Pointers into this buffer are not stable until after a message is complete.
41 header_bytes: std.ArrayListUnmanaged(u8),43 header_bytes_buffer: []u8,
42 /// The maximum allowed size of `header_bytes`.44 header_bytes_len: u32,
43 max_header_bytes: usize,45 next_chunk_length: u64,
44 next_chunk_length: u64 = 0,46 /// `false`: headers. `true`: trailers.
45 /// Whether this parser is done parsing a complete message.47 done: bool,
46 /// A message is only done when the entire payload has been read.
47 done: bool = false,
48
49 /// Initializes the parser with a dynamically growing header buffer of up to `max` bytes.
50 pub fn initDynamic(max: usize) HeadersParser {
51 return .{
52 .header_bytes = .{},
53 .max_header_bytes = max,
54 .header_bytes_owned = true,
55 };
56 }
5748
58 /// Initializes the parser with a provided buffer `buf`.49 /// Initializes the parser with a provided buffer `buf`.
59 pub fn initStatic(buf: []u8) HeadersParser {50 pub fn init(buf: []u8) HeadersParser {
60 return .{51 return .{
61 .header_bytes = .{ .items = buf[0..0], .capacity = buf.len },52 .header_bytes_buffer = buf,
62 .max_header_bytes = buf.len,53 .header_bytes_len = 0,
63 .header_bytes_owned = false,54 .done = false,
55 .next_chunk_length = 0,
64 };56 };
65 }57 }
6658
67 /// Completely resets the parser to it's initial state.59 /// Reinitialize the parser.
68 /// This must be called after a message is complete.60 /// Asserts the parser is in the "done" state.
69 pub fn reset(r: *HeadersParser) void {61 pub fn reset(hp: *HeadersParser) void {
70 assert(r.done); // The message must be completely read before reset, otherwise the parser is in an invalid state.62 assert(hp.done);
7163 hp.* = .{
72 r.header_bytes.clearRetainingCapacity();64 .state = .start,
7365 .header_bytes_buffer = hp.header_bytes_buffer,
74 r.* = .{66 .header_bytes_len = 0,
75 .header_bytes = r.header_bytes,67 .done = false,
76 .max_header_bytes = r.max_header_bytes,68 .next_chunk_length = 0,
77 .header_bytes_owned = r.header_bytes_owned,
78 };69 };
79 }70 }
8071
81 /// Returns the number of bytes consumed by headers. This is always less than or equal to `bytes.len`.72 pub fn get(hp: HeadersParser) []u8 {
82 /// You should check `r.state.isContent()` after this to check if the headers are done.73 return hp.header_bytes_buffer[0..hp.header_bytes_len];
83 ///74 }
84 /// If the amount returned is less than `bytes.len`, you may assume that the parser is in a content state and the
85 /// first byte of content is located at `bytes[result]`.
86 pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {
87 const vector_len: comptime_int = @max(std.simd.suggestVectorLength(u8) orelse 1, 8);
88 const len: u32 = @intCast(bytes.len);
89 var index: u32 = 0;
90
91 while (true) {
92 switch (r.state) {
93 .invalid => unreachable,
94 .finished => return index,
95 .start => switch (len - index) {
96 0 => return index,
97 1 => {
98 switch (bytes[index]) {
99 '\r' => r.state = .seen_r,
100 '\n' => r.state = .seen_n,
101 else => {},
102 }
103
104 return index + 1;
105 },
106 2 => {
107 const b16 = int16(bytes[index..][0..2]);
108 const b8 = intShift(u8, b16);
109
110 switch (b8) {
111 '\r' => r.state = .seen_r,
112 '\n' => r.state = .seen_n,
113 else => {},
114 }
115
116 switch (b16) {
117 int16("\r\n") => r.state = .seen_rn,
118 int16("\n\n") => r.state = .finished,
119 else => {},
120 }
121
122 return index + 2;
123 },
124 3 => {
125 const b24 = int24(bytes[index..][0..3]);
126 const b16 = intShift(u16, b24);
127 const b8 = intShift(u8, b24);
128
129 switch (b8) {
130 '\r' => r.state = .seen_r,
131 '\n' => r.state = .seen_n,
132 else => {},
133 }
134
135 switch (b16) {
136 int16("\r\n") => r.state = .seen_rn,
137 int16("\n\n") => r.state = .finished,
138 else => {},
139 }
140
141 switch (b24) {
142 int24("\r\n\r") => r.state = .seen_rnr,
143 else => {},
144 }
145
146 return index + 3;
147 },
148 4...vector_len - 1 => {
149 const b32 = int32(bytes[index..][0..4]);
150 const b24 = intShift(u24, b32);
151 const b16 = intShift(u16, b32);
152 const b8 = intShift(u8, b32);
153
154 switch (b8) {
155 '\r' => r.state = .seen_r,
156 '\n' => r.state = .seen_n,
157 else => {},
158 }
159
160 switch (b16) {
161 int16("\r\n") => r.state = .seen_rn,
162 int16("\n\n") => r.state = .finished,
163 else => {},
164 }
165
166 switch (b24) {
167 int24("\r\n\r") => r.state = .seen_rnr,
168 else => {},
169 }
170
171 switch (b32) {
172 int32("\r\n\r\n") => r.state = .finished,
173 else => {},
174 }
175
176 index += 4;
177 continue;
178 },
179 else => {
180 const chunk = bytes[index..][0..vector_len];
181 const matches = if (use_vectors) matches: {
182 const Vector = @Vector(vector_len, u8);
183 // const BoolVector = @Vector(vector_len, bool);
184 const BitVector = @Vector(vector_len, u1);
185 const SizeVector = @Vector(vector_len, u8);
186
187 const v: Vector = chunk.*;
188 const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
189 const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
190 const matches_or: SizeVector = matches_r | matches_n;
191
192 break :matches @reduce(.Add, matches_or);
193 } else matches: {
194 var matches: u8 = 0;
195 for (chunk) |byte| switch (byte) {
196 '\r', '\n' => matches += 1,
197 else => {},
198 };
199 break :matches matches;
200 };
201 switch (matches) {
202 0 => {},
203 1 => switch (chunk[vector_len - 1]) {
204 '\r' => r.state = .seen_r,
205 '\n' => r.state = .seen_n,
206 else => {},
207 },
208 2 => {
209 const b16 = int16(chunk[vector_len - 2 ..][0..2]);
210 const b8 = intShift(u8, b16);
211
212 switch (b8) {
213 '\r' => r.state = .seen_r,
214 '\n' => r.state = .seen_n,
215 else => {},
216 }
217
218 switch (b16) {
219 int16("\r\n") => r.state = .seen_rn,
220 int16("\n\n") => r.state = .finished,
221 else => {},
222 }
223 },
224 3 => {
225 const b24 = int24(chunk[vector_len - 3 ..][0..3]);
226 const b16 = intShift(u16, b24);
227 const b8 = intShift(u8, b24);
228
229 switch (b8) {
230 '\r' => r.state = .seen_r,
231 '\n' => r.state = .seen_n,
232 else => {},
233 }
234
235 switch (b16) {
236 int16("\r\n") => r.state = .seen_rn,
237 int16("\n\n") => r.state = .finished,
238 else => {},
239 }
240
241 switch (b24) {
242 int24("\r\n\r") => r.state = .seen_rnr,
243 else => {},
244 }
245 },
246 4...vector_len => {
247 inline for (0..vector_len - 3) |i_usize| {
248 const i = @as(u32, @truncate(i_usize));
249
250 const b32 = int32(chunk[i..][0..4]);
251 const b16 = intShift(u16, b32);
252
253 if (b32 == int32("\r\n\r\n")) {
254 r.state = .finished;
255 return index + i + 4;
256 } else if (b16 == int16("\n\n")) {
257 r.state = .finished;
258 return index + i + 2;
259 }
260 }
261
262 const b24 = int24(chunk[vector_len - 3 ..][0..3]);
263 const b16 = intShift(u16, b24);
264 const b8 = intShift(u8, b24);
265
266 switch (b8) {
267 '\r' => r.state = .seen_r,
268 '\n' => r.state = .seen_n,
269 else => {},
270 }
271
272 switch (b16) {
273 int16("\r\n") => r.state = .seen_rn,
274 int16("\n\n") => r.state = .finished,
275 else => {},
276 }
277
278 switch (b24) {
279 int24("\r\n\r") => r.state = .seen_rnr,
280 else => {},
281 }
282 },
283 else => unreachable,
284 }
285
286 index += vector_len;
287 continue;
288 },
289 },
290 .seen_n => switch (len - index) {
291 0 => return index,
292 else => {
293 switch (bytes[index]) {
294 '\n' => r.state = .finished,
295 else => r.state = .start,
296 }
297
298 index += 1;
299 continue;
300 },
301 },
302 .seen_r => switch (len - index) {
303 0 => return index,
304 1 => {
305 switch (bytes[index]) {
306 '\n' => r.state = .seen_rn,
307 '\r' => r.state = .seen_r,
308 else => r.state = .start,
309 }
310
311 return index + 1;
312 },
313 2 => {
314 const b16 = int16(bytes[index..][0..2]);
315 const b8 = intShift(u8, b16);
316
317 switch (b8) {
318 '\r' => r.state = .seen_r,
319 '\n' => r.state = .seen_rn,
320 else => r.state = .start,
321 }
322
323 switch (b16) {
324 int16("\r\n") => r.state = .seen_rn,
325 int16("\n\r") => r.state = .seen_rnr,
326 int16("\n\n") => r.state = .finished,
327 else => {},
328 }
329
330 return index + 2;
331 },
332 else => {
333 const b24 = int24(bytes[index..][0..3]);
334 const b16 = intShift(u16, b24);
335 const b8 = intShift(u8, b24);
336
337 switch (b8) {
338 '\r' => r.state = .seen_r,
339 '\n' => r.state = .seen_n,
340 else => r.state = .start,
341 }
342
343 switch (b16) {
344 int16("\r\n") => r.state = .seen_rn,
345 int16("\n\n") => r.state = .finished,
346 else => {},
347 }
348
349 switch (b24) {
350 int24("\n\r\n") => r.state = .finished,
351 else => {},
352 }
353
354 index += 3;
355 continue;
356 },
357 },
358 .seen_rn => switch (len - index) {
359 0 => return index,
360 1 => {
361 switch (bytes[index]) {
362 '\r' => r.state = .seen_rnr,
363 '\n' => r.state = .seen_n,
364 else => r.state = .start,
365 }
366
367 return index + 1;
368 },
369 else => {
370 const b16 = int16(bytes[index..][0..2]);
371 const b8 = intShift(u8, b16);
372
373 switch (b8) {
374 '\r' => r.state = .seen_rnr,
375 '\n' => r.state = .seen_n,
376 else => r.state = .start,
377 }
378
379 switch (b16) {
380 int16("\r\n") => r.state = .finished,
381 int16("\n\n") => r.state = .finished,
382 else => {},
383 }
384
385 index += 2;
386 continue;
387 },
388 },
389 .seen_rnr => switch (len - index) {
390 0 => return index,
391 else => {
392 switch (bytes[index]) {
393 '\n' => r.state = .finished,
394 else => r.state = .start,
395 }
396
397 index += 1;
398 continue;
399 },
400 },
401 .chunk_head_size => unreachable,
402 .chunk_head_ext => unreachable,
403 .chunk_head_r => unreachable,
404 .chunk_data => unreachable,
405 .chunk_data_suffix => unreachable,
406 .chunk_data_suffix_r => unreachable,
407 }
40875
409 return index;76 pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {
410 }77 var hp: std.http.HeadParser = .{
78 .state = switch (r.state) {
79 .start => .start,
80 .seen_n => .seen_n,
81 .seen_r => .seen_r,
82 .seen_rn => .seen_rn,
83 .seen_rnr => .seen_rnr,
84 .finished => .finished,
85 else => unreachable,
86 },
87 };
88 const result = hp.feed(bytes);
89 r.state = switch (hp.state) {
90 .start => .start,
91 .seen_n => .seen_n,
92 .seen_r => .seen_r,
93 .seen_rn => .seen_rn,
94 .seen_rnr => .seen_rnr,
95 .finished => .finished,
96 };
97 return @intCast(result);
411 }98 }
41299
413 /// Returns the number of bytes consumed by the chunk size. This is always less than or equal to `bytes.len`.
414 /// You should check `r.state == .chunk_data` after this to check if the chunk size has been fully parsed.
415 ///
416 /// If the amount returned is less than `bytes.len`, you may assume that the parser is in the `chunk_data` state
417 /// and that the first byte of the chunk is at `bytes[result]`.
418 pub fn findChunkedLen(r: *HeadersParser, bytes: []const u8) u32 {100 pub fn findChunkedLen(r: *HeadersParser, bytes: []const u8) u32 {
419 const len = @as(u32, @intCast(bytes.len));101 var cp: std.http.ChunkParser = .{
420102 .state = switch (r.state) {
421 for (bytes[0..], 0..) |c, i| {103 .chunk_head_size => .head_size,
422 const index = @as(u32, @intCast(i));104 .chunk_head_ext => .head_ext,
423 switch (r.state) {105 .chunk_head_r => .head_r,
424 .chunk_data_suffix => switch (c) {106 .chunk_data => .data,
425 '\r' => r.state = .chunk_data_suffix_r,107 .chunk_data_suffix => .data_suffix,
426 '\n' => r.state = .chunk_head_size,108 .chunk_data_suffix_r => .data_suffix_r,
427 else => {109 .invalid => .invalid,
428 r.state = .invalid;
429 return index;
430 },
431 },
432 .chunk_data_suffix_r => switch (c) {
433 '\n' => r.state = .chunk_head_size,
434 else => {
435 r.state = .invalid;
436 return index;
437 },
438 },
439 .chunk_head_size => {
440 const digit = switch (c) {
441 '0'...'9' => |b| b - '0',
442 'A'...'Z' => |b| b - 'A' + 10,
443 'a'...'z' => |b| b - 'a' + 10,
444 '\r' => {
445 r.state = .chunk_head_r;
446 continue;
447 },
448 '\n' => {
449 r.state = .chunk_data;
450 return index + 1;
451 },
452 else => {
453 r.state = .chunk_head_ext;
454 continue;
455 },
456 };
457
458 const new_len = r.next_chunk_length *% 16 +% digit;
459 if (new_len <= r.next_chunk_length and r.next_chunk_length != 0) {
460 r.state = .invalid;
461 return index;
462 }
463
464 r.next_chunk_length = new_len;
465 },
466 .chunk_head_ext => switch (c) {
467 '\r' => r.state = .chunk_head_r,
468 '\n' => {
469 r.state = .chunk_data;
470 return index + 1;
471 },
472 else => continue,
473 },
474 .chunk_head_r => switch (c) {
475 '\n' => {
476 r.state = .chunk_data;
477 return index + 1;
478 },
479 else => {
480 r.state = .invalid;
481 return index;
482 },
483 },
484 else => unreachable,110 else => unreachable,
485 }111 },
486 }112 .chunk_len = r.next_chunk_length,
487113 };
488 return len;114 const result = cp.feed(bytes);
115 r.state = switch (cp.state) {
116 .head_size => .chunk_head_size,
117 .head_ext => .chunk_head_ext,
118 .head_r => .chunk_head_r,
119 .data => .chunk_data,
120 .data_suffix => .chunk_data_suffix,
121 .data_suffix_r => .chunk_data_suffix_r,
122 .invalid => .invalid,
123 };
124 r.next_chunk_length = cp.chunk_len;
125 return @intCast(result);
489 }126 }
490127
491 /// Returns whether or not the parser has finished parsing a complete message. A message is only complete after the128 /// Returns whether or not the parser has finished parsing a complete
492 /// entire body has been read and any trailing headers have been parsed.129 /// message. A message is only complete after the entire body has been read
130 /// and any trailing headers have been parsed.
493 pub fn isComplete(r: *HeadersParser) bool {131 pub fn isComplete(r: *HeadersParser) bool {
494 return r.done and r.state == .finished;132 return r.done and r.state == .finished;
495 }133 }
496134
497 pub const CheckCompleteHeadError = mem.Allocator.Error || error{HttpHeadersExceededSizeLimit};135 pub const CheckCompleteHeadError = error{HttpHeadersOversize};
498136
499 /// Pushes `in` into the parser. Returns the number of bytes consumed by the header. Any header bytes are appended137 /// Pushes `in` into the parser. Returns the number of bytes consumed by
500 /// to the `header_bytes` buffer.138 /// the header. Any header bytes are appended to `header_bytes_buffer`.
501 ///139 pub fn checkCompleteHead(hp: *HeadersParser, in: []const u8) CheckCompleteHeadError!u32 {
502 /// This function only uses `allocator` if `r.header_bytes_owned` is true, and may be undefined otherwise.140 if (hp.state.isContent()) return 0;
503 pub fn checkCompleteHead(r: *HeadersParser, allocator: std.mem.Allocator, in: []const u8) CheckCompleteHeadError!u32 {
504 if (r.state.isContent()) return 0;
505141
506 const i = r.findHeadersEnd(in);142 const i = hp.findHeadersEnd(in);
507 const data = in[0..i];143 const data = in[0..i];
508 if (r.header_bytes.items.len + data.len > r.max_header_bytes) {144 if (hp.header_bytes_len + data.len > hp.header_bytes_buffer.len)
509 return error.HttpHeadersExceededSizeLimit;145 return error.HttpHeadersOversize;
510 } else {
511 if (r.header_bytes_owned) try r.header_bytes.ensureUnusedCapacity(allocator, data.len);
512146
513 r.header_bytes.appendSliceAssumeCapacity(data);147 @memcpy(hp.header_bytes_buffer[hp.header_bytes_len..][0..data.len], data);
514 }148 hp.header_bytes_len += @intCast(data.len);
515149
516 return i;150 return i;
517 }151 }
...@@ -520,7 +154,8 @@ pub const HeadersParser = struct {...@@ -520,7 +154,8 @@ pub const HeadersParser = struct {
520 HttpChunkInvalid,154 HttpChunkInvalid,
521 };155 };
522156
523 /// Reads the body of the message into `buffer`. Returns the number of bytes placed in the buffer.157 /// Reads the body of the message into `buffer`. Returns the number of
158 /// bytes placed in the buffer.
524 ///159 ///
525 /// If `skip` is true, the buffer will be unused and the body will be skipped.160 /// If `skip` is true, the buffer will be unused and the body will be skipped.
526 ///161 ///
...@@ -571,9 +206,10 @@ pub const HeadersParser = struct {...@@ -571,9 +206,10 @@ pub const HeadersParser = struct {
571 .chunk_data => if (r.next_chunk_length == 0) {206 .chunk_data => if (r.next_chunk_length == 0) {
572 if (std.mem.eql(u8, conn.peek(), "\r\n")) {207 if (std.mem.eql(u8, conn.peek(), "\r\n")) {
573 r.state = .finished;208 r.state = .finished;
574 r.done = true;209 conn.drop(2);
575 } else {210 } else {
576 // The trailer section is formatted identically to the header section.211 // The trailer section is formatted identically
212 // to the header section.
577 r.state = .seen_rn;213 r.state = .seen_rn;
578 }214 }
579 r.done = true;215 r.done = true;
...@@ -713,57 +349,11 @@ const MockBufferedConnection = struct {...@@ -713,57 +349,11 @@ const MockBufferedConnection = struct {
713 }349 }
714};350};
715351
716test "HeadersParser.findHeadersEnd" {
717 var r: HeadersParser = undefined;
718 const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\nHello";
719
720 for (0..36) |i| {
721 r = HeadersParser.initDynamic(0);
722 try std.testing.expectEqual(@as(u32, @intCast(i)), r.findHeadersEnd(data[0..i]));
723 try std.testing.expectEqual(@as(u32, @intCast(35 - i)), r.findHeadersEnd(data[i..]));
724 }
725}
726
727test "HeadersParser.findChunkedLen" {
728 var r: HeadersParser = undefined;
729 const data = "Ff\r\nf0f000 ; ext\n0\r\nffffffffffffffffffffffffffffffffffffffff\r\n";
730
731 r = HeadersParser.initDynamic(0);
732 r.state = .chunk_head_size;
733 r.next_chunk_length = 0;
734
735 const first = r.findChunkedLen(data[0..]);
736 try testing.expectEqual(@as(u32, 4), first);
737 try testing.expectEqual(@as(u64, 0xff), r.next_chunk_length);
738 try testing.expectEqual(State.chunk_data, r.state);
739 r.state = .chunk_head_size;
740 r.next_chunk_length = 0;
741
742 const second = r.findChunkedLen(data[first..]);
743 try testing.expectEqual(@as(u32, 13), second);
744 try testing.expectEqual(@as(u64, 0xf0f000), r.next_chunk_length);
745 try testing.expectEqual(State.chunk_data, r.state);
746 r.state = .chunk_head_size;
747 r.next_chunk_length = 0;
748
749 const third = r.findChunkedLen(data[first + second ..]);
750 try testing.expectEqual(@as(u32, 3), third);
751 try testing.expectEqual(@as(u64, 0), r.next_chunk_length);
752 try testing.expectEqual(State.chunk_data, r.state);
753 r.state = .chunk_head_size;
754 r.next_chunk_length = 0;
755
756 const fourth = r.findChunkedLen(data[first + second + third ..]);
757 try testing.expectEqual(@as(u32, 16), fourth);
758 try testing.expectEqual(@as(u64, 0xffffffffffffffff), r.next_chunk_length);
759 try testing.expectEqual(State.invalid, r.state);
760}
761
762test "HeadersParser.read length" {352test "HeadersParser.read length" {
763 // mock BufferedConnection for read353 // mock BufferedConnection for read
354 var headers_buf: [256]u8 = undefined;
764355
765 var r = HeadersParser.initDynamic(256);356 var r = HeadersParser.init(&headers_buf);
766 defer r.header_bytes.deinit(std.testing.allocator);
767 const data = "GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\nHello";357 const data = "GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\nHello";
768358
769 var conn: MockBufferedConnection = .{359 var conn: MockBufferedConnection = .{
...@@ -773,8 +363,8 @@ test "HeadersParser.read length" {...@@ -773,8 +363,8 @@ test "HeadersParser.read length" {
773 while (true) { // read headers363 while (true) { // read headers
774 try conn.fill();364 try conn.fill();
775365
776 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());366 const nchecked = try r.checkCompleteHead(conn.peek());
777 conn.drop(@as(u16, @intCast(nchecked)));367 conn.drop(@intCast(nchecked));
778368
779 if (r.state.isContent()) break;369 if (r.state.isContent()) break;
780 }370 }
...@@ -786,14 +376,14 @@ test "HeadersParser.read length" {...@@ -786,14 +376,14 @@ test "HeadersParser.read length" {
786 try std.testing.expectEqual(@as(usize, 5), len);376 try std.testing.expectEqual(@as(usize, 5), len);
787 try std.testing.expectEqualStrings("Hello", buf[0..len]);377 try std.testing.expectEqualStrings("Hello", buf[0..len]);
788378
789 try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\n", r.header_bytes.items);379 try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\n", r.get());
790}380}
791381
792test "HeadersParser.read chunked" {382test "HeadersParser.read chunked" {
793 // mock BufferedConnection for read383 // mock BufferedConnection for read
794384
795 var r = HeadersParser.initDynamic(256);385 var headers_buf: [256]u8 = undefined;
796 defer r.header_bytes.deinit(std.testing.allocator);386 var r = HeadersParser.init(&headers_buf);
797 const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n2\r\nHe\r\n2\r\nll\r\n1\r\no\r\n0\r\n\r\n";387 const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n2\r\nHe\r\n2\r\nll\r\n1\r\no\r\n0\r\n\r\n";
798388
799 var conn: MockBufferedConnection = .{389 var conn: MockBufferedConnection = .{
...@@ -803,8 +393,8 @@ test "HeadersParser.read chunked" {...@@ -803,8 +393,8 @@ test "HeadersParser.read chunked" {
803 while (true) { // read headers393 while (true) { // read headers
804 try conn.fill();394 try conn.fill();
805395
806 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());396 const nchecked = try r.checkCompleteHead(conn.peek());
807 conn.drop(@as(u16, @intCast(nchecked)));397 conn.drop(@intCast(nchecked));
808398
809 if (r.state.isContent()) break;399 if (r.state.isContent()) break;
810 }400 }
...@@ -815,14 +405,14 @@ test "HeadersParser.read chunked" {...@@ -815,14 +405,14 @@ test "HeadersParser.read chunked" {
815 try std.testing.expectEqual(@as(usize, 5), len);405 try std.testing.expectEqual(@as(usize, 5), len);
816 try std.testing.expectEqualStrings("Hello", buf[0..len]);406 try std.testing.expectEqualStrings("Hello", buf[0..len]);
817407
818 try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n", r.header_bytes.items);408 try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n", r.get());
819}409}
820410
821test "HeadersParser.read chunked trailer" {411test "HeadersParser.read chunked trailer" {
822 // mock BufferedConnection for read412 // mock BufferedConnection for read
823413
824 var r = HeadersParser.initDynamic(256);414 var headers_buf: [256]u8 = undefined;
825 defer r.header_bytes.deinit(std.testing.allocator);415 var r = HeadersParser.init(&headers_buf);
826 const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n2\r\nHe\r\n2\r\nll\r\n1\r\no\r\n0\r\nContent-Type: text/plain\r\n\r\n";416 const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n2\r\nHe\r\n2\r\nll\r\n1\r\no\r\n0\r\nContent-Type: text/plain\r\n\r\n";
827417
828 var conn: MockBufferedConnection = .{418 var conn: MockBufferedConnection = .{
...@@ -832,8 +422,8 @@ test "HeadersParser.read chunked trailer" {...@@ -832,8 +422,8 @@ test "HeadersParser.read chunked trailer" {
832 while (true) { // read headers422 while (true) { // read headers
833 try conn.fill();423 try conn.fill();
834424
835 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());425 const nchecked = try r.checkCompleteHead(conn.peek());
836 conn.drop(@as(u16, @intCast(nchecked)));426 conn.drop(@intCast(nchecked));
837427
838 if (r.state.isContent()) break;428 if (r.state.isContent()) break;
839 }429 }
...@@ -847,11 +437,11 @@ test "HeadersParser.read chunked trailer" {...@@ -847,11 +437,11 @@ test "HeadersParser.read chunked trailer" {
847 while (true) { // read headers437 while (true) { // read headers
848 try conn.fill();438 try conn.fill();
849439
850 const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());440 const nchecked = try r.checkCompleteHead(conn.peek());
851 conn.drop(@as(u16, @intCast(nchecked)));441 conn.drop(@intCast(nchecked));
852442
853 if (r.state.isContent()) break;443 if (r.state.isContent()) break;
854 }444 }
855445
856 try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\nContent-Type: text/plain\r\n\r\n", r.header_bytes.items);446 try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\nContent-Type: text/plain\r\n\r\n", r.get());
857}447}
lib/std/http/test.zig created+995
...@@ -0,0 +1,995 @@
1const builtin = @import("builtin");
2const std = @import("std");
3const http = std.http;
4const mem = std.mem;
5const native_endian = builtin.cpu.arch.endian();
6const expect = std.testing.expect;
7const expectEqual = std.testing.expectEqual;
8const expectEqualStrings = std.testing.expectEqualStrings;
9const expectError = std.testing.expectError;
10
11test "trailers" {
12 const test_server = try createTestServer(struct {
13 fn run(net_server: *std.net.Server) anyerror!void {
14 var header_buffer: [1024]u8 = undefined;
15 var remaining: usize = 1;
16 while (remaining != 0) : (remaining -= 1) {
17 const conn = try net_server.accept();
18 defer conn.stream.close();
19
20 var server = http.Server.init(conn, &header_buffer);
21
22 try expectEqual(.ready, server.state);
23 var request = try server.receiveHead();
24 try serve(&request);
25 try expectEqual(.ready, server.state);
26 }
27 }
28
29 fn serve(request: *http.Server.Request) !void {
30 try expectEqualStrings(request.head.target, "/trailer");
31
32 var send_buffer: [1024]u8 = undefined;
33 var response = request.respondStreaming(.{
34 .send_buffer = &send_buffer,
35 });
36 try response.writeAll("Hello, ");
37 try response.flush();
38 try response.writeAll("World!\n");
39 try response.flush();
40 try response.endChunked(.{
41 .trailers = &.{
42 .{ .name = "X-Checksum", .value = "aaaa" },
43 },
44 });
45 }
46 });
47 defer test_server.destroy();
48
49 const gpa = std.testing.allocator;
50
51 var client: http.Client = .{ .allocator = gpa };
52 defer client.deinit();
53
54 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/trailer", .{
55 test_server.port(),
56 });
57 defer gpa.free(location);
58 const uri = try std.Uri.parse(location);
59
60 {
61 var server_header_buffer: [1024]u8 = undefined;
62 var req = try client.open(.GET, uri, .{
63 .server_header_buffer = &server_header_buffer,
64 });
65 defer req.deinit();
66
67 try req.send(.{});
68 try req.wait();
69
70 const body = try req.reader().readAllAlloc(gpa, 8192);
71 defer gpa.free(body);
72
73 try expectEqualStrings("Hello, World!\n", body);
74
75 var it = req.response.iterateHeaders();
76 {
77 const header = it.next().?;
78 try expect(!it.is_trailer);
79 try expectEqualStrings("connection", header.name);
80 try expectEqualStrings("keep-alive", header.value);
81 }
82 {
83 const header = it.next().?;
84 try expect(!it.is_trailer);
85 try expectEqualStrings("transfer-encoding", header.name);
86 try expectEqualStrings("chunked", header.value);
87 }
88 {
89 const header = it.next().?;
90 try expect(it.is_trailer);
91 try expectEqualStrings("X-Checksum", header.name);
92 try expectEqualStrings("aaaa", header.value);
93 }
94 try expectEqual(null, it.next());
95 }
96
97 // connection has been kept alive
98 try expect(client.connection_pool.free_len == 1);
99}
100
101test "HTTP server handles a chunked transfer coding request" {
102 const test_server = try createTestServer(struct {
103 fn run(net_server: *std.net.Server) !void {
104 var header_buffer: [8192]u8 = undefined;
105 const conn = try net_server.accept();
106 defer conn.stream.close();
107
108 var server = http.Server.init(conn, &header_buffer);
109 var request = try server.receiveHead();
110
111 try expect(request.head.transfer_encoding == .chunked);
112
113 var buf: [128]u8 = undefined;
114 const n = try (try request.reader()).readAll(&buf);
115 try expect(mem.eql(u8, buf[0..n], "ABCD"));
116
117 try request.respond("message from server!\n", .{
118 .extra_headers = &.{
119 .{ .name = "content-type", .value = "text/plain" },
120 },
121 .keep_alive = false,
122 });
123 }
124 });
125 defer test_server.destroy();
126
127 const request_bytes =
128 "POST / HTTP/1.1\r\n" ++
129 "Content-Type: text/plain\r\n" ++
130 "Transfer-Encoding: chunked\r\n" ++
131 "\r\n" ++
132 "1\r\n" ++
133 "A\r\n" ++
134 "1\r\n" ++
135 "B\r\n" ++
136 "2\r\n" ++
137 "CD\r\n" ++
138 "0\r\n" ++
139 "\r\n";
140
141 const gpa = std.testing.allocator;
142 const stream = try std.net.tcpConnectToHost(gpa, "127.0.0.1", test_server.port());
143 defer stream.close();
144 try stream.writeAll(request_bytes);
145}
146
147test "echo content server" {
148 const test_server = try createTestServer(struct {
149 fn run(net_server: *std.net.Server) anyerror!void {
150 var read_buffer: [1024]u8 = undefined;
151
152 accept: while (true) {
153 const conn = try net_server.accept();
154 defer conn.stream.close();
155
156 var http_server = http.Server.init(conn, &read_buffer);
157
158 while (http_server.state == .ready) {
159 var request = http_server.receiveHead() catch |err| switch (err) {
160 error.HttpConnectionClosing => continue :accept,
161 else => |e| return e,
162 };
163 if (mem.eql(u8, request.head.target, "/end")) {
164 return request.respond("", .{ .keep_alive = false });
165 }
166 if (request.head.expect) |expect_header_value| {
167 if (mem.eql(u8, expect_header_value, "garbage")) {
168 try expectError(error.HttpExpectationFailed, request.reader());
169 try request.respond("", .{ .keep_alive = false });
170 continue;
171 }
172 }
173 handleRequest(&request) catch |err| {
174 // This message helps the person troubleshooting determine whether
175 // output comes from the server thread or the client thread.
176 std.debug.print("handleRequest failed with '{s}'\n", .{@errorName(err)});
177 return err;
178 };
179 }
180 }
181 }
182
183 fn handleRequest(request: *http.Server.Request) !void {
184 //std.debug.print("server received {s} {s} {s}\n", .{
185 // @tagName(request.head.method),
186 // @tagName(request.head.version),
187 // request.head.target,
188 //});
189
190 const body = try (try request.reader()).readAllAlloc(std.testing.allocator, 8192);
191 defer std.testing.allocator.free(body);
192
193 try expect(mem.startsWith(u8, request.head.target, "/echo-content"));
194 try expectEqualStrings("Hello, World!\n", body);
195 try expectEqualStrings("text/plain", request.head.content_type.?);
196
197 var send_buffer: [100]u8 = undefined;
198 var response = request.respondStreaming(.{
199 .send_buffer = &send_buffer,
200 .content_length = switch (request.head.transfer_encoding) {
201 .chunked => null,
202 .none => len: {
203 try expectEqual(14, request.head.content_length.?);
204 break :len 14;
205 },
206 },
207 });
208
209 try response.flush(); // Test an early flush to send the HTTP headers before the body.
210 const w = response.writer();
211 try w.writeAll("Hello, ");
212 try w.writeAll("World!\n");
213 try response.end();
214 //std.debug.print(" server finished responding\n", .{});
215 }
216 });
217 defer test_server.destroy();
218
219 {
220 var client: http.Client = .{ .allocator = std.testing.allocator };
221 defer client.deinit();
222
223 try echoTests(&client, test_server.port());
224 }
225}
226
227test "Server.Request.respondStreaming non-chunked, unknown content-length" {
228 // In this case, the response is expected to stream until the connection is
229 // closed, indicating the end of the body.
230 const test_server = try createTestServer(struct {
231 fn run(net_server: *std.net.Server) anyerror!void {
232 var header_buffer: [1000]u8 = undefined;
233 var remaining: usize = 1;
234 while (remaining != 0) : (remaining -= 1) {
235 const conn = try net_server.accept();
236 defer conn.stream.close();
237
238 var server = http.Server.init(conn, &header_buffer);
239
240 try expectEqual(.ready, server.state);
241 var request = try server.receiveHead();
242 try expectEqualStrings(request.head.target, "/foo");
243 var send_buffer: [500]u8 = undefined;
244 var response = request.respondStreaming(.{
245 .send_buffer = &send_buffer,
246 .respond_options = .{
247 .transfer_encoding = .none,
248 },
249 });
250 var total: usize = 0;
251 for (0..500) |i| {
252 var buf: [30]u8 = undefined;
253 const line = try std.fmt.bufPrint(&buf, "{d}, ah ha ha!\n", .{i});
254 try response.writeAll(line);
255 total += line.len;
256 }
257 try expectEqual(7390, total);
258 try response.end();
259 try expectEqual(.closing, server.state);
260 }
261 }
262 });
263 defer test_server.destroy();
264
265 const request_bytes = "GET /foo HTTP/1.1\r\n\r\n";
266 const gpa = std.testing.allocator;
267 const stream = try std.net.tcpConnectToHost(gpa, "127.0.0.1", test_server.port());
268 defer stream.close();
269 try stream.writeAll(request_bytes);
270
271 const response = try stream.reader().readAllAlloc(gpa, 8192);
272 defer gpa.free(response);
273
274 var expected_response = std.ArrayList(u8).init(gpa);
275 defer expected_response.deinit();
276
277 try expected_response.appendSlice("HTTP/1.1 200 OK\r\n\r\n");
278
279 {
280 var total: usize = 0;
281 for (0..500) |i| {
282 var buf: [30]u8 = undefined;
283 const line = try std.fmt.bufPrint(&buf, "{d}, ah ha ha!\n", .{i});
284 try expected_response.appendSlice(line);
285 total += line.len;
286 }
287 try expectEqual(7390, total);
288 }
289
290 try expectEqualStrings(expected_response.items, response);
291}
292
293test "receiving arbitrary http headers from the client" {
294 const test_server = try createTestServer(struct {
295 fn run(net_server: *std.net.Server) anyerror!void {
296 var read_buffer: [666]u8 = undefined;
297 var remaining: usize = 1;
298 while (remaining != 0) : (remaining -= 1) {
299 const conn = try net_server.accept();
300 defer conn.stream.close();
301
302 var server = http.Server.init(conn, &read_buffer);
303 try expectEqual(.ready, server.state);
304 var request = try server.receiveHead();
305 try expectEqualStrings("/bar", request.head.target);
306 var it = request.iterateHeaders();
307 {
308 const header = it.next().?;
309 try expectEqualStrings("CoNneCtIoN", header.name);
310 try expectEqualStrings("close", header.value);
311 try expect(!it.is_trailer);
312 }
313 {
314 const header = it.next().?;
315 try expectEqualStrings("aoeu", header.name);
316 try expectEqualStrings("asdf", header.value);
317 try expect(!it.is_trailer);
318 }
319 try request.respond("", .{});
320 }
321 }
322 });
323 defer test_server.destroy();
324
325 const request_bytes = "GET /bar HTTP/1.1\r\n" ++
326 "CoNneCtIoN: close\r\n" ++
327 "aoeu: asdf\r\n" ++
328 "\r\n";
329 const gpa = std.testing.allocator;
330 const stream = try std.net.tcpConnectToHost(gpa, "127.0.0.1", test_server.port());
331 defer stream.close();
332 try stream.writeAll(request_bytes);
333
334 const response = try stream.reader().readAllAlloc(gpa, 8192);
335 defer gpa.free(response);
336
337 var expected_response = std.ArrayList(u8).init(gpa);
338 defer expected_response.deinit();
339
340 try expected_response.appendSlice("HTTP/1.1 200 OK\r\n");
341 try expected_response.appendSlice("content-length: 0\r\n\r\n");
342 try expectEqualStrings(expected_response.items, response);
343}
344
345test "general client/server API coverage" {
346 if (builtin.os.tag == .windows) {
347 // This test was never passing on Windows.
348 return error.SkipZigTest;
349 }
350
351 const global = struct {
352 var handle_new_requests = true;
353 };
354 const test_server = try createTestServer(struct {
355 fn run(net_server: *std.net.Server) anyerror!void {
356 var client_header_buffer: [1024]u8 = undefined;
357 outer: while (global.handle_new_requests) {
358 var connection = try net_server.accept();
359 defer connection.stream.close();
360
361 var http_server = http.Server.init(connection, &client_header_buffer);
362
363 while (http_server.state == .ready) {
364 var request = http_server.receiveHead() catch |err| switch (err) {
365 error.HttpConnectionClosing => continue :outer,
366 else => |e| return e,
367 };
368
369 try handleRequest(&request, net_server.listen_address.getPort());
370 }
371 }
372 }
373
374 fn handleRequest(request: *http.Server.Request, listen_port: u16) !void {
375 const log = std.log.scoped(.server);
376
377 log.info("{} {s} {s}", .{
378 request.head.method,
379 @tagName(request.head.version),
380 request.head.target,
381 });
382
383 const gpa = std.testing.allocator;
384 const body = try (try request.reader()).readAllAlloc(gpa, 8192);
385 defer gpa.free(body);
386
387 var send_buffer: [100]u8 = undefined;
388
389 if (mem.startsWith(u8, request.head.target, "/get")) {
390 var response = request.respondStreaming(.{
391 .send_buffer = &send_buffer,
392 .content_length = if (mem.indexOf(u8, request.head.target, "?chunked") == null)
393 14
394 else
395 null,
396 .respond_options = .{
397 .extra_headers = &.{
398 .{ .name = "content-type", .value = "text/plain" },
399 },
400 },
401 });
402 const w = response.writer();
403 try w.writeAll("Hello, ");
404 try w.writeAll("World!\n");
405 try response.end();
406 // Writing again would cause an assertion failure.
407 } else if (mem.startsWith(u8, request.head.target, "/large")) {
408 var response = request.respondStreaming(.{
409 .send_buffer = &send_buffer,
410 .content_length = 14 * 1024 + 14 * 10,
411 });
412
413 try response.flush(); // Test an early flush to send the HTTP headers before the body.
414
415 const w = response.writer();
416
417 var i: u32 = 0;
418 while (i < 5) : (i += 1) {
419 try w.writeAll("Hello, World!\n");
420 }
421
422 try w.writeAll("Hello, World!\n" ** 1024);
423
424 i = 0;
425 while (i < 5) : (i += 1) {
426 try w.writeAll("Hello, World!\n");
427 }
428
429 try response.end();
430 } else if (mem.eql(u8, request.head.target, "/redirect/1")) {
431 var response = request.respondStreaming(.{
432 .send_buffer = &send_buffer,
433 .respond_options = .{
434 .status = .found,
435 .extra_headers = &.{
436 .{ .name = "location", .value = "../../get" },
437 },
438 },
439 });
440
441 const w = response.writer();
442 try w.writeAll("Hello, ");
443 try w.writeAll("Redirected!\n");
444 try response.end();
445 } else if (mem.eql(u8, request.head.target, "/redirect/2")) {
446 try request.respond("Hello, Redirected!\n", .{
447 .status = .found,
448 .extra_headers = &.{
449 .{ .name = "location", .value = "/redirect/1" },
450 },
451 });
452 } else if (mem.eql(u8, request.head.target, "/redirect/3")) {
453 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/redirect/2", .{
454 listen_port,
455 });
456 defer gpa.free(location);
457
458 try request.respond("Hello, Redirected!\n", .{
459 .status = .found,
460 .extra_headers = &.{
461 .{ .name = "location", .value = location },
462 },
463 });
464 } else if (mem.eql(u8, request.head.target, "/redirect/4")) {
465 try request.respond("Hello, Redirected!\n", .{
466 .status = .found,
467 .extra_headers = &.{
468 .{ .name = "location", .value = "/redirect/3" },
469 },
470 });
471 } else if (mem.eql(u8, request.head.target, "/redirect/invalid")) {
472 const invalid_port = try getUnusedTcpPort();
473 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}", .{invalid_port});
474 defer gpa.free(location);
475
476 try request.respond("", .{
477 .status = .found,
478 .extra_headers = &.{
479 .{ .name = "location", .value = location },
480 },
481 });
482 } else {
483 try request.respond("", .{ .status = .not_found });
484 }
485 }
486
487 fn getUnusedTcpPort() !u16 {
488 const addr = try std.net.Address.parseIp("127.0.0.1", 0);
489 var s = try addr.listen(.{});
490 defer s.deinit();
491 return s.listen_address.in.getPort();
492 }
493 });
494 defer test_server.destroy();
495
496 const log = std.log.scoped(.client);
497
498 const gpa = std.testing.allocator;
499 var client: http.Client = .{ .allocator = gpa };
500 errdefer client.deinit();
501 // defer client.deinit(); handled below
502
503 const port = test_server.port();
504
505 { // read content-length response
506 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/get", .{port});
507 defer gpa.free(location);
508 const uri = try std.Uri.parse(location);
509
510 log.info("{s}", .{location});
511 var server_header_buffer: [1024]u8 = undefined;
512 var req = try client.open(.GET, uri, .{
513 .server_header_buffer = &server_header_buffer,
514 });
515 defer req.deinit();
516
517 try req.send(.{});
518 try req.wait();
519
520 const body = try req.reader().readAllAlloc(gpa, 8192);
521 defer gpa.free(body);
522
523 try expectEqualStrings("Hello, World!\n", body);
524 try expectEqualStrings("text/plain", req.response.content_type.?);
525 }
526
527 // connection has been kept alive
528 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
529
530 { // read large content-length response
531 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/large", .{port});
532 defer gpa.free(location);
533 const uri = try std.Uri.parse(location);
534
535 log.info("{s}", .{location});
536 var server_header_buffer: [1024]u8 = undefined;
537 var req = try client.open(.GET, uri, .{
538 .server_header_buffer = &server_header_buffer,
539 });
540 defer req.deinit();
541
542 try req.send(.{});
543 try req.wait();
544
545 const body = try req.reader().readAllAlloc(gpa, 8192 * 1024);
546 defer gpa.free(body);
547
548 try expectEqual(@as(usize, 14 * 1024 + 14 * 10), body.len);
549 }
550
551 // connection has been kept alive
552 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
553
554 { // send head request and not read chunked
555 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/get", .{port});
556 defer gpa.free(location);
557 const uri = try std.Uri.parse(location);
558
559 log.info("{s}", .{location});
560 var server_header_buffer: [1024]u8 = undefined;
561 var req = try client.open(.HEAD, uri, .{
562 .server_header_buffer = &server_header_buffer,
563 });
564 defer req.deinit();
565
566 try req.send(.{});
567 try req.wait();
568
569 const body = try req.reader().readAllAlloc(gpa, 8192);
570 defer gpa.free(body);
571
572 try expectEqualStrings("", body);
573 try expectEqualStrings("text/plain", req.response.content_type.?);
574 try expectEqual(14, req.response.content_length.?);
575 }
576
577 // connection has been kept alive
578 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
579
580 { // read chunked response
581 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/get?chunked", .{port});
582 defer gpa.free(location);
583 const uri = try std.Uri.parse(location);
584
585 log.info("{s}", .{location});
586 var server_header_buffer: [1024]u8 = undefined;
587 var req = try client.open(.GET, uri, .{
588 .server_header_buffer = &server_header_buffer,
589 });
590 defer req.deinit();
591
592 try req.send(.{});
593 try req.wait();
594
595 const body = try req.reader().readAllAlloc(gpa, 8192);
596 defer gpa.free(body);
597
598 try expectEqualStrings("Hello, World!\n", body);
599 try expectEqualStrings("text/plain", req.response.content_type.?);
600 }
601
602 // connection has been kept alive
603 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
604
605 { // send head request and not read chunked
606 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/get?chunked", .{port});
607 defer gpa.free(location);
608 const uri = try std.Uri.parse(location);
609
610 log.info("{s}", .{location});
611 var server_header_buffer: [1024]u8 = undefined;
612 var req = try client.open(.HEAD, uri, .{
613 .server_header_buffer = &server_header_buffer,
614 });
615 defer req.deinit();
616
617 try req.send(.{});
618 try req.wait();
619
620 const body = try req.reader().readAllAlloc(gpa, 8192);
621 defer gpa.free(body);
622
623 try expectEqualStrings("", body);
624 try expectEqualStrings("text/plain", req.response.content_type.?);
625 try expect(req.response.transfer_encoding == .chunked);
626 }
627
628 // connection has been kept alive
629 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
630
631 { // read content-length response with connection close
632 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/get", .{port});
633 defer gpa.free(location);
634 const uri = try std.Uri.parse(location);
635
636 log.info("{s}", .{location});
637 var server_header_buffer: [1024]u8 = undefined;
638 var req = try client.open(.GET, uri, .{
639 .server_header_buffer = &server_header_buffer,
640 .keep_alive = false,
641 });
642 defer req.deinit();
643
644 try req.send(.{});
645 try req.wait();
646
647 const body = try req.reader().readAllAlloc(gpa, 8192);
648 defer gpa.free(body);
649
650 try expectEqualStrings("Hello, World!\n", body);
651 try expectEqualStrings("text/plain", req.response.content_type.?);
652 }
653
654 // connection has been closed
655 try expect(client.connection_pool.free_len == 0);
656
657 { // relative redirect
658 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/redirect/1", .{port});
659 defer gpa.free(location);
660 const uri = try std.Uri.parse(location);
661
662 log.info("{s}", .{location});
663 var server_header_buffer: [1024]u8 = undefined;
664 var req = try client.open(.GET, uri, .{
665 .server_header_buffer = &server_header_buffer,
666 });
667 defer req.deinit();
668
669 try req.send(.{});
670 try req.wait();
671
672 const body = try req.reader().readAllAlloc(gpa, 8192);
673 defer gpa.free(body);
674
675 try expectEqualStrings("Hello, World!\n", body);
676 }
677
678 // connection has been kept alive
679 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
680
681 { // redirect from root
682 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/redirect/2", .{port});
683 defer gpa.free(location);
684 const uri = try std.Uri.parse(location);
685
686 log.info("{s}", .{location});
687 var server_header_buffer: [1024]u8 = undefined;
688 var req = try client.open(.GET, uri, .{
689 .server_header_buffer = &server_header_buffer,
690 });
691 defer req.deinit();
692
693 try req.send(.{});
694 try req.wait();
695
696 const body = try req.reader().readAllAlloc(gpa, 8192);
697 defer gpa.free(body);
698
699 try expectEqualStrings("Hello, World!\n", body);
700 }
701
702 // connection has been kept alive
703 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
704
705 { // absolute redirect
706 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/redirect/3", .{port});
707 defer gpa.free(location);
708 const uri = try std.Uri.parse(location);
709
710 log.info("{s}", .{location});
711 var server_header_buffer: [1024]u8 = undefined;
712 var req = try client.open(.GET, uri, .{
713 .server_header_buffer = &server_header_buffer,
714 });
715 defer req.deinit();
716
717 try req.send(.{});
718 try req.wait();
719
720 const body = try req.reader().readAllAlloc(gpa, 8192);
721 defer gpa.free(body);
722
723 try expectEqualStrings("Hello, World!\n", body);
724 }
725
726 // connection has been kept alive
727 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
728
729 { // too many redirects
730 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/redirect/4", .{port});
731 defer gpa.free(location);
732 const uri = try std.Uri.parse(location);
733
734 log.info("{s}", .{location});
735 var server_header_buffer: [1024]u8 = undefined;
736 var req = try client.open(.GET, uri, .{
737 .server_header_buffer = &server_header_buffer,
738 });
739 defer req.deinit();
740
741 try req.send(.{});
742 req.wait() catch |err| switch (err) {
743 error.TooManyHttpRedirects => {},
744 else => return err,
745 };
746 }
747
748 // connection has been kept alive
749 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
750
751 { // check client without segfault by connection error after redirection
752 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/redirect/invalid", .{port});
753 defer gpa.free(location);
754 const uri = try std.Uri.parse(location);
755
756 log.info("{s}", .{location});
757 var server_header_buffer: [1024]u8 = undefined;
758 var req = try client.open(.GET, uri, .{
759 .server_header_buffer = &server_header_buffer,
760 });
761 defer req.deinit();
762
763 try req.send(.{});
764 const result = req.wait();
765
766 // a proxy without an upstream is likely to return a 5xx status.
767 if (client.http_proxy == null) {
768 try expectError(error.ConnectionRefused, result); // expects not segfault but the regular error
769 }
770 }
771
772 // connection has been kept alive
773 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
774
775 { // issue 16282 *** This test leaves the client in an invalid state, it must be last ***
776 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/get", .{port});
777 defer gpa.free(location);
778 const uri = try std.Uri.parse(location);
779
780 const total_connections = client.connection_pool.free_size + 64;
781 var requests = try gpa.alloc(http.Client.Request, total_connections);
782 defer gpa.free(requests);
783
784 var header_bufs = std.ArrayList([]u8).init(gpa);
785 defer header_bufs.deinit();
786 defer for (header_bufs.items) |item| gpa.free(item);
787
788 for (0..total_connections) |i| {
789 const headers_buf = try gpa.alloc(u8, 1024);
790 try header_bufs.append(headers_buf);
791 var req = try client.open(.GET, uri, .{
792 .server_header_buffer = headers_buf,
793 });
794 req.response.parser.done = true;
795 req.connection.?.closing = false;
796 requests[i] = req;
797 }
798
799 for (0..total_connections) |i| {
800 requests[i].deinit();
801 }
802
803 // free connections should be full now
804 try expect(client.connection_pool.free_len == client.connection_pool.free_size);
805 }
806
807 client.deinit();
808
809 {
810 global.handle_new_requests = false;
811
812 const conn = try std.net.tcpConnectToAddress(test_server.net_server.listen_address);
813 conn.close();
814 }
815}
816
817fn echoTests(client: *http.Client, port: u16) !void {
818 const gpa = std.testing.allocator;
819 var location_buffer: [100]u8 = undefined;
820
821 { // send content-length request
822 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content", .{port});
823 defer gpa.free(location);
824 const uri = try std.Uri.parse(location);
825
826 var server_header_buffer: [1024]u8 = undefined;
827 var req = try client.open(.POST, uri, .{
828 .server_header_buffer = &server_header_buffer,
829 .extra_headers = &.{
830 .{ .name = "content-type", .value = "text/plain" },
831 },
832 });
833 defer req.deinit();
834
835 req.transfer_encoding = .{ .content_length = 14 };
836
837 try req.send(.{});
838 try req.writeAll("Hello, ");
839 try req.writeAll("World!\n");
840 try req.finish();
841
842 try req.wait();
843
844 const body = try req.reader().readAllAlloc(gpa, 8192);
845 defer gpa.free(body);
846
847 try expectEqualStrings("Hello, World!\n", body);
848 }
849
850 // connection has been kept alive
851 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
852
853 { // send chunked request
854 const uri = try std.Uri.parse(try std.fmt.bufPrint(
855 &location_buffer,
856 "http://127.0.0.1:{d}/echo-content",
857 .{port},
858 ));
859
860 var server_header_buffer: [1024]u8 = undefined;
861 var req = try client.open(.POST, uri, .{
862 .server_header_buffer = &server_header_buffer,
863 .extra_headers = &.{
864 .{ .name = "content-type", .value = "text/plain" },
865 },
866 });
867 defer req.deinit();
868
869 req.transfer_encoding = .chunked;
870
871 try req.send(.{});
872 try req.writeAll("Hello, ");
873 try req.writeAll("World!\n");
874 try req.finish();
875
876 try req.wait();
877
878 const body = try req.reader().readAllAlloc(gpa, 8192);
879 defer gpa.free(body);
880
881 try expectEqualStrings("Hello, World!\n", body);
882 }
883
884 // connection has been kept alive
885 try expect(client.http_proxy != null or client.connection_pool.free_len == 1);
886
887 { // Client.fetch()
888
889 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
890 defer gpa.free(location);
891
892 var body = std.ArrayList(u8).init(gpa);
893 defer body.deinit();
894
895 const res = try client.fetch(.{
896 .location = .{ .url = location },
897 .method = .POST,
898 .payload = "Hello, World!\n",
899 .extra_headers = &.{
900 .{ .name = "content-type", .value = "text/plain" },
901 },
902 .response_storage = .{ .dynamic = &body },
903 });
904 try expectEqual(.ok, res.status);
905 try expectEqualStrings("Hello, World!\n", body.items);
906 }
907
908 { // expect: 100-continue
909 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#expect-100", .{port});
910 defer gpa.free(location);
911 const uri = try std.Uri.parse(location);
912
913 var server_header_buffer: [1024]u8 = undefined;
914 var req = try client.open(.POST, uri, .{
915 .server_header_buffer = &server_header_buffer,
916 .extra_headers = &.{
917 .{ .name = "expect", .value = "100-continue" },
918 .{ .name = "content-type", .value = "text/plain" },
919 },
920 });
921 defer req.deinit();
922
923 req.transfer_encoding = .chunked;
924
925 try req.send(.{});
926 try req.writeAll("Hello, ");
927 try req.writeAll("World!\n");
928 try req.finish();
929
930 try req.wait();
931 try expectEqual(.ok, req.response.status);
932
933 const body = try req.reader().readAllAlloc(gpa, 8192);
934 defer gpa.free(body);
935
936 try expectEqualStrings("Hello, World!\n", body);
937 }
938
939 { // expect: garbage
940 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#expect-garbage", .{port});
941 defer gpa.free(location);
942 const uri = try std.Uri.parse(location);
943
944 var server_header_buffer: [1024]u8 = undefined;
945 var req = try client.open(.POST, uri, .{
946 .server_header_buffer = &server_header_buffer,
947 .extra_headers = &.{
948 .{ .name = "content-type", .value = "text/plain" },
949 .{ .name = "expect", .value = "garbage" },
950 },
951 });
952 defer req.deinit();
953
954 req.transfer_encoding = .chunked;
955
956 try req.send(.{});
957 try req.wait();
958 try expectEqual(.expectation_failed, req.response.status);
959 }
960
961 _ = try client.fetch(.{
962 .location = .{
963 .url = try std.fmt.bufPrint(&location_buffer, "http://127.0.0.1:{d}/end", .{port}),
964 },
965 });
966}
967
968const TestServer = struct {
969 server_thread: std.Thread,
970 net_server: std.net.Server,
971
972 fn destroy(self: *@This()) void {
973 self.server_thread.join();
974 self.net_server.deinit();
975 std.testing.allocator.destroy(self);
976 }
977
978 fn port(self: @This()) u16 {
979 return self.net_server.listen_address.in.getPort();
980 }
981};
982
983fn createTestServer(S: type) !*TestServer {
984 if (builtin.single_threaded) return error.SkipZigTest;
985 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
986 // https://github.com/ziglang/zig/issues/13782
987 return error.SkipZigTest;
988 }
989
990 const address = try std.net.Address.parseIp("127.0.0.1", 0);
991 const test_server = try std.testing.allocator.create(TestServer);
992 test_server.net_server = try address.listen(.{ .reuse_address = true });
993 test_server.server_thread = try std.Thread.spawn(.{}, S.run, .{&test_server.net_server});
994 return test_server;
995}
lib/std/io/Reader.zig+12
...@@ -360,6 +360,18 @@ pub fn readEnum(self: Self, comptime Enum: type, endian: std.builtin.Endian) any...@@ -360,6 +360,18 @@ pub fn readEnum(self: Self, comptime Enum: type, endian: std.builtin.Endian) any
360 return E.InvalidValue;360 return E.InvalidValue;
361}361}
362362
363/// Reads the stream until the end, ignoring all the data.
364/// Returns the number of bytes discarded.
365pub fn discard(self: Self) anyerror!u64 {
366 var trash: [4096]u8 = undefined;
367 var index: u64 = 0;
368 while (true) {
369 const n = try self.read(&trash);
370 if (n == 0) return index;
371 index += n;
372 }
373}
374
363const std = @import("../std.zig");375const std = @import("../std.zig");
364const Self = @This();376const Self = @This();
365const math = std.math;377const math = std.math;
lib/std/mem.zig+2-2
...@@ -1338,7 +1338,7 @@ pub fn indexOf(comptime T: type, haystack: []const T, needle: []const T) ?usize...@@ -1338,7 +1338,7 @@ pub fn indexOf(comptime T: type, haystack: []const T, needle: []const T) ?usize
1338pub fn lastIndexOfLinear(comptime T: type, haystack: []const T, needle: []const T) ?usize {1338pub fn lastIndexOfLinear(comptime T: type, haystack: []const T, needle: []const T) ?usize {
1339 var i: usize = haystack.len - needle.len;1339 var i: usize = haystack.len - needle.len;
1340 while (true) : (i -= 1) {1340 while (true) : (i -= 1) {
1341 if (mem.eql(T, haystack[i .. i + needle.len], needle)) return i;1341 if (mem.eql(T, haystack[i..][0..needle.len], needle)) return i;
1342 if (i == 0) return null;1342 if (i == 0) return null;
1343 }1343 }
1344}1344}
...@@ -1349,7 +1349,7 @@ pub fn indexOfPosLinear(comptime T: type, haystack: []const T, start_index: usiz...@@ -1349,7 +1349,7 @@ pub fn indexOfPosLinear(comptime T: type, haystack: []const T, start_index: usiz
1349 var i: usize = start_index;1349 var i: usize = start_index;
1350 const end = haystack.len - needle.len;1350 const end = haystack.len - needle.len;
1351 while (i <= end) : (i += 1) {1351 while (i <= end) : (i += 1) {
1352 if (eql(T, haystack[i .. i + needle.len], needle)) return i;1352 if (eql(T, haystack[i..][0..needle.len], needle)) return i;
1353 }1353 }
1354 return null;1354 return null;
1355}1355}
lib/std/net.zig+95-155
...@@ -4,15 +4,17 @@ const assert = std.debug.assert;...@@ -4,15 +4,17 @@ const assert = std.debug.assert;
4const net = @This();4const net = @This();
5const mem = std.mem;5const mem = std.mem;
6const os = std.os;6const os = std.os;
7const posix = std.posix;
7const fs = std.fs;8const fs = std.fs;
8const io = std.io;9const io = std.io;
9const native_endian = builtin.target.cpu.arch.endian();10const native_endian = builtin.target.cpu.arch.endian();
1011
11// Windows 10 added support for unix sockets in build 17063, redstone 4 is the12// Windows 10 added support for unix sockets in build 17063, redstone 4 is the
12// first release to support them.13// first release to support them.
13pub const has_unix_sockets = @hasDecl(os.sockaddr, "un") and14pub const has_unix_sockets = switch (builtin.os.tag) {
14 (builtin.target.os.tag != .windows or15 .windows => builtin.os.version_range.windows.isAtLeast(.win10_rs4) orelse false,
15 builtin.os.version_range.windows.isAtLeast(.win10_rs4) orelse false);16 else => true,
17};
1618
17pub const IPParseError = error{19pub const IPParseError = error{
18 Overflow,20 Overflow,
...@@ -122,7 +124,7 @@ pub const Address = extern union {...@@ -122,7 +124,7 @@ pub const Address = extern union {
122 @memset(&sock_addr.path, 0);124 @memset(&sock_addr.path, 0);
123 @memcpy(sock_addr.path[0..path.len], path);125 @memcpy(sock_addr.path[0..path.len], path);
124126
125 return Address{ .un = sock_addr };127 return .{ .un = sock_addr };
126 }128 }
127129
128 /// Returns the port in native endian.130 /// Returns the port in native endian.
...@@ -206,6 +208,60 @@ pub const Address = extern union {...@@ -206,6 +208,60 @@ pub const Address = extern union {
206 else => unreachable,208 else => unreachable,
207 }209 }
208 }210 }
211
212 pub const ListenError = posix.SocketError || posix.BindError || posix.ListenError ||
213 posix.SetSockOptError || posix.GetSockNameError;
214
215 pub const ListenOptions = struct {
216 /// How many connections the kernel will accept on the application's behalf.
217 /// If more than this many connections pool in the kernel, clients will start
218 /// seeing "Connection refused".
219 kernel_backlog: u31 = 128,
220 /// Sets SO_REUSEADDR and SO_REUSEPORT on POSIX.
221 /// Sets SO_REUSEADDR on Windows, which is roughly equivalent.
222 reuse_address: bool = false,
223 /// Deprecated. Does the same thing as reuse_address.
224 reuse_port: bool = false,
225 force_nonblocking: bool = false,
226 };
227
228 /// The returned `Server` has an open `stream`.
229 pub fn listen(address: Address, options: ListenOptions) ListenError!Server {
230 const nonblock: u32 = if (options.force_nonblocking) posix.SOCK.NONBLOCK else 0;
231 const sock_flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | nonblock;
232 const proto: u32 = if (address.any.family == posix.AF.UNIX) 0 else posix.IPPROTO.TCP;
233
234 const sockfd = try posix.socket(address.any.family, sock_flags, proto);
235 var s: Server = .{
236 .listen_address = undefined,
237 .stream = .{ .handle = sockfd },
238 };
239 errdefer s.stream.close();
240
241 if (options.reuse_address or options.reuse_port) {
242 try posix.setsockopt(
243 sockfd,
244 posix.SOL.SOCKET,
245 posix.SO.REUSEADDR,
246 &mem.toBytes(@as(c_int, 1)),
247 );
248 switch (builtin.os.tag) {
249 .windows => {},
250 else => try posix.setsockopt(
251 sockfd,
252 posix.SOL.SOCKET,
253 posix.SO.REUSEPORT,
254 &mem.toBytes(@as(c_int, 1)),
255 ),
256 }
257 }
258
259 var socklen = address.getOsSockLen();
260 try posix.bind(sockfd, &address.any, socklen);
261 try posix.listen(sockfd, options.kernel_backlog);
262 try posix.getsockname(sockfd, &s.listen_address.any, &socklen);
263 return s;
264 }
209};265};
210266
211pub const Ip4Address = extern struct {267pub const Ip4Address = extern struct {
...@@ -657,7 +713,7 @@ pub fn connectUnixSocket(path: []const u8) !Stream {...@@ -657,7 +713,7 @@ pub fn connectUnixSocket(path: []const u8) !Stream {
657 os.SOCK.STREAM | os.SOCK.CLOEXEC | opt_non_block,713 os.SOCK.STREAM | os.SOCK.CLOEXEC | opt_non_block,
658 0,714 0,
659 );715 );
660 errdefer os.closeSocket(sockfd);716 errdefer Stream.close(.{ .handle = sockfd });
661717
662 var addr = try std.net.Address.initUnix(path);718 var addr = try std.net.Address.initUnix(path);
663 try os.connect(sockfd, &addr.any, addr.getOsSockLen());719 try os.connect(sockfd, &addr.any, addr.getOsSockLen());
...@@ -669,7 +725,7 @@ fn if_nametoindex(name: []const u8) IPv6InterfaceError!u32 {...@@ -669,7 +725,7 @@ fn if_nametoindex(name: []const u8) IPv6InterfaceError!u32 {
669 if (builtin.target.os.tag == .linux) {725 if (builtin.target.os.tag == .linux) {
670 var ifr: os.ifreq = undefined;726 var ifr: os.ifreq = undefined;
671 const sockfd = try os.socket(os.AF.UNIX, os.SOCK.DGRAM | os.SOCK.CLOEXEC, 0);727 const sockfd = try os.socket(os.AF.UNIX, os.SOCK.DGRAM | os.SOCK.CLOEXEC, 0);
672 defer os.closeSocket(sockfd);728 defer Stream.close(.{ .handle = sockfd });
673729
674 @memcpy(ifr.ifrn.name[0..name.len], name);730 @memcpy(ifr.ifrn.name[0..name.len], name);
675 ifr.ifrn.name[name.len] = 0;731 ifr.ifrn.name[name.len] = 0;
...@@ -738,7 +794,7 @@ pub fn tcpConnectToAddress(address: Address) TcpConnectToAddressError!Stream {...@@ -738,7 +794,7 @@ pub fn tcpConnectToAddress(address: Address) TcpConnectToAddressError!Stream {
738 const sock_flags = os.SOCK.STREAM | nonblock |794 const sock_flags = os.SOCK.STREAM | nonblock |
739 (if (builtin.target.os.tag == .windows) 0 else os.SOCK.CLOEXEC);795 (if (builtin.target.os.tag == .windows) 0 else os.SOCK.CLOEXEC);
740 const sockfd = try os.socket(address.any.family, sock_flags, os.IPPROTO.TCP);796 const sockfd = try os.socket(address.any.family, sock_flags, os.IPPROTO.TCP);
741 errdefer os.closeSocket(sockfd);797 errdefer Stream.close(.{ .handle = sockfd });
742798
743 try os.connect(sockfd, &address.any, address.getOsSockLen());799 try os.connect(sockfd, &address.any, address.getOsSockLen());
744800
...@@ -1068,7 +1124,7 @@ fn linuxLookupName(...@@ -1068,7 +1124,7 @@ fn linuxLookupName(
1068 var prefixlen: i32 = 0;1124 var prefixlen: i32 = 0;
1069 const sock_flags = os.SOCK.DGRAM | os.SOCK.CLOEXEC;1125 const sock_flags = os.SOCK.DGRAM | os.SOCK.CLOEXEC;
1070 if (os.socket(addr.addr.any.family, sock_flags, os.IPPROTO.UDP)) |fd| syscalls: {1126 if (os.socket(addr.addr.any.family, sock_flags, os.IPPROTO.UDP)) |fd| syscalls: {
1071 defer os.closeSocket(fd);1127 defer Stream.close(.{ .handle = fd });
1072 os.connect(fd, da, dalen) catch break :syscalls;1128 os.connect(fd, da, dalen) catch break :syscalls;
1073 key |= DAS_USABLE;1129 key |= DAS_USABLE;
1074 os.getsockname(fd, sa, &salen) catch break :syscalls;1130 os.getsockname(fd, sa, &salen) catch break :syscalls;
...@@ -1553,7 +1609,7 @@ fn resMSendRc(...@@ -1553,7 +1609,7 @@ fn resMSendRc(
1553 },1609 },
1554 else => |e| return e,1610 else => |e| return e,
1555 };1611 };
1556 defer os.closeSocket(fd);1612 defer Stream.close(.{ .handle = fd });
15571613
1558 // Past this point, there are no errors. Each individual query will1614 // Past this point, there are no errors. Each individual query will
1559 // yield either no reply (indicated by zero length) or an answer1615 // yield either no reply (indicated by zero length) or an answer
...@@ -1729,13 +1785,15 @@ fn dnsParseCallback(ctx: dpc_ctx, rr: u8, data: []const u8, packet: []const u8)...@@ -1729,13 +1785,15 @@ fn dnsParseCallback(ctx: dpc_ctx, rr: u8, data: []const u8, packet: []const u8)
1729}1785}
17301786
1731pub const Stream = struct {1787pub const Stream = struct {
1732 // Underlying socket descriptor.1788 /// Underlying platform-defined type which may or may not be
1733 // Note that on some platforms this may not be interchangeable with a1789 /// interchangeable with a file system file descriptor.
1734 // regular files descriptor.1790 handle: posix.socket_t,
1735 handle: os.socket_t,1791
17361792 pub fn close(s: Stream) void {
1737 pub fn close(self: Stream) void {1793 switch (builtin.os.tag) {
1738 os.closeSocket(self.handle);1794 .windows => std.os.windows.closesocket(s.handle) catch unreachable,
1795 else => posix.close(s.handle),
1796 }
1739 }1797 }
17401798
1741 pub const ReadError = os.ReadError;1799 pub const ReadError = os.ReadError;
...@@ -1839,156 +1897,38 @@ pub const Stream = struct {...@@ -1839,156 +1897,38 @@ pub const Stream = struct {
1839 }1897 }
1840};1898};
18411899
1842pub const StreamServer = struct {1900pub const Server = struct {
1843 /// Copied from `Options` on `init`.
1844 kernel_backlog: u31,
1845 reuse_address: bool,
1846 reuse_port: bool,
1847 force_nonblocking: bool,
1848
1849 /// `undefined` until `listen` returns successfully.
1850 listen_address: Address,1901 listen_address: Address,
1902 stream: std.net.Stream,
18511903
1852 sockfd: ?os.socket_t,1904 pub const Connection = struct {
18531905 stream: std.net.Stream,
1854 pub const Options = struct {1906 address: Address,
1855 /// How many connections the kernel will accept on the application's behalf.
1856 /// If more than this many connections pool in the kernel, clients will start
1857 /// seeing "Connection refused".
1858 kernel_backlog: u31 = 128,
1859
1860 /// Enable SO.REUSEADDR on the socket.
1861 reuse_address: bool = false,
1862
1863 /// Enable SO.REUSEPORT on the socket.
1864 reuse_port: bool = false,
1865
1866 /// Force non-blocking mode.
1867 force_nonblocking: bool = false,
1868 };1907 };
18691908
1870 /// After this call succeeds, resources have been acquired and must1909 pub fn deinit(s: *Server) void {
1871 /// be released with `deinit`.1910 s.stream.close();
1872 pub fn init(options: Options) StreamServer {1911 s.* = undefined;
1873 return StreamServer{
1874 .sockfd = null,
1875 .kernel_backlog = options.kernel_backlog,
1876 .reuse_address = options.reuse_address,
1877 .reuse_port = options.reuse_port,
1878 .force_nonblocking = options.force_nonblocking,
1879 .listen_address = undefined,
1880 };
1881 }
1882
1883 /// Release all resources. The `StreamServer` memory becomes `undefined`.
1884 pub fn deinit(self: *StreamServer) void {
1885 self.close();
1886 self.* = undefined;
1887 }
1888
1889 pub fn listen(self: *StreamServer, address: Address) !void {
1890 const nonblock = 0;
1891 const sock_flags = os.SOCK.STREAM | os.SOCK.CLOEXEC | nonblock;
1892 var use_sock_flags: u32 = sock_flags;
1893 if (self.force_nonblocking) use_sock_flags |= os.SOCK.NONBLOCK;
1894 const proto = if (address.any.family == os.AF.UNIX) @as(u32, 0) else os.IPPROTO.TCP;
1895
1896 const sockfd = try os.socket(address.any.family, use_sock_flags, proto);
1897 self.sockfd = sockfd;
1898 errdefer {
1899 os.closeSocket(sockfd);
1900 self.sockfd = null;
1901 }
1902
1903 if (self.reuse_address) {
1904 try os.setsockopt(
1905 sockfd,
1906 os.SOL.SOCKET,
1907 os.SO.REUSEADDR,
1908 &mem.toBytes(@as(c_int, 1)),
1909 );
1910 }
1911 if (@hasDecl(os.SO, "REUSEPORT") and self.reuse_port) {
1912 try os.setsockopt(
1913 sockfd,
1914 os.SOL.SOCKET,
1915 os.SO.REUSEPORT,
1916 &mem.toBytes(@as(c_int, 1)),
1917 );
1918 }
1919
1920 var socklen = address.getOsSockLen();
1921 try os.bind(sockfd, &address.any, socklen);
1922 try os.listen(sockfd, self.kernel_backlog);
1923 try os.getsockname(sockfd, &self.listen_address.any, &socklen);
1924 }
1925
1926 /// Stop listening. It is still necessary to call `deinit` after stopping listening.
1927 /// Calling `deinit` will automatically call `close`. It is safe to call `close` when
1928 /// not listening.
1929 pub fn close(self: *StreamServer) void {
1930 if (self.sockfd) |fd| {
1931 os.closeSocket(fd);
1932 self.sockfd = null;
1933 self.listen_address = undefined;
1934 }
1935 }1912 }
19361913
1937 pub const AcceptError = error{1914 pub const AcceptError = posix.AcceptError;
1938 ConnectionAborted,
19391915
1940 /// The per-process limit on the number of open file descriptors has been reached.1916 /// Blocks until a client connects to the server. The returned `Connection` has
1941 ProcessFdQuotaExceeded,1917 /// an open stream.
19421918 pub fn accept(s: *Server) AcceptError!Connection {
1943 /// The system-wide limit on the total number of open files has been reached.
1944 SystemFdQuotaExceeded,
1945
1946 /// Not enough free memory. This often means that the memory allocation
1947 /// is limited by the socket buffer limits, not by the system memory.
1948 SystemResources,
1949
1950 /// Socket is not listening for new connections.
1951 SocketNotListening,
1952
1953 ProtocolFailure,
1954
1955 /// Socket is in non-blocking mode and there is no connection to accept.
1956 WouldBlock,
1957
1958 /// Firewall rules forbid connection.
1959 BlockedByFirewall,
1960
1961 FileDescriptorNotASocket,
1962
1963 ConnectionResetByPeer,
1964
1965 NetworkSubsystemFailed,
1966
1967 OperationNotSupported,
1968 } || os.UnexpectedError;
1969
1970 pub const Connection = struct {
1971 stream: Stream,
1972 address: Address,
1973 };
1974
1975 /// If this function succeeds, the returned `Connection` is a caller-managed resource.
1976 pub fn accept(self: *StreamServer) AcceptError!Connection {
1977 var accepted_addr: Address = undefined;1919 var accepted_addr: Address = undefined;
1978 var adr_len: os.socklen_t = @sizeOf(Address);1920 var addr_len: posix.socklen_t = @sizeOf(Address);
1979 const accept_result = os.accept(self.sockfd.?, &accepted_addr.any, &adr_len, os.SOCK.CLOEXEC);1921 const fd = try posix.accept(s.stream.handle, &accepted_addr.any, &addr_len, posix.SOCK.CLOEXEC);
19801922 return .{
1981 if (accept_result) |fd| {1923 .stream = .{ .handle = fd },
1982 return Connection{1924 .address = accepted_addr,
1983 .stream = Stream{ .handle = fd },1925 };
1984 .address = accepted_addr,
1985 };
1986 } else |err| {
1987 return err;
1988 }
1989 }1926 }
1990};1927};
19911928
1992test {1929test {
1993 _ = @import("net/test.zig");1930 _ = @import("net/test.zig");
1931 _ = Server;
1932 _ = Stream;
1933 _ = Address;
1994}1934}
lib/std/net/test.zig+8-18
...@@ -181,11 +181,9 @@ test "listen on a port, send bytes, receive bytes" {...@@ -181,11 +181,9 @@ test "listen on a port, send bytes, receive bytes" {
181 // configured.181 // configured.
182 const localhost = try net.Address.parseIp("127.0.0.1", 0);182 const localhost = try net.Address.parseIp("127.0.0.1", 0);
183183
184 var server = net.StreamServer.init(.{});184 var server = try localhost.listen(.{});
185 defer server.deinit();185 defer server.deinit();
186186
187 try server.listen(localhost);
188
189 const S = struct {187 const S = struct {
190 fn clientFn(server_address: net.Address) !void {188 fn clientFn(server_address: net.Address) !void {
191 const socket = try net.tcpConnectToAddress(server_address);189 const socket = try net.tcpConnectToAddress(server_address);
...@@ -215,17 +213,11 @@ test "listen on an in use port" {...@@ -215,17 +213,11 @@ test "listen on an in use port" {
215213
216 const localhost = try net.Address.parseIp("127.0.0.1", 0);214 const localhost = try net.Address.parseIp("127.0.0.1", 0);
217215
218 var server1 = net.StreamServer.init(net.StreamServer.Options{216 var server1 = try localhost.listen(.{ .reuse_port = true });
219 .reuse_port = true,
220 });
221 defer server1.deinit();217 defer server1.deinit();
222 try server1.listen(localhost);
223218
224 var server2 = net.StreamServer.init(net.StreamServer.Options{219 var server2 = try server1.listen_address.listen(.{ .reuse_port = true });
225 .reuse_port = true,
226 });
227 defer server2.deinit();220 defer server2.deinit();
228 try server2.listen(server1.listen_address);
229}221}
230222
231fn testClientToHost(allocator: mem.Allocator, name: []const u8, port: u16) anyerror!void {223fn testClientToHost(allocator: mem.Allocator, name: []const u8, port: u16) anyerror!void {
...@@ -252,7 +244,7 @@ fn testClient(addr: net.Address) anyerror!void {...@@ -252,7 +244,7 @@ fn testClient(addr: net.Address) anyerror!void {
252 try testing.expect(mem.eql(u8, msg, "hello from server\n"));244 try testing.expect(mem.eql(u8, msg, "hello from server\n"));
253}245}
254246
255fn testServer(server: *net.StreamServer) anyerror!void {247fn testServer(server: *net.Server) anyerror!void {
256 if (builtin.os.tag == .wasi) return error.SkipZigTest;248 if (builtin.os.tag == .wasi) return error.SkipZigTest;
257249
258 var client = try server.accept();250 var client = try server.accept();
...@@ -274,15 +266,14 @@ test "listen on a unix socket, send bytes, receive bytes" {...@@ -274,15 +266,14 @@ test "listen on a unix socket, send bytes, receive bytes" {
274 }266 }
275 }267 }
276268
277 var server = net.StreamServer.init(.{});
278 defer server.deinit();
279
280 const socket_path = try generateFileName("socket.unix");269 const socket_path = try generateFileName("socket.unix");
281 defer testing.allocator.free(socket_path);270 defer testing.allocator.free(socket_path);
282271
283 const socket_addr = try net.Address.initUnix(socket_path);272 const socket_addr = try net.Address.initUnix(socket_path);
284 defer std.fs.cwd().deleteFile(socket_path) catch {};273 defer std.fs.cwd().deleteFile(socket_path) catch {};
285 try server.listen(socket_addr);274
275 var server = try socket_addr.listen(.{});
276 defer server.deinit();
286277
287 const S = struct {278 const S = struct {
288 fn clientFn(path: []const u8) !void {279 fn clientFn(path: []const u8) !void {
...@@ -323,9 +314,8 @@ test "non-blocking tcp server" {...@@ -323,9 +314,8 @@ test "non-blocking tcp server" {
323 }314 }
324315
325 const localhost = try net.Address.parseIp("127.0.0.1", 0);316 const localhost = try net.Address.parseIp("127.0.0.1", 0);
326 var server = net.StreamServer.init(.{ .force_nonblocking = true });317 var server = localhost.listen(.{ .force_nonblocking = true });
327 defer server.deinit();318 defer server.deinit();
328 try server.listen(localhost);
329319
330 const accept_err = server.accept();320 const accept_err = server.accept();
331 try testing.expectError(error.WouldBlock, accept_err);321 try testing.expectError(error.WouldBlock, accept_err);
lib/std/os.zig-8
...@@ -3598,14 +3598,6 @@ pub fn shutdown(sock: socket_t, how: ShutdownHow) ShutdownError!void {...@@ -3598,14 +3598,6 @@ pub fn shutdown(sock: socket_t, how: ShutdownHow) ShutdownError!void {
3598 }3598 }
3599}3599}
36003600
3601pub fn closeSocket(sock: socket_t) void {
3602 if (builtin.os.tag == .windows) {
3603 windows.closesocket(sock) catch unreachable;
3604 } else {
3605 close(sock);
3606 }
3607}
3608
3609pub const BindError = error{3601pub const BindError = error{
3610 /// The address is protected, and the user is not the superuser.3602 /// The address is protected, and the user is not the superuser.
3611 /// For UNIX domain sockets: Search permission is denied on a component3603 /// For UNIX domain sockets: Search permission is denied on a component
lib/std/os/linux/io_uring.zig+16-15
...@@ -4,6 +4,7 @@ const assert = std.debug.assert;...@@ -4,6 +4,7 @@ const assert = std.debug.assert;
4const mem = std.mem;4const mem = std.mem;
5const net = std.net;5const net = std.net;
6const os = std.os;6const os = std.os;
7const posix = std.posix;
7const linux = os.linux;8const linux = os.linux;
8const testing = std.testing;9const testing = std.testing;
910
...@@ -3730,8 +3731,8 @@ const SocketTestHarness = struct {...@@ -3730,8 +3731,8 @@ const SocketTestHarness = struct {
3730 client: os.socket_t,3731 client: os.socket_t,
37313732
3732 fn close(self: SocketTestHarness) void {3733 fn close(self: SocketTestHarness) void {
3733 os.closeSocket(self.client);3734 posix.close(self.client);
3734 os.closeSocket(self.listener);3735 posix.close(self.listener);
3735 }3736 }
3736};3737};
37373738
...@@ -3739,7 +3740,7 @@ fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {...@@ -3739,7 +3740,7 @@ fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {
3739 // Create a TCP server socket3740 // Create a TCP server socket
3740 var address = try net.Address.parseIp4("127.0.0.1", 0);3741 var address = try net.Address.parseIp4("127.0.0.1", 0);
3741 const listener_socket = try createListenerSocket(&address);3742 const listener_socket = try createListenerSocket(&address);
3742 errdefer os.closeSocket(listener_socket);3743 errdefer posix.close(listener_socket);
37433744
3744 // Submit 1 accept3745 // Submit 1 accept
3745 var accept_addr: os.sockaddr = undefined;3746 var accept_addr: os.sockaddr = undefined;
...@@ -3748,7 +3749,7 @@ fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {...@@ -3748,7 +3749,7 @@ fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {
37483749
3749 // Create a TCP client socket3750 // Create a TCP client socket
3750 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);3751 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3751 errdefer os.closeSocket(client);3752 errdefer posix.close(client);
3752 _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());3753 _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
37533754
3754 try testing.expectEqual(@as(u32, 2), try ring.submit());3755 try testing.expectEqual(@as(u32, 2), try ring.submit());
...@@ -3788,7 +3789,7 @@ fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {...@@ -3788,7 +3789,7 @@ fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {
3788fn createListenerSocket(address: *net.Address) !os.socket_t {3789fn createListenerSocket(address: *net.Address) !os.socket_t {
3789 const kernel_backlog = 1;3790 const kernel_backlog = 1;
3790 const listener_socket = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);3791 const listener_socket = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3791 errdefer os.closeSocket(listener_socket);3792 errdefer posix.close(listener_socket);
37923793
3793 try os.setsockopt(listener_socket, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));3794 try os.setsockopt(listener_socket, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
3794 try os.bind(listener_socket, &address.any, address.getOsSockLen());3795 try os.bind(listener_socket, &address.any, address.getOsSockLen());
...@@ -3813,7 +3814,7 @@ test "accept multishot" {...@@ -3813,7 +3814,7 @@ test "accept multishot" {
38133814
3814 var address = try net.Address.parseIp4("127.0.0.1", 0);3815 var address = try net.Address.parseIp4("127.0.0.1", 0);
3815 const listener_socket = try createListenerSocket(&address);3816 const listener_socket = try createListenerSocket(&address);
3816 defer os.closeSocket(listener_socket);3817 defer posix.close(listener_socket);
38173818
3818 // submit multishot accept operation3819 // submit multishot accept operation
3819 var addr: os.sockaddr = undefined;3820 var addr: os.sockaddr = undefined;
...@@ -3826,7 +3827,7 @@ test "accept multishot" {...@@ -3826,7 +3827,7 @@ test "accept multishot" {
3826 while (nr > 0) : (nr -= 1) {3827 while (nr > 0) : (nr -= 1) {
3827 // connect client3828 // connect client
3828 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);3829 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3829 errdefer os.closeSocket(client);3830 errdefer posix.close(client);
3830 try os.connect(client, &address.any, address.getOsSockLen());3831 try os.connect(client, &address.any, address.getOsSockLen());
38313832
3832 // test accept completion3833 // test accept completion
...@@ -3836,7 +3837,7 @@ test "accept multishot" {...@@ -3836,7 +3837,7 @@ test "accept multishot" {
3836 try testing.expect(cqe.user_data == userdata);3837 try testing.expect(cqe.user_data == userdata);
3837 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE > 0); // more flag is set3838 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE > 0); // more flag is set
38383839
3839 os.closeSocket(client);3840 posix.close(client);
3840 }3841 }
3841}3842}
38423843
...@@ -3909,7 +3910,7 @@ test "accept_direct" {...@@ -3909,7 +3910,7 @@ test "accept_direct" {
3909 try ring.register_files(registered_fds[0..]);3910 try ring.register_files(registered_fds[0..]);
39103911
3911 const listener_socket = try createListenerSocket(&address);3912 const listener_socket = try createListenerSocket(&address);
3912 defer os.closeSocket(listener_socket);3913 defer posix.close(listener_socket);
39133914
3914 const accept_userdata: u64 = 0xaaaaaaaa;3915 const accept_userdata: u64 = 0xaaaaaaaa;
3915 const read_userdata: u64 = 0xbbbbbbbb;3916 const read_userdata: u64 = 0xbbbbbbbb;
...@@ -3927,7 +3928,7 @@ test "accept_direct" {...@@ -3927,7 +3928,7 @@ test "accept_direct" {
3927 // connect3928 // connect
3928 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);3929 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3929 try os.connect(client, &address.any, address.getOsSockLen());3930 try os.connect(client, &address.any, address.getOsSockLen());
3930 defer os.closeSocket(client);3931 defer posix.close(client);
39313932
3932 // accept completion3933 // accept completion
3933 const cqe_accept = try ring.copy_cqe();3934 const cqe_accept = try ring.copy_cqe();
...@@ -3961,7 +3962,7 @@ test "accept_direct" {...@@ -3961,7 +3962,7 @@ test "accept_direct" {
3961 // connect3962 // connect
3962 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);3963 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3963 try os.connect(client, &address.any, address.getOsSockLen());3964 try os.connect(client, &address.any, address.getOsSockLen());
3964 defer os.closeSocket(client);3965 defer posix.close(client);
3965 // completion with error3966 // completion with error
3966 const cqe_accept = try ring.copy_cqe();3967 const cqe_accept = try ring.copy_cqe();
3967 try testing.expect(cqe_accept.user_data == accept_userdata);3968 try testing.expect(cqe_accept.user_data == accept_userdata);
...@@ -3989,7 +3990,7 @@ test "accept_multishot_direct" {...@@ -3989,7 +3990,7 @@ test "accept_multishot_direct" {
3989 try ring.register_files(registered_fds[0..]);3990 try ring.register_files(registered_fds[0..]);
39903991
3991 const listener_socket = try createListenerSocket(&address);3992 const listener_socket = try createListenerSocket(&address);
3992 defer os.closeSocket(listener_socket);3993 defer posix.close(listener_socket);
39933994
3994 const accept_userdata: u64 = 0xaaaaaaaa;3995 const accept_userdata: u64 = 0xaaaaaaaa;
39953996
...@@ -4003,7 +4004,7 @@ test "accept_multishot_direct" {...@@ -4003,7 +4004,7 @@ test "accept_multishot_direct" {
4003 // connect4004 // connect
4004 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);4005 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
4005 try os.connect(client, &address.any, address.getOsSockLen());4006 try os.connect(client, &address.any, address.getOsSockLen());
4006 defer os.closeSocket(client);4007 defer posix.close(client);
40074008
4008 // accept completion4009 // accept completion
4009 const cqe_accept = try ring.copy_cqe();4010 const cqe_accept = try ring.copy_cqe();
...@@ -4018,7 +4019,7 @@ test "accept_multishot_direct" {...@@ -4018,7 +4019,7 @@ test "accept_multishot_direct" {
4018 // connect4019 // connect
4019 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);4020 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
4020 try os.connect(client, &address.any, address.getOsSockLen());4021 try os.connect(client, &address.any, address.getOsSockLen());
4021 defer os.closeSocket(client);4022 defer posix.close(client);
4022 // completion with error4023 // completion with error
4023 const cqe_accept = try ring.copy_cqe();4024 const cqe_accept = try ring.copy_cqe();
4024 try testing.expect(cqe_accept.user_data == accept_userdata);4025 try testing.expect(cqe_accept.user_data == accept_userdata);
...@@ -4092,7 +4093,7 @@ test "socket_direct/socket_direct_alloc/close_direct" {...@@ -4092,7 +4093,7 @@ test "socket_direct/socket_direct_alloc/close_direct" {
4092 // use sockets from registered_fds in connect operation4093 // use sockets from registered_fds in connect operation
4093 var address = try net.Address.parseIp4("127.0.0.1", 0);4094 var address = try net.Address.parseIp4("127.0.0.1", 0);
4094 const listener_socket = try createListenerSocket(&address);4095 const listener_socket = try createListenerSocket(&address);
4095 defer os.closeSocket(listener_socket);4096 defer posix.close(listener_socket);
4096 const accept_userdata: u64 = 0xaaaaaaaa;4097 const accept_userdata: u64 = 0xaaaaaaaa;
4097 const connect_userdata: u64 = 0xbbbbbbbb;4098 const connect_userdata: u64 = 0xbbbbbbbb;
4098 const close_userdata: u64 = 0xcccccccc;4099 const close_userdata: u64 = 0xcccccccc;
lib/std/os/test.zig+1-1
...@@ -817,7 +817,7 @@ test "shutdown socket" {...@@ -817,7 +817,7 @@ test "shutdown socket" {
817 error.SocketNotConnected => {},817 error.SocketNotConnected => {},
818 else => |e| return e,818 else => |e| return e,
819 };819 };
820 os.closeSocket(sock);820 std.net.Stream.close(.{ .handle = sock });
821}821}
822822
823test "sigaction" {823test "sigaction" {
src/Package/Fetch.zig+38-47
...@@ -354,7 +354,8 @@ pub fn run(f: *Fetch) RunError!void {...@@ -354,7 +354,8 @@ pub fn run(f: *Fetch) RunError!void {
354 .{ path_or_url, @errorName(file_err), @errorName(uri_err) },354 .{ path_or_url, @errorName(file_err), @errorName(uri_err) },
355 ));355 ));
356 };356 };
357 var resource = try f.initResource(uri);357 var server_header_buffer: [header_buffer_size]u8 = undefined;
358 var resource = try f.initResource(uri, &server_header_buffer);
358 return runResource(f, uri.path, &resource, null);359 return runResource(f, uri.path, &resource, null);
359 }360 }
360 },361 },
...@@ -415,7 +416,8 @@ pub fn run(f: *Fetch) RunError!void {...@@ -415,7 +416,8 @@ pub fn run(f: *Fetch) RunError!void {
415 f.location_tok,416 f.location_tok,
416 try eb.printString("invalid URI: {s}", .{@errorName(err)}),417 try eb.printString("invalid URI: {s}", .{@errorName(err)}),
417 );418 );
418 var resource = try f.initResource(uri);419 var server_header_buffer: [header_buffer_size]u8 = undefined;
420 var resource = try f.initResource(uri, &server_header_buffer);
419 return runResource(f, uri.path, &resource, remote.hash);421 return runResource(f, uri.path, &resource, remote.hash);
420}422}
421423
...@@ -876,7 +878,9 @@ const FileType = enum {...@@ -876,7 +878,9 @@ const FileType = enum {
876 }878 }
877};879};
878880
879fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {881const header_buffer_size = 16 * 1024;
882
883fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Resource {
880 const gpa = f.arena.child_allocator;884 const gpa = f.arena.child_allocator;
881 const arena = f.arena.allocator();885 const arena = f.arena.allocator();
882 const eb = &f.error_bundle;886 const eb = &f.error_bundle;
...@@ -894,10 +898,9 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {...@@ -894,10 +898,9 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
894 if (ascii.eqlIgnoreCase(uri.scheme, "http") or898 if (ascii.eqlIgnoreCase(uri.scheme, "http") or
895 ascii.eqlIgnoreCase(uri.scheme, "https"))899 ascii.eqlIgnoreCase(uri.scheme, "https"))
896 {900 {
897 var h = std.http.Headers{ .allocator = gpa };901 var req = http_client.open(.GET, uri, .{
898 defer h.deinit();902 .server_header_buffer = server_header_buffer,
899903 }) catch |err| {
900 var req = http_client.open(.GET, uri, h, .{}) catch |err| {
901 return f.fail(f.location_tok, try eb.printString(904 return f.fail(f.location_tok, try eb.printString(
902 "unable to connect to server: {s}",905 "unable to connect to server: {s}",
903 .{@errorName(err)},906 .{@errorName(err)},
...@@ -935,7 +938,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {...@@ -935,7 +938,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
935 transport_uri.scheme = uri.scheme["git+".len..];938 transport_uri.scheme = uri.scheme["git+".len..];
936 var redirect_uri: []u8 = undefined;939 var redirect_uri: []u8 = undefined;
937 var session: git.Session = .{ .transport = http_client, .uri = transport_uri };940 var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
938 session.discoverCapabilities(gpa, &redirect_uri) catch |err| switch (err) {941 session.discoverCapabilities(gpa, &redirect_uri, server_header_buffer) catch |err| switch (err) {
939 error.Redirected => {942 error.Redirected => {
940 defer gpa.free(redirect_uri);943 defer gpa.free(redirect_uri);
941 return f.fail(f.location_tok, try eb.printString(944 return f.fail(f.location_tok, try eb.printString(
...@@ -961,6 +964,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {...@@ -961,6 +964,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
961 var ref_iterator = session.listRefs(gpa, .{964 var ref_iterator = session.listRefs(gpa, .{
962 .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },965 .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
963 .include_peeled = true,966 .include_peeled = true,
967 .server_header_buffer = server_header_buffer,
964 }) catch |err| {968 }) catch |err| {
965 return f.fail(f.location_tok, try eb.printString(969 return f.fail(f.location_tok, try eb.printString(
966 "unable to list refs: {s}",970 "unable to list refs: {s}",
...@@ -1003,7 +1007,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {...@@ -1003,7 +1007,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
1003 _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{1007 _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{
1004 std.fmt.fmtSliceHexLower(&want_oid),1008 std.fmt.fmtSliceHexLower(&want_oid),
1005 }) catch unreachable;1009 }) catch unreachable;
1006 var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch |err| {1010 var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}, server_header_buffer) catch |err| {
1007 return f.fail(f.location_tok, try eb.printString(1011 return f.fail(f.location_tok, try eb.printString(
1008 "unable to create fetch stream: {s}",1012 "unable to create fetch stream: {s}",
1009 .{@errorName(err)},1013 .{@errorName(err)},
...@@ -1036,7 +1040,7 @@ fn unpackResource(...@@ -1036,7 +1040,7 @@ fn unpackResource(
10361040
1037 .http_request => |req| ft: {1041 .http_request => |req| ft: {
1038 // Content-Type takes first precedence.1042 // Content-Type takes first precedence.
1039 const content_type = req.response.headers.getFirstValue("Content-Type") orelse1043 const content_type = req.response.content_type orelse
1040 return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));1044 return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
10411045
1042 // Extract the MIME type, ignoring charset and boundary directives1046 // Extract the MIME type, ignoring charset and boundary directives
...@@ -1069,7 +1073,7 @@ fn unpackResource(...@@ -1069,7 +1073,7 @@ fn unpackResource(
1069 }1073 }
10701074
1071 // Next, the filename from 'content-disposition: attachment' takes precedence.1075 // Next, the filename from 'content-disposition: attachment' takes precedence.
1072 if (req.response.headers.getFirstValue("Content-Disposition")) |cd_header| {1076 if (req.response.content_disposition) |cd_header| {
1073 break :ft FileType.fromContentDisposition(cd_header) orelse {1077 break :ft FileType.fromContentDisposition(cd_header) orelse {
1074 return f.fail(f.location_tok, try eb.printString(1078 return f.fail(f.location_tok, try eb.printString(
1075 "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",1079 "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
...@@ -1105,8 +1109,29 @@ fn unpackResource(...@@ -1105,8 +1109,29 @@ fn unpackResource(
1105 var dcp = std.compress.gzip.decompressor(br.reader());1109 var dcp = std.compress.gzip.decompressor(br.reader());
1106 try unpackTarball(f, tmp_directory.handle, dcp.reader());1110 try unpackTarball(f, tmp_directory.handle, dcp.reader());
1107 },1111 },
1108 .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),1112 .@"tar.xz" => {
1109 .@"tar.zst" => try unpackTarballCompressed(f, tmp_directory.handle, resource, ZstdWrapper),1113 const gpa = f.arena.child_allocator;
1114 const reader = resource.reader();
1115 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1116 var dcp = std.compress.xz.decompress(gpa, br.reader()) catch |err| {
1117 return f.fail(f.location_tok, try eb.printString(
1118 "unable to decompress tarball: {s}",
1119 .{@errorName(err)},
1120 ));
1121 };
1122 defer dcp.deinit();
1123 try unpackTarball(f, tmp_directory.handle, dcp.reader());
1124 },
1125 .@"tar.zst" => {
1126 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
1127 const window_buffer = try f.arena.allocator().create([window_size]u8);
1128 const reader = resource.reader();
1129 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1130 var dcp = std.compress.zstd.decompressor(br.reader(), .{
1131 .window_buffer = window_buffer,
1132 });
1133 return unpackTarball(f, tmp_directory.handle, dcp.reader());
1134 },
1110 .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {1135 .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {
1111 error.FetchFailed => return error.FetchFailed,1136 error.FetchFailed => return error.FetchFailed,
1112 error.OutOfMemory => return error.OutOfMemory,1137 error.OutOfMemory => return error.OutOfMemory,
...@@ -1118,40 +1143,6 @@ fn unpackResource(...@@ -1118,40 +1143,6 @@ fn unpackResource(
1118 }1143 }
1119}1144}
11201145
1121// due to slight differences in the API of std.compress.(gzip|xz) and std.compress.zstd, zstd is
1122// wrapped for generic use in unpackTarballCompressed: see github.com/ziglang/zig/issues/14739
1123const ZstdWrapper = struct {
1124 fn DecompressType(comptime T: type) type {
1125 return error{}!std.compress.zstd.DecompressStream(T, .{});
1126 }
1127
1128 fn decompress(allocator: Allocator, reader: anytype) DecompressType(@TypeOf(reader)) {
1129 return std.compress.zstd.decompressStream(allocator, reader);
1130 }
1131};
1132
1133fn unpackTarballCompressed(
1134 f: *Fetch,
1135 out_dir: fs.Dir,
1136 resource: *Resource,
1137 comptime Compression: type,
1138) RunError!void {
1139 const gpa = f.arena.child_allocator;
1140 const eb = &f.error_bundle;
1141 const reader = resource.reader();
1142 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1143
1144 var decompress = Compression.decompress(gpa, br.reader()) catch |err| {
1145 return f.fail(f.location_tok, try eb.printString(
1146 "unable to decompress tarball: {s}",
1147 .{@errorName(err)},
1148 ));
1149 };
1150 defer decompress.deinit();
1151
1152 return unpackTarball(f, out_dir, decompress.reader());
1153}
1154
1155fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {1146fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
1156 const eb = &f.error_bundle;1147 const eb = &f.error_bundle;
1157 const gpa = f.arena.child_allocator;1148 const gpa = f.arena.child_allocator;
src/Package/Fetch/git.zig+33-23
...@@ -494,8 +494,9 @@ pub const Session = struct {...@@ -494,8 +494,9 @@ pub const Session = struct {
494 session: *Session,494 session: *Session,
495 allocator: Allocator,495 allocator: Allocator,
496 redirect_uri: *[]u8,496 redirect_uri: *[]u8,
497 http_headers_buffer: []u8,
497 ) !void {498 ) !void {
498 var capability_iterator = try session.getCapabilities(allocator, redirect_uri);499 var capability_iterator = try session.getCapabilities(allocator, redirect_uri, http_headers_buffer);
499 defer capability_iterator.deinit();500 defer capability_iterator.deinit();
500 while (try capability_iterator.next()) |capability| {501 while (try capability_iterator.next()) |capability| {
501 if (mem.eql(u8, capability.key, "agent")) {502 if (mem.eql(u8, capability.key, "agent")) {
...@@ -521,6 +522,7 @@ pub const Session = struct {...@@ -521,6 +522,7 @@ pub const Session = struct {
521 session: Session,522 session: Session,
522 allocator: Allocator,523 allocator: Allocator,
523 redirect_uri: *[]u8,524 redirect_uri: *[]u8,
525 http_headers_buffer: []u8,
524 ) !CapabilityIterator {526 ) !CapabilityIterator {
525 var info_refs_uri = session.uri;527 var info_refs_uri = session.uri;
526 info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" });528 info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" });
...@@ -528,12 +530,13 @@ pub const Session = struct {...@@ -528,12 +530,13 @@ pub const Session = struct {
528 info_refs_uri.query = "service=git-upload-pack";530 info_refs_uri.query = "service=git-upload-pack";
529 info_refs_uri.fragment = null;531 info_refs_uri.fragment = null;
530532
531 var headers = std.http.Headers.init(allocator);533 const max_redirects = 3;
532 defer headers.deinit();534 var request = try session.transport.open(.GET, info_refs_uri, .{
533 try headers.append("Git-Protocol", "version=2");535 .redirect_behavior = @enumFromInt(max_redirects),
534536 .server_header_buffer = http_headers_buffer,
535 var request = try session.transport.open(.GET, info_refs_uri, headers, .{537 .extra_headers = &.{
536 .max_redirects = 3,538 .{ .name = "Git-Protocol", .value = "version=2" },
539 },
537 });540 });
538 errdefer request.deinit();541 errdefer request.deinit();
539 try request.send(.{});542 try request.send(.{});
...@@ -541,7 +544,8 @@ pub const Session = struct {...@@ -541,7 +544,8 @@ pub const Session = struct {
541544
542 try request.wait();545 try request.wait();
543 if (request.response.status != .ok) return error.ProtocolError;546 if (request.response.status != .ok) return error.ProtocolError;
544 if (request.redirects_left < 3) {547 const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects;
548 if (any_redirects_occurred) {
545 if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect;549 if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect;
546 var new_uri = request.uri;550 var new_uri = request.uri;
547 new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len];551 new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len];
...@@ -620,6 +624,7 @@ pub const Session = struct {...@@ -620,6 +624,7 @@ pub const Session = struct {
620 include_symrefs: bool = false,624 include_symrefs: bool = false,
621 /// Whether to include the peeled object ID for returned tag refs.625 /// Whether to include the peeled object ID for returned tag refs.
622 include_peeled: bool = false,626 include_peeled: bool = false,
627 server_header_buffer: []u8,
623 };628 };
624629
625 /// Returns an iterator over refs known to the server.630 /// Returns an iterator over refs known to the server.
...@@ -630,11 +635,6 @@ pub const Session = struct {...@@ -630,11 +635,6 @@ pub const Session = struct {
630 upload_pack_uri.query = null;635 upload_pack_uri.query = null;
631 upload_pack_uri.fragment = null;636 upload_pack_uri.fragment = null;
632637
633 var headers = std.http.Headers.init(allocator);
634 defer headers.deinit();
635 try headers.append("Content-Type", "application/x-git-upload-pack-request");
636 try headers.append("Git-Protocol", "version=2");
637
638 var body = std.ArrayListUnmanaged(u8){};638 var body = std.ArrayListUnmanaged(u8){};
639 defer body.deinit(allocator);639 defer body.deinit(allocator);
640 const body_writer = body.writer(allocator);640 const body_writer = body.writer(allocator);
...@@ -656,8 +656,13 @@ pub const Session = struct {...@@ -656,8 +656,13 @@ pub const Session = struct {
656 }656 }
657 try Packet.write(.flush, body_writer);657 try Packet.write(.flush, body_writer);
658658
659 var request = try session.transport.open(.POST, upload_pack_uri, headers, .{659 var request = try session.transport.open(.POST, upload_pack_uri, .{
660 .handle_redirects = false,660 .redirect_behavior = .unhandled,
661 .server_header_buffer = options.server_header_buffer,
662 .extra_headers = &.{
663 .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
664 .{ .name = "Git-Protocol", .value = "version=2" },
665 },
661 });666 });
662 errdefer request.deinit();667 errdefer request.deinit();
663 request.transfer_encoding = .{ .content_length = body.items.len };668 request.transfer_encoding = .{ .content_length = body.items.len };
...@@ -721,18 +726,18 @@ pub const Session = struct {...@@ -721,18 +726,18 @@ pub const Session = struct {
721726
722 /// Fetches the given refs from the server. A shallow fetch (depth 1) is727 /// Fetches the given refs from the server. A shallow fetch (depth 1) is
723 /// performed if the server supports it.728 /// performed if the server supports it.
724 pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream {729 pub fn fetch(
730 session: Session,
731 allocator: Allocator,
732 wants: []const []const u8,
733 http_headers_buffer: []u8,
734 ) !FetchStream {
725 var upload_pack_uri = session.uri;735 var upload_pack_uri = session.uri;
726 upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });736 upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
727 defer allocator.free(upload_pack_uri.path);737 defer allocator.free(upload_pack_uri.path);
728 upload_pack_uri.query = null;738 upload_pack_uri.query = null;
729 upload_pack_uri.fragment = null;739 upload_pack_uri.fragment = null;
730740
731 var headers = std.http.Headers.init(allocator);
732 defer headers.deinit();
733 try headers.append("Content-Type", "application/x-git-upload-pack-request");
734 try headers.append("Git-Protocol", "version=2");
735
736 var body = std.ArrayListUnmanaged(u8){};741 var body = std.ArrayListUnmanaged(u8){};
737 defer body.deinit(allocator);742 defer body.deinit(allocator);
738 const body_writer = body.writer(allocator);743 const body_writer = body.writer(allocator);
...@@ -756,8 +761,13 @@ pub const Session = struct {...@@ -756,8 +761,13 @@ pub const Session = struct {
756 try Packet.write(.{ .data = "done\n" }, body_writer);761 try Packet.write(.{ .data = "done\n" }, body_writer);
757 try Packet.write(.flush, body_writer);762 try Packet.write(.flush, body_writer);
758763
759 var request = try session.transport.open(.POST, upload_pack_uri, headers, .{764 var request = try session.transport.open(.POST, upload_pack_uri, .{
760 .handle_redirects = false,765 .redirect_behavior = .not_allowed,
766 .server_header_buffer = http_headers_buffer,
767 .extra_headers = &.{
768 .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
769 .{ .name = "Git-Protocol", .value = "version=2" },
770 },
761 });771 });
762 errdefer request.deinit();772 errdefer request.deinit();
763 request.transfer_encoding = .{ .content_length = body.items.len };773 request.transfer_encoding = .{ .content_length = body.items.len };
src/main.zig+5-5
...@@ -3322,13 +3322,13 @@ fn buildOutputType(...@@ -3322,13 +3322,13 @@ fn buildOutputType(
3322 .ip4 => |ip4_addr| {3322 .ip4 => |ip4_addr| {
3323 if (build_options.only_core_functionality) unreachable;3323 if (build_options.only_core_functionality) unreachable;
33243324
3325 var server = std.net.StreamServer.init(.{3325 const addr: std.net.Address = .{ .in = ip4_addr };
3326
3327 var server = try addr.listen(.{
3326 .reuse_address = true,3328 .reuse_address = true,
3327 });3329 });
3328 defer server.deinit();3330 defer server.deinit();
33293331
3330 try server.listen(.{ .in = ip4_addr });
3331
3332 const conn = try server.accept();3332 const conn = try server.accept();
3333 defer conn.stream.close();3333 defer conn.stream.close();
33343334
...@@ -5486,7 +5486,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5486,7 +5486,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5486 job_queue.read_only = true;5486 job_queue.read_only = true;
5487 cleanup_build_dir = job_queue.global_cache.handle;5487 cleanup_build_dir = job_queue.global_cache.handle;
5488 } else {5488 } else {
5489 try http_client.loadDefaultProxies();5489 try http_client.initDefaultProxies(arena);
5490 }5490 }
54915491
5492 try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);5492 try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
...@@ -7442,7 +7442,7 @@ fn cmdFetch(...@@ -7442,7 +7442,7 @@ fn cmdFetch(
7442 var http_client: std.http.Client = .{ .allocator = gpa };7442 var http_client: std.http.Client = .{ .allocator = gpa };
7443 defer http_client.deinit();7443 defer http_client.deinit();
74447444
7445 try http_client.loadDefaultProxies();7445 try http_client.initDefaultProxies(arena);
74467446
7447 var progress: std.Progress = .{ .dont_print_on_dumb = true };7447 var progress: std.Progress = .{ .dont_print_on_dumb = true };
7448 const root_prog_node = progress.start("Fetch", 0);7448 const root_prog_node = progress.start("Fetch", 0);
test/standalone.zig-4
...@@ -55,10 +55,6 @@ pub const simple_cases = [_]SimpleCase{...@@ -55,10 +55,6 @@ pub const simple_cases = [_]SimpleCase{
55 .os_filter = .windows,55 .os_filter = .windows,
56 .link_libc = true,56 .link_libc = true,
57 },57 },
58 .{
59 .src_path = "test/standalone/http.zig",
60 .all_modes = true,
61 },
6258
63 // Ensure the development tools are buildable. Alphabetically sorted.59 // Ensure the development tools are buildable. Alphabetically sorted.
64 // No need to build `tools/spirv/grammar.zig`.60 // No need to build `tools/spirv/grammar.zig`.
test/standalone/http.zig deleted-700
...@@ -1,700 +0,0 @@
1const std = @import("std");
2
3const http = std.http;
4const Server = http.Server;
5const Client = http.Client;
6
7const mem = std.mem;
8const testing = std.testing;
9
10pub const std_options = .{
11 .http_disable_tls = true,
12};
13
14const max_header_size = 8192;
15
16var gpa_server = std.heap.GeneralPurposeAllocator(.{ .stack_trace_frames = 12 }){};
17var gpa_client = std.heap.GeneralPurposeAllocator(.{ .stack_trace_frames = 12 }){};
18
19const salloc = gpa_server.allocator();
20const calloc = gpa_client.allocator();
21
22var server: Server = undefined;
23
24fn handleRequest(res: *Server.Response) !void {
25 const log = std.log.scoped(.server);
26
27 log.info("{} {s} {s}", .{ res.request.method, @tagName(res.request.version), res.request.target });
28
29 if (res.request.headers.contains("expect")) {
30 if (mem.eql(u8, res.request.headers.getFirstValue("expect").?, "100-continue")) {
31 res.status = .@"continue";
32 try res.send();
33 res.status = .ok;
34 } else {
35 res.status = .expectation_failed;
36 try res.send();
37 return;
38 }
39 }
40
41 const body = try res.reader().readAllAlloc(salloc, 8192);
42 defer salloc.free(body);
43
44 if (res.request.headers.contains("connection")) {
45 try res.headers.append("connection", "keep-alive");
46 }
47
48 if (mem.startsWith(u8, res.request.target, "/get")) {
49 if (std.mem.indexOf(u8, res.request.target, "?chunked") != null) {
50 res.transfer_encoding = .chunked;
51 } else {
52 res.transfer_encoding = .{ .content_length = 14 };
53 }
54
55 try res.headers.append("content-type", "text/plain");
56
57 try res.send();
58 if (res.request.method != .HEAD) {
59 try res.writeAll("Hello, ");
60 try res.writeAll("World!\n");
61 try res.finish();
62 } else {
63 try testing.expectEqual(res.writeAll("errors"), error.NotWriteable);
64 }
65 } else if (mem.startsWith(u8, res.request.target, "/large")) {
66 res.transfer_encoding = .{ .content_length = 14 * 1024 + 14 * 10 };
67
68 try res.send();
69
70 var i: u32 = 0;
71 while (i < 5) : (i += 1) {
72 try res.writeAll("Hello, World!\n");
73 }
74
75 try res.writeAll("Hello, World!\n" ** 1024);
76
77 i = 0;
78 while (i < 5) : (i += 1) {
79 try res.writeAll("Hello, World!\n");
80 }
81
82 try res.finish();
83 } else if (mem.startsWith(u8, res.request.target, "/echo-content")) {
84 try testing.expectEqualStrings("Hello, World!\n", body);
85 try testing.expectEqualStrings("text/plain", res.request.headers.getFirstValue("content-type").?);
86
87 if (res.request.headers.contains("transfer-encoding")) {
88 try testing.expectEqualStrings("chunked", res.request.headers.getFirstValue("transfer-encoding").?);
89 res.transfer_encoding = .chunked;
90 } else {
91 res.transfer_encoding = .{ .content_length = 14 };
92 try testing.expectEqualStrings("14", res.request.headers.getFirstValue("content-length").?);
93 }
94
95 try res.send();
96 try res.writeAll("Hello, ");
97 try res.writeAll("World!\n");
98 try res.finish();
99 } else if (mem.eql(u8, res.request.target, "/trailer")) {
100 res.transfer_encoding = .chunked;
101
102 try res.send();
103 try res.writeAll("Hello, ");
104 try res.writeAll("World!\n");
105 // try res.finish();
106 try res.connection.writeAll("0\r\nX-Checksum: aaaa\r\n\r\n");
107 } else if (mem.eql(u8, res.request.target, "/redirect/1")) {
108 res.transfer_encoding = .chunked;
109
110 res.status = .found;
111 try res.headers.append("location", "../../get");
112
113 try res.send();
114 try res.writeAll("Hello, ");
115 try res.writeAll("Redirected!\n");
116 try res.finish();
117 } else if (mem.eql(u8, res.request.target, "/redirect/2")) {
118 res.transfer_encoding = .chunked;
119
120 res.status = .found;
121 try res.headers.append("location", "/redirect/1");
122
123 try res.send();
124 try res.writeAll("Hello, ");
125 try res.writeAll("Redirected!\n");
126 try res.finish();
127 } else if (mem.eql(u8, res.request.target, "/redirect/3")) {
128 res.transfer_encoding = .chunked;
129
130 const location = try std.fmt.allocPrint(salloc, "http://127.0.0.1:{d}/redirect/2", .{server.socket.listen_address.getPort()});
131 defer salloc.free(location);
132
133 res.status = .found;
134 try res.headers.append("location", location);
135
136 try res.send();
137 try res.writeAll("Hello, ");
138 try res.writeAll("Redirected!\n");
139 try res.finish();
140 } else if (mem.eql(u8, res.request.target, "/redirect/4")) {
141 res.transfer_encoding = .chunked;
142
143 res.status = .found;
144 try res.headers.append("location", "/redirect/3");
145
146 try res.send();
147 try res.writeAll("Hello, ");
148 try res.writeAll("Redirected!\n");
149 try res.finish();
150 } else if (mem.eql(u8, res.request.target, "/redirect/invalid")) {
151 const invalid_port = try getUnusedTcpPort();
152 const location = try std.fmt.allocPrint(salloc, "http://127.0.0.1:{d}", .{invalid_port});
153 defer salloc.free(location);
154
155 res.status = .found;
156 try res.headers.append("location", location);
157 try res.send();
158 try res.finish();
159 } else {
160 res.status = .not_found;
161 try res.send();
162 }
163}
164
165var handle_new_requests = true;
166
167fn runServer(srv: *Server) !void {
168 outer: while (handle_new_requests) {
169 var res = try srv.accept(.{
170 .allocator = salloc,
171 .header_strategy = .{ .dynamic = max_header_size },
172 });
173 defer res.deinit();
174
175 while (res.reset() != .closing) {
176 res.wait() catch |err| switch (err) {
177 error.HttpHeadersInvalid => continue :outer,
178 error.EndOfStream => continue,
179 else => return err,
180 };
181
182 try handleRequest(&res);
183 }
184 }
185}
186
187fn serverThread(srv: *Server) void {
188 defer srv.deinit();
189 defer _ = gpa_server.deinit();
190
191 runServer(srv) catch |err| {
192 std.debug.print("server error: {}\n", .{err});
193
194 if (@errorReturnTrace()) |trace| {
195 std.debug.dumpStackTrace(trace.*);
196 }
197
198 _ = gpa_server.deinit();
199 std.os.exit(1);
200 };
201}
202
203fn killServer(addr: std.net.Address) void {
204 handle_new_requests = false;
205
206 const conn = std.net.tcpConnectToAddress(addr) catch return;
207 conn.close();
208}
209
210fn getUnusedTcpPort() !u16 {
211 const addr = try std.net.Address.parseIp("127.0.0.1", 0);
212 var s = std.net.StreamServer.init(.{});
213 defer s.deinit();
214 try s.listen(addr);
215 return s.listen_address.in.getPort();
216}
217
218pub fn main() !void {
219 const log = std.log.scoped(.client);
220
221 defer _ = gpa_client.deinit();
222
223 server = Server.init(.{ .reuse_address = true });
224
225 const addr = std.net.Address.parseIp("127.0.0.1", 0) catch unreachable;
226 try server.listen(addr);
227
228 const port = server.socket.listen_address.getPort();
229
230 const server_thread = try std.Thread.spawn(.{}, serverThread, .{&server});
231
232 var client = Client{ .allocator = calloc };
233 errdefer client.deinit();
234 // defer client.deinit(); handled below
235
236 try client.loadDefaultProxies();
237
238 { // read content-length response
239 var h = http.Headers{ .allocator = calloc };
240 defer h.deinit();
241
242 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
243 defer calloc.free(location);
244 const uri = try std.Uri.parse(location);
245
246 log.info("{s}", .{location});
247 var req = try client.open(.GET, uri, h, .{});
248 defer req.deinit();
249
250 try req.send(.{});
251 try req.wait();
252
253 const body = try req.reader().readAllAlloc(calloc, 8192);
254 defer calloc.free(body);
255
256 try testing.expectEqualStrings("Hello, World!\n", body);
257 try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
258 }
259
260 // connection has been kept alive
261 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
262
263 { // read large content-length response
264 var h = http.Headers{ .allocator = calloc };
265 defer h.deinit();
266
267 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/large", .{port});
268 defer calloc.free(location);
269 const uri = try std.Uri.parse(location);
270
271 log.info("{s}", .{location});
272 var req = try client.open(.GET, uri, h, .{});
273 defer req.deinit();
274
275 try req.send(.{});
276 try req.wait();
277
278 const body = try req.reader().readAllAlloc(calloc, 8192 * 1024);
279 defer calloc.free(body);
280
281 try testing.expectEqual(@as(usize, 14 * 1024 + 14 * 10), body.len);
282 }
283
284 // connection has been kept alive
285 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
286
287 { // send head request and not read chunked
288 var h = http.Headers{ .allocator = calloc };
289 defer h.deinit();
290
291 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
292 defer calloc.free(location);
293 const uri = try std.Uri.parse(location);
294
295 log.info("{s}", .{location});
296 var req = try client.open(.HEAD, uri, h, .{});
297 defer req.deinit();
298
299 try req.send(.{});
300 try req.wait();
301
302 const body = try req.reader().readAllAlloc(calloc, 8192);
303 defer calloc.free(body);
304
305 try testing.expectEqualStrings("", body);
306 try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
307 try testing.expectEqualStrings("14", req.response.headers.getFirstValue("content-length").?);
308 }
309
310 // connection has been kept alive
311 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
312
313 { // read chunked response
314 var h = http.Headers{ .allocator = calloc };
315 defer h.deinit();
316
317 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get?chunked", .{port});
318 defer calloc.free(location);
319 const uri = try std.Uri.parse(location);
320
321 log.info("{s}", .{location});
322 var req = try client.open(.GET, uri, h, .{});
323 defer req.deinit();
324
325 try req.send(.{});
326 try req.wait();
327
328 const body = try req.reader().readAllAlloc(calloc, 8192);
329 defer calloc.free(body);
330
331 try testing.expectEqualStrings("Hello, World!\n", body);
332 try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
333 }
334
335 // connection has been kept alive
336 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
337
338 { // send head request and not read chunked
339 var h = http.Headers{ .allocator = calloc };
340 defer h.deinit();
341
342 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get?chunked", .{port});
343 defer calloc.free(location);
344 const uri = try std.Uri.parse(location);
345
346 log.info("{s}", .{location});
347 var req = try client.open(.HEAD, uri, h, .{});
348 defer req.deinit();
349
350 try req.send(.{});
351 try req.wait();
352
353 const body = try req.reader().readAllAlloc(calloc, 8192);
354 defer calloc.free(body);
355
356 try testing.expectEqualStrings("", body);
357 try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
358 try testing.expectEqualStrings("chunked", req.response.headers.getFirstValue("transfer-encoding").?);
359 }
360
361 // connection has been kept alive
362 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
363
364 { // check trailing headers
365 var h = http.Headers{ .allocator = calloc };
366 defer h.deinit();
367
368 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/trailer", .{port});
369 defer calloc.free(location);
370 const uri = try std.Uri.parse(location);
371
372 log.info("{s}", .{location});
373 var req = try client.open(.GET, uri, h, .{});
374 defer req.deinit();
375
376 try req.send(.{});
377 try req.wait();
378
379 const body = try req.reader().readAllAlloc(calloc, 8192);
380 defer calloc.free(body);
381
382 try testing.expectEqualStrings("Hello, World!\n", body);
383 try testing.expectEqualStrings("aaaa", req.response.headers.getFirstValue("x-checksum").?);
384 }
385
386 // connection has been kept alive
387 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
388
389 { // send content-length request
390 var h = http.Headers{ .allocator = calloc };
391 defer h.deinit();
392
393 try h.append("content-type", "text/plain");
394
395 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content", .{port});
396 defer calloc.free(location);
397 const uri = try std.Uri.parse(location);
398
399 log.info("{s}", .{location});
400 var req = try client.open(.POST, uri, h, .{});
401 defer req.deinit();
402
403 req.transfer_encoding = .{ .content_length = 14 };
404
405 try req.send(.{});
406 try req.writeAll("Hello, ");
407 try req.writeAll("World!\n");
408 try req.finish();
409
410 try req.wait();
411
412 const body = try req.reader().readAllAlloc(calloc, 8192);
413 defer calloc.free(body);
414
415 try testing.expectEqualStrings("Hello, World!\n", body);
416 }
417
418 // connection has been kept alive
419 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
420
421 { // read content-length response with connection close
422 var h = http.Headers{ .allocator = calloc };
423 defer h.deinit();
424
425 try h.append("connection", "close");
426
427 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
428 defer calloc.free(location);
429 const uri = try std.Uri.parse(location);
430
431 log.info("{s}", .{location});
432 var req = try client.open(.GET, uri, h, .{});
433 defer req.deinit();
434
435 try req.send(.{});
436 try req.wait();
437
438 const body = try req.reader().readAllAlloc(calloc, 8192);
439 defer calloc.free(body);
440
441 try testing.expectEqualStrings("Hello, World!\n", body);
442 try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
443 }
444
445 // connection has been closed
446 try testing.expect(client.connection_pool.free_len == 0);
447
448 { // send chunked request
449 var h = http.Headers{ .allocator = calloc };
450 defer h.deinit();
451
452 try h.append("content-type", "text/plain");
453
454 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content", .{port});
455 defer calloc.free(location);
456 const uri = try std.Uri.parse(location);
457
458 log.info("{s}", .{location});
459 var req = try client.open(.POST, uri, h, .{});
460 defer req.deinit();
461
462 req.transfer_encoding = .chunked;
463
464 try req.send(.{});
465 try req.writeAll("Hello, ");
466 try req.writeAll("World!\n");
467 try req.finish();
468
469 try req.wait();
470
471 const body = try req.reader().readAllAlloc(calloc, 8192);
472 defer calloc.free(body);
473
474 try testing.expectEqualStrings("Hello, World!\n", body);
475 }
476
477 // connection has been kept alive
478 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
479
480 { // relative redirect
481 var h = http.Headers{ .allocator = calloc };
482 defer h.deinit();
483
484 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/1", .{port});
485 defer calloc.free(location);
486 const uri = try std.Uri.parse(location);
487
488 log.info("{s}", .{location});
489 var req = try client.open(.GET, uri, h, .{});
490 defer req.deinit();
491
492 try req.send(.{});
493 try req.wait();
494
495 const body = try req.reader().readAllAlloc(calloc, 8192);
496 defer calloc.free(body);
497
498 try testing.expectEqualStrings("Hello, World!\n", body);
499 }
500
501 // connection has been kept alive
502 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
503
504 { // redirect from root
505 var h = http.Headers{ .allocator = calloc };
506 defer h.deinit();
507
508 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/2", .{port});
509 defer calloc.free(location);
510 const uri = try std.Uri.parse(location);
511
512 log.info("{s}", .{location});
513 var req = try client.open(.GET, uri, h, .{});
514 defer req.deinit();
515
516 try req.send(.{});
517 try req.wait();
518
519 const body = try req.reader().readAllAlloc(calloc, 8192);
520 defer calloc.free(body);
521
522 try testing.expectEqualStrings("Hello, World!\n", body);
523 }
524
525 // connection has been kept alive
526 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
527
528 { // absolute redirect
529 var h = http.Headers{ .allocator = calloc };
530 defer h.deinit();
531
532 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/3", .{port});
533 defer calloc.free(location);
534 const uri = try std.Uri.parse(location);
535
536 log.info("{s}", .{location});
537 var req = try client.open(.GET, uri, h, .{});
538 defer req.deinit();
539
540 try req.send(.{});
541 try req.wait();
542
543 const body = try req.reader().readAllAlloc(calloc, 8192);
544 defer calloc.free(body);
545
546 try testing.expectEqualStrings("Hello, World!\n", body);
547 }
548
549 // connection has been kept alive
550 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
551
552 { // too many redirects
553 var h = http.Headers{ .allocator = calloc };
554 defer h.deinit();
555
556 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/4", .{port});
557 defer calloc.free(location);
558 const uri = try std.Uri.parse(location);
559
560 log.info("{s}", .{location});
561 var req = try client.open(.GET, uri, h, .{});
562 defer req.deinit();
563
564 try req.send(.{});
565 req.wait() catch |err| switch (err) {
566 error.TooManyHttpRedirects => {},
567 else => return err,
568 };
569 }
570
571 // connection has been kept alive
572 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
573
574 { // check client without segfault by connection error after redirection
575 var h = http.Headers{ .allocator = calloc };
576 defer h.deinit();
577
578 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/invalid", .{port});
579 defer calloc.free(location);
580 const uri = try std.Uri.parse(location);
581
582 log.info("{s}", .{location});
583 var req = try client.open(.GET, uri, h, .{});
584 defer req.deinit();
585
586 try req.send(.{});
587 const result = req.wait();
588
589 // a proxy without an upstream is likely to return a 5xx status.
590 if (client.http_proxy == null) {
591 try testing.expectError(error.ConnectionRefused, result); // expects not segfault but the regular error
592 }
593 }
594
595 // connection has been kept alive
596 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
597
598 { // Client.fetch()
599 var h = http.Headers{ .allocator = calloc };
600 defer h.deinit();
601
602 try h.append("content-type", "text/plain");
603
604 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
605 defer calloc.free(location);
606
607 log.info("{s}", .{location});
608 var res = try client.fetch(calloc, .{
609 .location = .{ .url = location },
610 .method = .POST,
611 .headers = h,
612 .payload = .{ .string = "Hello, World!\n" },
613 });
614 defer res.deinit();
615
616 try testing.expectEqualStrings("Hello, World!\n", res.body.?);
617 }
618
619 { // expect: 100-continue
620 var h = http.Headers{ .allocator = calloc };
621 defer h.deinit();
622
623 try h.append("expect", "100-continue");
624 try h.append("content-type", "text/plain");
625
626 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#expect-100", .{port});
627 defer calloc.free(location);
628 const uri = try std.Uri.parse(location);
629
630 log.info("{s}", .{location});
631 var req = try client.open(.POST, uri, h, .{});
632 defer req.deinit();
633
634 req.transfer_encoding = .chunked;
635
636 try req.send(.{});
637 try req.writeAll("Hello, ");
638 try req.writeAll("World!\n");
639 try req.finish();
640
641 try req.wait();
642 try testing.expectEqual(http.Status.ok, req.response.status);
643
644 const body = try req.reader().readAllAlloc(calloc, 8192);
645 defer calloc.free(body);
646
647 try testing.expectEqualStrings("Hello, World!\n", body);
648 }
649
650 { // expect: garbage
651 var h = http.Headers{ .allocator = calloc };
652 defer h.deinit();
653
654 try h.append("content-type", "text/plain");
655 try h.append("expect", "garbage");
656
657 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#expect-garbage", .{port});
658 defer calloc.free(location);
659 const uri = try std.Uri.parse(location);
660
661 log.info("{s}", .{location});
662 var req = try client.open(.POST, uri, h, .{});
663 defer req.deinit();
664
665 req.transfer_encoding = .chunked;
666
667 try req.send(.{});
668 try req.wait();
669 try testing.expectEqual(http.Status.expectation_failed, req.response.status);
670 }
671
672 { // issue 16282 *** This test leaves the client in an invalid state, it must be last ***
673 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
674 defer calloc.free(location);
675 const uri = try std.Uri.parse(location);
676
677 const total_connections = client.connection_pool.free_size + 64;
678 var requests = try calloc.alloc(http.Client.Request, total_connections);
679 defer calloc.free(requests);
680
681 for (0..total_connections) |i| {
682 var req = try client.open(.GET, uri, .{ .allocator = calloc }, .{});
683 req.response.parser.done = true;
684 req.connection.?.closing = false;
685 requests[i] = req;
686 }
687
688 for (0..total_connections) |i| {
689 requests[i].deinit();
690 }
691
692 // free connections should be full now
693 try testing.expect(client.connection_pool.free_len == client.connection_pool.free_size);
694 }
695
696 client.deinit();
697
698 killServer(server.socket.listen_address);
699 server_thread.join();
700}