authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-16 18:35:57-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log78192637fba21c1cc6fca364be716795e4e44020
treec8cd4fd1f06505cb49750a7893b216d1f26575e0
parentd574875f00db32c40019c69465d450a6a58da67f

std.http: parser fixes

* add API for iterating over custom HTTP headers * remove `trailing` flag from std.http.Client.parse. Instead, simply don't call parse() for trailers. * fix the logic inside that parse() function. it was using wrong std.mem functions, ignoring malformed data, and returned errors on dead branches. * simplify logic inside wait() * fix HeadersParser not dropping the 2 read bytes of \r\n after a chunked transfer * move the trailers test to be a std lib unit test and make it pass

3 files changed, 142 insertions(+), 64 deletions(-)

lib/std/http/Client.zig+35-41
......@@ -428,12 +428,14 @@ pub const Response = struct {
428428 CompressionUnsupported,
429429 };
430430
431 pub fn parse(res: *Response, bytes: []const u8, trailing: bool) ParseError!void {
432 var it = mem.tokenizeAny(u8, bytes, "\r\n");
431 pub fn parse(res: *Response, bytes: []const u8) ParseError!void {
432 var it = mem.splitSequence(u8, bytes, "\r\n");
433433
434 const first_line = it.next() orelse return error.HttpHeadersInvalid;
435 if (first_line.len < 12)
434 const first_line = it.next().?;
435 if (first_line.len < 12) {
436 std.debug.print("first line: '{s}'\n", .{first_line});
436437 return error.HttpHeadersInvalid;
438 }
437439
438440 const version: http.Version = switch (int64(first_line[0..8])) {
439441 int64("HTTP/1.0") => .@"HTTP/1.0",
......@@ -449,17 +451,16 @@ pub const Response = struct {
449451 res.reason = reason;
450452
451453 while (it.next()) |line| {
452 if (line.len == 0) return error.HttpHeadersInvalid;
454 if (line.len == 0) return;
453455 switch (line[0]) {
454456 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
455457 else => {},
456458 }
457459
458 var line_it = mem.tokenizeAny(u8, line, ": ");
459 const header_name = line_it.next() orelse return error.HttpHeadersInvalid;
460 var line_it = mem.splitSequence(u8, line, ": ");
461 const header_name = line_it.next().?;
460462 const header_value = line_it.rest();
461
462 if (trailing) continue;
463 if (header_value.len == 0) return error.HttpHeadersInvalid;
463464
464465 if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
465466 res.keep_alive = !std.ascii.eqlIgnoreCase(header_value, "close");
......@@ -538,6 +539,10 @@ pub const Response = struct {
538539 try expectEqual(@as(u10, 999), parseInt3("999"));
539540 }
540541
542 pub fn iterateHeaders(r: Response) proto.HeaderIterator {
543 return proto.HeaderIterator.init(r.parser.get());
544 }
545
541546 version: http.Version,
542547 status: http.Status,
543548 reason: []const u8,
......@@ -868,7 +873,7 @@ pub const Request = struct {
868873 if (req.response.parser.state.isContent()) break;
869874 }
870875
871 try req.response.parse(req.response.parser.get(), false);
876 try req.response.parse(req.response.parser.get());
872877
873878 if (req.response.status == .@"continue") {
874879 // We're done parsing the continue response; reset to prepare
......@@ -903,21 +908,21 @@ pub const Request = struct {
903908 return; // The response is empty; no further setup or redirection is necessary.
904909 }
905910
906 if (req.response.transfer_encoding != .none) {
907 switch (req.response.transfer_encoding) {
908 .none => unreachable,
909 .chunked => {
910 req.response.parser.next_chunk_length = 0;
911 req.response.parser.state = .chunk_head_size;
912 },
913 }
914 } else if (req.response.content_length) |cl| {
915 req.response.parser.next_chunk_length = cl;
911 switch (req.response.transfer_encoding) {
912 .none => {
913 if (req.response.content_length) |cl| {
914 req.response.parser.next_chunk_length = cl;
916915
917 if (cl == 0) req.response.parser.done = true;
918 } else {
919 // read until the connection is closed
920 req.response.parser.next_chunk_length = std.math.maxInt(u64);
916 if (cl == 0) req.response.parser.done = true;
917 } else {
918 // read until the connection is closed
919 req.response.parser.next_chunk_length = std.math.maxInt(u64);
920 }
921 },
922 .chunked => {
923 req.response.parser.next_chunk_length = 0;
924 req.response.parser.state = .chunk_head_size;
925 },
921926 }
922927
923928 if (req.response.status.class() == .redirect and req.redirect_behavior != .unhandled) {
......@@ -1014,27 +1019,16 @@ pub const Request = struct {
10141019 //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,
10151020 else => try req.transferRead(buffer),
10161021 };
1022 if (out_index > 0) return out_index;
10171023
1018 if (out_index == 0) {
1019 const has_trail = !req.response.parser.state.isContent();
1020
1021 while (!req.response.parser.state.isContent()) { // read trailing headers
1022 try req.connection.?.fill();
1024 while (!req.response.parser.state.isContent()) { // read trailing headers
1025 try req.connection.?.fill();
10231026
1024 const nchecked = try req.response.parser.checkCompleteHead(req.connection.?.peek());
1025 req.connection.?.drop(@intCast(nchecked));
1026 }
1027
1028 if (has_trail) {
1029 // The response headers before the trailers are already
1030 // guaranteed to be valid, so they will always be parsed again
1031 // and cannot return an error.
1032 // This will *only* fail for a malformed trailer.
1033 req.response.parse(req.response.parser.get(), true) catch return error.InvalidTrailers;
1034 }
1027 const nchecked = try req.response.parser.checkCompleteHead(req.connection.?.peek());
1028 req.connection.?.drop(@intCast(nchecked));
10351029 }
10361030
1037 return out_index;
1031 return 0;
10381032 }
10391033
10401034 /// Reads data from the response body. Must be called after `wait`.
lib/std/http/protocol.zig+65-2
......@@ -570,9 +570,10 @@ pub const HeadersParser = struct {
570570 .chunk_data => if (r.next_chunk_length == 0) {
571571 if (std.mem.eql(u8, conn.peek(), "\r\n")) {
572572 r.state = .finished;
573 r.done = true;
573 conn.drop(2);
574574 } else {
575 // The trailer section is formatted identically to the header section.
575 // The trailer section is formatted identically
576 // to the header section.
576577 r.state = .seen_rn;
577578 }
578579 r.done = true;
......@@ -613,6 +614,68 @@ pub const HeadersParser = struct {
613614 }
614615};
615616
617pub const HeaderIterator = struct {
618 bytes: []const u8,
619 index: usize,
620 is_trailer: bool,
621
622 pub fn init(bytes: []const u8) HeaderIterator {
623 return .{
624 .bytes = bytes,
625 .index = std.mem.indexOfPosLinear(u8, bytes, 0, "\r\n").? + 2,
626 .is_trailer = false,
627 };
628 }
629
630 pub fn next(it: *HeaderIterator) ?std.http.Header {
631 const end = std.mem.indexOfPosLinear(u8, it.bytes, it.index, "\r\n").?;
632 var kv_it = std.mem.splitSequence(u8, it.bytes[it.index..end], ": ");
633 const name = kv_it.next().?;
634 const value = kv_it.rest();
635 if (value.len == 0) {
636 if (it.is_trailer) return null;
637 const next_end = std.mem.indexOfPosLinear(u8, it.bytes, end + 2, "\r\n") orelse
638 return null;
639 it.is_trailer = true;
640 it.index = next_end + 2;
641 kv_it = std.mem.splitSequence(u8, it.bytes[end + 2 .. next_end], ": ");
642 return .{
643 .name = kv_it.next().?,
644 .value = kv_it.rest(),
645 };
646 }
647 it.index = end + 2;
648 return .{
649 .name = name,
650 .value = value,
651 };
652 }
653
654 test next {
655 var it = HeaderIterator.init("200 OK\r\na: b\r\nc: d\r\n\r\ne: f\r\n\r\n");
656 try std.testing.expect(!it.is_trailer);
657 {
658 const header = it.next().?;
659 try std.testing.expect(!it.is_trailer);
660 try std.testing.expectEqualStrings("a", header.name);
661 try std.testing.expectEqualStrings("b", header.value);
662 }
663 {
664 const header = it.next().?;
665 try std.testing.expect(!it.is_trailer);
666 try std.testing.expectEqualStrings("c", header.name);
667 try std.testing.expectEqualStrings("d", header.value);
668 }
669 {
670 const header = it.next().?;
671 try std.testing.expect(it.is_trailer);
672 try std.testing.expectEqualStrings("e", header.name);
673 try std.testing.expectEqualStrings("f", header.value);
674 }
675 try std.testing.expectEqual(null, it.next());
676 }
677};
678
616679inline fn int16(array: *const [2]u8) u16 {
617680 return @as(u16, @bitCast(array.*));
618681}
lib/std/http/test.zig+42-21
......@@ -1,7 +1,8 @@
11const std = @import("std");
2const testing = std.testing;
23
34test "trailers" {
4 const gpa = std.testing.allocator;
5 const gpa = testing.allocator;
56
67 var http_server = std.http.Server.init(.{
78 .reuse_address = true,
......@@ -21,28 +22,49 @@ test "trailers" {
2122 defer gpa.free(location);
2223 const uri = try std.Uri.parse(location);
2324
24 var server_header_buffer: [1024]u8 = undefined;
25 var req = try client.open(.GET, uri, .{
26 .server_header_buffer = &server_header_buffer,
27 });
28 defer req.deinit();
29
30 try req.send(.{});
31 try req.wait();
32
33 const body = try req.reader().readAllAlloc(gpa, 8192);
34 defer gpa.free(body);
35
36 try std.testing.expectEqualStrings("Hello, World!\n", body);
37 if (true) @panic("TODO implement inspecting custom headers in responses");
38 //try testing.expectEqualStrings("aaaa", req.response.headers.getFirstValue("x-checksum").?);
25 {
26 var server_header_buffer: [1024]u8 = undefined;
27 var req = try client.open(.GET, uri, .{
28 .server_header_buffer = &server_header_buffer,
29 });
30 defer req.deinit();
31
32 try req.send(.{});
33 try req.wait();
34
35 const body = try req.reader().readAllAlloc(gpa, 8192);
36 defer gpa.free(body);
37
38 try testing.expectEqualStrings("Hello, World!\n", body);
39
40 var it = req.response.iterateHeaders();
41 {
42 const header = it.next().?;
43 try testing.expect(!it.is_trailer);
44 try testing.expectEqualStrings("connection", header.name);
45 try testing.expectEqualStrings("keep-alive", header.value);
46 }
47 {
48 const header = it.next().?;
49 try testing.expect(!it.is_trailer);
50 try testing.expectEqualStrings("transfer-encoding", header.name);
51 try testing.expectEqualStrings("chunked", header.value);
52 }
53 {
54 const header = it.next().?;
55 try testing.expect(it.is_trailer);
56 try testing.expectEqualStrings("X-Checksum", header.name);
57 try testing.expectEqualStrings("aaaa", header.value);
58 }
59 try testing.expectEqual(null, it.next());
60 }
3961
4062 // connection has been kept alive
41 try std.testing.expect(client.connection_pool.free_len == 1);
63 try testing.expect(client.connection_pool.free_len == 1);
4264}
4365
4466fn serverThread(http_server: *std.http.Server) anyerror!void {
45 const gpa = std.testing.allocator;
67 const gpa = testing.allocator;
4668
4769 var header_buffer: [1024]u8 = undefined;
4870 var remaining: usize = 1;
......@@ -60,17 +82,16 @@ fn serverThread(http_server: *std.http.Server) anyerror!void {
6082 };
6183 try serve(&res);
6284
63 try std.testing.expectEqual(.reset, res.reset());
85 try testing.expectEqual(.reset, res.reset());
6486 }
6587}
6688
6789fn serve(res: *std.http.Server.Response) !void {
68 try std.testing.expectEqualStrings(res.request.target, "/trailer");
90 try testing.expectEqualStrings(res.request.target, "/trailer");
6991 res.transfer_encoding = .chunked;
7092
7193 try res.send();
7294 try res.writeAll("Hello, ");
7395 try res.writeAll("World!\n");
74 // try res.finish();
7596 try res.connection.writeAll("0\r\nX-Checksum: aaaa\r\n\r\n");
7697}