authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-16 18:35:57-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log78192637fba21c1cc6fca364be716795e4e44020
treec8cd4fd1f06505cb49750a7893b216d1f26575e0
parentd574875f00db32c40019c69465d450a6a58da67f

std.http: parser fixes

* add API for iterating over custom HTTP headers * remove `trailing` flag from std.http.Client.parse. Instead, simply don't call parse() for trailers. * fix the logic inside that parse() function. it was using wrong std.mem functions, ignoring malformed data, and returned errors on dead branches. * simplify logic inside wait() * fix HeadersParser not dropping the 2 read bytes of \r\n after a chunked transfer * move the trailers test to be a std lib unit test and make it pass

3 files changed, 142 insertions(+), 64 deletions(-)

lib/std/http/Client.zig+35-41
...@@ -428,12 +428,14 @@ pub const Response = struct {...@@ -428,12 +428,14 @@ pub const Response = struct {
428 CompressionUnsupported,428 CompressionUnsupported,
429 };429 };
430430
431 pub fn parse(res: *Response, bytes: []const u8, trailing: bool) ParseError!void {431 pub fn parse(res: *Response, bytes: []const u8) ParseError!void {
432 var it = mem.tokenizeAny(u8, bytes, "\r\n");432 var it = mem.splitSequence(u8, bytes, "\r\n");
433433
434 const first_line = it.next() orelse return error.HttpHeadersInvalid;434 const first_line = it.next().?;
435 if (first_line.len < 12)435 if (first_line.len < 12) {
436 std.debug.print("first line: '{s}'\n", .{first_line});
436 return error.HttpHeadersInvalid;437 return error.HttpHeadersInvalid;
438 }
437439
438 const version: http.Version = switch (int64(first_line[0..8])) {440 const version: http.Version = switch (int64(first_line[0..8])) {
439 int64("HTTP/1.0") => .@"HTTP/1.0",441 int64("HTTP/1.0") => .@"HTTP/1.0",
...@@ -449,17 +451,16 @@ pub const Response = struct {...@@ -449,17 +451,16 @@ pub const Response = struct {
449 res.reason = reason;451 res.reason = reason;
450452
451 while (it.next()) |line| {453 while (it.next()) |line| {
452 if (line.len == 0) return error.HttpHeadersInvalid;454 if (line.len == 0) return;
453 switch (line[0]) {455 switch (line[0]) {
454 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,456 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
455 else => {},457 else => {},
456 }458 }
457459
458 var line_it = mem.tokenizeAny(u8, line, ": ");460 var line_it = mem.splitSequence(u8, line, ": ");
459 const header_name = line_it.next() orelse return error.HttpHeadersInvalid;461 const header_name = line_it.next().?;
460 const header_value = line_it.rest();462 const header_value = line_it.rest();
461463 if (header_value.len == 0) return error.HttpHeadersInvalid;
462 if (trailing) continue;
463464
464 if (std.ascii.eqlIgnoreCase(header_name, "connection")) {465 if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
465 res.keep_alive = !std.ascii.eqlIgnoreCase(header_value, "close");466 res.keep_alive = !std.ascii.eqlIgnoreCase(header_value, "close");
...@@ -538,6 +539,10 @@ pub const Response = struct {...@@ -538,6 +539,10 @@ pub const Response = struct {
538 try expectEqual(@as(u10, 999), parseInt3("999"));539 try expectEqual(@as(u10, 999), parseInt3("999"));
539 }540 }
540541
542 pub fn iterateHeaders(r: Response) proto.HeaderIterator {
543 return proto.HeaderIterator.init(r.parser.get());
544 }
545
541 version: http.Version,546 version: http.Version,
542 status: http.Status,547 status: http.Status,
543 reason: []const u8,548 reason: []const u8,
...@@ -868,7 +873,7 @@ pub const Request = struct {...@@ -868,7 +873,7 @@ pub const Request = struct {
868 if (req.response.parser.state.isContent()) break;873 if (req.response.parser.state.isContent()) break;
869 }874 }
870875
871 try req.response.parse(req.response.parser.get(), false);876 try req.response.parse(req.response.parser.get());
872877
873 if (req.response.status == .@"continue") {878 if (req.response.status == .@"continue") {
874 // We're done parsing the continue response; reset to prepare879 // We're done parsing the continue response; reset to prepare
...@@ -903,21 +908,21 @@ pub const Request = struct {...@@ -903,21 +908,21 @@ pub const Request = struct {
903 return; // The response is empty; no further setup or redirection is necessary.908 return; // The response is empty; no further setup or redirection is necessary.
904 }909 }
905910
906 if (req.response.transfer_encoding != .none) {911 switch (req.response.transfer_encoding) {
907 switch (req.response.transfer_encoding) {912 .none => {
908 .none => unreachable,913 if (req.response.content_length) |cl| {
909 .chunked => {914 req.response.parser.next_chunk_length = cl;
910 req.response.parser.next_chunk_length = 0;
911 req.response.parser.state = .chunk_head_size;
912 },
913 }
914 } else if (req.response.content_length) |cl| {
915 req.response.parser.next_chunk_length = cl;
916915
917 if (cl == 0) req.response.parser.done = true;916 if (cl == 0) req.response.parser.done = true;
918 } else {917 } else {
919 // read until the connection is closed918 // read until the connection is closed
920 req.response.parser.next_chunk_length = std.math.maxInt(u64);919 req.response.parser.next_chunk_length = std.math.maxInt(u64);
920 }
921 },
922 .chunked => {
923 req.response.parser.next_chunk_length = 0;
924 req.response.parser.state = .chunk_head_size;
925 },
921 }926 }
922927
923 if (req.response.status.class() == .redirect and req.redirect_behavior != .unhandled) {928 if (req.response.status.class() == .redirect and req.redirect_behavior != .unhandled) {
...@@ -1014,27 +1019,16 @@ pub const Request = struct {...@@ -1014,27 +1019,16 @@ pub const Request = struct {
1014 //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,1019 //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,
1015 else => try req.transferRead(buffer),1020 else => try req.transferRead(buffer),
1016 };1021 };
1022 if (out_index > 0) return out_index;
10171023
1018 if (out_index == 0) {1024 while (!req.response.parser.state.isContent()) { // read trailing headers
1019 const has_trail = !req.response.parser.state.isContent();1025 try req.connection.?.fill();
1020
1021 while (!req.response.parser.state.isContent()) { // read trailing headers
1022 try req.connection.?.fill();
10231026
1024 const nchecked = try req.response.parser.checkCompleteHead(req.connection.?.peek());1027 const nchecked = try req.response.parser.checkCompleteHead(req.connection.?.peek());
1025 req.connection.?.drop(@intCast(nchecked));1028 req.connection.?.drop(@intCast(nchecked));
1026 }
1027
1028 if (has_trail) {
1029 // The response headers before the trailers are already
1030 // guaranteed to be valid, so they will always be parsed again
1031 // and cannot return an error.
1032 // This will *only* fail for a malformed trailer.
1033 req.response.parse(req.response.parser.get(), true) catch return error.InvalidTrailers;
1034 }
1035 }1029 }
10361030
1037 return out_index;1031 return 0;
1038 }1032 }
10391033
1040 /// Reads data from the response body. Must be called after `wait`.1034 /// Reads data from the response body. Must be called after `wait`.
lib/std/http/protocol.zig+65-2
...@@ -570,9 +570,10 @@ pub const HeadersParser = struct {...@@ -570,9 +570,10 @@ pub const HeadersParser = struct {
570 .chunk_data => if (r.next_chunk_length == 0) {570 .chunk_data => if (r.next_chunk_length == 0) {
571 if (std.mem.eql(u8, conn.peek(), "\r\n")) {571 if (std.mem.eql(u8, conn.peek(), "\r\n")) {
572 r.state = .finished;572 r.state = .finished;
573 r.done = true;573 conn.drop(2);
574 } else {574 } else {
575 // The trailer section is formatted identically to the header section.575 // The trailer section is formatted identically
576 // to the header section.
576 r.state = .seen_rn;577 r.state = .seen_rn;
577 }578 }
578 r.done = true;579 r.done = true;
...@@ -613,6 +614,68 @@ pub const HeadersParser = struct {...@@ -613,6 +614,68 @@ pub const HeadersParser = struct {
613 }614 }
614};615};
615616
617pub const HeaderIterator = struct {
618 bytes: []const u8,
619 index: usize,
620 is_trailer: bool,
621
622 pub fn init(bytes: []const u8) HeaderIterator {
623 return .{
624 .bytes = bytes,
625 .index = std.mem.indexOfPosLinear(u8, bytes, 0, "\r\n").? + 2,
626 .is_trailer = false,
627 };
628 }
629
630 pub fn next(it: *HeaderIterator) ?std.http.Header {
631 const end = std.mem.indexOfPosLinear(u8, it.bytes, it.index, "\r\n").?;
632 var kv_it = std.mem.splitSequence(u8, it.bytes[it.index..end], ": ");
633 const name = kv_it.next().?;
634 const value = kv_it.rest();
635 if (value.len == 0) {
636 if (it.is_trailer) return null;
637 const next_end = std.mem.indexOfPosLinear(u8, it.bytes, end + 2, "\r\n") orelse
638 return null;
639 it.is_trailer = true;
640 it.index = next_end + 2;
641 kv_it = std.mem.splitSequence(u8, it.bytes[end + 2 .. next_end], ": ");
642 return .{
643 .name = kv_it.next().?,
644 .value = kv_it.rest(),
645 };
646 }
647 it.index = end + 2;
648 return .{
649 .name = name,
650 .value = value,
651 };
652 }
653
654 test next {
655 var it = HeaderIterator.init("200 OK\r\na: b\r\nc: d\r\n\r\ne: f\r\n\r\n");
656 try std.testing.expect(!it.is_trailer);
657 {
658 const header = it.next().?;
659 try std.testing.expect(!it.is_trailer);
660 try std.testing.expectEqualStrings("a", header.name);
661 try std.testing.expectEqualStrings("b", header.value);
662 }
663 {
664 const header = it.next().?;
665 try std.testing.expect(!it.is_trailer);
666 try std.testing.expectEqualStrings("c", header.name);
667 try std.testing.expectEqualStrings("d", header.value);
668 }
669 {
670 const header = it.next().?;
671 try std.testing.expect(it.is_trailer);
672 try std.testing.expectEqualStrings("e", header.name);
673 try std.testing.expectEqualStrings("f", header.value);
674 }
675 try std.testing.expectEqual(null, it.next());
676 }
677};
678
616inline fn int16(array: *const [2]u8) u16 {679inline fn int16(array: *const [2]u8) u16 {
617 return @as(u16, @bitCast(array.*));680 return @as(u16, @bitCast(array.*));
618}681}
lib/std/http/test.zig+42-21
...@@ -1,7 +1,8 @@...@@ -1,7 +1,8 @@
1const std = @import("std");1const std = @import("std");
2const testing = std.testing;
23
3test "trailers" {4test "trailers" {
4 const gpa = std.testing.allocator;5 const gpa = testing.allocator;
56
6 var http_server = std.http.Server.init(.{7 var http_server = std.http.Server.init(.{
7 .reuse_address = true,8 .reuse_address = true,
...@@ -21,28 +22,49 @@ test "trailers" {...@@ -21,28 +22,49 @@ test "trailers" {
21 defer gpa.free(location);22 defer gpa.free(location);
22 const uri = try std.Uri.parse(location);23 const uri = try std.Uri.parse(location);
2324
24 var server_header_buffer: [1024]u8 = undefined;25 {
25 var req = try client.open(.GET, uri, .{26 var server_header_buffer: [1024]u8 = undefined;
26 .server_header_buffer = &server_header_buffer,27 var req = try client.open(.GET, uri, .{
27 });28 .server_header_buffer = &server_header_buffer,
28 defer req.deinit();29 });
2930 defer req.deinit();
30 try req.send(.{});31
31 try req.wait();32 try req.send(.{});
3233 try req.wait();
33 const body = try req.reader().readAllAlloc(gpa, 8192);34
34 defer gpa.free(body);35 const body = try req.reader().readAllAlloc(gpa, 8192);
3536 defer gpa.free(body);
36 try std.testing.expectEqualStrings("Hello, World!\n", body);37
37 if (true) @panic("TODO implement inspecting custom headers in responses");38 try testing.expectEqualStrings("Hello, World!\n", body);
38 //try testing.expectEqualStrings("aaaa", req.response.headers.getFirstValue("x-checksum").?);39
40 var it = req.response.iterateHeaders();
41 {
42 const header = it.next().?;
43 try testing.expect(!it.is_trailer);
44 try testing.expectEqualStrings("connection", header.name);
45 try testing.expectEqualStrings("keep-alive", header.value);
46 }
47 {
48 const header = it.next().?;
49 try testing.expect(!it.is_trailer);
50 try testing.expectEqualStrings("transfer-encoding", header.name);
51 try testing.expectEqualStrings("chunked", header.value);
52 }
53 {
54 const header = it.next().?;
55 try testing.expect(it.is_trailer);
56 try testing.expectEqualStrings("X-Checksum", header.name);
57 try testing.expectEqualStrings("aaaa", header.value);
58 }
59 try testing.expectEqual(null, it.next());
60 }
3961
40 // connection has been kept alive62 // connection has been kept alive
41 try std.testing.expect(client.connection_pool.free_len == 1);63 try testing.expect(client.connection_pool.free_len == 1);
42}64}
4365
44fn serverThread(http_server: *std.http.Server) anyerror!void {66fn serverThread(http_server: *std.http.Server) anyerror!void {
45 const gpa = std.testing.allocator;67 const gpa = testing.allocator;
4668
47 var header_buffer: [1024]u8 = undefined;69 var header_buffer: [1024]u8 = undefined;
48 var remaining: usize = 1;70 var remaining: usize = 1;
...@@ -60,17 +82,16 @@ fn serverThread(http_server: *std.http.Server) anyerror!void {...@@ -60,17 +82,16 @@ fn serverThread(http_server: *std.http.Server) anyerror!void {
60 };82 };
61 try serve(&res);83 try serve(&res);
6284
63 try std.testing.expectEqual(.reset, res.reset());85 try testing.expectEqual(.reset, res.reset());
64 }86 }
65}87}
6688
67fn serve(res: *std.http.Server.Response) !void {89fn serve(res: *std.http.Server.Response) !void {
68 try std.testing.expectEqualStrings(res.request.target, "/trailer");90 try testing.expectEqualStrings(res.request.target, "/trailer");
69 res.transfer_encoding = .chunked;91 res.transfer_encoding = .chunked;
7092
71 try res.send();93 try res.send();
72 try res.writeAll("Hello, ");94 try res.writeAll("Hello, ");
73 try res.writeAll("World!\n");95 try res.writeAll("World!\n");
74 // try res.finish();
75 try res.connection.writeAll("0\r\nX-Checksum: aaaa\r\n\r\n");96 try res.connection.writeAll("0\r\nX-Checksum: aaaa\r\n\r\n");
76}97}