authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-21 20:51:55-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log380916c0f8883746e4d84d5334f68d0569d76f38
treed308cd79fd6eeeb1027d82c4da0de5e88a487727
parent40ed3c4d2493835cd0a9217b947b115780a9ff27

std.http.Server.Request.Respond: support all transfer encodings

Before I mistakenly thought that missing content-length meant zero when it actually means to stream until the connection is closed. Now the respond() function accepts transfer_encoding which can be left as default (use content.len for content-length), set to none which makes it omit the content-length, or chunked, which makes it format the response as a chunked transfer even though the server has the entire contents already buffered. The echo-content tests are moved from test/standalone/http.zig to the standard library where they are actually run.

3 files changed, 322 insertions(+), 199 deletions(-)

lib/std/http/Server.zig+81-35
......@@ -279,13 +279,15 @@ pub const Request = struct {
279279 reason: ?[]const u8 = null,
280280 keep_alive: bool = true,
281281 extra_headers: []const http.Header = &.{},
282 transfer_encoding: ?http.TransferEncoding = null,
282283 };
283284
284285 /// Send an entire HTTP response to the client, including headers and body.
285286 ///
286287 /// Automatically handles HEAD requests by omitting the body.
287 /// Uses the "content-length" header unless `content` is empty in which
288 /// case it omits the content-length header.
288 ///
289 /// Unless `transfer_encoding` is specified, uses the "content-length"
290 /// header.
289291 ///
290292 /// If the request contains a body and the connection is to be reused,
291293 /// discards the request body, leaving the Server in the `ready` state. If
......@@ -303,7 +305,9 @@ pub const Request = struct {
303305 assert(options.status != .@"continue");
304306 assert(options.extra_headers.len <= max_extra_headers);
305307
306 const keep_alive = request.discardBody(options.keep_alive);
308 const transfer_encoding_none = (options.transfer_encoding orelse .chunked) == .none;
309 const server_keep_alive = !transfer_encoding_none and options.keep_alive;
310 const keep_alive = request.discardBody(server_keep_alive);
307311
308312 const phrase = options.reason orelse options.status.phrase() orelse "";
309313
......@@ -314,9 +318,15 @@ pub const Request = struct {
314318 }) catch unreachable;
315319 if (keep_alive)
316320 h.appendSliceAssumeCapacity("connection: keep-alive\r\n");
317 if (content.len > 0)
321
322 if (options.transfer_encoding) |transfer_encoding| switch (transfer_encoding) {
323 .none => {},
324 .chunked => h.appendSliceAssumeCapacity("transfer-encoding: chunked\r\n"),
325 } else {
318326 h.fixedWriter().print("content-length: {d}\r\n", .{content.len}) catch unreachable;
327 }
319328
329 var chunk_header_buffer: [18]u8 = undefined;
320330 var iovecs: [max_extra_headers * 4 + 3]std.posix.iovec_const = undefined;
321331 var iovecs_len: usize = 0;
322332
......@@ -358,12 +368,47 @@ pub const Request = struct {
358368 };
359369 iovecs_len += 1;
360370
361 if (request.head.method != .HEAD and content.len > 0) {
362 iovecs[iovecs_len] = .{
363 .iov_base = content.ptr,
364 .iov_len = content.len,
365 };
366 iovecs_len += 1;
371 if (request.head.method != .HEAD) {
372 const is_chunked = (options.transfer_encoding orelse .none) == .chunked;
373 if (is_chunked) {
374 if (content.len > 0) {
375 const chunk_header = std.fmt.bufPrint(
376 &chunk_header_buffer,
377 "{x}\r\n",
378 .{content.len},
379 ) catch unreachable;
380
381 iovecs[iovecs_len] = .{
382 .iov_base = chunk_header.ptr,
383 .iov_len = chunk_header.len,
384 };
385 iovecs_len += 1;
386
387 iovecs[iovecs_len] = .{
388 .iov_base = content.ptr,
389 .iov_len = content.len,
390 };
391 iovecs_len += 1;
392
393 iovecs[iovecs_len] = .{
394 .iov_base = "\r\n",
395 .iov_len = 2,
396 };
397 iovecs_len += 1;
398 }
399
400 iovecs[iovecs_len] = .{
401 .iov_base = "0\r\n\r\n",
402 .iov_len = 5,
403 };
404 iovecs_len += 1;
405 } else if (content.len > 0) {
406 iovecs[iovecs_len] = .{
407 .iov_base = content.ptr,
408 .iov_len = content.len,
409 };
410 iovecs_len += 1;
411 }
367412 }
368413
369414 try request.server.connection.stream.writevAll(iovecs[0..iovecs_len]);
......@@ -400,8 +445,9 @@ pub const Request = struct {
400445 pub fn respondStreaming(request: *Request, options: RespondStreamingOptions) Response {
401446 const o = options.respond_options;
402447 assert(o.status != .@"continue");
403
404 const keep_alive = request.discardBody(o.keep_alive);
448 const transfer_encoding_none = (o.transfer_encoding orelse .chunked) == .none;
449 const server_keep_alive = !transfer_encoding_none and o.keep_alive;
450 const keep_alive = request.discardBody(server_keep_alive);
405451 const phrase = o.reason orelse o.status.phrase() orelse "";
406452
407453 var h = std.ArrayListUnmanaged(u8).initBuffer(options.send_buffer);
......@@ -815,26 +861,32 @@ pub const Response = struct {
815861 };
816862 iovecs_len += 1;
817863
818 iovecs[iovecs_len] = .{
819 .iov_base = chunk_header.ptr,
820 .iov_len = chunk_header.len,
821 };
822 iovecs_len += 1;
864 if (r.chunk_len > 0) {
865 iovecs[iovecs_len] = .{
866 .iov_base = chunk_header.ptr,
867 .iov_len = chunk_header.len,
868 };
869 iovecs_len += 1;
823870
824 iovecs[iovecs_len] = .{
825 .iov_base = r.send_buffer.ptr + r.send_buffer_end - r.chunk_len,
826 .iov_len = r.chunk_len,
827 };
828 iovecs_len += 1;
871 iovecs[iovecs_len] = .{
872 .iov_base = r.send_buffer.ptr + r.send_buffer_end - r.chunk_len,
873 .iov_len = r.chunk_len,
874 };
875 iovecs_len += 1;
876
877 iovecs[iovecs_len] = .{
878 .iov_base = "\r\n",
879 .iov_len = 2,
880 };
881 iovecs_len += 1;
882 }
829883
830884 if (end_trailers) |trailers| {
831 if (r.chunk_len > 0) {
832 iovecs[iovecs_len] = .{
833 .iov_base = "\r\n0\r\n",
834 .iov_len = 5,
835 };
836 iovecs_len += 1;
837 }
885 iovecs[iovecs_len] = .{
886 .iov_base = "0\r\n",
887 .iov_len = 3,
888 };
889 iovecs_len += 1;
838890
839891 for (trailers) |trailer| {
840892 iovecs[iovecs_len] = .{
......@@ -862,12 +914,6 @@ pub const Response = struct {
862914 iovecs_len += 1;
863915 }
864916
865 iovecs[iovecs_len] = .{
866 .iov_base = "\r\n",
867 .iov_len = 2,
868 };
869 iovecs_len += 1;
870 } else if (r.chunk_len > 0) {
871917 iovecs[iovecs_len] = .{
872918 .iov_base = "\r\n",
873919 .iov_len = 2,
lib/std/http/test.zig+241-1
......@@ -1,6 +1,7 @@
11const builtin = @import("builtin");
22const std = @import("std");
33const testing = std.testing;
4const native_endian = builtin.cpu.arch.endian();
45
56test "trailers" {
67 if (builtin.single_threaded) return error.SkipZigTest;
......@@ -106,7 +107,6 @@ test "HTTP server handles a chunked transfer coding request" {
106107 return error.SkipZigTest;
107108 }
108109
109 const native_endian = comptime builtin.cpu.arch.endian();
110110 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
111111 // https://github.com/ziglang/zig/issues/13782
112112 return error.SkipZigTest;
......@@ -168,3 +168,243 @@ test "HTTP server handles a chunked transfer coding request" {
168168
169169 server_thread.join();
170170}
171
172test "echo content server" {
173 if (builtin.single_threaded) return error.SkipZigTest;
174 if (builtin.os.tag == .wasi) return error.SkipZigTest;
175
176 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
177 // https://github.com/ziglang/zig/issues/13782
178 return error.SkipZigTest;
179 }
180
181 const gpa = std.testing.allocator;
182
183 const address = try std.net.Address.parseIp("127.0.0.1", 0);
184 var socket_server = try address.listen(.{ .reuse_address = true });
185 defer socket_server.deinit();
186 const port = socket_server.listen_address.in.getPort();
187
188 const server_thread = try std.Thread.spawn(.{}, (struct {
189 fn handleRequest(request: *std.http.Server.Request) !void {
190 std.debug.print("server received {s} {s} {s}\n", .{
191 @tagName(request.head.method),
192 @tagName(request.head.version),
193 request.head.target,
194 });
195
196 const body = try request.reader().readAllAlloc(std.testing.allocator, 8192);
197 defer std.testing.allocator.free(body);
198
199 try testing.expect(std.mem.startsWith(u8, request.head.target, "/echo-content"));
200 try testing.expectEqualStrings("Hello, World!\n", body);
201 try testing.expectEqualStrings("text/plain", request.head.content_type.?);
202
203 var send_buffer: [100]u8 = undefined;
204 var response = request.respondStreaming(.{
205 .send_buffer = &send_buffer,
206 .content_length = switch (request.head.transfer_encoding) {
207 .chunked => null,
208 .none => len: {
209 try testing.expectEqual(14, request.head.content_length.?);
210 break :len 14;
211 },
212 },
213 });
214
215 try response.flush(); // Test an early flush to send the HTTP headers before the body.
216 const w = response.writer();
217 try w.writeAll("Hello, ");
218 try w.writeAll("World!\n");
219 try response.end();
220 std.debug.print(" server finished responding\n", .{});
221 }
222
223 fn run(net_server: *std.net.Server) anyerror!void {
224 var read_buffer: [1024]u8 = undefined;
225
226 accept: while (true) {
227 const conn = try net_server.accept();
228 defer conn.stream.close();
229
230 var http_server = std.http.Server.init(conn, &read_buffer);
231
232 while (http_server.state == .ready) {
233 var request = http_server.receiveHead() catch |err| switch (err) {
234 error.HttpConnectionClosing => continue :accept,
235 else => |e| return e,
236 };
237 if (std.mem.eql(u8, request.head.target, "/end")) {
238 return request.respond("", .{ .keep_alive = false });
239 }
240 handleRequest(&request) catch |err| {
241 // This message helps the person troubleshooting determine whether
242 // output comes from the server thread or the client thread.
243 std.debug.print("handleRequest failed with '{s}'\n", .{@errorName(err)});
244 return err;
245 };
246 }
247 }
248 }
249 }).run, .{&socket_server});
250
251 defer server_thread.join();
252
253 {
254 var client: std.http.Client = .{ .allocator = gpa };
255 defer client.deinit();
256
257 try echoTests(&client, port);
258 }
259}
260
261fn echoTests(client: *std.http.Client, port: u16) !void {
262 const gpa = testing.allocator;
263 var location_buffer: [100]u8 = undefined;
264
265 { // send content-length request
266 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content", .{port});
267 defer gpa.free(location);
268 const uri = try std.Uri.parse(location);
269
270 var server_header_buffer: [1024]u8 = undefined;
271 var req = try client.open(.POST, uri, .{
272 .server_header_buffer = &server_header_buffer,
273 .extra_headers = &.{
274 .{ .name = "content-type", .value = "text/plain" },
275 },
276 });
277 defer req.deinit();
278
279 req.transfer_encoding = .{ .content_length = 14 };
280
281 try req.send(.{});
282 try req.writeAll("Hello, ");
283 try req.writeAll("World!\n");
284 try req.finish();
285
286 try req.wait();
287
288 const body = try req.reader().readAllAlloc(gpa, 8192);
289 defer gpa.free(body);
290
291 try testing.expectEqualStrings("Hello, World!\n", body);
292 }
293
294 // connection has been kept alive
295 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
296
297 { // send chunked request
298 const uri = try std.Uri.parse(try std.fmt.bufPrint(
299 &location_buffer,
300 "http://127.0.0.1:{d}/echo-content",
301 .{port},
302 ));
303
304 var server_header_buffer: [1024]u8 = undefined;
305 var req = try client.open(.POST, uri, .{
306 .server_header_buffer = &server_header_buffer,
307 .extra_headers = &.{
308 .{ .name = "content-type", .value = "text/plain" },
309 },
310 });
311 defer req.deinit();
312
313 req.transfer_encoding = .chunked;
314
315 try req.send(.{});
316 try req.writeAll("Hello, ");
317 try req.writeAll("World!\n");
318 try req.finish();
319
320 try req.wait();
321
322 const body = try req.reader().readAllAlloc(gpa, 8192);
323 defer gpa.free(body);
324
325 try testing.expectEqualStrings("Hello, World!\n", body);
326 }
327
328 // connection has been kept alive
329 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
330
331 { // Client.fetch()
332
333 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
334 defer gpa.free(location);
335
336 var body = std.ArrayList(u8).init(gpa);
337 defer body.deinit();
338
339 const res = try client.fetch(.{
340 .location = .{ .url = location },
341 .method = .POST,
342 .payload = "Hello, World!\n",
343 .extra_headers = &.{
344 .{ .name = "content-type", .value = "text/plain" },
345 },
346 .response_storage = .{ .dynamic = &body },
347 });
348 try testing.expectEqual(.ok, res.status);
349 try testing.expectEqualStrings("Hello, World!\n", body.items);
350 }
351
352 { // expect: 100-continue
353 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#expect-100", .{port});
354 defer gpa.free(location);
355 const uri = try std.Uri.parse(location);
356
357 var server_header_buffer: [1024]u8 = undefined;
358 var req = try client.open(.POST, uri, .{
359 .server_header_buffer = &server_header_buffer,
360 .extra_headers = &.{
361 .{ .name = "expect", .value = "100-continue" },
362 .{ .name = "content-type", .value = "text/plain" },
363 },
364 });
365 defer req.deinit();
366
367 req.transfer_encoding = .chunked;
368
369 try req.send(.{});
370 try req.writeAll("Hello, ");
371 try req.writeAll("World!\n");
372 try req.finish();
373
374 try req.wait();
375 try testing.expectEqual(.ok, req.response.status);
376
377 const body = try req.reader().readAllAlloc(gpa, 8192);
378 defer gpa.free(body);
379
380 try testing.expectEqualStrings("Hello, World!\n", body);
381 }
382
383 { // expect: garbage
384 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#expect-garbage", .{port});
385 defer gpa.free(location);
386 const uri = try std.Uri.parse(location);
387
388 var server_header_buffer: [1024]u8 = undefined;
389 var req = try client.open(.POST, uri, .{
390 .server_header_buffer = &server_header_buffer,
391 .extra_headers = &.{
392 .{ .name = "content-type", .value = "text/plain" },
393 .{ .name = "expect", .value = "garbage" },
394 },
395 });
396 defer req.deinit();
397
398 req.transfer_encoding = .chunked;
399
400 try req.send(.{});
401 try req.wait();
402 try testing.expectEqual(.expectation_failed, req.response.status);
403 }
404
405 _ = try client.fetch(.{
406 .location = .{
407 .url = try std.fmt.bufPrint(&location_buffer, "http://127.0.0.1:{d}/end", .{port}),
408 },
409 });
410}
test/standalone/http.zig-163
......@@ -81,26 +81,6 @@ fn handleRequest(request: *http.Server.Request, listen_port: u16) !void {
8181 try w.writeAll("Hello, World!\n");
8282 }
8383
84 try response.end();
85 } else if (mem.startsWith(u8, request.head.target, "/echo-content")) {
86 try testing.expectEqualStrings("Hello, World!\n", body);
87 try testing.expectEqualStrings("text/plain", request.head.content_type.?);
88
89 var response = request.respondStreaming(.{
90 .send_buffer = &send_buffer,
91 .content_length = switch (request.head.transfer_encoding) {
92 .chunked => null,
93 .none => len: {
94 try testing.expectEqual(14, request.head.content_length.?);
95 break :len 14;
96 },
97 },
98 });
99
100 try response.flush(); // Test an early flush to send the HTTP headers before the body.
101 const w = response.writer();
102 try w.writeAll("Hello, ");
103 try w.writeAll("World!\n");
10484 try response.end();
10585 } else if (mem.eql(u8, request.head.target, "/redirect/1")) {
10686 var response = request.respondStreaming(.{
......@@ -351,39 +331,6 @@ pub fn main() !void {
351331 // connection has been kept alive
352332 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
353333
354 { // send content-length request
355 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content", .{port});
356 defer calloc.free(location);
357 const uri = try std.Uri.parse(location);
358
359 log.info("{s}", .{location});
360 var server_header_buffer: [1024]u8 = undefined;
361 var req = try client.open(.POST, uri, .{
362 .server_header_buffer = &server_header_buffer,
363 .extra_headers = &.{
364 .{ .name = "content-type", .value = "text/plain" },
365 },
366 });
367 defer req.deinit();
368
369 req.transfer_encoding = .{ .content_length = 14 };
370
371 try req.send(.{});
372 try req.writeAll("Hello, ");
373 try req.writeAll("World!\n");
374 try req.finish();
375
376 try req.wait();
377
378 const body = try req.reader().readAllAlloc(calloc, 8192);
379 defer calloc.free(body);
380
381 try testing.expectEqualStrings("Hello, World!\n", body);
382 }
383
384 // connection has been kept alive
385 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
386
387334 { // read content-length response with connection close
388335 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
389336 defer calloc.free(location);
......@@ -410,39 +357,6 @@ pub fn main() !void {
410357 // connection has been closed
411358 try testing.expect(client.connection_pool.free_len == 0);
412359
413 { // send chunked request
414 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content", .{port});
415 defer calloc.free(location);
416 const uri = try std.Uri.parse(location);
417
418 log.info("{s}", .{location});
419 var server_header_buffer: [1024]u8 = undefined;
420 var req = try client.open(.POST, uri, .{
421 .server_header_buffer = &server_header_buffer,
422 .extra_headers = &.{
423 .{ .name = "content-type", .value = "text/plain" },
424 },
425 });
426 defer req.deinit();
427
428 req.transfer_encoding = .chunked;
429
430 try req.send(.{});
431 try req.writeAll("Hello, ");
432 try req.writeAll("World!\n");
433 try req.finish();
434
435 try req.wait();
436
437 const body = try req.reader().readAllAlloc(calloc, 8192);
438 defer calloc.free(body);
439
440 try testing.expectEqualStrings("Hello, World!\n", body);
441 }
442
443 // connection has been kept alive
444 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
445
446360 { // relative redirect
447361 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/1", .{port});
448362 defer calloc.free(location);
......@@ -561,83 +475,6 @@ pub fn main() !void {
561475 // connection has been kept alive
562476 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
563477
564 { // Client.fetch()
565
566 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
567 defer calloc.free(location);
568
569 log.info("{s}", .{location});
570 var body = std.ArrayList(u8).init(calloc);
571 defer body.deinit();
572
573 const res = try client.fetch(.{
574 .location = .{ .url = location },
575 .method = .POST,
576 .payload = "Hello, World!\n",
577 .extra_headers = &.{
578 .{ .name = "content-type", .value = "text/plain" },
579 },
580 .response_storage = .{ .dynamic = &body },
581 });
582 try testing.expectEqual(.ok, res.status);
583 try testing.expectEqualStrings("Hello, World!\n", body.items);
584 }
585
586 { // expect: 100-continue
587 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#expect-100", .{port});
588 defer calloc.free(location);
589 const uri = try std.Uri.parse(location);
590
591 log.info("{s}", .{location});
592 var server_header_buffer: [1024]u8 = undefined;
593 var req = try client.open(.POST, uri, .{
594 .server_header_buffer = &server_header_buffer,
595 .extra_headers = &.{
596 .{ .name = "expect", .value = "100-continue" },
597 .{ .name = "content-type", .value = "text/plain" },
598 },
599 });
600 defer req.deinit();
601
602 req.transfer_encoding = .chunked;
603
604 try req.send(.{});
605 try req.writeAll("Hello, ");
606 try req.writeAll("World!\n");
607 try req.finish();
608
609 try req.wait();
610 try testing.expectEqual(http.Status.ok, req.response.status);
611
612 const body = try req.reader().readAllAlloc(calloc, 8192);
613 defer calloc.free(body);
614
615 try testing.expectEqualStrings("Hello, World!\n", body);
616 }
617
618 { // expect: garbage
619 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#expect-garbage", .{port});
620 defer calloc.free(location);
621 const uri = try std.Uri.parse(location);
622
623 log.info("{s}", .{location});
624 var server_header_buffer: [1024]u8 = undefined;
625 var req = try client.open(.POST, uri, .{
626 .server_header_buffer = &server_header_buffer,
627 .extra_headers = &.{
628 .{ .name = "content-type", .value = "text/plain" },
629 .{ .name = "expect", .value = "garbage" },
630 },
631 });
632 defer req.deinit();
633
634 req.transfer_encoding = .chunked;
635
636 try req.send(.{});
637 try req.wait();
638 try testing.expectEqual(http.Status.expectation_failed, req.response.status);
639 }
640
641478 { // issue 16282 *** This test leaves the client in an invalid state, it must be last ***
642479 const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
643480 defer calloc.free(location);