authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-09-10 00:25:37-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-09-10 10:26:52-04:00
log0489d06c249d16457d66523b5c407fc7ddeca45a
tree799b4abcf50830158975f8d408c7ce52ed401a59
parenta29ce78651c05029dbd72064752a099885edfd0c
signaturelock-open Commit is signed but in an unrecognized format.

make the std lib support event-based I/O

also add -fstack-report

14 files changed, 392 insertions(+), 307 deletions(-)

CMakeLists.txt+1
......@@ -449,6 +449,7 @@ set(ZIG_SOURCES
449449 "${CMAKE_SOURCE_DIR}/src/os.cpp"
450450 "${CMAKE_SOURCE_DIR}/src/parser.cpp"
451451 "${CMAKE_SOURCE_DIR}/src/range_set.cpp"
452 "${CMAKE_SOURCE_DIR}/src/stack_report.cpp"
452453 "${CMAKE_SOURCE_DIR}/src/target.cpp"
453454 "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp"
454455 "${CMAKE_SOURCE_DIR}/src/translate_c.cpp"
src/all_types.hpp+3
......@@ -1972,6 +1972,8 @@ struct CodeGen {
19721972 ZigFn *panic_fn;
19731973 TldFn *panic_tld_fn;
19741974
1975 ZigFn *largest_frame_fn;
1976
19751977 WantPIC want_pic;
19761978 WantStackCheck want_stack_check;
19771979 CacheHash cache_hash;
......@@ -2004,6 +2006,7 @@ struct CodeGen {
20042006 bool generate_error_name_table;
20052007 bool enable_cache; // mutually exclusive with output_dir
20062008 bool enable_time_report;
2009 bool enable_stack_report;
20072010 bool system_linker_hack;
20082011 bool reported_bad_link_libc_error;
20092012 bool is_dynamic; // shared library rather than static library. dynamic musl rather than static musl.
src/analyze.cpp+14-9
......@@ -5737,11 +5737,19 @@ static void mark_suspension_point(Scope *scope) {
57375737 return;
57385738 case ScopeIdVarDecl:
57395739 case ScopeIdDefer:
5740 case ScopeIdBlock:
57405741 looking_for_exprs = false;
57415742 continue;
5742 case ScopeIdLoop:
57435743 case ScopeIdRuntime:
57445744 continue;
5745 case ScopeIdLoop: {
5746 ScopeLoop *loop_scope = reinterpret_cast<ScopeLoop *>(scope);
5747 if (loop_scope->spill_scope != nullptr) {
5748 loop_scope->spill_scope->need_spill = MemoizedBoolTrue;
5749 }
5750 looking_for_exprs = false;
5751 continue;
5752 }
57455753 case ScopeIdExpr: {
57465754 if (!looking_for_exprs) {
57475755 // Now we're only looking for a block, to see if it's in a loop (see the case ScopeIdBlock)
......@@ -5758,14 +5766,6 @@ static void mark_suspension_point(Scope *scope) {
57585766 child_expr_scope = parent_expr_scope;
57595767 continue;
57605768 }
5761 case ScopeIdBlock:
5762 if (scope->parent->parent->id == ScopeIdLoop) {
5763 ScopeLoop *loop_scope = reinterpret_cast<ScopeLoop *>(scope->parent->parent);
5764 if (loop_scope->spill_scope != nullptr) {
5765 loop_scope->spill_scope->need_spill = MemoizedBoolTrue;
5766 }
5767 }
5768 return;
57695769 }
57705770 }
57715771}
......@@ -6082,6 +6082,11 @@ static Error resolve_async_frame(CodeGen *g, ZigType *frame_type) {
60826082 frame_type->abi_size = frame_type->data.frame.locals_struct->abi_size;
60836083 frame_type->abi_align = frame_type->data.frame.locals_struct->abi_align;
60846084 frame_type->size_in_bits = frame_type->data.frame.locals_struct->size_in_bits;
6085
6086 if (g->largest_frame_fn == nullptr || frame_type->abi_size > g->largest_frame_fn->frame_type->abi_size) {
6087 g->largest_frame_fn = fn;
6088 }
6089
60856090 return ErrorNone;
60866091}
60876092
src/main.cpp+12
......@@ -16,6 +16,7 @@
1616#include "libc_installation.hpp"
1717#include "userland.h"
1818#include "glibc.hpp"
19#include "stack_report.hpp"
1920
2021#include <stdio.h>
2122
......@@ -62,6 +63,7 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) {
6263 " -fPIC enable Position Independent Code\n"
6364 " -fno-PIC disable Position Independent Code\n"
6465 " -ftime-report print timing diagnostics\n"
66 " -fstack-report print stack size diagnostics\n"
6567 " --libc [file] Provide a file which specifies libc paths\n"
6668 " --name [name] override output name\n"
6769 " --output-dir [dir] override output directory (defaults to cwd)\n"
......@@ -476,6 +478,7 @@ int main(int argc, char **argv) {
476478 size_t ver_minor = 0;
477479 size_t ver_patch = 0;
478480 bool timing_info = false;
481 bool stack_report = false;
479482 const char *cache_dir = nullptr;
480483 CliPkg *cur_pkg = allocate<CliPkg>(1);
481484 BuildMode build_mode = BuildModeDebug;
......@@ -664,6 +667,8 @@ int main(int argc, char **argv) {
664667 each_lib_rpath = true;
665668 } else if (strcmp(arg, "-ftime-report") == 0) {
666669 timing_info = true;
670 } else if (strcmp(arg, "-fstack-report") == 0) {
671 stack_report = true;
667672 } else if (strcmp(arg, "--enable-valgrind") == 0) {
668673 valgrind_support = ValgrindSupportEnabled;
669674 } else if (strcmp(arg, "--disable-valgrind") == 0) {
......@@ -1136,6 +1141,7 @@ int main(int argc, char **argv) {
11361141 g->subsystem = subsystem;
11371142
11381143 g->enable_time_report = timing_info;
1144 g->enable_stack_report = stack_report;
11391145 codegen_set_out_name(g, buf_out_name);
11401146 codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
11411147 g->want_single_threaded = want_single_threaded;
......@@ -1223,6 +1229,8 @@ int main(int argc, char **argv) {
12231229 codegen_build_and_link(g);
12241230 if (timing_info)
12251231 codegen_print_timing_report(g, stdout);
1232 if (stack_report)
1233 zig_print_stack_report(g, stdout);
12261234
12271235 if (cmd == CmdRun) {
12281236 const char *exec_path = buf_ptr(&g->output_file_path);
......@@ -1272,6 +1280,10 @@ int main(int argc, char **argv) {
12721280 codegen_print_timing_report(g, stdout);
12731281 }
12741282
1283 if (stack_report) {
1284 zig_print_stack_report(g, stdout);
1285 }
1286
12751287 Buf *test_exe_path_unresolved = &g->output_file_path;
12761288 Buf *test_exe_path = buf_alloc();
12771289 *test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1);
src/stack_report.cpp created+78
......@@ -0,0 +1,78 @@
1/*
2 * Copyright (c) 2019 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8#include "stack_report.hpp"
9
10static void tree_print(FILE *f, ZigType *ty, size_t indent);
11
12static void pretty_print_bytes(FILE *f, double n) {
13 if (n > 1024.0 * 1024.0 * 1024.0) {
14 fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0);
15 return;
16 }
17 if (n > 1024.0 * 1024.0) {
18 fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0);
19 return;
20 }
21 if (n > 1024.0) {
22 fprintf(f, "%.02f KiB", n / 1024.0);
23 return;
24 }
25 fprintf(f, "%.02f bytes", n );
26 return;
27}
28
29static int compare_type_abi_sizes_desc(const void *a, const void *b) {
30 uint64_t size_a = (*(ZigType * const*)(a))->abi_size;
31 uint64_t size_b = (*(ZigType * const*)(b))->abi_size;
32 if (size_a > size_b)
33 return -1;
34 if (size_a < size_b)
35 return 1;
36 return 0;
37}
38
39static void tree_print_struct(FILE *f, ZigType *struct_type, size_t indent) {
40 ZigList<ZigType *> children = {};
41 uint64_t sum_from_fields = 0;
42 for (size_t i = 0; i < struct_type->data.structure.src_field_count; i += 1) {
43 TypeStructField *field = &struct_type->data.structure.fields[i];
44 children.append(field->type_entry);
45 sum_from_fields += field->type_entry->abi_size;
46 }
47 qsort(children.items, children.length, sizeof(ZigType *), compare_type_abi_sizes_desc);
48 fprintf(f, " (padding = %" ZIG_PRI_u64 ")\n", struct_type->abi_size - sum_from_fields);
49 for (size_t i = 0; i < children.length; i += 1) {
50 ZigType *child_type = children.at(i);
51 tree_print(f, child_type, indent + 1);
52 }
53}
54
55static void tree_print(FILE *f, ZigType *ty, size_t indent) {
56 for (size_t i = 0; i < indent; i += 1) {
57 fprintf(f, " ");
58 }
59 fprintf(f, "%s: ", buf_ptr(&ty->name));
60 pretty_print_bytes(f, ty->abi_size);
61 switch (ty->id) {
62 case ZigTypeIdFnFrame:
63 return tree_print_struct(f, ty->data.frame.locals_struct, indent);
64 case ZigTypeIdStruct:
65 return tree_print_struct(f, ty, indent);
66 default:
67 fprintf(f, "\n");
68 return;
69 }
70}
71
72void zig_print_stack_report(CodeGen *g, FILE *f) {
73 if (g->largest_frame_fn == nullptr) {
74 fprintf(f, "No async function frames in entire compilation.\n");
75 return;
76 }
77 tree_print(f, g->largest_frame_fn->frame_type, 0);
78}
src/stack_report.hpp created+16
......@@ -0,0 +1,16 @@
1/*
2 * Copyright (c) 2019 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8#ifndef ZIG_STACK_REPORT_HPP
9#define ZIG_STACK_REPORT_HPP
10
11#include "all_types.hpp"
12#include <stdio.h>
13
14void zig_print_stack_report(CodeGen *g, FILE *f);
15
16#endif
std/c.zig+1
......@@ -68,6 +68,7 @@ pub extern "c" fn open(path: [*]const u8, oflag: c_uint, ...) c_int;
6868pub extern "c" fn openat(fd: c_int, path: [*]const u8, oflag: c_uint, ...) c_int;
6969pub extern "c" fn raise(sig: c_int) c_int;
7070pub extern "c" fn read(fd: fd_t, buf: [*]u8, nbyte: usize) isize;
71pub extern "c" fn readv(fd: c_int, iov: [*]const iovec, iovcnt: c_uint) isize;
7172pub extern "c" fn pread(fd: fd_t, buf: [*]u8, nbyte: usize, offset: u64) isize;
7273pub extern "c" fn preadv(fd: c_int, iov: [*]const iovec, iovcnt: c_uint, offset: usize) isize;
7374pub extern "c" fn writev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint) isize;
std/debug.zig+30-13
......@@ -330,14 +330,16 @@ pub fn writeCurrentStackTraceWindows(
330330 }
331331}
332332
333/// TODO once https://github.com/ziglang/zig/issues/3157 is fully implemented,
334/// make this `noasync fn` and remove the individual noasync calls.
333335pub fn printSourceAtAddress(debug_info: *DebugInfo, out_stream: var, address: usize, tty_color: bool) !void {
334336 if (windows.is_the_target) {
335 return printSourceAtAddressWindows(debug_info, out_stream, address, tty_color);
337 return noasync printSourceAtAddressWindows(debug_info, out_stream, address, tty_color);
336338 }
337339 if (os.darwin.is_the_target) {
338 return printSourceAtAddressMacOs(debug_info, out_stream, address, tty_color);
340 return noasync printSourceAtAddressMacOs(debug_info, out_stream, address, tty_color);
339341 }
340 return printSourceAtAddressPosix(debug_info, out_stream, address, tty_color);
342 return noasync printSourceAtAddressPosix(debug_info, out_stream, address, tty_color);
341343}
342344
343345fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, relocated_address: usize, tty_color: bool) !void {
......@@ -793,7 +795,7 @@ fn printLineInfo(
793795 try out_stream.write(GREEN ++ "^" ++ RESET ++ "\n");
794796 }
795797 } else |err| switch (err) {
796 error.EndOfFile, error.FileNotFound => {},
798 error.EndOfFile, error.FileNotFound => {},
797799 else => return err,
798800 }
799801 } else {
......@@ -816,16 +818,18 @@ pub const OpenSelfDebugInfoError = error{
816818 UnsupportedOperatingSystem,
817819};
818820
821/// TODO once https://github.com/ziglang/zig/issues/3157 is fully implemented,
822/// make this `noasync fn` and remove the individual noasync calls.
819823pub fn openSelfDebugInfo(allocator: *mem.Allocator) !DebugInfo {
820824 if (builtin.strip_debug_info)
821825 return error.MissingDebugInfo;
822826 if (windows.is_the_target) {
823 return openSelfDebugInfoWindows(allocator);
827 return noasync openSelfDebugInfoWindows(allocator);
824828 }
825829 if (os.darwin.is_the_target) {
826 return openSelfDebugInfoMacOs(allocator);
830 return noasync openSelfDebugInfoMacOs(allocator);
827831 }
828 return openSelfDebugInfoPosix(allocator);
832 return noasync openSelfDebugInfoPosix(allocator);
829833}
830834
831835fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo {
......@@ -1508,15 +1512,25 @@ fn parseFormValueBlock(allocator: *mem.Allocator, in_stream: var, size: usize) !
15081512}
15091513
15101514fn parseFormValueConstant(allocator: *mem.Allocator, in_stream: var, signed: bool, comptime size: i32) !FormValue {
1515 // TODO: Please forgive me, I've worked around zig not properly spilling some intermediate values here.
1516 // `noasync` should be removed from all the function calls once it is fixed.
15111517 return FormValue{
15121518 .Const = Constant{
15131519 .signed = signed,
15141520 .payload = switch (size) {
1515 1 => try in_stream.readIntLittle(u8),
1516 2 => try in_stream.readIntLittle(u16),
1517 4 => try in_stream.readIntLittle(u32),
1518 8 => try in_stream.readIntLittle(u64),
1519 -1 => if (signed) @bitCast(u64, try leb.readILEB128(i64, in_stream)) else try leb.readULEB128(u64, in_stream),
1521 1 => try noasync in_stream.readIntLittle(u8),
1522 2 => try noasync in_stream.readIntLittle(u16),
1523 4 => try noasync in_stream.readIntLittle(u32),
1524 8 => try noasync in_stream.readIntLittle(u64),
1525 -1 => blk: {
1526 if (signed) {
1527 const x = try noasync leb.readILEB128(i64, in_stream);
1528 break :blk @bitCast(u64, x);
1529 } else {
1530 const x = try noasync leb.readULEB128(u64, in_stream);
1531 break :blk x;
1532 }
1533 },
15201534 else => @compileError("Invalid size"),
15211535 },
15221536 },
......@@ -1584,7 +1598,10 @@ fn parseFormValue(allocator: *mem.Allocator, in_stream: var, form_id: u64, is_64
15841598 DW.FORM_strp => FormValue{ .StrPtr = try parseFormValueDwarfOffsetSize(in_stream, is_64) },
15851599 DW.FORM_indirect => {
15861600 const child_form_id = try leb.readULEB128(u64, in_stream);
1587 return parseFormValue(allocator, in_stream, child_form_id, is_64);
1601 const F = @typeOf(async parseFormValue(allocator, in_stream, child_form_id, is_64));
1602 var frame = try allocator.create(F);
1603 defer allocator.destroy(frame);
1604 return await @asyncCall(frame, {}, parseFormValue, allocator, in_stream, child_form_id, is_64);
15881605 },
15891606 else => error.InvalidDebugInfo,
15901607 };
std/event.zig-2
......@@ -6,7 +6,6 @@ pub const Locked = @import("event/locked.zig").Locked;
66pub const RwLock = @import("event/rwlock.zig").RwLock;
77pub const RwLocked = @import("event/rwlocked.zig").RwLocked;
88pub const Loop = @import("event/loop.zig").Loop;
9pub const io = @import("event/io.zig");
109pub const fs = @import("event/fs.zig");
1110pub const net = @import("event/net.zig");
1211
......@@ -15,7 +14,6 @@ test "import event tests" {
1514 _ = @import("event/fs.zig");
1615 _ = @import("event/future.zig");
1716 _ = @import("event/group.zig");
18 _ = @import("event/io.zig");
1917 _ = @import("event/lock.zig");
2018 _ = @import("event/locked.zig");
2119 _ = @import("event/rwlock.zig");
std/event/io.zig deleted-76
......@@ -1,76 +0,0 @@
1const std = @import("../std.zig");
2const builtin = @import("builtin");
3const assert = std.debug.assert;
4const mem = std.mem;
5
6pub fn InStream(comptime ReadError: type) type {
7 return struct {
8 const Self = @This();
9 pub const Error = ReadError;
10
11 /// Return the number of bytes read. It may be less than buffer.len.
12 /// If the number of bytes read is 0, it means end of stream.
13 /// End of stream is not an error condition.
14 readFn: async fn (self: *Self, buffer: []u8) Error!usize,
15
16 /// Return the number of bytes read. It may be less than buffer.len.
17 /// If the number of bytes read is 0, it means end of stream.
18 /// End of stream is not an error condition.
19 pub async fn read(self: *Self, buffer: []u8) !usize {
20 return self.readFn(self, buffer);
21 }
22
23 /// Return the number of bytes read. If it is less than buffer.len
24 /// it means end of stream.
25 pub async fn readFull(self: *Self, buffer: []u8) !usize {
26 var index: usize = 0;
27 while (index != buf.len) {
28 const amt_read = try self.read(buf[index..]);
29 if (amt_read == 0) return index;
30 index += amt_read;
31 }
32 return index;
33 }
34
35 /// Same as `readFull` but end of stream returns `error.EndOfStream`.
36 pub async fn readNoEof(self: *Self, buf: []u8) !void {
37 const amt_read = try self.readFull(buf[index..]);
38 if (amt_read < buf.len) return error.EndOfStream;
39 }
40
41 pub async fn readIntLittle(self: *Self, comptime T: type) !T {
42 var bytes: [@sizeOf(T)]u8 = undefined;
43 try self.readNoEof(bytes[0..]);
44 return mem.readIntLittle(T, &bytes);
45 }
46
47 pub async fn readIntBe(self: *Self, comptime T: type) !T {
48 var bytes: [@sizeOf(T)]u8 = undefined;
49 try self.readNoEof(bytes[0..]);
50 return mem.readIntBig(T, &bytes);
51 }
52
53 pub async fn readInt(self: *Self, comptime T: type, endian: builtin.Endian) !T {
54 var bytes: [@sizeOf(T)]u8 = undefined;
55 try self.readNoEof(bytes[0..]);
56 return mem.readInt(T, &bytes, endian);
57 }
58
59 pub async fn readStruct(self: *Self, comptime T: type) !T {
60 // Only extern and packed structs have defined in-memory layout.
61 comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);
62 var res: [1]T = undefined;
63 try self.readNoEof(@sliceToBytes(res[0..]));
64 return res[0];
65 }
66 };
67}
68
69pub fn OutStream(comptime WriteError: type) type {
70 return struct {
71 const Self = @This();
72 pub const Error = WriteError;
73
74 writeFn: async fn (self: *Self, buffer: []u8) Error!void,
75 };
76}
std/event/loop.zig+5-9
......@@ -86,18 +86,10 @@ pub const Loop = struct {
8686 };
8787 };
8888
89 pub const IoMode = enum {
90 blocking,
91 evented,
92 mixed,
93 };
94 pub const io_mode: IoMode = if (@hasDecl(root, "io_mode")) root.io_mode else IoMode.blocking;
9589 var global_instance_state: Loop = undefined;
96 threadlocal var per_thread_instance: ?*Loop = null;
97 const default_instance: ?*Loop = switch (io_mode) {
90 const default_instance: ?*Loop = switch (std.io.mode) {
9891 .blocking => null,
9992 .evented => &global_instance_state,
100 .mixed => per_thread_instance,
10193 };
10294 pub const instance: ?*Loop = if (@hasDecl(root, "event_loop")) root.event_loop else default_instance;
10395
......@@ -470,6 +462,10 @@ pub const Loop = struct {
470462 }
471463 }
472464
465 pub fn waitUntilFdReadable(self: *Loop, fd: os.fd_t) !void {
466 return self.linuxWaitFd(fd, os.EPOLLET | os.EPOLLIN);
467 }
468
473469 pub async fn bsdWaitKev(self: *Loop, ident: usize, filter: i16, fflags: u32) !os.Kevent {
474470 var resume_node = ResumeNode.Basic{
475471 .base = ResumeNode{
std/io.zig+14-174
......@@ -1,5 +1,6 @@
11const std = @import("std.zig");
22const builtin = @import("builtin");
3const root = @import("root");
34const c = std.c;
45
56const math = std.math;
......@@ -15,6 +16,18 @@ const fmt = std.fmt;
1516const File = std.fs.File;
1617const testing = std.testing;
1718
19pub const Mode = enum {
20 blocking,
21 evented,
22};
23pub const mode: Mode = if (@hasDecl(root, "io_mode"))
24 root.io_mode
25else if (@hasDecl(root, "event_loop"))
26 Mode.evented
27else
28 Mode.blocking;
29pub const is_async = mode != .blocking;
30
1831pub const GetStdIoError = os.windows.GetStdHandleError;
1932
2033pub fn getStdOut() GetStdIoError!File {
......@@ -44,180 +57,7 @@ pub fn getStdIn() GetStdIoError!File {
4457pub const SeekableStream = @import("io/seekable_stream.zig").SeekableStream;
4558pub const SliceSeekableInStream = @import("io/seekable_stream.zig").SliceSeekableInStream;
4659pub const COutStream = @import("io/c_out_stream.zig").COutStream;
47
48pub fn InStream(comptime ReadError: type) type {
49 return struct {
50 const Self = @This();
51 pub const Error = ReadError;
52
53 /// Return the number of bytes read. If the number read is smaller than buf.len, it
54 /// means the stream reached the end. Reaching the end of a stream is not an error
55 /// condition.
56 readFn: fn (self: *Self, buffer: []u8) Error!usize,
57
58 /// Replaces `buffer` contents by reading from the stream until it is finished.
59 /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and
60 /// the contents read from the stream are lost.
61 pub fn readAllBuffer(self: *Self, buffer: *Buffer, max_size: usize) !void {
62 try buffer.resize(0);
63
64 var actual_buf_len: usize = 0;
65 while (true) {
66 const dest_slice = buffer.toSlice()[actual_buf_len..];
67 const bytes_read = try self.readFull(dest_slice);
68 actual_buf_len += bytes_read;
69
70 if (bytes_read != dest_slice.len) {
71 buffer.shrink(actual_buf_len);
72 return;
73 }
74
75 const new_buf_size = math.min(max_size, actual_buf_len + mem.page_size);
76 if (new_buf_size == actual_buf_len) return error.StreamTooLong;
77 try buffer.resize(new_buf_size);
78 }
79 }
80
81 /// Allocates enough memory to hold all the contents of the stream. If the allocated
82 /// memory would be greater than `max_size`, returns `error.StreamTooLong`.
83 /// Caller owns returned memory.
84 /// If this function returns an error, the contents from the stream read so far are lost.
85 pub fn readAllAlloc(self: *Self, allocator: *mem.Allocator, max_size: usize) ![]u8 {
86 var buf = Buffer.initNull(allocator);
87 defer buf.deinit();
88
89 try self.readAllBuffer(&buf, max_size);
90 return buf.toOwnedSlice();
91 }
92
93 /// Replaces `buffer` contents by reading from the stream until `delimiter` is found.
94 /// Does not include the delimiter in the result.
95 /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and the contents
96 /// read from the stream so far are lost.
97 pub fn readUntilDelimiterBuffer(self: *Self, buffer: *Buffer, delimiter: u8, max_size: usize) !void {
98 try buffer.resize(0);
99
100 while (true) {
101 var byte: u8 = try self.readByte();
102
103 if (byte == delimiter) {
104 return;
105 }
106
107 if (buffer.len() == max_size) {
108 return error.StreamTooLong;
109 }
110
111 try buffer.appendByte(byte);
112 }
113 }
114
115 /// Allocates enough memory to read until `delimiter`. If the allocated
116 /// memory would be greater than `max_size`, returns `error.StreamTooLong`.
117 /// Caller owns returned memory.
118 /// If this function returns an error, the contents from the stream read so far are lost.
119 pub fn readUntilDelimiterAlloc(self: *Self, allocator: *mem.Allocator, delimiter: u8, max_size: usize) ![]u8 {
120 var buf = Buffer.initNull(allocator);
121 defer buf.deinit();
122
123 try self.readUntilDelimiterBuffer(&buf, delimiter, max_size);
124 return buf.toOwnedSlice();
125 }
126
127 /// Returns the number of bytes read. It may be less than buffer.len.
128 /// If the number of bytes read is 0, it means end of stream.
129 /// End of stream is not an error condition.
130 pub fn read(self: *Self, buffer: []u8) Error!usize {
131 return self.readFn(self, buffer);
132 }
133
134 /// Returns the number of bytes read. If the number read is smaller than buf.len, it
135 /// means the stream reached the end. Reaching the end of a stream is not an error
136 /// condition.
137 pub fn readFull(self: *Self, buffer: []u8) Error!usize {
138 var index: usize = 0;
139 while (index != buffer.len) {
140 const amt = try self.read(buffer[index..]);
141 if (amt == 0) return index;
142 index += amt;
143 }
144 return index;
145 }
146
147 /// Same as `readFull` but end of stream returns `error.EndOfStream`.
148 pub fn readNoEof(self: *Self, buf: []u8) !void {
149 const amt_read = try self.readFull(buf);
150 if (amt_read < buf.len) return error.EndOfStream;
151 }
152
153 /// Reads 1 byte from the stream or returns `error.EndOfStream`.
154 pub fn readByte(self: *Self) !u8 {
155 var result: [1]u8 = undefined;
156 try self.readNoEof(result[0..]);
157 return result[0];
158 }
159
160 /// Same as `readByte` except the returned byte is signed.
161 pub fn readByteSigned(self: *Self) !i8 {
162 return @bitCast(i8, try self.readByte());
163 }
164
165 /// Reads a native-endian integer
166 pub fn readIntNative(self: *Self, comptime T: type) !T {
167 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
168 try self.readNoEof(bytes[0..]);
169 return mem.readIntNative(T, &bytes);
170 }
171
172 /// Reads a foreign-endian integer
173 pub fn readIntForeign(self: *Self, comptime T: type) !T {
174 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
175 try self.readNoEof(bytes[0..]);
176 return mem.readIntForeign(T, &bytes);
177 }
178
179 pub fn readIntLittle(self: *Self, comptime T: type) !T {
180 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
181 try self.readNoEof(bytes[0..]);
182 return mem.readIntLittle(T, &bytes);
183 }
184
185 pub fn readIntBig(self: *Self, comptime T: type) !T {
186 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
187 try self.readNoEof(bytes[0..]);
188 return mem.readIntBig(T, &bytes);
189 }
190
191 pub fn readInt(self: *Self, comptime T: type, endian: builtin.Endian) !T {
192 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
193 try self.readNoEof(bytes[0..]);
194 return mem.readInt(T, &bytes, endian);
195 }
196
197 pub fn readVarInt(self: *Self, comptime ReturnType: type, endian: builtin.Endian, size: usize) !ReturnType {
198 assert(size <= @sizeOf(ReturnType));
199 var bytes_buf: [@sizeOf(ReturnType)]u8 = undefined;
200 const bytes = bytes_buf[0..size];
201 try self.readNoEof(bytes);
202 return mem.readVarInt(ReturnType, bytes, endian);
203 }
204
205 pub fn skipBytes(self: *Self, num_bytes: u64) !void {
206 var i: u64 = 0;
207 while (i < num_bytes) : (i += 1) {
208 _ = try self.readByte();
209 }
210 }
211
212 pub fn readStruct(self: *Self, comptime T: type) !T {
213 // Only extern and packed structs have defined in-memory layout.
214 comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);
215 var res: [1]T = undefined;
216 try self.readNoEof(@sliceToBytes(res[0..]));
217 return res[0];
218 }
219 };
220}
60pub const InStream = @import("io/in_stream.zig").InStream;
22161
22262pub fn OutStream(comptime WriteError: type) type {
22363 return struct {
std/io/in_stream.zig created+200
......@@ -0,0 +1,200 @@
1const std = @import("../std.zig");
2const builtin = @import("builtin");
3const root = @import("root");
4const math = std.math;
5const assert = std.debug.assert;
6const mem = std.mem;
7const Buffer = std.Buffer;
8
9pub const default_stack_size = 4 * 1024 * 1024;
10pub const stack_size: usize = if (@hasDecl(root, "stack_size_std_io_InStream"))
11 root.stack_size_std_io_InStream
12else
13 default_stack_size;
14pub const stack_align = 16;
15
16pub fn InStream(comptime ReadError: type) type {
17 return struct {
18 const Self = @This();
19 pub const Error = ReadError;
20 pub const ReadFn = if (std.io.is_async)
21 async fn (self: *Self, buffer: []u8) Error!usize
22 else
23 fn (self: *Self, buffer: []u8) Error!usize;
24
25 /// Returns the number of bytes read. It may be less than buffer.len.
26 /// If the number of bytes read is 0, it means end of stream.
27 /// End of stream is not an error condition.
28 readFn: ReadFn,
29
30 /// Returns the number of bytes read. It may be less than buffer.len.
31 /// If the number of bytes read is 0, it means end of stream.
32 /// End of stream is not an error condition.
33 pub fn read(self: *Self, buffer: []u8) Error!usize {
34 if (std.io.is_async) {
35 var stack_frame: [stack_size]u8 align(stack_align) = undefined;
36 // TODO https://github.com/ziglang/zig/issues/3068
37 var result: Error!usize = undefined;
38 return await @asyncCall(&stack_frame, &result, self.readFn, self, buffer);
39 } else {
40 return self.readFn(self, buffer);
41 }
42 }
43
44 /// Returns the number of bytes read. If the number read is smaller than buf.len, it
45 /// means the stream reached the end. Reaching the end of a stream is not an error
46 /// condition.
47 pub fn readFull(self: *Self, buffer: []u8) Error!usize {
48 var index: usize = 0;
49 while (index != buffer.len) {
50 const amt = try self.read(buffer[index..]);
51 if (amt == 0) return index;
52 index += amt;
53 }
54 return index;
55 }
56
57 /// Returns the number of bytes read. If the number read would be smaller than buf.len,
58 /// error.EndOfStream is returned instead.
59 pub fn readNoEof(self: *Self, buf: []u8) !void {
60 const amt_read = try self.readFull(buf);
61 if (amt_read < buf.len) return error.EndOfStream;
62 }
63
64 /// Replaces `buffer` contents by reading from the stream until it is finished.
65 /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and
66 /// the contents read from the stream are lost.
67 pub fn readAllBuffer(self: *Self, buffer: *Buffer, max_size: usize) !void {
68 try buffer.resize(0);
69
70 var actual_buf_len: usize = 0;
71 while (true) {
72 const dest_slice = buffer.toSlice()[actual_buf_len..];
73 const bytes_read = try self.readFull(dest_slice);
74 actual_buf_len += bytes_read;
75
76 if (bytes_read != dest_slice.len) {
77 buffer.shrink(actual_buf_len);
78 return;
79 }
80
81 const new_buf_size = math.min(max_size, actual_buf_len + mem.page_size);
82 if (new_buf_size == actual_buf_len) return error.StreamTooLong;
83 try buffer.resize(new_buf_size);
84 }
85 }
86
87 /// Allocates enough memory to hold all the contents of the stream. If the allocated
88 /// memory would be greater than `max_size`, returns `error.StreamTooLong`.
89 /// Caller owns returned memory.
90 /// If this function returns an error, the contents from the stream read so far are lost.
91 pub fn readAllAlloc(self: *Self, allocator: *mem.Allocator, max_size: usize) ![]u8 {
92 var buf = Buffer.initNull(allocator);
93 defer buf.deinit();
94
95 try self.readAllBuffer(&buf, max_size);
96 return buf.toOwnedSlice();
97 }
98
99 /// Replaces `buffer` contents by reading from the stream until `delimiter` is found.
100 /// Does not include the delimiter in the result.
101 /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and the contents
102 /// read from the stream so far are lost.
103 pub fn readUntilDelimiterBuffer(self: *Self, buffer: *Buffer, delimiter: u8, max_size: usize) !void {
104 try buffer.resize(0);
105
106 while (true) {
107 var byte: u8 = try self.readByte();
108
109 if (byte == delimiter) {
110 return;
111 }
112
113 if (buffer.len() == max_size) {
114 return error.StreamTooLong;
115 }
116
117 try buffer.appendByte(byte);
118 }
119 }
120
121 /// Allocates enough memory to read until `delimiter`. If the allocated
122 /// memory would be greater than `max_size`, returns `error.StreamTooLong`.
123 /// Caller owns returned memory.
124 /// If this function returns an error, the contents from the stream read so far are lost.
125 pub fn readUntilDelimiterAlloc(self: *Self, allocator: *mem.Allocator, delimiter: u8, max_size: usize) ![]u8 {
126 var buf = Buffer.initNull(allocator);
127 defer buf.deinit();
128
129 try self.readUntilDelimiterBuffer(&buf, delimiter, max_size);
130 return buf.toOwnedSlice();
131 }
132
133 /// Reads 1 byte from the stream or returns `error.EndOfStream`.
134 pub fn readByte(self: *Self) !u8 {
135 var result: [1]u8 = undefined;
136 try self.readNoEof(result[0..]);
137 return result[0];
138 }
139
140 /// Same as `readByte` except the returned byte is signed.
141 pub fn readByteSigned(self: *Self) !i8 {
142 return @bitCast(i8, try self.readByte());
143 }
144
145 /// Reads a native-endian integer
146 pub fn readIntNative(self: *Self, comptime T: type) !T {
147 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
148 try self.readNoEof(bytes[0..]);
149 return mem.readIntNative(T, &bytes);
150 }
151
152 /// Reads a foreign-endian integer
153 pub fn readIntForeign(self: *Self, comptime T: type) !T {
154 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
155 try self.readNoEof(bytes[0..]);
156 return mem.readIntForeign(T, &bytes);
157 }
158
159 pub fn readIntLittle(self: *Self, comptime T: type) !T {
160 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
161 try self.readNoEof(bytes[0..]);
162 return mem.readIntLittle(T, &bytes);
163 }
164
165 pub fn readIntBig(self: *Self, comptime T: type) !T {
166 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
167 try self.readNoEof(bytes[0..]);
168 return mem.readIntBig(T, &bytes);
169 }
170
171 pub fn readInt(self: *Self, comptime T: type, endian: builtin.Endian) !T {
172 var bytes: [(T.bit_count + 7) / 8]u8 = undefined;
173 try self.readNoEof(bytes[0..]);
174 return mem.readInt(T, &bytes, endian);
175 }
176
177 pub fn readVarInt(self: *Self, comptime ReturnType: type, endian: builtin.Endian, size: usize) !ReturnType {
178 assert(size <= @sizeOf(ReturnType));
179 var bytes_buf: [@sizeOf(ReturnType)]u8 = undefined;
180 const bytes = bytes_buf[0..size];
181 try self.readNoEof(bytes);
182 return mem.readVarInt(ReturnType, bytes, endian);
183 }
184
185 pub fn skipBytes(self: *Self, num_bytes: u64) !void {
186 var i: u64 = 0;
187 while (i < num_bytes) : (i += 1) {
188 _ = try self.readByte();
189 }
190 }
191
192 pub fn readStruct(self: *Self, comptime T: type) !T {
193 // Only extern and packed structs have defined in-memory layout.
194 comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);
195 var res: [1]T = undefined;
196 try self.readNoEof(@sliceToBytes(res[0..]));
197 return res[0];
198 }
199 };
200}
std/os.zig+18-24
......@@ -254,13 +254,18 @@ pub const ReadError = error{
254254 IsDir,
255255 OperationAborted,
256256 BrokenPipe,
257
258 /// This error occurs when no global event loop is configured,
259 /// and reading from the file descriptor would block.
260 WouldBlock,
261
257262 Unexpected,
258263};
259264
260265/// Returns the number of bytes that were read, which can be less than
261266/// buf.len. If 0 bytes were read, that means EOF.
262/// This function is for blocking file descriptors only. For non-blocking, see
263/// `readAsync`.
267/// If the application has a global event loop enabled, EAGAIN is handled
268/// via the event loop. Otherwise EAGAIN results in error.WouldBlock.
264269pub fn read(fd: fd_t, buf: []u8) ReadError!usize {
265270 if (windows.is_the_target) {
266271 return windows.ReadFile(fd, buf);
......@@ -279,28 +284,19 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize {
279284 }
280285 }
281286
282 // Linux can return EINVAL when read amount is > 0x7ffff000
283 // See https://github.com/ziglang/zig/pull/743#issuecomment-363158274
284 // TODO audit this. Shawn Landden says that this is not actually true.
285 // if this logic should stay, move it to std.os.linux
286 const max_buf_len = 0x7ffff000;
287
288 var index: usize = 0;
289 while (index < buf.len) {
290 const want_to_read = math.min(buf.len - index, usize(max_buf_len));
291 const rc = system.read(fd, buf.ptr + index, want_to_read);
287 while (true) {
288 const rc = system.read(fd, buf.ptr, buf.len);
292289 switch (errno(rc)) {
293 0 => {
294 const amt_read = @intCast(usize, rc);
295 index += amt_read;
296 if (amt_read == want_to_read) continue;
297 // Read returned less than buf.len.
298 return index;
299 },
290 0 => return @intCast(usize, rc),
300291 EINTR => continue,
301292 EINVAL => unreachable,
302293 EFAULT => unreachable,
303 EAGAIN => unreachable, // This function is for blocking reads.
294 EAGAIN => if (std.event.Loop.instance) |loop| {
295 loop.waitUntilFdReadable(fd) catch return error.WouldBlock;
296 continue;
297 } else {
298 return error.WouldBlock;
299 },
304300 EBADF => unreachable, // Always a race condition.
305301 EIO => return error.InputOutput,
306302 EISDIR => return error.IsDir,
......@@ -313,8 +309,7 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize {
313309}
314310
315311/// Number of bytes read is returned. Upon reading end-of-file, zero is returned.
316/// This function is for blocking file descriptors only. For non-blocking, see
317/// `preadvAsync`.
312/// This function is for blocking file descriptors only.
318313pub fn preadv(fd: fd_t, iov: []const iovec, offset: u64) ReadError!usize {
319314 if (darwin.is_the_target) {
320315 // Darwin does not have preadv but it does have pread.
......@@ -386,8 +381,7 @@ pub const WriteError = error{
386381};
387382
388383/// Write to a file descriptor. Keeps trying if it gets interrupted.
389/// This function is for blocking file descriptors only. For non-blocking, see
390/// `writeAsync`.
384/// This function is for blocking file descriptors only.
391385pub fn write(fd: fd_t, bytes: []const u8) WriteError!void {
392386 if (windows.is_the_target) {
393387 return windows.WriteFile(fd, bytes);