From f597e0d0923e914ec4690016a7aab8ebdaaab4db Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Thu, 10 Oct 2019 18:03:57 +0300 Subject: [PATCH 01/31] Change layout and styles --- lib/std/special/docs/index.html | 507 ++++++++++++++++++++++---------- 1 file changed, 359 insertions(+), 148 deletions(-) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index ec226f220b372e287d01ebb6f44c1ccd39aeb8fa..f0d68ffdab70448461b1a01f87ae5deff64a9d71 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -5,28 +5,28 @@ Documentation - Zig - - -
- -

Loading...

- - -

- - - - - - - - - - - - - -
-- 2.54.0 From a34b697c0c6655407d08aee1b7f56ce5ca5205a2 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Thu, 10 Oct 2019 18:55:14 +0300 Subject: [PATCH 02/31] Add transtition to search field --- lib/std/special/docs/index.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index f0d68ffdab70448461b1a01f87ae5deff64a9d71..c26468be177f2942233a6aa9f9b0e6f00147e96d 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -428,10 +428,13 @@ border-bottom-style: solid; border-bottom-color: #000; outline: none; + transition: border-bottom-color 0.35s, background 0.35s, box-shadow 0.35s; } .search:focus { border-bottom-color: #ffbb4d; + background: white; + box-shadow: 0 0.3em 1em 0.125em rgba(0, 0, 0, 0.18); } .search::placeholder { -- 2.54.0 From 238ea85b7efcef797d7cb88c97dc677ecacd4938 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Fri, 11 Oct 2019 00:39:08 +0300 Subject: [PATCH 03/31] Fix dark mode Additionally introduces css variables --- lib/std/special/docs/index.html | 681 +++++++++++++++----------------- 1 file changed, 317 insertions(+), 364 deletions(-) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index c26468be177f2942233a6aa9f9b0e6f00147e96d..81668fd11ecfab240e1cd68d3a08bee14c8db272 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -5,158 +5,243 @@ Documentation - Zig - -
+ +
- -
-
-
- +
+ +
+
+
+
+

Loading...

-- 2.54.0 From c95a9e978582ca96bf5462c6bfdd40b934e9ba92 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Thu, 17 Oct 2019 01:54:39 +0300 Subject: [PATCH 08/31] Cleanup --- lib/std/special/docs/index.html | 41 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index 8bb6ad414d136c91dbea45dc7dd2645256458247..8ce1033235a4ea4362fb0bbff789988753278d5c 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -13,6 +13,13 @@ --bg-color: #ffffff; --link-color: #2A6286; --sidebar-sh-color: rgba(0, 0, 0, 0.09); + --sidebar-pkg-bg-color: #f1f1f1; + --sidebar-pkglnk-tx-color: #141414; + --sidebar-pkglnk-tx-color-hover: #fff; + --sidebar-pkglnk-tx-color-active: #000; + --sidebar-pkglnk-bg-color: transparent; + --sidebar-pkglnk-bg-color-hover: #555; + --sidebar-pkglnk-bg-color-active: #FFBB4D; --search-bg-color: #f3f3f3; --search-bg-color-focus: #ffffff; --search-sh-color: rgba(0, 0, 0, 0.18); @@ -118,24 +125,25 @@ list-style-type: none; margin: 0; padding: 0; - background-color: #f1f1f1; + background-color: var(--sidebar-pkg-bg-color); } .sidebar .packages > li > a { display: block; padding: 0.5rem 1rem; - color: var(--tx-color); + color: var(--sidebar-pkglnk-tx-color); + background-color: var(--sidebar-pkglnk-bg-color); text-decoration: none; } .sidebar .packages > li > a:hover { - color: #fff; - background-color: #555; + color: var(--sidebar-pkglnk-tx-color-hover); + background-color: var(--sidebar-pkglnk-bg-color-hover); } .sidebar .packages > li > a.active { - color: #000; - background-color: #FFBB4D; + color: var(--sidebar-pkglnk-tx-color-active); + background-color: var(--sidebar-pkglnk-bg-color-active); } .sidebar p.str { @@ -367,6 +375,13 @@ --bg-color: #111; --link-color: #88f; --sidebar-sh-color: rgba(128, 128, 128, 0.09); + --sidebar-pkg-bg-color: #333; + --sidebar-pkglnk-tx-color: #fff; + --sidebar-pkglnk-tx-color-hover: #fff; + --sidebar-pkglnk-tx-color-active: #000; + --sidebar-pkglnk-bg-color: transparent; + --sidebar-pkglnk-bg-color-hover: #555; + --sidebar-pkglnk-bg-color-active: #FFBB4D; --search-bg-color: #3c3c3c; --search-bg-color-focus: #000; --search-sh-color: rgba(255, 255, 255, 0.28); @@ -390,20 +405,6 @@ background-color: #FFBB4D; color: #000; } - #listPkgs { - background-color: #333; - } - #listPkgs li a { - color: #fff; - } - #listPkgs li a:hover { - background-color: #555; - color: #fff; - } - #listPkgs li a.active { - background-color: #FFBB4D; - color: #000; - } #listSearchResults li.selected { background-color: #000; } -- 2.54.0 From 8774b1d9478139a8da0f43c49fc33e01c74a6a3f Mon Sep 17 00:00:00 2001 From: Nick Erdmann Date: Fri, 18 Oct 2019 19:49:02 +0200 Subject: [PATCH 09/31] std/os/windows: remove subsystem detection for uefi --- lib/std/os/windows.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 7c1761a4b8fb21a8c9aa15645cb0f68ac5ee5703..c35d9d592b62893a160cf9a19d90834e7e7114bc 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -42,7 +42,6 @@ pub const subsystem: ?builtin.SubSystem = blk: { break :blk builtin.SubSystem.Console; } }, - .uefi => break :blk builtin.SubSystem.EfiApplication, else => break :blk null, } }; -- 2.54.0 From c8dec5729cabb90fa56fad2b6d77171189e01703 Mon Sep 17 00:00:00 2001 From: Nick Erdmann Date: Fri, 18 Oct 2019 20:11:28 +0200 Subject: [PATCH 10/31] std/build: make subsystem configurable --- lib/std/build.zig | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/std/build.zig b/lib/std/build.zig index 68d3f2a31512428f13ad59a0e757b4e7203aaa05..cb02c8f1317dc97d1f69ec4c46cb88ad9b3d36f5 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1491,6 +1491,8 @@ pub const LibExeObjStep = struct { /// Position Independent Code force_pic: ?bool = null, + subsystem: ?builtin.SubSystem = null, + const LinkObject = union(enum) { StaticPath: []const u8, OtherStep: *LibExeObjStep, @@ -2325,6 +2327,20 @@ pub const LibExeObjStep = struct { } } + if (self.subsystem) |subsystem| { + try zig_args.append("--subsystem"); + try zig_args.append(switch (subsystem) { + .Console => "console", + .Windows => "windows", + .Posix => "posix", + .Native => "native", + .EfiApplication => "efi_application", + .EfiBootServiceDriver => "efi_boot_service_driver", + .EfiRom => "efi_rom", + .EfiRuntimeDriver => "efi_runtime_driver", + }); + } + if (self.kind == Kind.Test) { try builder.spawnChild(zig_args.toSliceConst()); } else { -- 2.54.0 From fa9f1d23968d3b9fe3ae35ed1306aec0cbcfd908 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Oct 2019 03:23:23 -0400 Subject: [PATCH 11/31] add -DZIG_ENABLE_MEM_PROFILE option and -fmem-report flag This can be used to find what's taking up memory in the compiler. Here's an example of how to use it: ``` ./zig test ../lib/std/std.zig --cache off -fmem-report ``` And here's the output I get for this on x86_64-linux-gnu today: ``` Const: 6462833 items, 152 bytes each, total 936.84 MiB ConstGlobalRefs: 17236534 items, 24 bytes each, total 394.51 MiB ResetResult: 1698108 items, 160 bytes each, total 259.11 MiB ConstExprValue: 3118299 items, 80 bytes each, total 237.91 MiB EndExpr: 1345395 items, 168 bytes each, total 215.56 MiB Unknown_8: 27370821 items, 8 bytes each, total 208.82 MiB VarPtr: 1127866 items, 168 bytes each, total 180.70 MiB IrBasicBlock: 794834 items, 120 bytes each, total 90.96 MiB LoadPtr: 554024 items, 160 bytes each, total 84.54 MiB Unknown_64: 1245715 items, 64 bytes each, total 76.03 MiB Unknown_40: 1879218 items, 40 bytes each, total 71.69 MiB Unknown_72: 989117 items, 72 bytes each, total 67.92 MiB Return: 423783 items, 160 bytes each, total 64.66 MiB Unknown_168: 332480 items, 168 bytes each, total 53.27 MiB Unknown_152: 336890 items, 152 bytes each, total 48.84 MiB AddImplicitReturnType: 230819 items, 168 bytes each, total 36.98 MiB Br: 217835 items, 168 bytes each, total 34.90 MiB Unknown_184: 179529 items, 184 bytes each, total 31.50 MiB FieldPtr: 179388 items, 184 bytes each, total 31.48 MiB BinOp: 171004 items, 176 bytes each, total 28.70 MiB LoadPtrGen: 173287 items, 168 bytes each, total 27.76 MiB CondBr: 137864 items, 192 bytes each, total 25.24 MiB Unknown_720: 30918 items, 720 bytes each, total 21.23 MiB CallSrc: 99320 items, 216 bytes each, total 20.46 MiB Unknown_160: 129243 items, 160 bytes each, total 19.72 MiB Unknown_1: 19339456 items, 1 bytes each, total 18.44 MiB CheckStatementIsVoid: 119838 items, 160 bytes each, total 18.29 MiB Unknown_48: 371178 items, 48 bytes each, total 16.99 MiB TestComptime: 101443 items, 160 bytes each, total 15.48 MiB DeclVarSrc: 72578 items, 184 bytes each, total 12.74 MiB StorePtr: 72776 items, 176 bytes each, total 12.22 MiB ZigVar: 79201 items, 160 bytes each, total 12.09 MiB Unknown_16: 770643 items, 16 bytes each, total 11.76 MiB Phi: 60482 items, 184 bytes each, total 10.61 MiB TestErrSrc: 66177 items, 168 bytes each, total 10.60 MiB Unknown_240: 45164 items, 240 bytes each, total 10.34 MiB ElemPtr: 58232 items, 184 bytes each, total 10.22 MiB AllocaSrc: 60053 items, 176 bytes each, total 10.08 MiB CallGen: 44873 items, 224 bytes each, total 9.59 MiB SaveErrRetAddr: 63787 items, 152 bytes each, total 9.25 MiB Unknown_112: 82283 items, 112 bytes each, total 8.79 MiB AllocaGen: 51909 items, 176 bytes each, total 8.71 MiB Unknown_24: 373599 items, 24 bytes each, total 8.55 MiB ResultLocPeer: 113683 items, 72 bytes each, total 7.81 MiB DeclRef: 36343 items, 168 bytes each, total 5.82 MiB UnwrapErrPayload: 34603 items, 168 bytes each, total 5.54 MiB Ref: 33414 items, 168 bytes each, total 5.35 MiB Unknown_104: 53882 items, 104 bytes each, total 5.34 MiB DeclVarGen: 32540 items, 168 bytes each, total 5.21 MiB StructFieldPtr: 30449 items, 176 bytes each, total 5.11 MiB UnwrapErrCode: 31508 items, 168 bytes each, total 5.05 MiB Unknown_56: 90256 items, 56 bytes each, total 4.82 MiB SpillBegin: 28722 items, 168 bytes each, total 4.60 MiB SpillEnd: 28722 items, 160 bytes each, total 4.38 MiB ResultLocReturn: 64573 items, 48 bytes each, total 2.96 MiB PtrType: 14702 items, 184 bytes each, total 2.58 MiB SliceType: 15005 items, 176 bytes each, total 2.52 MiB Unknown_176: 13326 items, 176 bytes each, total 2.24 MiB RefGen: 12881 items, 168 bytes each, total 2.06 MiB UnOp: 12102 items, 176 bytes each, total 2.03 MiB SwitchBr: 9453 items, 200 bytes each, total 1.80 MiB TestErrGen: 11143 items, 160 bytes each, total 1.70 MiB Unknown_32: 52359 items, 32 bytes each, total 1.60 MiB CheckSwitchProngs: 9094 items, 184 bytes each, total 1.60 MiB TypeOf: 9259 items, 160 bytes each, total 1.41 MiB IntCast: 8772 items, 168 bytes each, total 1.41 MiB OptionalUnwrapPtr: 8755 items, 168 bytes each, total 1.40 MiB SwitchTarget: 9094 items, 160 bytes each, total 1.39 MiB Cast: 8198 items, 176 bytes each, total 1.38 MiB WidenOrShorten: 8448 items, 160 bytes each, total 1.29 MiB ErrorUnion: 7613 items, 176 bytes each, total 1.28 MiB SliceSrc: 6249 items, 192 bytes each, total 1.14 MiB ErrWrapCode: 7133 items, 168 bytes each, total 1.14 MiB TypeName: 7328 items, 160 bytes each, total 1.12 MiB ImplicitCast: 5480 items, 176 bytes each, total 941.88 KiB ResolveResult: 5638 items, 168 bytes each, total 924.98 KiB ResultLocInstruction: 22696 items, 40 bytes each, total 886.56 KiB BitCastSrc: 4947 items, 168 bytes each, total 811.62 KiB CompileErr: 5148 items, 160 bytes each, total 804.38 KiB ReturnPtr: 5305 items, 152 bytes each, total 787.46 KiB Unreachable: 5038 items, 152 bytes each, total 747.83 KiB TestNonNull: 4716 items, 160 bytes each, total 736.88 KiB BitCastGen: 4431 items, 160 bytes each, total 692.34 KiB PtrToInt: 4289 items, 160 bytes each, total 670.16 KiB SliceGen: 3573 items, 192 bytes each, total 669.94 KiB ArrayType: 4081 items, 168 bytes each, total 669.54 KiB IntType: 3868 items, 168 bytes each, total 634.59 KiB Unknown_88: 7213 items, 88 bytes each, total 619.87 KiB Truncate: 3771 items, 168 bytes each, total 618.68 KiB TypeInfo: 3740 items, 160 bytes each, total 584.38 KiB SwitchVar: 3385 items, 176 bytes each, total 581.80 KiB ContainerInitFields: 3223 items, 184 bytes each, total 579.13 KiB ContainerInitList: 2309 items, 192 bytes each, total 432.94 KiB PtrCastGen: 2626 items, 168 bytes each, total 430.83 KiB BoolNot: 2457 items, 160 bytes each, total 383.91 KiB FnProto: 2054 items, 184 bytes each, total 369.08 KiB MergeErrSets: 1927 items, 176 bytes each, total 331.20 KiB Unknown_136: 2486 items, 136 bytes each, total 330.17 KiB Unknown_80: 4059 items, 80 bytes each, total 317.11 KiB Bswap: 1670 items, 168 bytes each, total 273.98 KiB TypeId: 1680 items, 160 bytes each, total 262.50 KiB PtrCastSrc: 1371 items, 176 bytes each, total 235.64 KiB ErrName: 1193 items, 160 bytes each, total 186.41 KiB UnionTag: 1120 items, 160 bytes each, total 175.00 KiB TagName: 1050 items, 160 bytes each, total 164.06 KiB SizeOf: 942 items, 160 bytes each, total 147.19 KiB MemberName: 871 items, 168 bytes each, total 142.90 KiB Import: 881 items, 160 bytes each, total 137.66 KiB PtrOfArrayToSlice: 758 items, 168 bytes each, total 124.36 KiB UnionFieldPtr: 710 items, 176 bytes each, total 122.03 KiB EnumToInt: 778 items, 160 bytes each, total 121.56 KiB CheckRuntimeScope: 700 items, 168 bytes each, total 114.84 KiB FieldParentPtr: 632 items, 184 bytes each, total 113.56 KiB BoolToInt: 719 items, 160 bytes each, total 112.34 KiB ResultLocPeerParent: 904 items, 104 bytes each, total 91.81 KiB IntToPtr: 537 items, 168 bytes each, total 88.10 KiB AlignOf: 561 items, 160 bytes each, total 87.66 KiB AtomicRmw: 356 items, 208 bytes each, total 72.31 KiB MemberCount: 441 items, 160 bytes each, total 68.91 KiB Memset: 342 items, 176 bytes each, total 58.78 KiB PopCount: 321 items, 168 bytes each, total 52.66 KiB AlignCast: 251 items, 168 bytes each, total 41.18 KiB IrInstruction *: 5230 items, 8 bytes each, total 40.86 KiB IrBasicBlock *: 5230 items, 8 bytes each, total 40.86 KiB TagType: 261 items, 160 bytes each, total 40.78 KiB HasDecl: 234 items, 168 bytes each, total 38.39 KiB OverflowOp: 191 items, 200 bytes each, total 37.30 KiB Export: 209 items, 176 bytes each, total 35.92 KiB SetCold: 219 items, 160 bytes each, total 34.22 KiB ReturnAddress: 216 items, 152 bytes each, total 32.06 KiB FromBytes: 178 items, 176 bytes each, total 30.59 KiB SetRuntimeSafety: 188 items, 160 bytes each, total 29.38 KiB OptionalWrap: 151 items, 168 bytes each, total 24.77 KiB Clz: 143 items, 168 bytes each, total 23.46 KiB ResizeSlice: 135 items, 168 bytes each, total 22.15 KiB UnionInitNamedField: 106 items, 184 bytes each, total 19.05 KiB Panic: 102 items, 160 bytes each, total 15.94 KiB SwitchElseVar: 93 items, 168 bytes each, total 15.26 KiB ToBytes: 89 items, 168 bytes each, total 14.60 KiB IntToFloat: 78 items, 168 bytes each, total 12.80 KiB Unknown_4360: 3 items, 4360 bytes each, total 12.77 KiB ErrWrapPayload: 72 items, 168 bytes each, total 11.81 KiB FloatOp: 62 items, 176 bytes each, total 10.66 KiB FloatToInt: 47 items, 168 bytes each, total 7.71 KiB FloatCast: 46 items, 168 bytes each, total 7.55 KiB ErrToInt: 47 items, 160 bytes each, total 7.34 KiB Asm: 33 items, 216 bytes each, total 6.96 KiB ErrSetCast: 40 items, 168 bytes each, total 6.56 KiB Memcpy: 34 items, 176 bytes each, total 5.84 KiB AtomicLoad: 17 items, 184 bytes each, total 3.05 KiB AwaitSrc: 16 items, 168 bytes each, total 2.62 KiB Resume: 14 items, 160 bytes each, total 2.19 KiB AwaitGen: 12 items, 176 bytes each, total 2.06 KiB ArgType: 12 items, 168 bytes each, total 1.97 KiB AnyFrameType: 12 items, 160 bytes each, total 1.88 KiB SuspendFinish: 10 items, 160 bytes each, total 1.56 KiB SuspendBegin: 10 items, 160 bytes each, total 1.56 KiB Ctz: 9 items, 168 bytes each, total 1.48 KiB FrameHandle: 8 items, 152 bytes each, total 1.19 KiB SetEvalBranchQuota: 7 items, 160 bytes each, total 1.09 KiB AssertZero: 7 items, 160 bytes each, total 1.09 KiB UndeclaredIdent: 7 items, 160 bytes each, total 1.09 KiB CmpxchgSrc: 5 items, 216 bytes each, total 1.05 KiB CmpxchgGen: 5 items, 200 bytes each, total 1000.00 bytes IntToEnum: 4 items, 168 bytes each, total 672.00 bytes VectorType: 4 items, 168 bytes each, total 672.00 bytes ErrorReturnTrace: 2 items, 160 bytes each, total 320.00 bytes Breakpoint: 2 items, 152 bytes each, total 304.00 bytes FrameAddress: 2 items, 152 bytes each, total 304.00 bytes Unknown_4: 61 items, 4 bytes each, total 244.00 bytes VectorToArray: 1 items, 168 bytes each, total 168.00 bytes SetAlignStack: 1 items, 160 bytes each, total 160.00 bytes Unknown_12: 2 items, 12 bytes each, total 24.00 bytes ErrorTableEntry *: 0 items, 8 bytes each, total 0.00 bytes AstNode *: 0 items, 8 bytes each, total 0.00 bytes Total bytes used: 3.51 GiB ``` You can see that most of the memory is taken up by IR instructions, as well as comptime values. This points toward 2 changes which will greatly reduce memory usage: * Rework semantic analysis so that IR instructions can be freed. Currently the comptime value struct (ConstExprValue) is embedded directly into the IrInstruction struct. If this is made to be separate, at the very least pass 1 IR instructions can be freed. This includes `Const` which is the largest usage of memory currently. * Rework the ConstExprValue struct to no longer be a tagged union. For example, there's no need for an integer comptime value to be 80 bytes. From this you can also see that this eliminates some things from being the culprit. Before doing this analysis, I considered whether doing string interning would help. From the above output, you can see that all strings in the compiler account for only 18 MiB, so string interning would have been a dead end. --- CMakeLists.txt | 2 + src/analyze.cpp | 4 +- src/config.h.in | 5 +- src/dump_analysis.cpp | 19 +----- src/ir.cpp | 52 +++++++++------ src/ir_print.cpp | 6 +- src/ir_print.hpp | 2 + src/main.cpp | 75 ++++++++++++++------- src/memory_profiling.cpp | 139 +++++++++++++++++++++++++++++++++++++++ src/memory_profiling.hpp | 22 +++++++ src/util.cpp | 18 +++++ src/util.hpp | 35 ++++++++-- 12 files changed, 305 insertions(+), 74 deletions(-) create mode 100644 src/memory_profiling.cpp create mode 100644 src/memory_profiling.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fbc34f80511b580413579b84288996efe0c81605..3a34d034ddaaf8f5f106dc2062fd4e1715e067f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,7 @@ message("Configuring zig version ${ZIG_VERSION}") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries") set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL "Disable copying lib/ files to install prefix") +set(ZIG_ENABLE_MEM_PROFILE off CACHE BOOL "Activate memory usage instrumentation") if(ZIG_STATIC) set(ZIG_STATIC_LLVM "on") @@ -455,6 +456,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ir_print.cpp" "${CMAKE_SOURCE_DIR}/src/libc_installation.cpp" "${CMAKE_SOURCE_DIR}/src/link.cpp" + "${CMAKE_SOURCE_DIR}/src/memory_profiling.cpp" "${CMAKE_SOURCE_DIR}/src/os.cpp" "${CMAKE_SOURCE_DIR}/src/parser.cpp" "${CMAKE_SOURCE_DIR}/src/range_set.cpp" diff --git a/src/analyze.cpp b/src/analyze.cpp index 92b221a2ecdb3bc5370b85eee769f20b74ec9400..ed4547957bd424c5223b154fa54bc8374c663495 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -5783,8 +5783,8 @@ ConstExprValue *create_const_arg_tuple(CodeGen *g, size_t arg_index_start, size_ ConstExprValue *create_const_vals(size_t count) { - ConstGlobalRefs *global_refs = allocate(count); - ConstExprValue *vals = allocate(count); + ConstGlobalRefs *global_refs = allocate(count, "ConstGlobalRefs"); + ConstExprValue *vals = allocate(count, "ConstExprValue"); for (size_t i = 0; i < count; i += 1) { vals[i].global_refs = &global_refs[i]; } diff --git a/src/config.h.in b/src/config.h.in index a99aab0d7235cf238d10aedfa6b3d0dfb7068216..7a0ea3536a94df0dab99f7bfc634ac4bdc10ee2e 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -13,9 +13,6 @@ #define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@ #define ZIG_VERSION_STRING "@ZIG_VERSION@" -// Only used for running tests before installing. -#define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test" - // Used for communicating build information to self hosted build. #define ZIG_CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@" #define ZIG_CXX_COMPILER "@CMAKE_CXX_COMPILER@" @@ -24,4 +21,6 @@ #define ZIG_LLVM_CONFIG_EXE "@LLVM_CONFIG_EXE@" #define ZIG_DIA_GUIDS_LIB "@ZIG_DIA_GUIDS_LIB_ESCAPED@" +#cmakedefine ZIG_ENABLE_MEM_PROFILE + #endif diff --git a/src/dump_analysis.cpp b/src/dump_analysis.cpp index d5b109fbcb03db934eedb20bbacc1556d2a08aff..9703d3e57d107b157e5de956c048c24942fe473f 100644 --- a/src/dump_analysis.cpp +++ b/src/dump_analysis.cpp @@ -240,23 +240,6 @@ static void jw_string(JsonWriter *jw, const char *s) { static void tree_print(FILE *f, ZigType *ty, size_t indent); -static void pretty_print_bytes(FILE *f, double n) { - if (n > 1024.0 * 1024.0 * 1024.0) { - fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0); - return; - } - if (n > 1024.0 * 1024.0) { - fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0); - return; - } - if (n > 1024.0) { - fprintf(f, "%.02f KiB", n / 1024.0); - return; - } - fprintf(f, "%.02f bytes", n ); - return; -} - static int compare_type_abi_sizes_desc(const void *a, const void *b) { uint64_t size_a = (*(ZigType * const*)(a))->abi_size; uint64_t size_b = (*(ZigType * const*)(b))->abi_size; @@ -322,7 +305,7 @@ static void tree_print(FILE *f, ZigType *ty, size_t indent) { start_peer(f, indent); fprintf(f, "\"sizef\": \""); - pretty_print_bytes(f, ty->abi_size); + zig_pretty_print_bytes(f, ty->abi_size); fprintf(f, "\""); start_peer(f, indent); diff --git a/src/ir.cpp b/src/ir.cpp index 7dd141423c0c720a2009d90a5245b900bd50a2d1..b71a48dc2b7ba8121a4362b91d58f739d261c232 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -413,7 +413,7 @@ ZigType *ir_analyze_type_expr(IrAnalyze *ira, Scope *scope, AstNode *node) { } static IrBasicBlock *ir_create_basic_block(IrBuilder *irb, Scope *scope, const char *name_hint) { - IrBasicBlock *result = allocate(1); + IrBasicBlock *result = allocate(1, "IrBasicBlock"); result->scope = scope; result->name_hint = name_hint; result->debug_id = exec_next_debug_id(irb->exec); @@ -1085,13 +1085,18 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSpillEnd *) { template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { - T *special_instruction = allocate(1); + const char *name = nullptr; +#ifdef ZIG_ENABLE_MEM_PROFILE + T *dummy = nullptr; + name = ir_instruction_type_str(ir_instruction_id(dummy)); +#endif + T *special_instruction = allocate(1, name); special_instruction->base.id = ir_instruction_id(special_instruction); special_instruction->base.scope = scope; special_instruction->base.source_node = source_node; special_instruction->base.debug_id = exec_next_debug_id(irb->exec); special_instruction->base.owner_bb = irb->current_basic_block; - special_instruction->base.value.global_refs = allocate(1); + special_instruction->base.value.global_refs = allocate(1, "ConstGlobalRefs"); return special_instruction; } @@ -3569,7 +3574,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, switch (node->data.return_expr.kind) { case ReturnKindUnconditional: { - ResultLocReturn *result_loc_ret = allocate(1); + ResultLocReturn *result_loc_ret = allocate(1, "ResultLocReturn"); result_loc_ret->base.id = ResultLocIdReturn; ir_build_reset_result(irb, scope, node, &result_loc_ret->base); @@ -3664,7 +3669,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val, nullptr)); IrInstructionSpillBegin *spill_begin = ir_build_spill_begin(irb, scope, node, err_val, SpillIdRetErrCode); - ResultLocReturn *result_loc_ret = allocate(1); + ResultLocReturn *result_loc_ret = allocate(1, "ResultLocReturn"); result_loc_ret->base.id = ResultLocIdReturn; ir_build_reset_result(irb, scope, node, &result_loc_ret->base); ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base); @@ -3692,7 +3697,7 @@ static ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_s Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime, bool skip_name_check) { - ZigVar *variable_entry = allocate(1); + ZigVar *variable_entry = allocate(1, "ZigVar"); variable_entry->parent_scope = parent_scope; variable_entry->shadowable = is_shadowable; variable_entry->mem_slot_index = SIZE_MAX; @@ -3767,7 +3772,7 @@ static ZigVar *ir_create_var(IrBuilder *irb, AstNode *node, Scope *scope, Buf *n } static ResultLocPeer *create_peer_result(ResultLocPeerParent *peer_parent) { - ResultLocPeer *result = allocate(1); + ResultLocPeer *result = allocate(1, "ResultLocPeer"); result->base.id = ResultLocIdPeer; result->base.source_instruction = peer_parent->base.source_instruction; result->parent = peer_parent; @@ -3806,7 +3811,7 @@ static IrInstruction *ir_gen_block(IrBuilder *irb, Scope *parent_scope, AstNode scope_block->is_comptime = ir_build_const_bool(irb, parent_scope, block_node, ir_should_inline(irb->exec, parent_scope)); - scope_block->peer_parent = allocate(1); + scope_block->peer_parent = allocate(1, "ResultLocPeerParent"); scope_block->peer_parent->base.id = ResultLocIdPeerParent; scope_block->peer_parent->base.source_instruction = scope_block->is_comptime; scope_block->peer_parent->end_bb = scope_block->end_block; @@ -3933,7 +3938,7 @@ static IrInstruction *ir_gen_assign(IrBuilder *irb, Scope *scope, AstNode *node) if (lvalue == irb->codegen->invalid_instruction) return irb->codegen->invalid_instruction; - ResultLocInstruction *result_loc_inst = allocate(1); + ResultLocInstruction *result_loc_inst = allocate(1, "ResultLocInstruction"); result_loc_inst->base.id = ResultLocIdInstruction; result_loc_inst->base.source_instruction = lvalue; ir_ref_instruction(lvalue, irb->current_basic_block); @@ -4005,10 +4010,10 @@ static IrInstruction *ir_gen_bool_or(IrBuilder *irb, Scope *scope, AstNode *node ir_set_cursor_at_end_and_append_block(irb, true_block); - IrInstruction **incoming_values = allocate(2); + IrInstruction **incoming_values = allocate(2, "IrInstruction *"); incoming_values[0] = val1; incoming_values[1] = val2; - IrBasicBlock **incoming_blocks = allocate(2); + IrBasicBlock **incoming_blocks = allocate(2, "IrBasicBlock *"); incoming_blocks[0] = post_val1_block; incoming_blocks[1] = post_val2_block; @@ -8017,7 +8022,8 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A err_set_type->abi_size = irb->codegen->builtin_types.entry_global_error_set->abi_size; err_set_type->data.error_set.errors = allocate(err_count); - ErrorTableEntry **errors = allocate(irb->codegen->errors_by_index.length + err_count); + size_t errors_count = irb->codegen->errors_by_index.length + err_count; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0; i < err_count; i += 1) { AstNode *field_node = node->data.err_set_decl.decls.at(i); @@ -8048,7 +8054,7 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A } errors[err->value] = err; } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return ir_build_const_type(irb, parent_scope, node, err_set_type); } @@ -9574,7 +9580,8 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp if (type_is_global_error_set(set2)) { return set1; } - ErrorTableEntry **errors = allocate(ira->codegen->errors_by_index.length); + size_t errors_count = ira->codegen->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); populate_error_set_table(errors, set1); ZigList intersection_list = {}; @@ -9595,7 +9602,7 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp buf_appendf(&err_set_type->name, "%s%s", comma, buf_ptr(&existing_entry_with_docs->name)); } } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); err_set_type->data.error_set.err_count = intersection_list.length; err_set_type->data.error_set.errors = intersection_list.items; @@ -9792,7 +9799,8 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted return result; } - ErrorTableEntry **errors = allocate(g->errors_by_index.length); + size_t errors_count = g->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0; i < container_set->data.error_set.err_count; i += 1) { ErrorTableEntry *error_entry = container_set->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); @@ -9809,7 +9817,7 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted result.data.error_set_mismatch->missing_errors.append(contained_error_entry); } } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return result; } @@ -14686,14 +14694,15 @@ static IrInstruction *ir_analyze_instruction_merge_err_sets(IrAnalyze *ira, return ira->codegen->invalid_instruction; } - ErrorTableEntry **errors = allocate(ira->codegen->errors_by_index.length); + size_t errors_count = ira->codegen->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0, count = op1_type->data.error_set.err_count; i < count; i += 1) { ErrorTableEntry *error_entry = op1_type->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); errors[error_entry->value] = error_entry; } ZigType *result_type = get_error_set_union(ira->codegen, errors, op1_type, op2_type, instruction->type_name); - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return ir_const_type(ira, &instruction->base, result_type); } @@ -24034,7 +24043,8 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, return ira->codegen->invalid_instruction; } - AstNode **field_prev_uses = allocate(ira->codegen->errors_by_index.length); + size_t field_prev_uses_count = ira->codegen->errors_by_index.length; + AstNode **field_prev_uses = allocate(field_prev_uses_count, "AstNode *"); for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) { IrInstructionCheckSwitchProngsRange *range = &instruction->ranges[range_i]; @@ -24091,7 +24101,7 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, } } - free(field_prev_uses); + deallocate(field_prev_uses, field_prev_uses_count, "AstNode *"); } else if (switch_type->id == ZigTypeIdInt) { RangeSet rs = {0}; for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) { diff --git a/src/ir_print.cpp b/src/ir_print.cpp index ecd8248d69ee36c5dbb8d92047631a05dcbafd24..441808e834fd91210e78bd65bd2cfd075773ca15 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -38,8 +38,8 @@ struct IrPrint { static void ir_print_other_instruction(IrPrint *irp, IrInstruction *instruction); -static const char* ir_instruction_type_str(IrInstruction* instruction) { - switch (instruction->id) { +const char* ir_instruction_type_str(IrInstructionId id) { + switch (id) { case IrInstructionIdInvalid: return "Invalid"; case IrInstructionIdShuffleVector: @@ -387,7 +387,7 @@ static void ir_print_prefix(IrPrint *irp, IrInstruction *instruction, bool trail const char *ref_count = ir_has_side_effects(instruction) ? "-" : buf_ptr(buf_sprintf("%" ZIG_PRI_usize "", instruction->ref_count)); fprintf(irp->f, "%c%-3zu| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, - ir_instruction_type_str(instruction), type_name, ref_count); + ir_instruction_type_str(instruction->id), type_name, ref_count); } static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) { diff --git a/src/ir_print.hpp b/src/ir_print.hpp index 0960af4e6fe8c103cab6d42c3a7b8f003d0a7bdc..e3947077c8f018a98da79922ef00559237c2308f 100644 --- a/src/ir_print.hpp +++ b/src/ir_print.hpp @@ -15,4 +15,6 @@ void ir_print(CodeGen *codegen, FILE *f, IrExecutable *executable, int indent_size, IrPass pass); void ir_print_instruction(CodeGen *codegen, FILE *f, IrInstruction *instruction, int indent_size, IrPass pass); +const char* ir_instruction_type_str(IrInstructionId id); + #endif diff --git a/src/main.cpp b/src/main.cpp index b398eec9911229015ed7b442107f3c2353411561..4709035859e3f17b7d197faffffd730ae3e8a5eb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -64,6 +64,9 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " -fno-PIC disable Position Independent Code\n" " -ftime-report print timing diagnostics\n" " -fstack-report print stack size diagnostics\n" +#ifdef ZIG_ENABLE_MEM_PROFILE + " -fmem-report print memory usage diagnostics\n" +#endif " -fdump-analysis write analysis.json file with type information\n" " -femit-docs create a docs/ dir with html documentation\n" " -fno-emit-bin skip emitting machine code\n" @@ -306,9 +309,29 @@ static int zig_error_no_build_file(void) { extern "C" int ZigClang_main(int argc, char **argv); +#ifdef ZIG_ENABLE_MEM_PROFILE +bool mem_report = false; +#endif + +int main_exit(Stage2ProgressNode *root_progress_node, int exit_code) { + if (root_progress_node != nullptr) { + stage2_progress_end(root_progress_node); + } +#ifdef ZIG_ENABLE_MEM_PROFILE + if (mem_report) { + memprof_dump_stats(stderr); + } +#endif + return exit_code; +} + int main(int argc, char **argv) { stage2_attach_segfault_handler(); +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_init(); +#endif + char *arg0 = argv[0]; Error err; @@ -670,6 +693,13 @@ int main(int argc, char **argv) { timing_info = true; } else if (strcmp(arg, "-fstack-report") == 0) { stack_report = true; + } else if (strcmp(arg, "-fmem-report") == 0) { +#ifdef ZIG_ENABLE_MEM_PROFILE + mem_report = true; +#else + fprintf(stderr, "-fmem-report requires configuring with -DZIG_ENABLE_MEM_PROFILE=ON\n"); + return print_error_usage(arg0); +#endif } else if (strcmp(arg, "-fdump-analysis") == 0) { enable_dump_analysis = true; } else if (strcmp(arg, "-femit-docs") == 0) { @@ -1038,16 +1068,14 @@ int main(int argc, char **argv) { if (in_file) { ZigLibCInstallation libc; if ((err = zig_libc_parse(&libc, buf_create_from_str(in_file), &target, true))) - return EXIT_FAILURE; - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_FAILURE); + return main_exit(root_progress_node, EXIT_SUCCESS); } ZigLibCInstallation libc; if ((err = zig_libc_find_native(&libc, true))) - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); zig_libc_render(&libc, stdout); - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdBuiltin: { CodeGen *g = codegen_create(main_pkg_path, nullptr, &target, @@ -1065,10 +1093,9 @@ int main(int argc, char **argv) { Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdRun: case CmdBuild: @@ -1142,7 +1169,7 @@ int main(int argc, char **argv) { libc = allocate(1); if ((err = zig_libc_parse(libc, buf_create_from_str(libc_txt), &target, true))) { fprintf(stderr, "Unable to parse --libc text file: %s\n", err_str(err)); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } } Buf *cache_dir_buf; @@ -1219,7 +1246,7 @@ int main(int argc, char **argv) { codegen_set_rdynamic(g, rdynamic); if (mmacosx_version_min && mios_version_min) { fprintf(stderr, "-mmacosx-version-min and -mios-version-min options not allowed together\n"); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } if (mmacosx_version_min) { @@ -1259,6 +1286,11 @@ int main(int argc, char **argv) { zig_print_stack_report(g, stdout); if (cmd == CmdRun) { + stage2_progress_end(root_progress_node); +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dump_stats(stderr); +#endif + const char *exec_path = buf_ptr(&g->output_file_path); ZigList args = {0}; @@ -1282,10 +1314,9 @@ int main(int argc, char **argv) { buf_replace(&g->output_file_path, '/', '\\'); #endif if (printf("%s\n", buf_ptr(&g->output_file_path)) < 0) - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } else { zig_unreachable(); } @@ -1293,8 +1324,7 @@ int main(int argc, char **argv) { codegen_translate_c(g, in_file_buf, stdout, cmd == CmdTranslateCUserland); if (timing_info) codegen_print_timing_report(g, stderr); - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } else if (cmd == CmdTest) { codegen_set_emit_file_type(g, emit_file_type); @@ -1314,7 +1344,7 @@ int main(int argc, char **argv) { if (g->disable_bin_generation) { fprintf(stderr, "Semantic analysis complete. No binary produced due to -fno-emit-bin.\n"); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } Buf *test_exe_path_unresolved = &g->output_file_path; @@ -1324,7 +1354,7 @@ int main(int argc, char **argv) { if (emit_file_type != EmitFileTypeBinary) { fprintf(stderr, "Created %s but skipping execution because it is non executable.\n", buf_ptr(test_exe_path)); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } for (size_t i = 0; i < test_exec_args.length; i += 1) { @@ -1336,7 +1366,7 @@ int main(int argc, char **argv) { if (!target_can_exec(&native, &target) && test_exec_args.length == 0) { fprintf(stderr, "Created %s but skipping execution because it is non-native.\n", buf_ptr(test_exe_path)); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } Termination term; @@ -1348,21 +1378,20 @@ int main(int argc, char **argv) { fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n"); fprintf(stderr, "%s\n", buf_ptr(test_exe_path)); } - stage2_progress_end(root_progress_node); - return (term.how == TerminationIdClean) ? term.code : -1; + return main_exit(root_progress_node, (term.how == TerminationIdClean) ? term.code : -1); } else { zig_unreachable(); } } case CmdVersion: printf("%s\n", ZIG_VERSION_STRING); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); case CmdZen: { const char *ptr; size_t len; stage2_zen(&ptr, &len); fwrite(ptr, len, 1, stdout); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdTargets: return print_target_list(stdout); diff --git a/src/memory_profiling.cpp b/src/memory_profiling.cpp new file mode 100644 index 0000000000000000000000000000000000000000..494ffae117ac82776922d2e091d1f43404e0eb48 --- /dev/null +++ b/src/memory_profiling.cpp @@ -0,0 +1,139 @@ +#include "memory_profiling.hpp" +#include "hash_map.hpp" +#include "list.hpp" +#include "util.hpp" +#include + +#ifdef ZIG_ENABLE_MEM_PROFILE + +static bool str_eql_str(const char *a, const char *b) { + return strcmp(a, b) == 0; +} + +static uint32_t str_hash(const char *s) { + // FNV 32-bit hash + uint32_t h = 2166136261; + for (; *s; s += 1) { + h = h ^ *s; + h = h * 16777619; + } + return h; +} + +struct CountAndSize { + size_t item_count; + size_t type_size; +}; + +ZigList unknown_names = {}; +HashMap usage_table = {}; +bool table_active = false; + + +static const char *get_default_name(const char *name_or_null, size_t type_size) { + if (name_or_null != nullptr) return name_or_null; + if (type_size >= unknown_names.length) { + table_active = false; + unknown_names.resize(type_size + 1); + table_active = true; + } + if (unknown_names.at(type_size) == nullptr) { + char buf[100]; + sprintf(buf, "Unknown_%zu%c", type_size, 0); + unknown_names.at(type_size) = strdup(buf); + } + return unknown_names.at(type_size); +} + +void memprof_alloc(const char *name, size_t count, size_t type_size) { + if (!table_active) return; + if (count == 0) return; + // temporarily disable during table put + table_active = false; + name = get_default_name(name, type_size); + auto existing_entry = usage_table.put_unique(name, {count, type_size}); + if (existing_entry != nullptr) { + assert(existing_entry->value.type_size == type_size); // allocated name does not match type + existing_entry->value.item_count += count; + } + table_active = true; +} + +void memprof_dealloc(const char *name, size_t count, size_t type_size) { + if (!table_active) return; + if (count == 0) return; + name = get_default_name(name, type_size); + auto existing_entry = usage_table.maybe_get(name); + if (existing_entry == nullptr) { + zig_panic("deallocated more than allocated; compromised memory usage stats"); + } + if (existing_entry->value.type_size != type_size) { + zig_panic("deallocated name '%s' does not match expected type size %zu", name, type_size); + } + existing_entry->value.item_count -= count; +} + +void memprof_init(void) { + usage_table.init(1024); + table_active = true; +} + +struct MemItem { + const char *type_name; + CountAndSize count_and_size; +}; + +static size_t get_bytes(const MemItem *item) { + return item->count_and_size.item_count * item->count_and_size.type_size; +} + +static int compare_bytes_desc(const void *a, const void *b) { + size_t size_a = get_bytes((const MemItem *)(a)); + size_t size_b = get_bytes((const MemItem *)(b)); + if (size_a > size_b) + return -1; + if (size_a < size_b) + return 1; + return 0; +} + +void memprof_dump_stats(FILE *file) { + assert(table_active); + // disable modifications from this function + table_active = false; + + ZigList list = {}; + + auto it = usage_table.entry_iterator(); + for (;;) { + auto *entry = it.next(); + if (!entry) + break; + + list.append({entry->key, entry->value}); + } + + qsort(list.items, list.length, sizeof(MemItem), compare_bytes_desc); + + size_t total_bytes_used = 0; + + for (size_t i = 0; i < list.length; i += 1) { + const MemItem *item = &list.at(i); + fprintf(file, "%s: %zu items, %zu bytes each, total ", item->type_name, + item->count_and_size.item_count, item->count_and_size.type_size); + size_t bytes = get_bytes(item); + zig_pretty_print_bytes(file, bytes); + fprintf(file, "\n"); + + total_bytes_used += bytes; + } + + fprintf(stderr, "Total bytes used: "); + zig_pretty_print_bytes(file, total_bytes_used); + fprintf(file, "\n"); + + list.deinit(); + table_active = true; +} + +#endif diff --git a/src/memory_profiling.hpp b/src/memory_profiling.hpp new file mode 100644 index 0000000000000000000000000000000000000000..6d43d81e3ccaaa6e2a2394d6cdd9223307962433 --- /dev/null +++ b/src/memory_profiling.hpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_MEMORY_PROFILING_HPP +#define ZIG_MEMORY_PROFILING_HPP + +#include "config.h" + +#include +#include + +void memprof_init(void); + +void memprof_alloc(const char *name, size_t item_count, size_t type_size); +void memprof_dealloc(const char *name, size_t item_count, size_t type_size); + +void memprof_dump_stats(FILE *file); +#endif diff --git a/src/util.cpp b/src/util.cpp index 13bfbbcd4715ad74b038e3b431863f700d18ba7f..055d5720100077299efdc453a639132b9f46f8ff 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -119,3 +119,21 @@ Slice SplitIterator_rest(SplitIterator *self) { SplitIterator memSplit(Slice buffer, Slice split_bytes) { return SplitIterator{0, buffer, split_bytes}; } + +void zig_pretty_print_bytes(FILE *f, double n) { + if (n > 1024.0 * 1024.0 * 1024.0) { + fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0); + return; + } + if (n > 1024.0 * 1024.0) { + fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0); + return; + } + if (n > 1024.0) { + fprintf(f, "%.02f KiB", n / 1024.0); + return; + } + fprintf(f, "%.02f bytes", n ); + return; +} + diff --git a/src/util.hpp b/src/util.hpp index 8abcef32ceec78241148729f4cbed96c228c13a2..79bebd3355695ad81974157d691fdf211eeda110 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -8,6 +8,8 @@ #ifndef ZIG_UTIL_HPP #define ZIG_UTIL_HPP +#include "memory_profiling.hpp" + #include #include #include @@ -96,7 +98,10 @@ static inline int ctzll(unsigned long long mask) { template -ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) { +ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_alloc(name, count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (count == 0) @@ -109,7 +114,10 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) { } template -ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) { +ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_alloc(name, count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (count == 0) @@ -122,7 +130,7 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) { } template -static inline T *reallocate(T *old, size_t old_count, size_t new_count) { +static inline T *reallocate(T *old, size_t old_count, size_t new_count, const char *name = nullptr) { T *ptr = reallocate_nonzero(old, old_count, new_count); if (new_count > old_count) { memset(&ptr[old_count], 0, (new_count - old_count) * sizeof(T)); @@ -131,7 +139,11 @@ static inline T *reallocate(T *old, size_t old_count, size_t new_count) { } template -static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) { +static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dealloc(name, old_count, sizeof(T)); + memprof_alloc(name, new_count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (new_count == 0 && old == nullptr) @@ -143,6 +155,19 @@ static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) return ptr; } +template +static inline void deallocate(T *old, size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dealloc(name, count, sizeof(T)); +#endif + free(old); +} + +template +static inline void destroy(T *old, const char *name = nullptr) { + return deallocate(old, 1); +} + template constexpr size_t array_length(const T (&)[n]) { return n; @@ -225,6 +250,8 @@ static inline double zig_f16_to_double(float16_t x) { return z; } +void zig_pretty_print_bytes(FILE *f, double n); + template struct Optional { T value; -- 2.54.0 From 4e985123447dba40e282e9663e3ca669d3f9243a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Oct 2019 03:51:04 -0400 Subject: [PATCH 12/31] reduce the size of IrInstruction by 8 bytes on 64 bit targets This brings the std lib tests down from 3.51 GiB memory usage to 3.41 GiB, by making two fields that were 64 bits 32 bits. This is a small thing; the bigger wins will come from the strategy outlined in the previous commit. --- src/all_types.hpp | 6 +++--- src/ir.cpp | 7 ++++--- src/ir_print.cpp | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index f2e614f192b9daa3ac8848d40c7eed6375d4eba1..e2e1db48d49897df588d83e7a177e9528bc10dbb 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2569,12 +2569,12 @@ enum IrInstructionId { struct IrInstruction { Scope *scope; AstNode *source_node; + LLVMValueRef llvm_value; ConstExprValue value; - size_t debug_id; - LLVMValueRef llvm_value; + uint32_t debug_id; // if ref_count is zero and the instruction has no side effects, // the instruction can be omitted in codegen - size_t ref_count; + uint32_t ref_count; // When analyzing IR, instructions that point to this instruction in the "old ir" // can find the instruction that corresponds to this value in the "new ir" // with this child field. diff --git a/src/ir.cpp b/src/ir.cpp index b71a48dc2b7ba8121a4362b91d58f739d261c232..981aa55b2a6e3d32cc8585867c90fd80535eac10 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -10822,7 +10822,8 @@ static IrInstruction *ira_suspend(IrAnalyze *ira, IrInstruction *old_instruction IrSuspendPosition *suspend_pos) { if (ira->codegen->verbose_ir) { - fprintf(stderr, "suspend %s_%zu %s_%zu #%zu (%zu,%zu)\n", ira->old_irb.current_basic_block->name_hint, + fprintf(stderr, "suspend %s_%zu %s_%zu #%" PRIu32 " (%zu,%zu)\n", + ira->old_irb.current_basic_block->name_hint, ira->old_irb.current_basic_block->debug_id, ira->old_irb.exec->basic_block_list.at(ira->old_bb_index)->name_hint, ira->old_irb.exec->basic_block_list.at(ira->old_bb_index)->debug_id, @@ -10860,7 +10861,7 @@ static IrInstruction *ira_resume(IrAnalyze *ira) { ira->instruction_index = pos.instruction_index; assert(pos.instruction_index < ira->old_irb.current_basic_block->instruction_list.length); if (ira->codegen->verbose_ir) { - fprintf(stderr, "%s_%zu #%zu\n", ira->old_irb.current_basic_block->name_hint, + fprintf(stderr, "%s_%zu #%" PRIu32 "\n", ira->old_irb.current_basic_block->name_hint, ira->old_irb.current_basic_block->debug_id, ira->old_irb.current_basic_block->instruction_list.at(pos.instruction_index)->debug_id); } @@ -26328,7 +26329,7 @@ ZigType *ir_analyze(CodeGen *codegen, IrExecutable *old_exec, IrExecutable *new_ } if (ira->codegen->verbose_ir) { - fprintf(stderr, "analyze #%zu\n", old_instruction->debug_id); + fprintf(stderr, "analyze #%" PRIu32 "\n", old_instruction->debug_id); } IrInstruction *new_instruction = ir_analyze_instruction_base(ira, old_instruction); if (new_instruction != nullptr) { diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 441808e834fd91210e78bd65bd2cfd075773ca15..d3c77f3638d36aa2a6de225337ea368a154405ca 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -385,8 +385,8 @@ static void ir_print_prefix(IrPrint *irp, IrInstruction *instruction, bool trail const char mark = trailing ? ':' : '#'; const char *type_name = instruction->value.type ? buf_ptr(&instruction->value.type->name) : "(unknown)"; const char *ref_count = ir_has_side_effects(instruction) ? - "-" : buf_ptr(buf_sprintf("%" ZIG_PRI_usize "", instruction->ref_count)); - fprintf(irp->f, "%c%-3zu| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, + "-" : buf_ptr(buf_sprintf("%" PRIu32 "", instruction->ref_count)); + fprintf(irp->f, "%c%-3" PRIu32 "| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, ir_instruction_type_str(instruction->id), type_name, ref_count); } @@ -398,7 +398,7 @@ static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) { } static void ir_print_var_instruction(IrPrint *irp, IrInstruction *instruction) { - fprintf(irp->f, "#%" ZIG_PRI_usize "", instruction->debug_id); + fprintf(irp->f, "#%" PRIu32 "", instruction->debug_id); if (irp->pass != IrPassSrc && irp->printed.maybe_get(instruction) == nullptr) { irp->printed.put(instruction, 0); irp->pending.append(instruction); -- 2.54.0 From bab93e75611de79a1a1e898b0c9e45905817189f Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 19 Oct 2019 16:54:49 +0200 Subject: [PATCH 13/31] Fix crash when generating constant unions with single field --- src/codegen.cpp | 9 +++++++-- test/stage1/behavior/union.zig | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 18af74afd74eceaa612b3bc511d240eee6e3e05f..82d2480eb9a0a1a583f626bc904dac468b87e62d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6355,12 +6355,17 @@ static LLVMValueRef gen_const_ptr_union_recursive(CodeGen *g, ConstExprValue *un ConstParent *parent = &union_const_val->parent; LLVMValueRef base_ptr = gen_parent_ptr(g, union_const_val, parent); + // Slot in the structure where the payload is stored, if equal to SIZE_MAX + // the union has no tag and a single field and is collapsed into the field + // itself + size_t union_payload_index = union_const_val->type->data.unionation.gen_union_index; + ZigType *u32 = g->builtin_types.entry_u32; LLVMValueRef indices[] = { LLVMConstNull(get_llvm_type(g, u32)), - LLVMConstInt(get_llvm_type(g, u32), 0, false), // TODO test const union with more aligned tag type than payload + LLVMConstInt(get_llvm_type(g, u32), union_payload_index, false), }; - return LLVMConstInBoundsGEP(base_ptr, indices, 2); + return LLVMConstInBoundsGEP(base_ptr, indices, (union_payload_index != SIZE_MAX) ? 2 : 1); } static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, ConstExprValue *const_val) { diff --git a/test/stage1/behavior/union.zig b/test/stage1/behavior/union.zig index f5e9ec09a89e174a9a495832a06184614dab3bd1..40adf601c454703e65510813ca3613eb18787bb1 100644 --- a/test/stage1/behavior/union.zig +++ b/test/stage1/behavior/union.zig @@ -521,3 +521,17 @@ test "extern union doesn't trigger field check at comptime" { const x = U{ .x = 0x55AAAA55 }; comptime expect(x.y == 0x55); } + +const Foo1 = union(enum) { + f: struct { + x: usize, + }, +}; +var glbl: Foo1 = undefined; + +test "global union with single field is correctly initialized" { + glbl = Foo1{ + .f = @memberType(Foo1, 0){ .x = 123 }, + }; + expect(glbl.f.x == 123); +} -- 2.54.0 From 6f7939a452e69b77581f5390b8075e9dfa81b03e Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 19 Oct 2019 16:20:30 +0200 Subject: [PATCH 14/31] Prevent too eager constant-folding of switch expression A pointer was wrongly assumed to be comptime-available causing the analysis pass to assume its initial value was constant. Fixes #3481 --- src/ir.cpp | 2 +- test/stage1/behavior/switch.zig | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/ir.cpp b/src/ir.cpp index 981aa55b2a6e3d32cc8585867c90fd80535eac10..2ceabb91b03db1eaa0f108c213c9d14ec7e412df 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -19251,7 +19251,7 @@ static IrInstruction *ir_analyze_instruction_switch_target(IrAnalyze *ira, ZigType *target_type = target_value_ptr->value.type->data.pointer.child_type; ConstExprValue *pointee_val = nullptr; - if (instr_is_comptime(target_value_ptr)) { + if (instr_is_comptime(target_value_ptr) && target_value_ptr->value.data.x_ptr.mut != ConstPtrMutRuntimeVar) { pointee_val = const_ptr_pointee(ira, ira->codegen, &target_value_ptr->value, target_value_ptr->source_node); if (pointee_val == nullptr) return ira->codegen->invalid_instruction; diff --git a/test/stage1/behavior/switch.zig b/test/stage1/behavior/switch.zig index bc1b4a7a094674dc01fed71412610b6d37318a0d..936dbed786f7d1eedc7029657705439212832492 100644 --- a/test/stage1/behavior/switch.zig +++ b/test/stage1/behavior/switch.zig @@ -434,3 +434,21 @@ test "switch with disjoint range" { 126...126 => {}, } } + +var state: u32 = 0; +fn poll() void { + switch (state) { + 0 => { + state = 1; + }, + else => { + state += 1; + }, + } +} + +test "switch on global mutable var isn't constant-folded" { + while (state < 2) { + poll(); + } +} -- 2.54.0 From e78d3750c58d26bac0e24c40eb89c2f4796bc15c Mon Sep 17 00:00:00 2001 From: stratact Date: Wed, 18 Sep 2019 23:56:45 -0700 Subject: [PATCH 15/31] Use 8192 sized buffers and remove allocator parameters --- doc/docgen.zig | 2 +- lib/std/build.zig | 4 +-- lib/std/event/fs.zig | 2 +- lib/std/fs.zig | 61 +++++++++++--------------------------- lib/std/os/test.zig | 6 ++-- src-self-hosted/main.zig | 2 +- src-self-hosted/stage1.zig | 2 +- src-self-hosted/test.zig | 4 +-- test/cli.zig | 2 +- tools/process_headers.zig | 2 +- 10 files changed, 31 insertions(+), 56 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 73289e5edc05ce1e51160a55af636d2126ad6fd9..bdc260421380acd7e0f8d6f18957ccb6a3f7dc12 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -51,7 +51,7 @@ pub fn main() !void { var toc = try genToc(allocator, &tokenizer); try fs.makePath(allocator, tmp_dir_name); - defer fs.deleteTree(allocator, tmp_dir_name) catch {}; + defer fs.deleteTree(tmp_dir_name) catch {}; try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream, zig_exe); try buffered_out_stream.flush(); diff --git a/lib/std/build.zig b/lib/std/build.zig index 68d3f2a31512428f13ad59a0e757b4e7203aaa05..32974da92a0e302347e0a8de4e02ec310af138e0 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -331,7 +331,7 @@ pub const Builder = struct { if (self.verbose) { warn("rm {}\n", full_path); } - fs.deleteTree(self.allocator, full_path) catch {}; + fs.deleteTree(full_path) catch {}; } // TODO remove empty directories @@ -2671,7 +2671,7 @@ pub const RemoveDirStep = struct { const self = @fieldParentPtr(RemoveDirStep, "step", step); const full_path = self.builder.pathFromRoot(self.dir_path); - fs.deleteTree(self.builder.allocator, full_path) catch |err| { + fs.deleteTree(full_path) catch |err| { warn("Unable to remove {}: {}\n", full_path, @errorName(err)); return err; }; diff --git a/lib/std/event/fs.zig b/lib/std/event/fs.zig index 4490e1deae8a9d6d7c4da17914722fce1c40e9f1..81fd950e446aa8e6088c7b2a9509610e19eacd1a 100644 --- a/lib/std/event/fs.zig +++ b/lib/std/event/fs.zig @@ -1312,7 +1312,7 @@ const test_tmp_dir = "std_event_fs_test"; // // // TODO move this into event loop too // try os.makePath(allocator, test_tmp_dir); -// defer os.deleteTree(allocator, test_tmp_dir) catch {}; +// defer os.deleteTree(test_tmp_dir) catch {}; // // var loop: Loop = undefined; // try loop.initMultiThreaded(allocator); diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 287c0316b74d3f10abef4fc5021abe41d81a7715..1552f65ecdc7cab0f6aae12e69b9e4b3eba9d8ad 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -37,6 +37,8 @@ pub const MAX_PATH_BYTES = switch (builtin.os) { else => @compileError("Unsupported OS"), }; +pub const MAX_BUF_BYTES: usize = 8192; + // here we replace the standard +/ with -_ so that it can be used in a file name const b64_fs_encoder = base64.Base64Encoder.init("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", base64.standard_pad_char); @@ -371,7 +373,7 @@ const DeleteTreeError = error{ /// this function recursively removes its entries and then tries again. /// TODO determine if we can remove the allocator requirement /// https://github.com/ziglang/zig/issues/2886 -pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError!void { +pub fn deleteTree(full_path: []const u8) DeleteTreeError!void { start_over: while (true) { var got_access_denied = false; // First, try deleting the item as a file. This way we don't follow sym links. @@ -395,7 +397,7 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! => return err, } { - var dir = Dir.open(allocator, full_path) catch |err| switch (err) { + var dir = Dir.open(full_path) catch |err| switch (err) { error.NotDir => { if (got_access_denied) { return error.AccessDenied; @@ -424,17 +426,14 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! }; defer dir.close(); - var full_entry_buf = std.ArrayList(u8).init(allocator); - defer full_entry_buf.deinit(); - while (try dir.next()) |entry| { - try full_entry_buf.resize(full_path.len + entry.name.len + 1); - const full_entry_path = full_entry_buf.toSlice(); + var full_entry_buf: [MAX_BUF_BYTES]u8 = undefined; + const full_entry_path = full_entry_buf[0..]; mem.copy(u8, full_entry_path, full_path); full_entry_path[full_path.len] = path.sep; mem.copy(u8, full_entry_path[full_path.len + 1 ..], entry.name); - try deleteTree(allocator, full_entry_path); + try deleteTree(full_entry_path[0..full_path.len + entry.name.len + 1]); } } return deleteDir(full_path); @@ -445,19 +444,18 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! /// files, and into the one that reads files from an open directory handle. pub const Dir = struct { handle: Handle, - allocator: *Allocator, pub const Handle = switch (builtin.os) { .macosx, .ios, .freebsd, .netbsd => struct { fd: i32, seek: i64, - buf: []u8, + buf: [MAX_BUF_BYTES]u8, index: usize, end_index: usize, }, .linux => struct { fd: i32, - buf: []u8, + buf: [MAX_BUF_BYTES]u8, index: usize, end_index: usize, }, @@ -512,9 +510,8 @@ pub const Dir = struct { /// Call close when done. /// TODO remove the allocator requirement from this API /// https://github.com/ziglang/zig/issues/2885 - pub fn open(allocator: *Allocator, dir_path: []const u8) OpenError!Dir { + pub fn open(dir_path: []const u8) OpenError!Dir { return Dir{ - .allocator = allocator, .handle = switch (builtin.os) { .windows => blk: { var find_file_data: os.windows.WIN32_FIND_DATAW = undefined; @@ -548,7 +545,6 @@ pub const Dir = struct { if (os.windows.is_the_target) { return os.windows.FindClose(self.handle.handle); } - self.allocator.free(self.handle.buf); os.close(self.handle.fd); } @@ -579,14 +575,10 @@ pub const Dir = struct { fn nextDarwin(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { const rc = os.system.__getdirentries64( self.handle.fd, - self.handle.buf.ptr, + self.handle.buf[0..].ptr, self.handle.buf.len, &self.handle.seek, ); @@ -596,10 +588,7 @@ pub const Dir = struct { os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } } @@ -666,21 +655,14 @@ pub const Dir = struct { fn nextLinux(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { - const rc = os.linux.getdents64(self.handle.fd, self.handle.buf.ptr, self.handle.buf.len); + const rc = os.linux.getdents64(self.handle.fd, self.handle.buf[0..].ptr, self.handle.buf.len); switch (os.linux.getErrno(rc)) { 0 => {}, os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } if (rc == 0) return null; @@ -720,14 +702,10 @@ pub const Dir = struct { fn nextBsd(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { const rc = os.system.getdirentries( self.handle.fd, - self.handle.buf.ptr, + self.handle.buf[0..].ptr, self.handle.buf.len, &self.handle.seek, ); @@ -736,10 +714,7 @@ pub const Dir = struct { os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } if (rc == 0) return null; @@ -807,7 +782,7 @@ pub const Walker = struct { try self.name_buffer.append(base.name); if (base.kind == .Directory) { // TODO https://github.com/ziglang/zig/issues/2888 - var new_dir = try Dir.open(self.stack.allocator, self.name_buffer.toSliceConst()); + var new_dir = try Dir.open(self.name_buffer.toSliceConst()); { errdefer new_dir.close(); try self.stack.append(StackItem{ @@ -841,7 +816,7 @@ pub const Walker = struct { pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); - var dir_it = try Dir.open(allocator, dir_path); + var dir_it = try Dir.open(dir_path); errdefer dir_it.close(); var name_buffer = try std.Buffer.init(allocator, dir_path); diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 8d6b437ff382f83eb5ec78ae4bd8a31e1f684a1b..b6e6728142ffae0f9a2b47c0a91b70bd36fb4333 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -19,8 +19,8 @@ test "makePath, put some files in it, deleteTree" { try fs.makePath(a, "os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c"); try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c" ++ fs.path.sep_str ++ "file.txt", "nonsense"); try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "file2.txt", "blah"); - try fs.deleteTree(a, "os_test_tmp"); - if (fs.Dir.open(a, "os_test_tmp")) |dir| { + try fs.deleteTree("os_test_tmp"); + if (fs.Dir.open("os_test_tmp")) |dir| { @panic("expected error"); } else |err| { expect(err == error.FileNotFound); @@ -37,7 +37,7 @@ test "access file" { try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "file.txt", ""); try os.access("os_test_tmp" ++ fs.path.sep_str ++ "file.txt", os.F_OK); - try fs.deleteTree(a, "os_test_tmp"); + try fs.deleteTree("os_test_tmp"); } fn testThreadIdFn(thread_id: *Thread.Id) void { diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 52eda5824aef40a55f2e8f926071eb3a43d36f7d..a2e77394ce2008e177a50cac923bc59822d6990a 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -747,7 +747,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtErro )) catch |err| switch (err) { error.IsDir, error.AccessDenied => { // TODO make event based (and dir.next()) - var dir = try fs.Dir.open(fmt.loop.allocator, file_path); + var dir = try fs.Dir.open(file_path); defer dir.close(); var group = event.Group(FmtError!void).init(fmt.loop); diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index 48b14ebcacda2a9b990bc84266240a17b2970fa6..d36073abf308375e48872111d45f9919d26bd19d 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -283,7 +283,7 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void const source_code = io.readFileAlloc(fmt.allocator, file_path) catch |err| switch (err) { error.IsDir, error.AccessDenied => { // TODO make event based (and dir.next()) - var dir = try fs.Dir.open(fmt.allocator, file_path); + var dir = try fs.Dir.open(file_path); defer dir.close(); while (try dir.next()) |entry| { diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 526518ca47016b921310e9ab97ee6b71957b0bc4..ce51de9a7b6315e9f142029b09ba4c5bcc3faf55 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -56,11 +56,11 @@ pub const TestContext = struct { errdefer allocator.free(self.zig_lib_dir); try std.fs.makePath(allocator, tmp_dir_name); - errdefer std.fs.deleteTree(allocator, tmp_dir_name) catch {}; + errdefer std.fs.deleteTree(tmp_dir_name) catch {}; } fn deinit(self: *TestContext) void { - std.fs.deleteTree(allocator, tmp_dir_name) catch {}; + std.fs.deleteTree(tmp_dir_name) catch {}; allocator.free(self.zig_lib_dir); self.zig_compiler.deinit(); self.loop.deinit(); diff --git a/test/cli.zig b/test/cli.zig index 63a116a81154ebfa42c9c0e20340be0ce048ec56..2999df14b0d1a39046c11541f058e9d04cce2531 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -37,7 +37,7 @@ pub fn main() !void { testMissingOutputPath, }; for (test_fns) |testFn| { - try fs.deleteTree(a, dir_path); + try fs.deleteTree(dir_path); try fs.makeDir(dir_path); try testFn(zig_exe, dir_path); } diff --git a/tools/process_headers.zig b/tools/process_headers.zig index 420c118cb976ccbbe697ce2a25609d40caaab8da..667da5f7c9c0592606afd7eac05114f9a19619c6 100644 --- a/tools/process_headers.zig +++ b/tools/process_headers.zig @@ -340,7 +340,7 @@ pub fn main() !void { try dir_stack.append(target_include_dir); while (dir_stack.popOrNull()) |full_dir_name| { - var dir = std.fs.Dir.open(allocator, full_dir_name) catch |err| switch (err) { + var dir = std.fs.Dir.open(full_dir_name) catch |err| switch (err) { error.FileNotFound => continue :search, error.AccessDenied => continue :search, else => return err, -- 2.54.0 From 771dafbabaaf4cda43d722dbc4bcb7d8464b576d Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 20 Oct 2019 16:58:00 +0200 Subject: [PATCH 16/31] Resolve lazy slice start/end values Fixes #3489 --- src/ir.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 2ceabb91b03db1eaa0f108c213c9d14ec7e412df..6bdfd86b29941f7add7fcf85dda7e6d475122eec 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -23084,17 +23084,22 @@ static IrInstruction *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstruction zig_unreachable(); } - uint64_t start_scalar = bigint_as_u64(&casted_start->value.data.x_bigint); + ConstExprValue *start_val = ir_resolve_const(ira, casted_start, UndefBad); + if (!start_val) + return ira->codegen->invalid_instruction; + + uint64_t start_scalar = bigint_as_u64(&start_val->data.x_bigint); if (!ptr_is_undef && start_scalar > rel_end) { ir_add_error(ira, &instruction->base, buf_sprintf("out of bounds slice")); return ira->codegen->invalid_instruction; } - uint64_t end_scalar; + uint64_t end_scalar = rel_end; if (end) { - end_scalar = bigint_as_u64(&end->value.data.x_bigint); - } else { - end_scalar = rel_end; + ConstExprValue *end_val = ir_resolve_const(ira, end, UndefBad); + if (!end_val) + return ira->codegen->invalid_instruction; + end_scalar = bigint_as_u64(&end_val->data.x_bigint); } if (!ptr_is_undef) { if (end_scalar > rel_end) { -- 2.54.0 From 5b1a492012241276a4b7539ca6664234f0629c79 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 20 Oct 2019 21:48:23 -0400 Subject: [PATCH 17/31] breaking: improve std.fs directory handling API * Added `std.c.unlinkat` and `std.os.unlinkat`. * Removed `std.fs.MAX_BUF_BYTES` (this declaration never made it to master branch) * Added `std.fs.Dir.deleteTree` to be used on an open directory handle. * `std.fs.deleteTree` has better behavior for both relative and absolute paths. For absolute paths, it opens the base directory and uses that handle for subsequent operations. For relative paths, it does a similar strategy, using the cwd handle. * The error set of `std.fs.deleteTree` is improved to no longer have these possible errors: - OutOfMemory - FileTooBig - IsDir - DirNotEmpty - PathAlreadyExists - NoSpaceLeft * Added `std.fs.Dir.posix_cwd` which is a statically initialized directory representing the current working directory. * The error set of `std.Dir.open` is improved to no longer have these possible errors: - FileTooBig - IsDir - NoSpaceLeft - PathAlreadyExists - OutOfMemory * Added more alternative functions to `std.fs` for when the path parameter is a null terminated string. This can sometimes be more effecient on systems which have an ABI based on null terminated strings. * Added `std.fs.Dir.openDir`, `std.fs.Dir.deleteFile`, and `std.fs.Dir.deleteDir` which all operate on an open directory handle. * `std.fs.Walker.Entry` now has a `dir` field, which can be used to do operations directly on `std.fs.Walker.Entry.basename`, avoiding `error.NameTooLong` for deeply nested paths. * Added more docs to `std.os.OpenError` This commit does the POSIX components for these changes. I plan to follow up shortly with a commit for Windows. --- lib/std/c.zig | 1 + lib/std/fs.zig | 845 ++++++++++++++++++++++--------------- lib/std/os.zig | 56 ++- src-self-hosted/stage1.zig | 6 +- 4 files changed, 551 insertions(+), 357 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 9c7ec2f0e5e50336f4466a30ef30daac95f69213..f591481d047fc0a3b93afa9aca9d853effd680dc 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -80,6 +80,7 @@ pub extern "c" fn mmap(addr: ?*align(page_size) c_void, len: usize, prot: c_uint pub extern "c" fn munmap(addr: *align(page_size) c_void, len: usize) c_int; pub extern "c" fn mprotect(addr: *align(page_size) c_void, len: usize, prot: c_uint) c_int; pub extern "c" fn unlink(path: [*]const u8) c_int; +pub extern "c" fn unlinkat(dirfd: fd_t, path: [*]const u8, flags: c_uint) c_int; pub extern "c" fn getcwd(buf: [*]u8, size: usize) ?[*]u8; pub extern "c" fn waitpid(pid: c_int, stat_loc: *c_uint, options: c_uint) c_int; pub extern "c" fn fork() c_int; diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 1552f65ecdc7cab0f6aae12e69b9e4b3eba9d8ad..89071b407fcd6932c3be1219044fa79251d1560c 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -37,8 +37,6 @@ pub const MAX_PATH_BYTES = switch (builtin.os) { else => @compileError("Unsupported OS"), }; -pub const MAX_BUF_BYTES: usize = 8192; - // here we replace the standard +/ with -_ so that it can be used in a file name const b64_fs_encoder = base64.Base64Encoder.init("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", base64.standard_pad_char); @@ -337,136 +335,35 @@ pub fn deleteDirW(dir_path: [*]const u16) !void { return os.rmdirW(dir_path); } -const DeleteTreeError = error{ - OutOfMemory, - AccessDenied, - FileTooBig, - IsDir, - SymLinkLoop, - ProcessFdQuotaExceeded, - NameTooLong, - SystemFdQuotaExceeded, - NoDevice, - SystemResources, - NoSpaceLeft, - PathAlreadyExists, - ReadOnlyFileSystem, - NotDir, - FileNotFound, - FileSystem, - FileBusy, - DirNotEmpty, - DeviceBusy, - - /// On Windows, file paths must be valid Unicode. - InvalidUtf8, - - /// On Windows, file paths cannot contain these characters: - /// '/', '*', '?', '"', '<', '>', '|' - BadPathName, - - Unexpected, -}; - -/// Whether `full_path` describes a symlink, file, or directory, this function -/// removes it. If it cannot be removed because it is a non-empty directory, -/// this function recursively removes its entries and then tries again. -/// TODO determine if we can remove the allocator requirement -/// https://github.com/ziglang/zig/issues/2886 -pub fn deleteTree(full_path: []const u8) DeleteTreeError!void { - start_over: while (true) { - var got_access_denied = false; - // First, try deleting the item as a file. This way we don't follow sym links. - if (deleteFile(full_path)) { - return; - } else |err| switch (err) { - error.FileNotFound => return, - error.IsDir => {}, - error.AccessDenied => got_access_denied = true, - - error.InvalidUtf8, - error.SymLinkLoop, - error.NameTooLong, - error.SystemResources, - error.ReadOnlyFileSystem, - error.NotDir, - error.FileSystem, - error.FileBusy, - error.BadPathName, - error.Unexpected, - => return err, - } - { - var dir = Dir.open(full_path) catch |err| switch (err) { - error.NotDir => { - if (got_access_denied) { - return error.AccessDenied; - } - continue :start_over; - }, - - error.OutOfMemory, - error.AccessDenied, - error.FileTooBig, - error.IsDir, - error.SymLinkLoop, - error.ProcessFdQuotaExceeded, - error.NameTooLong, - error.SystemFdQuotaExceeded, - error.NoDevice, - error.FileNotFound, - error.SystemResources, - error.NoSpaceLeft, - error.PathAlreadyExists, - error.Unexpected, - error.InvalidUtf8, - error.BadPathName, - error.DeviceBusy, - => return err, - }; - defer dir.close(); - - while (try dir.next()) |entry| { - var full_entry_buf: [MAX_BUF_BYTES]u8 = undefined; - const full_entry_path = full_entry_buf[0..]; - mem.copy(u8, full_entry_path, full_path); - full_entry_path[full_path.len] = path.sep; - mem.copy(u8, full_entry_path[full_path.len + 1 ..], entry.name); - - try deleteTree(full_entry_path[0..full_path.len + entry.name.len + 1]); - } - } - return deleteDir(full_path); +/// Removes a symlink, file, or directory. +/// If `full_path` is relative, this is equivalent to `Dir.deleteTree` with the +/// current working directory as the open directory handle. +/// If `full_path` is absolute, this is equivalent to `Dir.deleteTree` with the +/// base directory. +pub fn deleteTree(full_path: []const u8) !void { + if (path.isAbsolute(full_path)) { + const dirname = path.dirname(full_path) orelse return error{ + /// Attempt to remove the root file system path. + /// This error is unreachable if `full_path` is relative. + CannotDeleteRootDirectory, + }.CannotDeleteRootDirectory; + + var dir = try Dir.open(dirname); + defer dir.close(); + + return dir.deleteTree(path.basename(full_path)); + } else { + return Dir.posix_cwd.deleteTree(full_path); } } -/// TODO: separate this API into the one that opens directory handles to then subsequently open -/// files, and into the one that reads files from an open directory handle. pub const Dir = struct { - handle: Handle, + fd: os.fd_t, - pub const Handle = switch (builtin.os) { - .macosx, .ios, .freebsd, .netbsd => struct { - fd: i32, - seek: i64, - buf: [MAX_BUF_BYTES]u8, - index: usize, - end_index: usize, - }, - .linux => struct { - fd: i32, - buf: [MAX_BUF_BYTES]u8, - index: usize, - end_index: usize, - }, - .windows => struct { - handle: os.windows.HANDLE, - find_file_data: os.windows.WIN32_FIND_DATAW, - first: bool, - name_data: [256]u8, - }, - else => @compileError("unimplemented"), - }; + /// An open handle to the current working directory. + /// Closing this directory is safety-checked illegal behavior. + /// Not available on Windows. + pub const posix_cwd = Dir{ .fd = os.AT_FDCWD }; pub const Entry = struct { name: []const u8, @@ -485,269 +382,504 @@ pub const Dir = struct { }; }; + pub const Iterator = switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => struct { + dir: Dir, + seek: i64, + buf: [buffer_len]u8, + index: usize, + end_index: usize, + + pub const buffer_len = 8192; + + const Self = @This(); + + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. + pub fn next(self: *Self) !?Entry { + switch (builtin.os) { + .macosx, .ios => return self.nextDarwin(), + .freebsd, .netbsd => return self.nextBsd(), + else => @compileError("unimplemented"), + } + } + + fn nextDarwin(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.system.__getdirentries64( + self.dir.fd, + &self.buf, + self.buf.len, + &self.seek, + ); + if (rc == 0) return null; + if (rc < 0) { + switch (os.errno(rc)) { + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + } + self.index = 0; + self.end_index = @intCast(usize, rc); + break; + } + } + const darwin_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); + const next_index = self.index + darwin_entry.d_reclen; + self.index = next_index; + + const name = @ptrCast([*]u8, &darwin_entry.d_name)[0..darwin_entry.d_namlen]; + + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (darwin_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + os.DT_WHT => Entry.Kind.Whiteout, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + + fn nextBsd(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.system.getdirentries( + self.dir.fd, + self.buf[0..].ptr, + self.buf.len, + &self.seek, + ); + switch (os.errno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + if (rc == 0) return null; + self.index = 0; + self.end_index = @intCast(usize, rc); + break; + } + } + const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); + const next_index = self.index + freebsd_entry.d_reclen; + self.index = next_index; + + const name = @ptrCast([*]u8, &freebsd_entry.d_name)[0..freebsd_entry.d_namlen]; + + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (freebsd_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + os.DT_WHT => Entry.Kind.Whiteout, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + }, + .linux => struct { + dir: Dir, + buf: [buffer_len]u8, + index: usize, + end_index: usize, + + pub const buffer_len = 8192; + + const Self = @This(); + + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. + pub fn next(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); + switch (os.linux.getErrno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + if (rc == 0) return null; + self.index = 0; + self.end_index = rc; + break; + } + } + const linux_entry = @ptrCast(*align(1) os.dirent64, &self.buf[self.index]); + const next_index = self.index + linux_entry.d_reclen; + self.index = next_index; + + const name = mem.toSlice(u8, @ptrCast([*]u8, &linux_entry.d_name)); + + // skip . and .. entries + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (linux_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + }, + .windows => struct { + dir: Dir, + find_file_data: os.windows.WIN32_FIND_DATAW, + first: bool, + name_data: [256]u8, + }, + else => @compileError("unimplemented"), + }; + pub const OpenError = error{ FileNotFound, NotDir, AccessDenied, - FileTooBig, - IsDir, SymLinkLoop, ProcessFdQuotaExceeded, NameTooLong, SystemFdQuotaExceeded, NoDevice, SystemResources, - NoSpaceLeft, - PathAlreadyExists, - OutOfMemory, InvalidUtf8, BadPathName, DeviceBusy, + } || os.UnexpectedError; - Unexpected, - }; - - /// Call close when done. - /// TODO remove the allocator requirement from this API - /// https://github.com/ziglang/zig/issues/2885 + /// Call `close` to free the directory handle. pub fn open(dir_path: []const u8) OpenError!Dir { - return Dir{ - .handle = switch (builtin.os) { - .windows => blk: { - var find_file_data: os.windows.WIN32_FIND_DATAW = undefined; - const handle = try os.windows.FindFirstFile(dir_path, &find_file_data); - break :blk Handle{ - .handle = handle, - .find_file_data = find_file_data, // TODO guaranteed copy elision - .first = true, - .name_data = undefined, - }; - }, - .macosx, .ios, .freebsd, .netbsd => Handle{ - .fd = try os.open(dir_path, os.O_RDONLY | os.O_NONBLOCK | os.O_DIRECTORY | os.O_CLOEXEC, 0), - .seek = 0, - .index = 0, - .end_index = 0, - .buf = [_]u8{}, - }, - .linux => Handle{ - .fd = try os.open(dir_path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC, 0), - .index = 0, - .end_index = 0, - .buf = [_]u8{}, - }, - else => @compileError("unimplemented"), - }, - }; + return posix_cwd.openDir(dir_path); + } + + /// Same as `open` except the parameter is null-terminated. + pub fn openC(dir_path_c: [*]const u8) OpenError!Dir { + return posix_cwd.openDirC(dir_path_c); } pub fn close(self: *Dir) void { if (os.windows.is_the_target) { - return os.windows.FindClose(self.handle.handle); + @panic("TODO"); } - os.close(self.handle.fd); + os.close(self.fd); + self.* = undefined; } - /// Memory such as file names referenced in this returned entry becomes invalid - /// with subsequent calls to next, as well as when this `Dir` is deinitialized. - pub fn next(self: *Dir) !?Entry { - switch (builtin.os) { - .linux => return self.nextLinux(), - .macosx, .ios => return self.nextDarwin(), - .windows => return self.nextWindows(), - .freebsd => return self.nextBsd(), - .netbsd => return self.nextBsd(), - else => @compileError("unimplemented"), - } - } - - pub fn openRead(self: Dir, file_path: []const u8) os.OpenError!File { - const path_c = try os.toPosixPath(file_path); + /// Call `File.close` on the result when done. + pub fn openRead(self: Dir, sub_path: []const u8) File.OpenError!File { + const path_c = try os.toPosixPath(sub_path); return self.openReadC(&path_c); } - pub fn openReadC(self: Dir, file_path: [*]const u8) OpenError!File { + /// Call `File.close` on the result when done. + pub fn openReadC(self: Dir, sub_path: [*]const u8) File.OpenError!File { const flags = os.O_LARGEFILE | os.O_RDONLY; - const fd = try os.openatC(self.handle.fd, file_path, flags, 0); + const fd = try os.openatC(self.fd, sub_path, flags, 0); return File.openHandle(fd); } - fn nextDarwin(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.system.__getdirentries64( - self.handle.fd, - self.handle.buf[0..].ptr, - self.handle.buf.len, - &self.handle.seek, - ); - if (rc == 0) return null; - if (rc < 0) { - switch (os.errno(rc)) { - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - } - self.handle.index = 0; - self.handle.end_index = @intCast(usize, rc); - break; - } - } - const darwin_entry = @ptrCast(*align(1) os.dirent, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + darwin_entry.d_reclen; - self.handle.index = next_index; + /// Call `close` on the result when done. + pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { + const sub_path_c = try os.toPosixPath(sub_path); + return self.openDirC(&sub_path_c); + } - const name = @ptrCast([*]u8, &darwin_entry.d_name)[0..darwin_entry.d_namlen]; + /// Call `close` on the result when done. + pub fn openDirC(self: Dir, sub_path: [*]const u8) OpenError!Dir { + const flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC; + const fd = os.openatC(self.fd, sub_path, flags, 0) catch |err| switch (err) { + error.FileTooBig => unreachable, // can't happen for directories + error.IsDir => unreachable, // we're providing O_DIRECTORY + error.NoSpaceLeft => unreachable, // not providing O_CREAT + error.PathAlreadyExists => unreachable, // not providing O_CREAT + else => |e| return e, + }; + return Dir{ .fd = fd }; + } - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } + pub const DeleteFileError = os.UnlinkError; - const entry_kind = switch (darwin_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - os.DT_WHT => Entry.Kind.Whiteout, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; - } + /// Delete a file name and possibly the file it refers to, based on an open directory handle. + pub fn deleteFile(self: Dir, sub_path: []const u8) DeleteFileError!void { + const sub_path_c = try os.toPosixPath(sub_path); + return self.deleteFileC(&sub_path_c); } - fn nextWindows(self: *Dir) !?Entry { - while (true) { - if (self.handle.first) { - self.handle.first = false; - } else { - if (!try os.windows.FindNextFile(self.handle.handle, &self.handle.find_file_data)) - return null; - } - const name_utf16le = mem.toSlice(u16, self.handle.find_file_data.cFileName[0..].ptr); - if (mem.eql(u16, name_utf16le, [_]u16{'.'}) or mem.eql(u16, name_utf16le, [_]u16{ '.', '.' })) - continue; - // Trust that Windows gives us valid UTF-16LE - const name_utf8_len = std.unicode.utf16leToUtf8(self.handle.name_data[0..], name_utf16le) catch unreachable; - const name_utf8 = self.handle.name_data[0..name_utf8_len]; - const kind = blk: { - const attrs = self.handle.find_file_data.dwFileAttributes; - if (attrs & os.windows.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; - if (attrs & os.windows.FILE_ATTRIBUTE_REPARSE_POINT != 0) break :blk Entry.Kind.SymLink; - break :blk Entry.Kind.File; - }; - return Entry{ - .name = name_utf8, - .kind = kind, - }; - } + /// Same as `deleteFile` except the parameter is null-terminated. + pub fn deleteFileC(self: Dir, sub_path_c: [*]const u8) DeleteFileError!void { + os.unlinkatC(self.fd, sub_path_c, 0) catch |err| switch (err) { + error.DirNotEmpty => unreachable, // not passing AT_REMOVEDIR + else => |e| return e, + }; } - fn nextLinux(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.linux.getdents64(self.handle.fd, self.handle.buf[0..].ptr, self.handle.buf.len); - switch (os.linux.getErrno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.handle.index = 0; - self.handle.end_index = rc; - break; - } - } - const linux_entry = @ptrCast(*align(1) os.dirent64, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + linux_entry.d_reclen; - self.handle.index = next_index; + pub const DeleteDirError = error{ + DirNotEmpty, + FileNotFound, + AccessDenied, + FileBusy, + FileSystem, + SymLinkLoop, + NameTooLong, + NotDir, + SystemResources, + ReadOnlyFileSystem, + InvalidUtf8, + BadPathName, + Unexpected, + }; - const name = mem.toSlice(u8, @ptrCast([*]u8, &linux_entry.d_name)); + /// Returns `error.DirNotEmpty` if the directory is not empty. + /// To delete a directory recursively, see `deleteTree`. + pub fn deleteDir(self: Dir, sub_path: []const u8) DeleteDirError!void { + const sub_path_c = try os.toPosixPath(sub_path); + return self.deleteDirC(&sub_path_c); + } - // skip . and .. entries - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } + /// Same as `deleteDir` except the parameter is null-terminated. + pub fn deleteDirC(self: Dir, sub_path_c: [*]const u8) DeleteDirError!void { + os.unlinkatC(self.fd, sub_path_c, os.AT_REMOVEDIR) catch |err| switch (err) { + error.IsDir => unreachable, // not possible since we pass AT_REMOVEDIR + else => |e| return e, + }; + } - const entry_kind = switch (linux_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; + pub fn iterate(self: Dir) Iterator { + switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => return Iterator{ + .dir = self, + .seek = 0, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .linux => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .windows => @panic("TODO"), + else => @compileError("unimplemented"), } } - fn nextBsd(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.system.getdirentries( - self.handle.fd, - self.handle.buf[0..].ptr, - self.handle.buf.len, - &self.handle.seek, - ); - switch (os.errno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.handle.index = 0; - self.handle.end_index = @intCast(usize, rc); - break; - } - } - const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + freebsd_entry.d_reclen; - self.handle.index = next_index; + pub const DeleteTreeError = error{ + AccessDenied, + FileTooBig, + SymLinkLoop, + ProcessFdQuotaExceeded, + NameTooLong, + SystemFdQuotaExceeded, + NoDevice, + SystemResources, + ReadOnlyFileSystem, + FileSystem, + FileBusy, + DeviceBusy, + + /// One of the path components was not a directory. + /// This error is unreachable if `sub_path` does not contain a path separator. + NotDir, + + /// On Windows, file paths must be valid Unicode. + InvalidUtf8, + + /// On Windows, file paths cannot contain these characters: + /// '/', '*', '?', '"', '<', '>', '|' + BadPathName, + } || os.UnexpectedError; - const name = @ptrCast([*]u8, &freebsd_entry.d_name)[0..freebsd_entry.d_namlen]; + /// Whether `full_path` describes a symlink, file, or directory, this function + /// removes it. If it cannot be removed because it is a non-empty directory, + /// this function recursively removes its entries and then tries again. + /// This operation is not atomic on most file systems. + pub fn deleteTree(self: Dir, sub_path: []const u8) DeleteTreeError!void { + start_over: while (true) { + var got_access_denied = false; + // First, try deleting the item as a file. This way we don't follow sym links. + if (self.deleteFile(sub_path)) { + return; + } else |err| switch (err) { + error.FileNotFound => return, + error.IsDir => {}, + error.AccessDenied => got_access_denied = true, - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; + error.InvalidUtf8, + error.SymLinkLoop, + error.NameTooLong, + error.SystemResources, + error.ReadOnlyFileSystem, + error.NotDir, + error.FileSystem, + error.FileBusy, + error.BadPathName, + error.Unexpected, + => |e| return e, } + var dir = self.openDir(sub_path) catch |err| switch (err) { + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; + } + continue :start_over; + }, + error.FileNotFound => { + // That's fine, we were trying to remove this directory anyway. + continue :start_over; + }, - const entry_kind = switch (freebsd_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - os.DT_WHT => Entry.Kind.Whiteout, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, + error.AccessDenied, + error.SymLinkLoop, + error.ProcessFdQuotaExceeded, + error.NameTooLong, + error.SystemFdQuotaExceeded, + error.NoDevice, + error.SystemResources, + error.Unexpected, + error.InvalidUtf8, + error.BadPathName, + error.DeviceBusy, + => |e| return e, }; + var cleanup_dir_parent: ?Dir = null; + defer if (cleanup_dir_parent) |*d| d.close(); + + var cleanup_dir = true; + defer if (cleanup_dir) dir.close(); + + var dir_name_buf: [MAX_PATH_BYTES]u8 = undefined; + var dir_name: []const u8 = sub_path; + var parent_dir = self; + + // Here we must avoid recursion, in order to provide O(1) memory guarantee of this function. + // Go through each entry and if it is not a directory, delete it. If it is a directory, + // open it, and close the original directory. Repeat. Then start the entire operation over. + + scan_dir: while (true) { + var dir_it = dir.iterate(); + while (try dir_it.next()) |entry| { + if (dir.deleteFile(entry.name)) { + continue; + } else |err| switch (err) { + error.FileNotFound => continue, + + // Impossible because we do not pass any path separators. + error.NotDir => unreachable, + + error.IsDir => {}, + error.AccessDenied => got_access_denied = true, + + error.InvalidUtf8, + error.SymLinkLoop, + error.NameTooLong, + error.SystemResources, + error.ReadOnlyFileSystem, + error.FileSystem, + error.FileBusy, + error.BadPathName, + error.Unexpected, + => |e| return e, + } + + const new_dir = dir.openDir(entry.name) catch |err| switch (err) { + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; + } + continue :scan_dir; + }, + error.FileNotFound => { + // That's fine, we were trying to remove this directory anyway. + continue :scan_dir; + }, + + error.AccessDenied, + error.SymLinkLoop, + error.ProcessFdQuotaExceeded, + error.NameTooLong, + error.SystemFdQuotaExceeded, + error.NoDevice, + error.SystemResources, + error.Unexpected, + error.InvalidUtf8, + error.BadPathName, + error.DeviceBusy, + => |e| return e, + }; + if (cleanup_dir_parent) |*d| d.close(); + cleanup_dir_parent = dir; + dir = new_dir; + mem.copy(u8, &dir_name_buf, entry.name); + dir_name = dir_name_buf[0..entry.name.len]; + continue :scan_dir; + } + // Reached the end of the directory entries, which means we successfully deleted all of them. + // Now to remove the directory itself. + dir.close(); + cleanup_dir = false; + + if (cleanup_dir_parent) |d| { + d.deleteDir(dir_name) catch |err| switch (err) { + // These two things can happen due to file system race conditions. + error.FileNotFound, error.DirNotEmpty => continue :start_over, + else => |e| return e, + }; + continue :start_over; + } else { + self.deleteDir(sub_path) catch |err| switch (err) { + error.FileNotFound => return, + error.DirNotEmpty => continue :start_over, + else => |e| return e, + }; + return; + } + } } } }; @@ -757,13 +889,18 @@ pub const Walker = struct { name_buffer: std.Buffer, pub const Entry = struct { - path: []const u8, + /// The containing directory. This can be used to operate directly on `basename` + /// rather than `path`, avoiding `error.NameTooLong` for deeply nested paths. + /// The directory remains open until `next` or `deinit` is called. + dir: Dir, basename: []const u8, + + path: []const u8, kind: Dir.Entry.Kind, }; const StackItem = struct { - dir_it: Dir, + dir_it: Dir.Iterator, dirname_len: usize, }; @@ -781,23 +918,26 @@ pub const Walker = struct { try self.name_buffer.appendByte(path.sep); try self.name_buffer.append(base.name); if (base.kind == .Directory) { - // TODO https://github.com/ziglang/zig/issues/2888 - var new_dir = try Dir.open(self.name_buffer.toSliceConst()); + var new_dir = top.dir_it.dir.openDir(base.name) catch |err| switch (err) { + error.NameTooLong => unreachable, // no path sep in base.name + else => |e| return e, + }; { errdefer new_dir.close(); try self.stack.append(StackItem{ - .dir_it = new_dir, + .dir_it = new_dir.iterate(), .dirname_len = self.name_buffer.len(), }); } } return Entry{ + .dir = top.dir_it.dir, .basename = self.name_buffer.toSliceConst()[dirname_len + 1 ..], .path = self.name_buffer.toSliceConst(), .kind = base.kind, }; } else { - self.stack.pop().dir_it.close(); + self.stack.pop().dir_it.dir.close(); } } } @@ -812,12 +952,13 @@ pub const Walker = struct { /// Recursively iterates over a directory. /// Must call `Walker.deinit` when done. /// `dir_path` must not end in a path separator. +/// The order of returned file system entries is undefined. /// TODO: https://github.com/ziglang/zig/issues/2888 pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); - var dir_it = try Dir.open(dir_path); - errdefer dir_it.close(); + var dir = try Dir.open(dir_path); + errdefer dir.close(); var name_buffer = try std.Buffer.init(allocator, dir_path); errdefer name_buffer.deinit(); @@ -828,7 +969,7 @@ pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { }; try walker.stack.append(Walker.StackItem{ - .dir_it = dir_it, + .dir_it = dir.iterate(), .dirname_len = dir_path.len, }); diff --git a/lib/std/os.zig b/lib/std/os.zig index de01da2fa5241b0ce0521b11fec2e3826a2de5ac..e7319095cb9acf31a52a7068b6654e1ebf260390 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -529,22 +529,36 @@ pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) WriteError!void pub const OpenError = error{ AccessDenied, - FileTooBig, - IsDir, SymLinkLoop, ProcessFdQuotaExceeded, - NameTooLong, SystemFdQuotaExceeded, NoDevice, FileNotFound, + /// The path exceeded `MAX_PATH_BYTES` bytes. + NameTooLong, + /// Insufficient kernel memory was available, or /// the named file is a FIFO and per-user hard limit on /// memory allocation for pipes has been reached. SystemResources, + /// The file is too large to be opened. This error is unreachable + /// for 64-bit targets, as well as when opening directories. + FileTooBig, + + /// The path refers to directory but the `O_DIRECTORY` flag was not provided. + IsDir, + + /// A new path cannot be created because the device has no room for the new file. + /// This error is only reachable when the `O_CREAT` flag is provided. NoSpaceLeft, + + /// A component used as a directory in the path was not, in fact, a directory, or + /// `O_DIRECTORY` was specified and the path was not a directory. NotDir, + + /// The path already exists and the `O_CREAT` and `O_EXCL` flags were provided. PathAlreadyExists, DeviceBusy, } || UnexpectedError; @@ -978,6 +992,42 @@ pub fn unlinkC(file_path: [*]const u8) UnlinkError!void { } } +pub const UnlinkatError = UnlinkError || error{ + /// When passing `AT_REMOVEDIR`, this error occurs when the named directory is not empty. + DirNotEmpty, +}; + +/// Delete a file name and possibly the file it refers to, based on an open directory handle. +pub fn unlinkat(dirfd: fd_t, file_path: []const u8, flags: u32) UnlinkatError!void { + const file_path_c = try toPosixPath(file_path); + return unlinkatC(dirfd, &file_path_c, flags); +} + +/// Same as `unlinkat` but `file_path` is a null-terminated string. +pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatError!void { + switch (errno(system.unlinkat(dirfd, file_path_c, flags))) { + 0 => return, + EACCES => return error.AccessDenied, + EPERM => return error.AccessDenied, + EBUSY => return error.FileBusy, + EFAULT => unreachable, + EIO => return error.FileSystem, + EISDIR => return error.IsDir, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOTDIR => return error.NotDir, + ENOMEM => return error.SystemResources, + EROFS => return error.ReadOnlyFileSystem, + ENOTEMPTY => return error.DirNotEmpty, + + EINVAL => unreachable, // invalid flags, or pathname has . as last component + EBADF => unreachable, // always a race condition + + else => |err| return unexpectedErrno(err), + } +} + const RenameError = error{ AccessDenied, FileBusy, diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index d36073abf308375e48872111d45f9919d26bd19d..5a0294fa7d276a50142b46bf7688ebc4bebe6eb7 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -286,8 +286,10 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void var dir = try fs.Dir.open(file_path); defer dir.close(); - while (try dir.next()) |entry| { - if (entry.kind == fs.Dir.Entry.Kind.Directory or mem.endsWith(u8, entry.name, ".zig")) { + var dir_it = dir.iterate(); + + while (try dir_it.next()) |entry| { + if (entry.kind == .Directory or mem.endsWith(u8, entry.name, ".zig")) { const full_path = try fs.path.join(fmt.allocator, [_][]const u8{ file_path, entry.name }); try fmtPath(fmt, full_path, check_mode); } -- 2.54.0 From ddb87237de60cb0feacc606d21c9dd48c5c3bd4b Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 20 Oct 2019 22:56:58 +0200 Subject: [PATCH 18/31] Fix crash with peer type resolution & error unions Fixes #3353 --- src/ir.cpp | 12 ++++++++++++ test/compile_errors.zig | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/ir.cpp b/src/ir.cpp index 6bdfd86b29941f7add7fcf85dda7e6d475122eec..4cce464a4c72a2095445e53d54542e66f9b66e4c 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -10120,6 +10120,18 @@ static ZigType *ir_resolve_peer_types(IrAnalyze *ira, AstNode *source_node, ZigT } else { err_set_type = cur_type; } + + if (!resolve_inferred_error_set(ira->codegen, err_set_type, cur_inst->source_node)) { + return ira->codegen->builtin_types.entry_invalid; + } + + if (type_is_global_error_set(err_set_type)) { + err_set_type = ira->codegen->builtin_types.entry_global_error_set; + continue; + } + + update_errors_helper(ira->codegen, &errors, &errors_count); + for (uint32_t i = 0; i < err_set_type->data.error_set.err_count; i += 1) { ErrorTableEntry *error_entry = err_set_type->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 288fd501cf691e34e21b2123ade31e9b8b220bfa..35c6ded4a4a47a22c7e5c345497bc079c9eb46f4 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -2,6 +2,16 @@ const tests = @import("tests.zig"); const builtin = @import("builtin"); pub fn addCases(cases: *tests.CompileErrorContext) void { + cases.add( + "comparison with error union and error value", + \\export fn entry() void { + \\ var number_or_error: anyerror!i32 = error.SomethingAwful; + \\ _ = number_or_error == error.SomethingAwful; + \\} + , + "tmp.zig:3:25: error: operator not allowed for type 'anyerror!i32'", + ); + cases.add( "switch with overlapping case ranges", \\export fn entry() void { -- 2.54.0 From ef67c497856fb97f7d2854991a72237ad4332810 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 14:18:01 -0400 Subject: [PATCH 19/31] [wip] use NtDll APIs on Windows to implement std.fs.Dir --- lib/std/fs.zig | 365 +++++++++++++++++++++++++--------- lib/std/fs/file.zig | 1 + lib/std/fs/path.zig | 19 ++ lib/std/io.zig | 5 +- lib/std/os.zig | 79 ++++++++ lib/std/os/bits/windows.zig | 3 + lib/std/os/windows.zig | 23 ++- lib/std/os/windows/bits.zig | 146 +++++++++++++- lib/std/os/windows/ntdll.zig | 25 ++- lib/std/os/windows/status.zig | 6 + 10 files changed, 566 insertions(+), 106 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 89071b407fcd6932c3be1219044fa79251d1560c..51623d3057981f5779ed21c3079a300f08014fe8 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -353,18 +353,13 @@ pub fn deleteTree(full_path: []const u8) !void { return dir.deleteTree(path.basename(full_path)); } else { - return Dir.posix_cwd.deleteTree(full_path); + return Dir.cwd().deleteTree(full_path); } } pub const Dir = struct { fd: os.fd_t, - /// An open handle to the current working directory. - /// Closing this directory is safety-checked illegal behavior. - /// Not available on Windows. - pub const posix_cwd = Dir{ .fd = os.AT_FDCWD }; - pub const Entry = struct { name: []const u8, kind: Kind, @@ -386,12 +381,10 @@ pub const Dir = struct { .macosx, .ios, .freebsd, .netbsd => struct { dir: Dir, seek: i64, - buf: [buffer_len]u8, + buf: [8192]u8, // TODO align(@alignOf(os.dirent)), index: usize, end_index: usize, - pub const buffer_len = 8192; - const Self = @This(); /// Memory such as file names referenced in this returned entry becomes invalid @@ -407,27 +400,24 @@ pub const Dir = struct { fn nextDarwin(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.system.__getdirentries64( - self.dir.fd, - &self.buf, - self.buf.len, - &self.seek, - ); - if (rc == 0) return null; - if (rc < 0) { - switch (os.errno(rc)) { - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } + const rc = os.system.__getdirentries64( + self.dir.fd, + &self.buf, + self.buf.len, + &self.seek, + ); + if (rc == 0) return null; + if (rc < 0) { + switch (os.errno(rc)) { + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } - self.index = 0; - self.end_index = @intCast(usize, rc); - break; } + self.index = 0; + self.end_index = @intCast(usize, rc); } const darwin_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); const next_index = self.index + darwin_entry.d_reclen; @@ -460,26 +450,23 @@ pub const Dir = struct { fn nextBsd(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.system.getdirentries( - self.dir.fd, - self.buf[0..].ptr, - self.buf.len, - &self.seek, - ); - switch (os.errno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.index = 0; - self.end_index = @intCast(usize, rc); - break; + const rc = os.system.getdirentries( + self.dir.fd, + self.buf[0..].ptr, + self.buf.len, + &self.seek, + ); + switch (os.errno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } + if (rc == 0) return null; + self.index = 0; + self.end_index = @intCast(usize, rc); } const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); const next_index = self.index + freebsd_entry.d_reclen; @@ -511,12 +498,10 @@ pub const Dir = struct { }, .linux => struct { dir: Dir, - buf: [buffer_len]u8, + buf: [8192]u8, // TODO align(@alignOf(os.dirent64)), index: usize, end_index: usize, - pub const buffer_len = 8192; - const Self = @This(); /// Memory such as file names referenced in this returned entry becomes invalid @@ -524,21 +509,18 @@ pub const Dir = struct { pub fn next(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); - switch (os.linux.getErrno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.index = 0; - self.end_index = rc; - break; + const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); + switch (os.linux.getErrno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } + if (rc == 0) return null; + self.index = 0; + self.end_index = rc; } const linux_entry = @ptrCast(*align(1) os.dirent64, &self.buf[self.index]); const next_index = self.index + linux_entry.d_reclen; @@ -570,13 +552,117 @@ pub const Dir = struct { }, .windows => struct { dir: Dir, - find_file_data: os.windows.WIN32_FIND_DATAW, + buf: [8192]u8 align(@alignOf(os.windows.FILE_BOTH_DIR_INFORMATION)), + index: usize, + end_index: usize, first: bool, name_data: [256]u8, + + const Self = @This(); + + pub fn next(self: *Self) !?Entry { + start_over: while (true) { + const w = os.windows; + if (self.index >= self.end_index) { + var io: w.IO_STATUS_BLOCK = undefined; + //var mask_buf = [2]u16{ 'a', 0 }; + //var mask = w.UNICODE_STRING{ + // .Length = 2, + // .MaximumLength = 2, + // .Buffer = &mask_buf, + //}; + const rc = w.ntdll.NtQueryDirectoryFile( + self.dir.fd, + null, + null, + null, + &io, + &self.buf, + self.buf.len, + .FileBothDirectoryInformation, + w.FALSE, + null, + if (self.first) w.BOOLEAN(w.TRUE) else w.BOOLEAN(w.FALSE), + ); + self.first = false; + if (io.Information == 0) return null; + self.index = 0; + self.end_index = io.Information; + switch (rc) { + w.STATUS.SUCCESS => {}, + else => return w.unexpectedStatus(rc), + } + } + + const aligned_ptr = @alignCast(@alignOf(w.FILE_BOTH_DIR_INFORMATION), &self.buf[self.index]); + const dir_info = @ptrCast(*w.FILE_BOTH_DIR_INFORMATION, aligned_ptr); + if (dir_info.NextEntryOffset != 0) { + self.index += dir_info.NextEntryOffset; + } else { + self.index = self.buf.len; + } + + const name_utf16le = @ptrCast([*]u16, &dir_info.FileName)[0 .. dir_info.FileNameLength / 2]; + + if (mem.eql(u16, name_utf16le, [_]u16{'.'}) or mem.eql(u16, name_utf16le, [_]u16{ '.', '.' })) + continue; + // Trust that Windows gives us valid UTF-16LE + const name_utf8_len = std.unicode.utf16leToUtf8(self.name_data[0..], name_utf16le) catch unreachable; + const name_utf8 = self.name_data[0..name_utf8_len]; + const kind = blk: { + const attrs = dir_info.FileAttributes; + if (attrs & w.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; + if (attrs & w.FILE_ATTRIBUTE_REPARSE_POINT != 0) break :blk Entry.Kind.SymLink; + break :blk Entry.Kind.File; + }; + return Entry{ + .name = name_utf8, + .kind = kind, + }; + } + } }, else => @compileError("unimplemented"), }; + pub fn iterate(self: Dir) Iterator { + switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => return Iterator{ + .dir = self, + .seek = 0, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .linux => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .windows => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .first = true, + .buf = undefined, + .name_data = undefined, + }, + else => @compileError("unimplemented"), + } + } + + /// Returns an open handle to the current working directory. + /// Closing the returned `Dir` is checked illegal behavior. + /// On POSIX targets, this function is comptime-callable. + pub fn cwd() Dir { + if (os.windows.is_the_target) { + return Dir{ .fd = os.windows.peb().ProcessParameters.CurrentDirectory.Handle }; + } else { + return Dir{ .fd = os.AT_FDCWD }; + } + } + pub const OpenError = error{ FileNotFound, NotDir, @@ -594,18 +680,15 @@ pub const Dir = struct { /// Call `close` to free the directory handle. pub fn open(dir_path: []const u8) OpenError!Dir { - return posix_cwd.openDir(dir_path); + return cwd().openDir(dir_path); } /// Same as `open` except the parameter is null-terminated. pub fn openC(dir_path_c: [*]const u8) OpenError!Dir { - return posix_cwd.openDirC(dir_path_c); + return cwd().openDirC(dir_path_c); } pub fn close(self: *Dir) void { - if (os.windows.is_the_target) { - @panic("TODO"); - } os.close(self.fd); self.* = undefined; } @@ -625,14 +708,25 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { + std.debug.warn("openDir {}\n", sub_path); + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); + return self.openDirW(&sub_path_w); + } + const sub_path_c = try os.toPosixPath(sub_path); return self.openDirC(&sub_path_c); } - /// Call `close` on the result when done. - pub fn openDirC(self: Dir, sub_path: [*]const u8) OpenError!Dir { + /// Same as `openDir` except the parameter is null-terminated. + pub fn openDirC(self: Dir, sub_path_c: [*]const u8) OpenError!Dir { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.cStrToPrefixedFileW(sub_path_c); + return self.openDirW(&sub_path_w); + } + const flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC; - const fd = os.openatC(self.fd, sub_path, flags, 0) catch |err| switch (err) { + const fd = os.openatC(self.fd, sub_path_c, flags, 0) catch |err| switch (err) { error.FileTooBig => unreachable, // can't happen for directories error.IsDir => unreachable, // we're providing O_DIRECTORY error.NoSpaceLeft => unreachable, // not providing O_CREAT @@ -642,6 +736,79 @@ pub const Dir = struct { return Dir{ .fd = fd }; } + /// Same as `openDir` except the path parameter is UTF16LE, NT-prefixed. + /// This function is Windows-only. + pub fn openDirW(self: Dir, sub_path_w: [*]const u16) OpenError!Dir { + const w = os.windows; + var result = Dir{ + .fd = undefined, + }; + //var mask: ?[*]const u16 = undefined; + //var nt_name: w.UNICODE_STRING = undefined; + //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { + // return error.FileNotFound; + //} + //defer w.ntdll.RtlFreeUnicodeString(&nt_name); + //if (mask) |m| { + // if (m[0] == 0) { + // return error.FileNotFound; + // } else { + // nt_name.Length = @intCast(u16, @ptrToInt(mask) - @ptrToInt(nt_name.Buffer)); + // } + //} else { + // return error.FileNotFound; + //} + + const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); + std.debug.warn("path_len_bytes = {}\n", path_len_bytes); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = if (path.isAbsoluteW(sub_path_w)) null else self.fd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + var io: w.IO_STATUS_BLOCK = undefined; + const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; + //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); + var buf: [200]u8 = undefined; + //var buf2: [200]u8 = undefined; + const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; + //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; + std.debug.warn("path: {}\n", buf[0..len]); + //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); + const rc = w.ntdll.NtCreateFile( + &result.fd, + w.GENERIC_READ | w.SYNCHRONIZE, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE, + w.FILE_OPEN, + w.FILE_DIRECTORY_FILE | w.FILE_SYNCHRONOUS_IO_NONALERT | w.FILE_OPEN_FOR_BACKUP_INTENT, + null, + 0, + ); + std.debug.warn("result.fd = {}\n", result.fd); + switch (rc) { + w.STATUS.SUCCESS => return result, + w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), + w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.INVALID_PARAMETER => { + @panic("invalid parameter"); + }, + else => return w.unexpectedStatus(rc), + } + } + pub const DeleteFileError = os.UnlinkError; /// Delete a file name and possibly the file it refers to, based on an open directory handle. @@ -677,6 +844,10 @@ pub const Dir = struct { /// Returns `error.DirNotEmpty` if the directory is not empty. /// To delete a directory recursively, see `deleteTree`. pub fn deleteDir(self: Dir, sub_path: []const u8) DeleteDirError!void { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); + return self.deleteDirW(&sub_path_w); + } const sub_path_c = try os.toPosixPath(sub_path); return self.deleteDirC(&sub_path_c); } @@ -689,24 +860,25 @@ pub const Dir = struct { }; } - pub fn iterate(self: Dir) Iterator { - switch (builtin.os) { - .macosx, .ios, .freebsd, .netbsd => return Iterator{ - .dir = self, - .seek = 0, - .index = 0, - .end_index = 0, - .buf = undefined, - }, - .linux => return Iterator{ - .dir = self, - .index = 0, - .end_index = 0, - .buf = undefined, - }, - .windows => @panic("TODO"), - else => @compileError("unimplemented"), - } + /// Same as `deleteDir` except the parameter is UTF16LE, NT prefixed. + /// This function is Windows-only. + pub fn deleteDirW(self: Dir, sub_path_w: [*]const u16) DeleteDirError!void { + os.unlinkatW(self.fd, sub_path_w, os.AT_REMOVEDIR) catch |err| switch (err) { + error.IsDir => unreachable, // not possible since we pass AT_REMOVEDIR + else => |e| return e, + }; + } + + /// Read value of a symbolic link. + /// The return value is a slice of `buffer`, from index `0`. + pub fn readLink(self: Dir, sub_path: []const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + const sub_path_c = try os.toPosixPath(sub_path); + return self.readLinkC(&sub_path_c, buffer); + } + + /// Same as `readLink`, except the `pathname` parameter is null-terminated. + pub fn readLinkC(self: Dir, sub_path_c: [*]const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + return os.readlinkatC(self.fd, sub_path_c, buffer); } pub const DeleteTreeError = error{ @@ -953,7 +1125,6 @@ pub const Walker = struct { /// Must call `Walker.deinit` when done. /// `dir_path` must not end in a path separator. /// The order of returned file system entries is undefined. -/// TODO: https://github.com/ziglang/zig/issues/2888 pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); @@ -978,15 +1149,13 @@ pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { /// Read value of a symbolic link. /// The return value is a slice of buffer, from index `0`. -/// TODO https://github.com/ziglang/zig/issues/2888 -pub fn readLink(pathname: []const u8, buffer: *[os.PATH_MAX]u8) ![]u8 { +pub fn readLink(pathname: []const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { return os.readlink(pathname, buffer); } -/// Same as `readLink`, except the `pathname` parameter is null-terminated. -/// TODO https://github.com/ziglang/zig/issues/2888 -pub fn readLinkC(pathname: [*]const u8, buffer: *[os.PATH_MAX]u8) ![]u8 { - return os.readlinkC(pathname, buffer); +/// Same as `readLink`, except the parameter is null-terminated. +pub fn readLinkC(pathname_c: [*]const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + return os.readlinkC(pathname_c, buffer); } pub const OpenSelfExeError = os.OpenError || os.windows.CreateFileError || SelfExePathError; diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 32c221d063a2f05c3282ad05e797db51f1125ed7..a9212d69d4bd98490b27eb4a586b393690f8ffe1 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -243,6 +243,7 @@ pub const File = struct { switch (rc) { windows.STATUS.SUCCESS => {}, windows.STATUS.BUFFER_OVERFLOW => {}, + windows.STATUS.INVALID_PARAMETER => unreachable, else => return windows.unexpectedStatus(rc), } return Stat{ diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig index 2bb23f04cedd84f7b83cce491fb4ddee8109e2ee..560d9b58f3440382f5e7b1050f44d3347d7f6ffc 100644 --- a/lib/std/fs/path.zig +++ b/lib/std/fs/path.zig @@ -136,6 +136,25 @@ pub fn isAbsolute(path: []const u8) bool { } } +pub fn isAbsoluteW(path_w: [*]const u16) bool { + if (path_w[0] == '/') + return true; + + if (path_w[0] == '\\') { + return true; + } + if (path_w[0] == 0 or path_w[1] == 0 or path_w[2] == 0) { + return false; + } + if (path_w[1] == ':') { + if (path_w[2] == '/') + return true; + if (path_w[2] == '\\') + return true; + } + return false; +} + pub fn isAbsoluteWindows(path: []const u8) bool { if (path[0] == '/') return true; diff --git a/lib/std/io.zig b/lib/std/io.zig index 4407e6b2b39afad0d6cb71c30394ec31ea48281b..19623d93ee4ff2dc19474ce94c2dc043e640c6fe 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -127,6 +127,7 @@ pub fn OutStream(comptime WriteError: type) type { }; } +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn writeFile(path: []const u8, data: []const u8) !void { var file = try File.openWrite(path); defer file.close(); @@ -134,11 +135,13 @@ pub fn writeFile(path: []const u8, data: []const u8) !void { } /// On success, caller owns returned buffer. +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn readFileAlloc(allocator: *mem.Allocator, path: []const u8) ![]u8 { return readFileAllocAligned(allocator, path, @alignOf(u8)); } /// On success, caller owns returned buffer. +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn readFileAllocAligned(allocator: *mem.Allocator, path: []const u8, comptime A: u29) ![]align(A) u8 { var file = try File.openRead(path); defer file.close(); @@ -1084,7 +1087,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, // safety. If it is bad, it will be caught anyway. const TagInt = @TagType(TagType); const tag = try self.deserializeInt(TagInt); - + inline for (info.fields) |field_info| { if (field_info.enum_field.?.value == tag) { const name = field_info.name; diff --git a/lib/std/os.zig b/lib/std/os.zig index e7319095cb9acf31a52a7068b6654e1ebf260390..2d8ddb0abcb3eca2b00f75e1fba888b705b3853b 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -999,12 +999,20 @@ pub const UnlinkatError = UnlinkError || error{ /// Delete a file name and possibly the file it refers to, based on an open directory handle. pub fn unlinkat(dirfd: fd_t, file_path: []const u8, flags: u32) UnlinkatError!void { + if (windows.is_the_target) { + const file_path_w = try windows.sliceToPrefixedFileW(file_path); + return unlinkatW(dirfd, &file_path_w, flags); + } const file_path_c = try toPosixPath(file_path); return unlinkatC(dirfd, &file_path_c, flags); } /// Same as `unlinkat` but `file_path` is a null-terminated string. pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatError!void { + if (windows.is_the_target) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path_c); + return unlinkatW(dirfd, &file_path_w, flags); + } switch (errno(system.unlinkat(dirfd, file_path_c, flags))) { 0 => return, EACCES => return error.AccessDenied, @@ -1028,6 +1036,56 @@ pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatErro } } +/// Same as `unlinkat` but `sub_path_w` is UTF16LE, NT prefixed. Windows only. +pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatError!void { + const w = windows; + + const want_rmdir_behavior = (flags & AT_REMOVEDIR) != 0; + const create_options_flags = if (want_rmdir_behavior) + w.ULONG(w.FILE_DELETE_ON_CLOSE) + else + w.ULONG(w.FILE_DELETE_ON_CLOSE | w.FILE_NON_DIRECTORY_FILE); + var nt_name: w.UNICODE_STRING = undefined; + if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { + return error.FileNotFound; + } + defer w.ntdll.RtlFreeUnicodeString(&nt_name); + + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = dirfd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + var io: w.IO_STATUS_BLOCK = undefined; + var tmp_handle: w.HANDLE = undefined; + var rc = w.ntdll.NtCreateFile( + &tmp_handle, + w.SYNCHRONIZE | w.DELETE, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + w.FILE_OPEN, + create_options_flags, + null, + 0, + ); + if (rc == w.STATUS.SUCCESS) { + rc = w.ntdll.NtClose(tmp_handle); + } + switch (rc) { + w.STATUS.SUCCESS => return, + w.STATUS.OBJECT_NAME_INVALID => unreachable, + w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.INVALID_PARAMETER => unreachable, + else => return w.unexpectedStatus(rc), + } +} + const RenameError = error{ AccessDenied, FileBusy, @@ -1287,6 +1345,27 @@ pub fn readlinkC(file_path: [*]const u8, out_buffer: []u8) ReadLinkError![]u8 { } } +pub fn readlinkatC(dirfd: fd_t, file_path: [*]const u8, out_buffer: []u8) ReadLinkError![]u8 { + if (windows.is_the_target) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path); + @compileError("TODO implement readlink for Windows"); + } + const rc = system.readlinkat(dirfd, file_path, out_buffer.ptr, out_buffer.len); + switch (errno(rc)) { + 0 => return out_buffer[0..@bitCast(usize, rc)], + EACCES => return error.AccessDenied, + EFAULT => unreachable, + EINVAL => unreachable, + EIO => return error.FileSystem, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOMEM => return error.SystemResources, + ENOTDIR => return error.NotDir, + else => |err| return unexpectedErrno(err), + } +} + pub const SetIdError = error{ ResourceLimitReached, InvalidUserId, diff --git a/lib/std/os/bits/windows.zig b/lib/std/os/bits/windows.zig index fc148d812ff417e6954e475993f378c3c87f4db2..c261b52bed764befef804567ab22094ab86e7d00 100644 --- a/lib/std/os/bits/windows.zig +++ b/lib/std/os/bits/windows.zig @@ -158,3 +158,6 @@ pub const EWOULDBLOCK = 140; pub const EDQUOT = 10069; pub const F_OK = 0; + +/// Remove directory instead of unlinking file +pub const AT_REMOVEDIR = 0x200; diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 7c1761a4b8fb21a8c9aa15645cb0f68ac5ee5703..6a5d03b52945f209121dea2d9f1bc22c86523096 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -792,6 +792,25 @@ pub fn SetFileTime( } } +pub fn peb() *PEB { + switch (builtin.arch) { + .i386 => { + return asm ( + \\ mov %%fs:0x18, %[ptr] + \\ mov %%ds:0x30(%[ptr]), %[ptr] + : [ptr] "=r" (-> *PEB) + ); + }, + .x86_64 => { + return asm ( + \\ mov %%gs:0x60, %[ptr] + : [ptr] "=r" (-> *PEB) + ); + }, + else => @compileError("unsupported architecture"), + } +} + /// A file time is a 64-bit value that represents the number of 100-nanosecond /// intervals that have elapsed since 12:00 A.M. January 1, 1601 Coordinated /// Universal Time (UTC). @@ -844,8 +863,8 @@ pub fn sliceToPrefixedSuffixedFileW(s: []const u8, comptime suffix: []const u16) else => {}, } } - const start_index = if (mem.startsWith(u8, s, "\\\\") or !std.fs.path.isAbsolute(s)) 0 else blk: { - const prefix = [_]u16{ '\\', '\\', '?', '\\' }; + const start_index = if (mem.startsWith(u8, s, "\\?") or !std.fs.path.isAbsolute(s)) 0 else blk: { + const prefix = [_]u16{ '\\', '?', '?', '\\' }; mem.copy(u16, result[0..], prefix); break :blk prefix.len; }; diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig index ddfdd27e1bdde68bc4b7601a57a878904fc38547..68d5ab6b87a081e0a95c708859827baa99e86a25 100644 --- a/lib/std/os/windows/bits.zig +++ b/lib/std/os/windows/bits.zig @@ -300,6 +300,44 @@ pub const FILE_SHARE_DELETE = 0x00000004; pub const FILE_SHARE_READ = 0x00000001; pub const FILE_SHARE_WRITE = 0x00000002; +pub const DELETE = 0x00010000; +pub const READ_CONTROL = 0x00020000; +pub const WRITE_DAC = 0x00040000; +pub const WRITE_OWNER = 0x00080000; +pub const SYNCHRONIZE = 0x00100000; +pub const STANDARD_RIGHTS_REQUIRED = 0x000f0000; + +// disposition for NtCreateFile +pub const FILE_SUPERSEDE = 0; +pub const FILE_OPEN = 1; +pub const FILE_CREATE = 2; +pub const FILE_OPEN_IF = 3; +pub const FILE_OVERWRITE = 4; +pub const FILE_OVERWRITE_IF = 5; +pub const FILE_MAXIMUM_DISPOSITION = 5; + +// flags for NtCreateFile and NtOpenFile +pub const FILE_DIRECTORY_FILE = 0x00000001; +pub const FILE_WRITE_THROUGH = 0x00000002; +pub const FILE_SEQUENTIAL_ONLY = 0x00000004; +pub const FILE_NO_INTERMEDIATE_BUFFERING = 0x00000008; +pub const FILE_SYNCHRONOUS_IO_ALERT = 0x00000010; +pub const FILE_SYNCHRONOUS_IO_NONALERT = 0x00000020; +pub const FILE_NON_DIRECTORY_FILE = 0x00000040; +pub const FILE_CREATE_TREE_CONNECTION = 0x00000080; +pub const FILE_COMPLETE_IF_OPLOCKED = 0x00000100; +pub const FILE_NO_EA_KNOWLEDGE = 0x00000200; +pub const FILE_OPEN_FOR_RECOVERY = 0x00000400; +pub const FILE_RANDOM_ACCESS = 0x00000800; +pub const FILE_DELETE_ON_CLOSE = 0x00001000; +pub const FILE_OPEN_BY_FILE_ID = 0x00002000; +pub const FILE_OPEN_FOR_BACKUP_INTENT = 0x00004000; +pub const FILE_NO_COMPRESSION = 0x00008000; +pub const FILE_RESERVE_OPFILTER = 0x00100000; +pub const FILE_TRANSACTED_MODE = 0x00200000; +pub const FILE_OPEN_OFFLINE_FILE = 0x00400000; +pub const FILE_OPEN_FOR_FREE_SPACE_QUERY = 0x00800000; + pub const CREATE_ALWAYS = 2; pub const CREATE_NEW = 1; pub const OPEN_ALWAYS = 4; @@ -720,15 +758,117 @@ pub const VECTORED_EXCEPTION_HANDLER = stdcallcc fn (ExceptionInfo: *EXCEPTION_P pub const OBJECT_ATTRIBUTES = extern struct { Length: ULONG, - RootDirectory: HANDLE, + RootDirectory: ?HANDLE, ObjectName: *UNICODE_STRING, Attributes: ULONG, SecurityDescriptor: ?*c_void, SecurityQualityOfService: ?*c_void, }; +pub const OBJ_INHERIT = 0x00000002; +pub const OBJ_PERMANENT = 0x00000010; +pub const OBJ_EXCLUSIVE = 0x00000020; +pub const OBJ_CASE_INSENSITIVE = 0x00000040; +pub const OBJ_OPENIF = 0x00000080; +pub const OBJ_OPENLINK = 0x00000100; +pub const OBJ_KERNEL_HANDLE = 0x00000200; +pub const OBJ_VALID_ATTRIBUTES = 0x000003F2; + pub const UNICODE_STRING = extern struct { - Length: USHORT, - MaximumLength: USHORT, + Length: c_ushort, + MaximumLength: c_ushort, Buffer: [*]WCHAR, }; + +pub const PEB = extern struct { + Reserved1: [2]BYTE, + BeingDebugged: BYTE, + Reserved2: [1]BYTE, + Reserved3: [2]PVOID, + Ldr: *PEB_LDR_DATA, + ProcessParameters: *RTL_USER_PROCESS_PARAMETERS, + Reserved4: [3]PVOID, + AtlThunkSListPtr: PVOID, + Reserved5: PVOID, + Reserved6: ULONG, + Reserved7: PVOID, + Reserved8: ULONG, + AtlThunkSListPtr32: ULONG, + Reserved9: [45]PVOID, + Reserved10: [96]BYTE, + PostProcessInitRoutine: PPS_POST_PROCESS_INIT_ROUTINE, + Reserved11: [128]BYTE, + Reserved12: [1]PVOID, + SessionId: ULONG, +}; + +pub const PEB_LDR_DATA = extern struct { + Reserved1: [8]BYTE, + Reserved2: [3]PVOID, + InMemoryOrderModuleList: LIST_ENTRY, +}; + +pub const RTL_USER_PROCESS_PARAMETERS = extern struct { + AllocationSize: ULONG, + Size: ULONG, + Flags: ULONG, + DebugFlags: ULONG, + ConsoleHandle: HANDLE, + ConsoleFlags: ULONG, + hStdInput: HANDLE, + hStdOutput: HANDLE, + hStdError: HANDLE, + CurrentDirectory: CURDIR, + DllPath: UNICODE_STRING, + ImagePathName: UNICODE_STRING, + CommandLine: UNICODE_STRING, + Environment: [*]WCHAR, + dwX: ULONG, + dwY: ULONG, + dwXSize: ULONG, + dwYSize: ULONG, + dwXCountChars: ULONG, + dwYCountChars: ULONG, + dwFillAttribute: ULONG, + dwFlags: ULONG, + dwShowWindow: ULONG, + WindowTitle: UNICODE_STRING, + Desktop: UNICODE_STRING, + ShellInfo: UNICODE_STRING, + RuntimeInfo: UNICODE_STRING, + DLCurrentDirectory: [0x20]RTL_DRIVE_LETTER_CURDIR, +}; + +pub const RTL_DRIVE_LETTER_CURDIR = extern struct { + Flags: c_ushort, + Length: c_ushort, + TimeStamp: ULONG, + DosPath: UNICODE_STRING, +}; + +pub const PPS_POST_PROCESS_INIT_ROUTINE = ?extern fn () void; + +pub const FILE_BOTH_DIR_INFORMATION = extern struct { + NextEntryOffset: ULONG, + FileIndex: ULONG, + CreationTime: LARGE_INTEGER, + LastAccessTime: LARGE_INTEGER, + LastWriteTime: LARGE_INTEGER, + ChangeTime: LARGE_INTEGER, + EndOfFile: LARGE_INTEGER, + AllocationSize: LARGE_INTEGER, + FileAttributes: ULONG, + FileNameLength: ULONG, + EaSize: ULONG, + ShortNameLength: CHAR, + ShortName: [12]WCHAR, + FileName: [1]WCHAR, +}; +pub const FILE_BOTH_DIRECTORY_INFORMATION = FILE_BOTH_DIR_INFORMATION; + +pub const IO_APC_ROUTINE = extern fn (PVOID, *IO_STATUS_BLOCK, ULONG) void; + +pub const CURDIR = extern struct { + DosPath: UNICODE_STRING, + Handle: HANDLE, +}; diff --git a/lib/std/os/windows/ntdll.zig b/lib/std/os/windows/ntdll.zig index bfc98aba8af81de7a1beb31aa736f032b37b0da7..f9149200930d7d2de9f458ec162a30272111e019 100644 --- a/lib/std/os/windows/ntdll.zig +++ b/lib/std/os/windows/ntdll.zig @@ -13,12 +13,33 @@ pub extern "NtDll" stdcallcc fn NtCreateFile( DesiredAccess: ACCESS_MASK, ObjectAttributes: *OBJECT_ATTRIBUTES, IoStatusBlock: *IO_STATUS_BLOCK, - AllocationSize: *LARGE_INTEGER, + AllocationSize: ?*LARGE_INTEGER, FileAttributes: ULONG, ShareAccess: ULONG, CreateDisposition: ULONG, CreateOptions: ULONG, - EaBuffer: *c_void, + EaBuffer: ?*c_void, EaLength: ULONG, ) NTSTATUS; pub extern "NtDll" stdcallcc fn NtClose(Handle: HANDLE) NTSTATUS; +pub extern "NtDll" stdcallcc fn RtlDosPathNameToNtPathName_U( + DosPathName: [*]const u16, + NtPathName: *UNICODE_STRING, + NtFileNamePart: ?*?[*]const u16, + DirectoryInfo: ?*CURDIR, +) BOOL; +pub extern "NtDll" stdcallcc fn RtlFreeUnicodeString(UnicodeString: *UNICODE_STRING) void; + +pub extern "NtDll" stdcallcc fn NtQueryDirectoryFile( + FileHandle: HANDLE, + Event: ?HANDLE, + ApcRoutine: ?IO_APC_ROUTINE, + ApcContext: ?*c_void, + IoStatusBlock: *IO_STATUS_BLOCK, + FileInformation: *c_void, + Length: ULONG, + FileInformationClass: FILE_INFORMATION_CLASS, + ReturnSingleEntry: BOOLEAN, + FileName: ?*UNICODE_STRING, + RestartScan: BOOLEAN, +) NTSTATUS; diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index b9fd2b495f177b0c0b8e66d601d5685f2ae9e937..d381e3b2e1ba960c81968ef246d1758c87d75cf6 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -3,3 +3,9 @@ pub const SUCCESS = 0x00000000; /// The data was too large to fit into the specified buffer. pub const BUFFER_OVERFLOW = 0x80000005; + +pub const INVALID_PARAMETER = 0xC000000D; +pub const ACCESS_DENIED = 0xC0000022; +pub const OBJECT_NAME_INVALID = 0xC0000033; +pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; +pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; -- 2.54.0 From 16a91e6a4591d3d1198c0f0eaee9266333afd35e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 18:35:14 -0400 Subject: [PATCH 20/31] adjust test runner to print the test name when error occurs --- lib/std/progress.zig | 4 ++-- lib/std/special/test_runner.zig | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/std/progress.zig b/lib/std/progress.zig index 1d29763c10791dc6f99733b7cd77910dd7c76264..fba77092b30ae47cb8fe92a4e245f601b0e50748 100644 --- a/lib/std/progress.zig +++ b/lib/std/progress.zig @@ -155,11 +155,11 @@ pub const Progress = struct { } if (node.estimated_total_items) |total| { if (need_ellipse) self.bufWrite(&end, " "); - self.bufWrite(&end, "[{}/{}] ", node.completed_items, total); + self.bufWrite(&end, "[{}/{}] ", node.completed_items + 1, total); need_ellipse = false; } else if (node.completed_items != 0) { if (need_ellipse) self.bufWrite(&end, " "); - self.bufWrite(&end, "[{}] ", node.completed_items); + self.bufWrite(&end, "[{}] ", node.completed_items + 1); need_ellipse = false; } } diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 8fb1b5842d09236309ff21cd81341b1a4f52e15e..c16484d162c2b0de24ed9a982eb8476a23e9dac6 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -24,11 +24,14 @@ pub fn main() anyerror!void { test_node.end(); progress.log("{}...SKIP\n", test_fn.name); }, - else => return err, + else => { + progress.log(""); + return err; + }, } } root_node.end(); if (ok_count != test_fn_list.len) { - progress.log("{} passed; {} skipped.\n", ok_count, skip_count); + std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } -- 2.54.0 From 859cbef8a41dfbfde9c8caab00272b77912d3f0b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 19:01:08 -0400 Subject: [PATCH 21/31] test runner: refresh display before running test This makes it so that when a test fails quickly, and writes output, it will do so after the test name is printed. --- lib/std/special/test_runner.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index c16484d162c2b0de24ed9a982eb8476a23e9dac6..bc57b3b925feea3bae80c61ec8c12fc525564ede 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -15,6 +15,7 @@ pub fn main() anyerror!void { for (test_fn_list) |test_fn, i| { var test_node = root_node.start(test_fn.name, null); test_node.activate(); + progress.refresh(); if (test_fn.func()) |_| { ok_count += 1; test_node.end(); -- 2.54.0 From 87f632b08a04f4a03facfa220df1cbc466a5de39 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 19:17:33 -0400 Subject: [PATCH 22/31] fs.Dir.openDir: use empty object name for "." on Windows --- CMakeLists.txt | 3 +++ lib/std/fs.zig | 35 +++++++++++++++++++++++++++------ lib/std/os/windows.zig | 4 +++- lib/std/os/windows/bits.zig | 2 ++ lib/std/os/windows/kernel32.zig | 2 ++ lib/std/os/windows/status.zig | 1 + 6 files changed, 40 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a34d034ddaaf8f5f106dc2062fd4e1715e067f0..cd1ba9d3f8af0be9f556019e6bd4528487b8ee1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -630,5 +630,8 @@ set_target_properties(zig PROPERTIES LINK_FLAGS ${EXE_LDFLAGS} ) target_link_libraries(zig compiler "${LIBUSERLAND}") +if(MSVC) + target_link_libraries(zig ntdll.lib) +endif() add_dependencies(zig zig_build_libuserland) install(TARGETS zig DESTINATION bin) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 51623d3057981f5779ed21c3079a300f08014fe8..8f9f8c17d962ec0122157fb9a173c2b3c8c25229 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -590,6 +590,7 @@ pub const Dir = struct { self.end_index = io.Information; switch (rc) { w.STATUS.SUCCESS => {}, + w.STATUS.ACCESS_DENIED => return error.AccessDenied, else => return w.unexpectedStatus(rc), } } @@ -708,7 +709,7 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { - std.debug.warn("openDir {}\n", sub_path); + // std.debug.warn("openDir {}\n", sub_path); if (os.windows.is_the_target) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); return self.openDirW(&sub_path_w); @@ -740,9 +741,26 @@ pub const Dir = struct { /// This function is Windows-only. pub fn openDirW(self: Dir, sub_path_w: [*]const u16) OpenError!Dir { const w = os.windows; + var result = Dir{ .fd = undefined, }; + + const desired_access = w.GENERIC_READ | w.SYNCHRONIZE; + + // if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // // Windows gives me STATUS_OBJECT_NAME_INVALID with "." as the object name. + + // if (w.kernel32.DuplicateHandle(w.self_process_handle, self.fd, w.self_process_handle, &result.fd, desired_access, w.TRUE, 0) == 0) { + // switch (w.kernel32.GetLastError()) { + // else => |err| return w.unexpectedError(err), + // } + + // @panic("handle DuplicateHandle error"); + // } + // return result; + // } + //var mask: ?[*]const u16 = undefined; //var nt_name: w.UNICODE_STRING = undefined; //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { @@ -760,7 +778,7 @@ pub const Dir = struct { //} const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); - std.debug.warn("path_len_bytes = {}\n", path_len_bytes); + // std.debug.warn("path_len_bytes = {}\n", path_len_bytes); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, @@ -774,7 +792,11 @@ pub const Dir = struct { .SecurityDescriptor = null, .SecurityQualityOfService = null, }; - std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // Windows does not recognize this, but it does work with empty string. + nt_name.Length = 0; + } + // std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); var io: w.IO_STATUS_BLOCK = undefined; const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); @@ -782,11 +804,11 @@ pub const Dir = struct { //var buf2: [200]u8 = undefined; const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; - std.debug.warn("path: {}\n", buf[0..len]); + // std.debug.warn("path: {}\n", buf[0..len]); //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); const rc = w.ntdll.NtCreateFile( &result.fd, - w.GENERIC_READ | w.SYNCHRONIZE, + desired_access, &attr, &io, null, @@ -797,11 +819,12 @@ pub const Dir = struct { null, 0, ); - std.debug.warn("result.fd = {}\n", result.fd); + // std.debug.warn("result.fd = {}\n", result.fd); switch (rc) { w.STATUS.SUCCESS => return result, w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.OBJECT_PATH_NOT_FOUND => return error.FileNotFound, w.STATUS.INVALID_PARAMETER => { @panic("invalid parameter"); }, diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 6a5d03b52945f209121dea2d9f1bc22c86523096..93a5a65a920c9adc2f9834950b88859c7ba0483f 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -20,6 +20,8 @@ pub const shell32 = @import("windows/shell32.zig"); pub usingnamespace @import("windows/bits.zig"); +pub const self_process_handle = @intToPtr(HANDLE, maxInt(usize)); + /// `builtin` is missing `subsystem` when the subsystem is automatically detected, /// so Zig standard library has the subsystem detection logic here. This should generally be /// used rather than `builtin.subsystem`. @@ -898,7 +900,7 @@ pub fn unexpectedError(err: DWORD) std.os.UnexpectedError { /// and you get an unexpected status. pub fn unexpectedStatus(status: NTSTATUS) std.os.UnexpectedError { if (std.os.unexpected_error_tracing) { - std.debug.warn("error.Unexpected NTSTATUS={}\n", status); + std.debug.warn("error.Unexpected NTSTATUS=0x{x}\n", status); std.debug.dumpCurrentStackTrace(null); } return error.Unexpected; diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig index 68d5ab6b87a081e0a95c708859827baa99e86a25..214f75186f2e98e2c3a5e3f5c868a83cffda0c3d 100644 --- a/lib/std/os/windows/bits.zig +++ b/lib/std/os/windows/bits.zig @@ -872,3 +872,5 @@ pub const CURDIR = extern struct { DosPath: UNICODE_STRING, Handle: HANDLE, }; + +pub const DUPLICATE_SAME_ACCESS = 2; \ No newline at end of file diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index 2ae73ad45a63bce7a4bf3e503591ec71a2097d09..736d81ae584ae2c456e145b4b66fc7182d183ba5 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -47,6 +47,8 @@ pub extern "kernel32" stdcallcc fn CreateThread(lpThreadAttributes: ?LPSECURITY_ pub extern "kernel32" stdcallcc fn DeleteFileW(lpFileName: [*]const u16) BOOL; +pub extern "kernel32" stdcallcc fn DuplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, hTargetProcessHandle: HANDLE, lpTargetHandle: *HANDLE, dwDesiredAccess: DWORD, bInheritHandle: BOOL, dwOptions: DWORD) BOOL; + pub extern "kernel32" stdcallcc fn ExitProcess(exit_code: UINT) noreturn; pub extern "kernel32" stdcallcc fn FindFirstFileW(lpFileName: [*]const u16, lpFindFileData: *WIN32_FIND_DATAW) HANDLE; diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index d381e3b2e1ba960c81968ef246d1758c87d75cf6..424cae5e293c8e317ce4327855eb012bf013dbaa 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -8,4 +8,5 @@ pub const INVALID_PARAMETER = 0xC000000D; pub const ACCESS_DENIED = 0xC0000022; pub const OBJECT_NAME_INVALID = 0xC0000033; pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; +pub const OBJECT_PATH_NOT_FOUND = 0xC000003A; pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; -- 2.54.0 From 98c4365b66e6184aa867ec7d5d1d01435ec99798 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 19:39:30 -0400 Subject: [PATCH 23/31] darwin: add AT_* bits --- lib/std/os/bits/darwin.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/std/os/bits/darwin.zig b/lib/std/os/bits/darwin.zig index b076f95884828bccf2a55527d42becf14844246b..fe3156bb902a2635e189671dcc2c1d40ab052358 100644 --- a/lib/std/os/bits/darwin.zig +++ b/lib/std/os/bits/darwin.zig @@ -1178,3 +1178,17 @@ pub fn S_IWHT(m: u32) bool { return m & S_IFMT == S_IFWHT; } pub const HOST_NAME_MAX = 72; + +pub const AT_FDCWD = -2; + +/// Use effective ids in access check +pub const AT_EACCESS = 0x0010; + +/// Act on the symlink itself not the target +pub const AT_SYMLINK_NOFOLLOW = 0x0020; + +/// Act on target of symlink +pub const AT_SYMLINK_FOLLOW = 0x0040; + +/// Path refers to directory +pub const AT_REMOVEDIR = 0x0080; -- 2.54.0 From 5917572e59ff68e33ae7fb0ec7b8d592e2915463 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:16:43 -0400 Subject: [PATCH 24/31] cleanup --- lib/std/fs.zig | 76 +++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 8f9f8c17d962ec0122157fb9a173c2b3c8c25229..3fdd175e77d514df1c119575b9addb581ab66577 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -377,6 +377,8 @@ pub const Dir = struct { }; }; + const IteratorError = error{AccessDenied} || os.UnexpectedError; + pub const Iterator = switch (builtin.os) { .macosx, .ios, .freebsd, .netbsd => struct { dir: Dir, @@ -387,9 +389,11 @@ pub const Dir = struct { const Self = @This(); + pub const Error = IteratorError; + /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. - pub fn next(self: *Self) !?Entry { + pub fn next(self: *Self) Error!?Entry { switch (builtin.os) { .macosx, .ios => return self.nextDarwin(), .freebsd, .netbsd => return self.nextBsd(), @@ -504,9 +508,11 @@ pub const Dir = struct { const Self = @This(); + pub const Error = IteratorError; + /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. - pub fn next(self: *Self) !?Entry { + pub fn next(self: *Self) Error!?Entry { start_over: while (true) { if (self.index >= self.end_index) { const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); @@ -560,17 +566,13 @@ pub const Dir = struct { const Self = @This(); - pub fn next(self: *Self) !?Entry { + pub const Error = IteratorError; + + pub fn next(self: *Self) Error!?Entry { start_over: while (true) { const w = os.windows; if (self.index >= self.end_index) { var io: w.IO_STATUS_BLOCK = undefined; - //var mask_buf = [2]u16{ 'a', 0 }; - //var mask = w.UNICODE_STRING{ - // .Length = 2, - // .MaximumLength = 2, - // .Buffer = &mask_buf, - //}; const rc = w.ntdll.NtQueryDirectoryFile( self.dir.fd, null, @@ -709,7 +711,6 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { - // std.debug.warn("openDir {}\n", sub_path); if (os.windows.is_the_target) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); return self.openDirW(&sub_path_w); @@ -746,39 +747,7 @@ pub const Dir = struct { .fd = undefined, }; - const desired_access = w.GENERIC_READ | w.SYNCHRONIZE; - - // if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { - // // Windows gives me STATUS_OBJECT_NAME_INVALID with "." as the object name. - - // if (w.kernel32.DuplicateHandle(w.self_process_handle, self.fd, w.self_process_handle, &result.fd, desired_access, w.TRUE, 0) == 0) { - // switch (w.kernel32.GetLastError()) { - // else => |err| return w.unexpectedError(err), - // } - - // @panic("handle DuplicateHandle error"); - // } - // return result; - // } - - //var mask: ?[*]const u16 = undefined; - //var nt_name: w.UNICODE_STRING = undefined; - //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { - // return error.FileNotFound; - //} - //defer w.ntdll.RtlFreeUnicodeString(&nt_name); - //if (mask) |m| { - // if (m[0] == 0) { - // return error.FileNotFound; - // } else { - // nt_name.Length = @intCast(u16, @ptrToInt(mask) - @ptrToInt(nt_name.Buffer)); - // } - //} else { - // return error.FileNotFound; - //} - const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); - // std.debug.warn("path_len_bytes = {}\n", path_len_bytes); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, @@ -796,19 +765,15 @@ pub const Dir = struct { // Windows does not recognize this, but it does work with empty string. nt_name.Length = 0; } - // std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + // If you're looking to contribute to zig and fix this, see here for an example of how to + // implement this: https://git.midipix.org/ntapi/tree/src/fs/ntapi_tt_open_physical_parent_directory.c + @panic("TODO opening '..' with a relative directory handle is not yet implemented on Windows"); + } var io: w.IO_STATUS_BLOCK = undefined; - const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; - //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); - var buf: [200]u8 = undefined; - //var buf2: [200]u8 = undefined; - const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; - //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; - // std.debug.warn("path: {}\n", buf[0..len]); - //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); const rc = w.ntdll.NtCreateFile( &result.fd, - desired_access, + w.GENERIC_READ | w.SYNCHRONIZE, &attr, &io, null, @@ -819,15 +784,12 @@ pub const Dir = struct { null, 0, ); - // std.debug.warn("result.fd = {}\n", result.fd); switch (rc) { w.STATUS.SUCCESS => return result, - w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), + w.STATUS.OBJECT_NAME_INVALID => unreachable, w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, w.STATUS.OBJECT_PATH_NOT_FOUND => return error.FileNotFound, - w.STATUS.INVALID_PARAMETER => { - @panic("invalid parameter"); - }, + w.STATUS.INVALID_PARAMETER => unreachable, else => return w.unexpectedStatus(rc), } } -- 2.54.0 From 9a82f00a4fb897acf50446a725768c3aaa562513 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:16:52 -0400 Subject: [PATCH 25/31] add freebsd AT_* bits --- lib/std/os/bits/freebsd.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/std/os/bits/freebsd.zig b/lib/std/os/bits/freebsd.zig index 3d07e92e018db0ddda2e0eef6e3d91b573460373..4427acb3343a64acffe6ac032ed70958651b3e78 100644 --- a/lib/std/os/bits/freebsd.zig +++ b/lib/std/os/bits/freebsd.zig @@ -939,3 +939,23 @@ pub fn S_IWHT(m: u32) bool { } pub const HOST_NAME_MAX = 255; + +/// Magic value that specify the use of the current working directory +/// to determine the target of relative file paths in the openat() and +/// similar syscalls. +pub const AT_FDCWD = -100; + +/// Check access using effective user and group ID +pub const AT_EACCESS = 0x0100; + +/// Do not follow symbolic links +pub const AT_SYMLINK_NOFOLLOW = 0x0200; + +/// Follow symbolic link +pub const AT_SYMLINK_FOLLOW = 0x0400; + +/// Remove directory instead of file +pub const AT_REMOVEDIR = 0x0800; + +/// Fail if not under dirfd +pub const AT_BENEATH = 0x1000; -- 2.54.0 From 65165554d0c6cf39ff091ace5ab5d48106a935fd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:49:35 -0400 Subject: [PATCH 26/31] test runner: restore previous behavior when... ...stderr does not support ansi escape codes --- lib/std/special/test_runner.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 8fb1b5842d09236309ff21cd81341b1a4f52e15e..78db63e4d6c793c71c287297cb30415f9603cb64 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -15,14 +15,17 @@ pub fn main() anyerror!void { for (test_fn_list) |test_fn, i| { var test_node = root_node.start(test_fn.name, null); test_node.activate(); + if (progress.terminal == null) std.debug.warn("{}/{} {}...", i + 1, test_fn_list.len, test_fn.name); if (test_fn.func()) |_| { ok_count += 1; test_node.end(); + if (progress.terminal == null) std.debug.warn("OK\n"); } else |err| switch (err) { error.SkipZigTest => { skip_count += 1; test_node.end(); progress.log("{}...SKIP\n", test_fn.name); + if (progress.terminal == null) std.debug.warn("SKIP\n"); }, else => return err, } @@ -30,5 +33,6 @@ pub fn main() anyerror!void { root_node.end(); if (ok_count != test_fn_list.len) { progress.log("{} passed; {} skipped.\n", ok_count, skip_count); + if (progress.terminal == null) std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } -- 2.54.0 From fc6f84f3f08aca0d4e2352091cc2db2ca5779c7d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:58:53 -0400 Subject: [PATCH 27/31] fix os.unlinkatW implementation --- lib/std/os.zig | 22 ++++++++++++++++++---- lib/std/os/windows/status.zig | 1 + 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 2d8ddb0abcb3eca2b00f75e1fba888b705b3853b..7a5eeda82c2df57334a24136a49c98e2d020b91d 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1045,11 +1045,24 @@ pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatErro w.ULONG(w.FILE_DELETE_ON_CLOSE) else w.ULONG(w.FILE_DELETE_ON_CLOSE | w.FILE_NON_DIRECTORY_FILE); - var nt_name: w.UNICODE_STRING = undefined; - if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { - return error.FileNotFound; + + const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + // The Windows API makes this mutable, but it will not mutate here. + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // Windows does not recognize this, but it does work with empty string. + nt_name.Length = 0; } - defer w.ntdll.RtlFreeUnicodeString(&nt_name); + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + // Can't remove the parent directory with an open handle. + return error.FileBusy; + } + var attr = w.OBJECT_ATTRIBUTES{ .Length = @sizeOf(w.OBJECT_ATTRIBUTES), @@ -1082,6 +1095,7 @@ pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatErro w.STATUS.OBJECT_NAME_INVALID => unreachable, w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, w.STATUS.INVALID_PARAMETER => unreachable, + w.STATUS.FILE_IS_A_DIRECTORY => return error.IsDir, else => return w.unexpectedStatus(rc), } } diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index 424cae5e293c8e317ce4327855eb012bf013dbaa..5f38948620a52f07c138ec18dcda3804320020d9 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -10,3 +10,4 @@ pub const OBJECT_NAME_INVALID = 0xC0000033; pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; pub const OBJECT_PATH_NOT_FOUND = 0xC000003A; pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; +pub const FILE_IS_A_DIRECTORY = 0xC00000BA; -- 2.54.0 From 064377be9aaf409bf483f512354ef22f656bf213 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 22:20:45 -0400 Subject: [PATCH 28/31] test runner: restore previous behavior of printing all tests passed --- lib/std/special/test_runner.zig | 7 ++++--- test/cli.zig | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 78db63e4d6c793c71c287297cb30415f9603cb64..f835401e1ca4de8952c9dec8da6207992f8c8983 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -31,8 +31,9 @@ pub fn main() anyerror!void { } } root_node.end(); - if (ok_count != test_fn_list.len) { - progress.log("{} passed; {} skipped.\n", ok_count, skip_count); - if (progress.terminal == null) std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); + if (ok_count == test_fn_list.len) { + std.debug.warn("All tests passed.\n"); + } else { + std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } diff --git a/test/cli.zig b/test/cli.zig index 2999df14b0d1a39046c11541f058e9d04cce2531..6095536422e4fa050ceda6c81a4fcd2cfc749797 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -87,7 +87,7 @@ fn exec(cwd: []const u8, argv: []const []const u8) !ChildProcess.ExecResult { fn testZigInitLib(zig_exe: []const u8, dir_path: []const u8) !void { _ = try exec(dir_path, [_][]const u8{ zig_exe, "init-lib" }); const test_result = try exec(dir_path, [_][]const u8{ zig_exe, "build", "test" }); - testing.expect(std.mem.eql(u8, test_result.stderr, "")); + testing.expect(std.mem.endsWith(u8, test_result.stderr, "All tests passed.\n")); } fn testZigInitExe(zig_exe: []const u8, dir_path: []const u8) !void { -- 2.54.0 From 2550cb4638e12e94073cbca5c0ee3aa263c25dbe Mon Sep 17 00:00:00 2001 From: Vexu <15308111+Vexu@users.noreply.github.com> Date: Mon, 21 Oct 2019 22:22:08 +0300 Subject: [PATCH 29/31] remove pub syntax for container fields --- doc/langref.html.in | 4 ++-- lib/std/child_process.zig | 32 ++++++++++++++++---------------- lib/std/heap.zig | 2 +- lib/std/http/headers.zig | 6 +++--- lib/std/io.zig | 20 ++++++++++---------- lib/std/io/seekable_stream.zig | 4 ++-- lib/std/zig/ast.zig | 4 +--- lib/std/zig/parse.zig | 16 +++++++--------- lib/std/zig/parser_test.zig | 6 +++--- lib/std/zig/render.zig | 8 ++------ src/all_types.hpp | 2 -- src/parser.cpp | 16 +++++++--------- 12 files changed, 54 insertions(+), 66 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index a30670b0c70154a9469e0b7e264fe0fa7b72f565..17792d6470ed9c0cae71b58040c5f6ea12b22959 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -10086,8 +10086,8 @@ ContainerMembers <- TestDecl ContainerMembers / TopLevelComptime ContainerMembers / KEYWORD_pub? TopLevelDecl ContainerMembers - / KEYWORD_pub? ContainerField COMMA ContainerMembers - / KEYWORD_pub? ContainerField + / ContainerField COMMA ContainerMembers + / ContainerField / TestDecl <- KEYWORD_test STRINGLITERAL Block diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 7fb663ac28d336cc9abbf0bb5fc6de399c8c53b7..d786c3f4de356fdce76d44671e4a71c6213020f0 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -17,35 +17,35 @@ const TailQueue = std.TailQueue; const maxInt = std.math.maxInt; pub const ChildProcess = struct { - pub pid: if (os.windows.is_the_target) void else i32, - pub handle: if (os.windows.is_the_target) windows.HANDLE else void, - pub thread_handle: if (os.windows.is_the_target) windows.HANDLE else void, + pid: if (os.windows.is_the_target) void else i32, + handle: if (os.windows.is_the_target) windows.HANDLE else void, + thread_handle: if (os.windows.is_the_target) windows.HANDLE else void, - pub allocator: *mem.Allocator, + allocator: *mem.Allocator, - pub stdin: ?File, - pub stdout: ?File, - pub stderr: ?File, + stdin: ?File, + stdout: ?File, + stderr: ?File, - pub term: ?(SpawnError!Term), + term: ?(SpawnError!Term), - pub argv: []const []const u8, + argv: []const []const u8, /// Leave as null to use the current env map using the supplied allocator. - pub env_map: ?*const BufMap, + env_map: ?*const BufMap, - pub stdin_behavior: StdIo, - pub stdout_behavior: StdIo, - pub stderr_behavior: StdIo, + stdin_behavior: StdIo, + stdout_behavior: StdIo, + stderr_behavior: StdIo, /// Set to change the user id when spawning the child process. - pub uid: if (os.windows.is_the_target) void else ?u32, + uid: if (os.windows.is_the_target) void else ?u32, /// Set to change the group id when spawning the child process. - pub gid: if (os.windows.is_the_target) void else ?u32, + gid: if (os.windows.is_the_target) void else ?u32, /// Set to change the current working directory when spawning the child process. - pub cwd: ?[]const u8, + cwd: ?[]const u8, err_pipe: if (os.windows.is_the_target) void else [2]os.fd_t, llnode: if (os.windows.is_the_target) void else TailQueue(*ChildProcess).Node, diff --git a/lib/std/heap.zig b/lib/std/heap.zig index b968b6242f6a634e9f33d83a35dec88aa43da1b6..f3a0d457d2a0a2403bdd047bfc3c6c932ab2168f 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -338,7 +338,7 @@ pub const HeapAllocator = switch (builtin.os) { /// This allocator takes an existing allocator, wraps it, and provides an interface /// where you can allocate without freeing, and then free it all together. pub const ArenaAllocator = struct { - pub allocator: Allocator, + allocator: Allocator, child_allocator: *Allocator, buffer_list: std.SinglyLinkedList([]u8), diff --git a/lib/std/http/headers.zig b/lib/std/http/headers.zig index a8dfa686298fa8c71f0a6643e1e605b5af44d5ec..7ee035ce80cebc09c66dfb6ce37f38d8c3f11d04 100644 --- a/lib/std/http/headers.zig +++ b/lib/std/http/headers.zig @@ -28,9 +28,9 @@ fn never_index_default(name: []const u8) bool { const HeaderEntry = struct { allocator: *Allocator, - pub name: []const u8, - pub value: []u8, - pub never_index: bool, + name: []const u8, + value: []u8, + never_index: bool, const Self = @This(); diff --git a/lib/std/io.zig b/lib/std/io.zig index 4407e6b2b39afad0d6cb71c30394ec31ea48281b..f9082ee952e4b79cc548c02c76649548abb1873a 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -161,7 +161,7 @@ pub fn BufferedInStreamCustom(comptime buffer_size: usize, comptime Error: type) const Self = @This(); const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, unbuffered_in_stream: *Stream, @@ -273,7 +273,7 @@ pub fn PeekStream(comptime buffer_size: usize, comptime InStreamError: type) typ pub const Error = InStreamError; pub const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, base: *Stream, // Right now the look-ahead space is statically allocated, but a version with dynamic allocation @@ -336,7 +336,7 @@ pub const SliceInStream = struct { pub const Error = error{}; pub const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, pos: usize, slice: []const u8, @@ -514,9 +514,9 @@ pub const SliceOutStream = struct { pub const Error = error{OutOfSpace}; pub const Stream = OutStream(Error); - pub stream: Stream, + stream: Stream, - pub pos: usize, + pos: usize, slice: []u8, pub fn init(slice: []u8) SliceOutStream { @@ -571,7 +571,7 @@ pub const NullOutStream = struct { pub const Error = error{}; pub const Stream = OutStream(Error); - pub stream: Stream, + stream: Stream, pub fn init() NullOutStream { return NullOutStream{ @@ -595,8 +595,8 @@ pub fn CountingOutStream(comptime OutStreamError: type) type { pub const Stream = OutStream(Error); pub const Error = OutStreamError; - pub stream: Stream, - pub bytes_written: u64, + stream: Stream, + bytes_written: u64, child_stream: *Stream, pub fn init(child_stream: *Stream) Self { @@ -635,7 +635,7 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamEr pub const Stream = OutStream(Error); pub const Error = OutStreamError; - pub stream: Stream, + stream: Stream, unbuffered_out_stream: *Stream, @@ -1084,7 +1084,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, // safety. If it is bad, it will be caught anyway. const TagInt = @TagType(TagType); const tag = try self.deserializeInt(TagInt); - + inline for (info.fields) |field_info| { if (field_info.enum_field.?.value == tag) { const name = field_info.name; diff --git a/lib/std/io/seekable_stream.zig b/lib/std/io/seekable_stream.zig index 86f76d8c1431c1d1d266ee76aa0cd7eb7fea1914..48dc31b785ccaad7a10048d6a51e7885ca72a63c 100644 --- a/lib/std/io/seekable_stream.zig +++ b/lib/std/io/seekable_stream.zig @@ -39,8 +39,8 @@ pub const SliceSeekableInStream = struct { pub const Stream = InStream(Error); pub const SeekableInStream = SeekableStream(SeekError, GetSeekPosError); - pub stream: Stream, - pub seekable_stream: SeekableInStream, + stream: Stream, + seekable_stream: SeekableInStream, pos: usize, slice: []const u8, diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 79e66471449218723bb698861f85ea6f261c6bf2..9f32c6e54b5531297160f54b877d5088b337879b 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -290,7 +290,7 @@ pub const Error = union(enum) { pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found '{}'"); pub const ExpectedParamType = SimpleError("Expected parameter type"); - pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member"); + pub const ExpectedPubItem = SimpleError("Expected function or variable declaration after pub"); pub const UnattachedDocComment = SimpleError("Unattached documentation comment"); pub const ExtraAlignQualifier = SimpleError("Extra align qualifier"); pub const ExtraConstQualifier = SimpleError("Extra const qualifier"); @@ -757,7 +757,6 @@ pub const Node = struct { pub const ContainerField = struct { base: Node, doc_comments: ?*DocComment, - visib_token: ?TokenIndex, name_token: TokenIndex, type_expr: ?*Node, value_expr: ?*Node, @@ -780,7 +779,6 @@ pub const Node = struct { } pub fn firstToken(self: *const ContainerField) TokenIndex { - if (self.visib_token) |visib_token| return visib_token; return self.name_token; } diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 151784284d256bd005421c131e0dfb5a7200aa2b..71fa376361708f5f1cce0eb36a35266101d63fb7 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -138,9 +138,15 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No continue; } + if (visib_token != null) { + try tree.errors.push(AstError{ + .ExpectedPubItem = AstError.ExpectedPubItem{ .token = it.index }, + }); + return error.ParseError; + } + if (try parseContainerField(arena, it, tree)) |node| { const field = node.cast(Node.ContainerField).?; - field.visib_token = visib_token; field.doc_comments = doc_comments; try list.push(node); const comma = eatToken(it, .Comma) orelse break; @@ -149,13 +155,6 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No continue; } - // Dangling pub - if (visib_token != null) { - try tree.errors.push(AstError{ - .ExpectedPubItem = AstError.ExpectedPubItem{ .token = it.index }, - }); - } - break; } @@ -407,7 +406,6 @@ fn parseContainerField(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No node.* = Node.ContainerField{ .base = Node{ .id = .ContainerField }, .doc_comments = null, - .visib_token = null, .name_token = name_token, .type_expr = type_expr, .value_expr = value_expr, diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 78f80786c5da964433f4dda195b231b83adbffd0..28bd287c8a55501fba71912d1cbcc2830e770b28 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -1766,7 +1766,7 @@ test "zig fmt: struct declaration" { \\const S = struct { \\ const Self = @This(); \\ f1: u8, - \\ pub f3: u8, + \\ f3: u8, \\ \\ fn method(self: *Self) Self { \\ return self.*; @@ -1777,14 +1777,14 @@ test "zig fmt: struct declaration" { \\ \\const Ps = packed struct { \\ a: u8, - \\ pub b: u8, + \\ b: u8, \\ \\ c: u8, \\}; \\ \\const Es = extern struct { \\ a: u8, - \\ pub b: u8, + \\ b: u8, \\ \\ c: u8, \\}; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 1be4410f382386e6421058890b32ce25a344a6fb..e5875fd04f05716a69be8aadfb767793506be441 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -254,10 +254,6 @@ fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, i try renderDocComments(tree, stream, field, indent, start_col); - if (field.visib_token) |visib_token| { - try renderToken(tree, stream, visib_token, indent, start_col, Space.Space); // pub - } - if (field.type_expr == null and field.value_expr == null) { return renderToken(tree, stream, field.name_token, indent, start_col, Space.Comma); // name, } else if (field.type_expr != null and field.value_expr == null) { @@ -2206,8 +2202,8 @@ const FindByteOutStream = struct { pub const Error = error{}; pub const Stream = std.io.OutStream(Error); - pub stream: Stream, - pub byte_found: bool, + stream: Stream, + byte_found: bool, byte: u8, pub fn init(byte: u8) Self { diff --git a/src/all_types.hpp b/src/all_types.hpp index e2e1db48d49897df588d83e7a177e9528bc10dbb..267c5479be329aa13564b14d3c34fb1e90e5bfc8 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -990,8 +990,6 @@ struct AstNodeStructField { // populated if the "align(A)" is present AstNode *align_expr; Buf doc_comments; - - VisibMod visib_mod; }; struct AstNodeStringLiteral { diff --git a/src/parser.cpp b/src/parser.cpp index 9fcf233e2d4c4ef3d75e71bb9801702f5e765d71..b6742ef1f17f888e732437c4ab9364a5a6d22d43 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -518,8 +518,8 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) { // <- TestDecl ContainerMembers // / TopLevelComptime ContainerMembers // / KEYWORD_pub? TopLevelDecl ContainerMembers -// / KEYWORD_pub? ContainerField COMMA ContainerMembers -// / KEYWORD_pub? ContainerField +// / ContainerField COMMA ContainerMembers +// / ContainerField // / static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { AstNodeContainerDecl res = {}; @@ -548,10 +548,13 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { continue; } + if (visib_token != nullptr) { + ast_error(pc, peek_token(pc), "expected function or variable declaration after pub"); + } + AstNode *container_field = ast_parse_container_field(pc); if (container_field != nullptr) { assert(container_field->type == NodeTypeStructField); - container_field->data.struct_field.visib_mod = visib_mod; container_field->data.struct_field.doc_comments = doc_comment_buf; res.fields.append(container_field); if (eat_token_if(pc, TokenIdComma) != nullptr) { @@ -561,12 +564,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { } } - // We visib_token wasn't eaten, then we haven't consumed the first token in this rule yet. - // It is therefore safe to return and let the caller continue parsing. - if (visib_token == nullptr) - break; - - ast_invalid_token_error(pc, peek_token(pc)); + break; } return res; -- 2.54.0 From 87b11617b55254abab267e8739ee2f0d926fbcf5 Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Mon, 21 Oct 2019 14:26:15 -0600 Subject: [PATCH 30/31] Fix accept function API The sockaddr pointer and size of the accept function points to a data structure that can only be determined at runtime. The only requirement is that it must be large enough to hold 2 bytes for the address family value. Typeical usage of the socket API is for UDP/TCP IPv4 and IPv6 sockets, which use sockaddr_in and sockaddr_in6. And some sockets can actually support both simultaneously in which case the app may want to have access to the size of the returned sockaddr. Operating systems can even support custom protocols where they use custom sockaddr data structures. In this case the standard library would have no knowledge of the actual size of the sockaddr being passed into the accept function. In this case the standard library should defer to the app to pass in the size of their structure. --- lib/std/os.zig | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index de01da2fa5241b0ce0521b11fec2e3826a2de5ac..4d0dac896ce810d86984926946738cd773ef2140 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1478,10 +1478,9 @@ pub const AcceptError = error{ /// Accept a connection on a socket. `fd` must be opened in blocking mode. /// See also `accept4_async`. -pub fn accept4(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { +pub fn accept4(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptError!i32 { while (true) { - var sockaddr_size = u32(@sizeOf(sockaddr)); - const rc = system.accept4(fd, addr, &sockaddr_size, flags); + const rc = system.accept4(fd, addr, addrSize, flags); switch (errno(rc)) { 0 => return @intCast(i32, rc), EINTR => continue, @@ -1506,10 +1505,9 @@ pub fn accept4(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { /// This is the same as `accept4` except `fd` is expected to be non-blocking. /// Returns -1 if would block. -pub fn accept4_async(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { +pub fn accept4_async(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptError!i32 { while (true) { - var sockaddr_size = u32(@sizeOf(sockaddr)); - const rc = system.accept4(fd, addr, &sockaddr_size, flags); + const rc = system.accept4(fd, addr, addrSize, flags); switch (errno(rc)) { 0 => return @intCast(i32, rc), EINTR => continue, -- 2.54.0 From a5cc758036720babeb13ec8cdec68b720c6af1eb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 23:23:11 -0400 Subject: [PATCH 31/31] std.os.accept4: improve docs and integrate with evented I/O --- lib/std/os.zig | 66 ++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 4d0dac896ce810d86984926946738cd773ef2140..3395ef669dd277e0a36d0ff458f24251ed7f14bc 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1476,17 +1476,46 @@ pub const AcceptError = error{ BlockedByFirewall, } || UnexpectedError; -/// Accept a connection on a socket. `fd` must be opened in blocking mode. -/// See also `accept4_async`. -pub fn accept4(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptError!i32 { +/// Accept a connection on a socket. +/// If the application has a global event loop enabled, EAGAIN is handled +/// via the event loop. Otherwise EAGAIN results in error.WouldBlock. +pub fn accept4( + /// This argument is a socket that has been created with `socket`, bound to a local address + /// with `bind`, and is listening for connections after a `listen`. + sockfd: i32, + /// This argument is a pointer to a sockaddr structure. This structure is filled in with the + /// address of the peer socket, as known to the communications layer. The exact format of the + /// address returned addr is determined by the socket's address family (see `socket` and the + /// respective protocol man pages). + addr: *sockaddr, + /// This argument is a value-result argument: the caller must initialize it to contain the + /// size (in bytes) of the structure pointed to by addr; on return it will contain the actual size + /// of the peer address. + /// + /// The returned address is truncated if the buffer provided is too small; in this case, `addr_size` + /// will return a value greater than was supplied to the call. + addr_size: *usize, + /// If flags is 0, then `accept4` is the same as `accept`. The following values can be bitwise + /// ORed in flags to obtain different behavior: + /// * `SOCK_NONBLOCK` - Set the `O_NONBLOCK` file status flag on the open file description (see `open`) + /// referred to by the new file descriptor. Using this flag saves extra calls to `fcntl` to achieve + /// the same result. + /// * `SOCK_CLOEXEC` - Set the close-on-exec (`FD_CLOEXEC`) flag on the new file descriptor. See the + /// description of the `O_CLOEXEC` flag in `open` for reasons why this may be useful. + flags: u32, +) AcceptError!i32 { while (true) { - const rc = system.accept4(fd, addr, addrSize, flags); + const rc = system.accept4(sockfd, addr, addr_size, flags); switch (errno(rc)) { 0 => return @intCast(i32, rc), EINTR => continue, - else => |err| return unexpectedErrno(err), - EAGAIN => unreachable, // This function is for blocking only. + EAGAIN => if (std.event.Loop.instance) |loop| { + loop.waitUntilFdReadable(sockfd) catch return error.WouldBlock; + continue; + } else { + return error.WouldBlock; + }, EBADF => unreachable, // always a race condition ECONNABORTED => return error.ConnectionAborted, EFAULT => unreachable, @@ -1499,33 +1528,8 @@ pub fn accept4(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptErr EOPNOTSUPP => return error.OperationNotSupported, EPROTO => return error.ProtocolFailure, EPERM => return error.BlockedByFirewall, - } - } -} -/// This is the same as `accept4` except `fd` is expected to be non-blocking. -/// Returns -1 if would block. -pub fn accept4_async(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptError!i32 { - while (true) { - const rc = system.accept4(fd, addr, addrSize, flags); - switch (errno(rc)) { - 0 => return @intCast(i32, rc), - EINTR => continue, else => |err| return unexpectedErrno(err), - - EAGAIN => return -1, - EBADF => unreachable, // always a race condition - ECONNABORTED => return error.ConnectionAborted, - EFAULT => unreachable, - EINVAL => unreachable, - EMFILE => return error.ProcessFdQuotaExceeded, - ENFILE => return error.SystemFdQuotaExceeded, - ENOBUFS => return error.SystemResources, - ENOMEM => return error.SystemResources, - ENOTSOCK => return error.FileDescriptorNotASocket, - EOPNOTSUPP => return error.OperationNotSupported, - EPROTO => return error.ProtocolFailure, - EPERM => return error.BlockedByFirewall, } } } -- 2.54.0