From fa9f1d23968d3b9fe3ae35ed1306aec0cbcfd908 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Oct 2019 03:23:23 -0400 Subject: [PATCH] add -DZIG_ENABLE_MEM_PROFILE option and -fmem-report flag This can be used to find what's taking up memory in the compiler. Here's an example of how to use it: ``` ./zig test ../lib/std/std.zig --cache off -fmem-report ``` And here's the output I get for this on x86_64-linux-gnu today: ``` Const: 6462833 items, 152 bytes each, total 936.84 MiB ConstGlobalRefs: 17236534 items, 24 bytes each, total 394.51 MiB ResetResult: 1698108 items, 160 bytes each, total 259.11 MiB ConstExprValue: 3118299 items, 80 bytes each, total 237.91 MiB EndExpr: 1345395 items, 168 bytes each, total 215.56 MiB Unknown_8: 27370821 items, 8 bytes each, total 208.82 MiB VarPtr: 1127866 items, 168 bytes each, total 180.70 MiB IrBasicBlock: 794834 items, 120 bytes each, total 90.96 MiB LoadPtr: 554024 items, 160 bytes each, total 84.54 MiB Unknown_64: 1245715 items, 64 bytes each, total 76.03 MiB Unknown_40: 1879218 items, 40 bytes each, total 71.69 MiB Unknown_72: 989117 items, 72 bytes each, total 67.92 MiB Return: 423783 items, 160 bytes each, total 64.66 MiB Unknown_168: 332480 items, 168 bytes each, total 53.27 MiB Unknown_152: 336890 items, 152 bytes each, total 48.84 MiB AddImplicitReturnType: 230819 items, 168 bytes each, total 36.98 MiB Br: 217835 items, 168 bytes each, total 34.90 MiB Unknown_184: 179529 items, 184 bytes each, total 31.50 MiB FieldPtr: 179388 items, 184 bytes each, total 31.48 MiB BinOp: 171004 items, 176 bytes each, total 28.70 MiB LoadPtrGen: 173287 items, 168 bytes each, total 27.76 MiB CondBr: 137864 items, 192 bytes each, total 25.24 MiB Unknown_720: 30918 items, 720 bytes each, total 21.23 MiB CallSrc: 99320 items, 216 bytes each, total 20.46 MiB Unknown_160: 129243 items, 160 bytes each, total 19.72 MiB Unknown_1: 19339456 items, 1 bytes each, total 18.44 MiB CheckStatementIsVoid: 119838 items, 160 bytes each, total 18.29 MiB Unknown_48: 371178 items, 48 bytes each, total 16.99 MiB TestComptime: 101443 items, 160 bytes each, total 15.48 MiB DeclVarSrc: 72578 items, 184 bytes each, total 12.74 MiB StorePtr: 72776 items, 176 bytes each, total 12.22 MiB ZigVar: 79201 items, 160 bytes each, total 12.09 MiB Unknown_16: 770643 items, 16 bytes each, total 11.76 MiB Phi: 60482 items, 184 bytes each, total 10.61 MiB TestErrSrc: 66177 items, 168 bytes each, total 10.60 MiB Unknown_240: 45164 items, 240 bytes each, total 10.34 MiB ElemPtr: 58232 items, 184 bytes each, total 10.22 MiB AllocaSrc: 60053 items, 176 bytes each, total 10.08 MiB CallGen: 44873 items, 224 bytes each, total 9.59 MiB SaveErrRetAddr: 63787 items, 152 bytes each, total 9.25 MiB Unknown_112: 82283 items, 112 bytes each, total 8.79 MiB AllocaGen: 51909 items, 176 bytes each, total 8.71 MiB Unknown_24: 373599 items, 24 bytes each, total 8.55 MiB ResultLocPeer: 113683 items, 72 bytes each, total 7.81 MiB DeclRef: 36343 items, 168 bytes each, total 5.82 MiB UnwrapErrPayload: 34603 items, 168 bytes each, total 5.54 MiB Ref: 33414 items, 168 bytes each, total 5.35 MiB Unknown_104: 53882 items, 104 bytes each, total 5.34 MiB DeclVarGen: 32540 items, 168 bytes each, total 5.21 MiB StructFieldPtr: 30449 items, 176 bytes each, total 5.11 MiB UnwrapErrCode: 31508 items, 168 bytes each, total 5.05 MiB Unknown_56: 90256 items, 56 bytes each, total 4.82 MiB SpillBegin: 28722 items, 168 bytes each, total 4.60 MiB SpillEnd: 28722 items, 160 bytes each, total 4.38 MiB ResultLocReturn: 64573 items, 48 bytes each, total 2.96 MiB PtrType: 14702 items, 184 bytes each, total 2.58 MiB SliceType: 15005 items, 176 bytes each, total 2.52 MiB Unknown_176: 13326 items, 176 bytes each, total 2.24 MiB RefGen: 12881 items, 168 bytes each, total 2.06 MiB UnOp: 12102 items, 176 bytes each, total 2.03 MiB SwitchBr: 9453 items, 200 bytes each, total 1.80 MiB TestErrGen: 11143 items, 160 bytes each, total 1.70 MiB Unknown_32: 52359 items, 32 bytes each, total 1.60 MiB CheckSwitchProngs: 9094 items, 184 bytes each, total 1.60 MiB TypeOf: 9259 items, 160 bytes each, total 1.41 MiB IntCast: 8772 items, 168 bytes each, total 1.41 MiB OptionalUnwrapPtr: 8755 items, 168 bytes each, total 1.40 MiB SwitchTarget: 9094 items, 160 bytes each, total 1.39 MiB Cast: 8198 items, 176 bytes each, total 1.38 MiB WidenOrShorten: 8448 items, 160 bytes each, total 1.29 MiB ErrorUnion: 7613 items, 176 bytes each, total 1.28 MiB SliceSrc: 6249 items, 192 bytes each, total 1.14 MiB ErrWrapCode: 7133 items, 168 bytes each, total 1.14 MiB TypeName: 7328 items, 160 bytes each, total 1.12 MiB ImplicitCast: 5480 items, 176 bytes each, total 941.88 KiB ResolveResult: 5638 items, 168 bytes each, total 924.98 KiB ResultLocInstruction: 22696 items, 40 bytes each, total 886.56 KiB BitCastSrc: 4947 items, 168 bytes each, total 811.62 KiB CompileErr: 5148 items, 160 bytes each, total 804.38 KiB ReturnPtr: 5305 items, 152 bytes each, total 787.46 KiB Unreachable: 5038 items, 152 bytes each, total 747.83 KiB TestNonNull: 4716 items, 160 bytes each, total 736.88 KiB BitCastGen: 4431 items, 160 bytes each, total 692.34 KiB PtrToInt: 4289 items, 160 bytes each, total 670.16 KiB SliceGen: 3573 items, 192 bytes each, total 669.94 KiB ArrayType: 4081 items, 168 bytes each, total 669.54 KiB IntType: 3868 items, 168 bytes each, total 634.59 KiB Unknown_88: 7213 items, 88 bytes each, total 619.87 KiB Truncate: 3771 items, 168 bytes each, total 618.68 KiB TypeInfo: 3740 items, 160 bytes each, total 584.38 KiB SwitchVar: 3385 items, 176 bytes each, total 581.80 KiB ContainerInitFields: 3223 items, 184 bytes each, total 579.13 KiB ContainerInitList: 2309 items, 192 bytes each, total 432.94 KiB PtrCastGen: 2626 items, 168 bytes each, total 430.83 KiB BoolNot: 2457 items, 160 bytes each, total 383.91 KiB FnProto: 2054 items, 184 bytes each, total 369.08 KiB MergeErrSets: 1927 items, 176 bytes each, total 331.20 KiB Unknown_136: 2486 items, 136 bytes each, total 330.17 KiB Unknown_80: 4059 items, 80 bytes each, total 317.11 KiB Bswap: 1670 items, 168 bytes each, total 273.98 KiB TypeId: 1680 items, 160 bytes each, total 262.50 KiB PtrCastSrc: 1371 items, 176 bytes each, total 235.64 KiB ErrName: 1193 items, 160 bytes each, total 186.41 KiB UnionTag: 1120 items, 160 bytes each, total 175.00 KiB TagName: 1050 items, 160 bytes each, total 164.06 KiB SizeOf: 942 items, 160 bytes each, total 147.19 KiB MemberName: 871 items, 168 bytes each, total 142.90 KiB Import: 881 items, 160 bytes each, total 137.66 KiB PtrOfArrayToSlice: 758 items, 168 bytes each, total 124.36 KiB UnionFieldPtr: 710 items, 176 bytes each, total 122.03 KiB EnumToInt: 778 items, 160 bytes each, total 121.56 KiB CheckRuntimeScope: 700 items, 168 bytes each, total 114.84 KiB FieldParentPtr: 632 items, 184 bytes each, total 113.56 KiB BoolToInt: 719 items, 160 bytes each, total 112.34 KiB ResultLocPeerParent: 904 items, 104 bytes each, total 91.81 KiB IntToPtr: 537 items, 168 bytes each, total 88.10 KiB AlignOf: 561 items, 160 bytes each, total 87.66 KiB AtomicRmw: 356 items, 208 bytes each, total 72.31 KiB MemberCount: 441 items, 160 bytes each, total 68.91 KiB Memset: 342 items, 176 bytes each, total 58.78 KiB PopCount: 321 items, 168 bytes each, total 52.66 KiB AlignCast: 251 items, 168 bytes each, total 41.18 KiB IrInstruction *: 5230 items, 8 bytes each, total 40.86 KiB IrBasicBlock *: 5230 items, 8 bytes each, total 40.86 KiB TagType: 261 items, 160 bytes each, total 40.78 KiB HasDecl: 234 items, 168 bytes each, total 38.39 KiB OverflowOp: 191 items, 200 bytes each, total 37.30 KiB Export: 209 items, 176 bytes each, total 35.92 KiB SetCold: 219 items, 160 bytes each, total 34.22 KiB ReturnAddress: 216 items, 152 bytes each, total 32.06 KiB FromBytes: 178 items, 176 bytes each, total 30.59 KiB SetRuntimeSafety: 188 items, 160 bytes each, total 29.38 KiB OptionalWrap: 151 items, 168 bytes each, total 24.77 KiB Clz: 143 items, 168 bytes each, total 23.46 KiB ResizeSlice: 135 items, 168 bytes each, total 22.15 KiB UnionInitNamedField: 106 items, 184 bytes each, total 19.05 KiB Panic: 102 items, 160 bytes each, total 15.94 KiB SwitchElseVar: 93 items, 168 bytes each, total 15.26 KiB ToBytes: 89 items, 168 bytes each, total 14.60 KiB IntToFloat: 78 items, 168 bytes each, total 12.80 KiB Unknown_4360: 3 items, 4360 bytes each, total 12.77 KiB ErrWrapPayload: 72 items, 168 bytes each, total 11.81 KiB FloatOp: 62 items, 176 bytes each, total 10.66 KiB FloatToInt: 47 items, 168 bytes each, total 7.71 KiB FloatCast: 46 items, 168 bytes each, total 7.55 KiB ErrToInt: 47 items, 160 bytes each, total 7.34 KiB Asm: 33 items, 216 bytes each, total 6.96 KiB ErrSetCast: 40 items, 168 bytes each, total 6.56 KiB Memcpy: 34 items, 176 bytes each, total 5.84 KiB AtomicLoad: 17 items, 184 bytes each, total 3.05 KiB AwaitSrc: 16 items, 168 bytes each, total 2.62 KiB Resume: 14 items, 160 bytes each, total 2.19 KiB AwaitGen: 12 items, 176 bytes each, total 2.06 KiB ArgType: 12 items, 168 bytes each, total 1.97 KiB AnyFrameType: 12 items, 160 bytes each, total 1.88 KiB SuspendFinish: 10 items, 160 bytes each, total 1.56 KiB SuspendBegin: 10 items, 160 bytes each, total 1.56 KiB Ctz: 9 items, 168 bytes each, total 1.48 KiB FrameHandle: 8 items, 152 bytes each, total 1.19 KiB SetEvalBranchQuota: 7 items, 160 bytes each, total 1.09 KiB AssertZero: 7 items, 160 bytes each, total 1.09 KiB UndeclaredIdent: 7 items, 160 bytes each, total 1.09 KiB CmpxchgSrc: 5 items, 216 bytes each, total 1.05 KiB CmpxchgGen: 5 items, 200 bytes each, total 1000.00 bytes IntToEnum: 4 items, 168 bytes each, total 672.00 bytes VectorType: 4 items, 168 bytes each, total 672.00 bytes ErrorReturnTrace: 2 items, 160 bytes each, total 320.00 bytes Breakpoint: 2 items, 152 bytes each, total 304.00 bytes FrameAddress: 2 items, 152 bytes each, total 304.00 bytes Unknown_4: 61 items, 4 bytes each, total 244.00 bytes VectorToArray: 1 items, 168 bytes each, total 168.00 bytes SetAlignStack: 1 items, 160 bytes each, total 160.00 bytes Unknown_12: 2 items, 12 bytes each, total 24.00 bytes ErrorTableEntry *: 0 items, 8 bytes each, total 0.00 bytes AstNode *: 0 items, 8 bytes each, total 0.00 bytes Total bytes used: 3.51 GiB ``` You can see that most of the memory is taken up by IR instructions, as well as comptime values. This points toward 2 changes which will greatly reduce memory usage: * Rework semantic analysis so that IR instructions can be freed. Currently the comptime value struct (ConstExprValue) is embedded directly into the IrInstruction struct. If this is made to be separate, at the very least pass 1 IR instructions can be freed. This includes `Const` which is the largest usage of memory currently. * Rework the ConstExprValue struct to no longer be a tagged union. For example, there's no need for an integer comptime value to be 80 bytes. From this you can also see that this eliminates some things from being the culprit. Before doing this analysis, I considered whether doing string interning would help. From the above output, you can see that all strings in the compiler account for only 18 MiB, so string interning would have been a dead end. --- CMakeLists.txt | 2 + src/analyze.cpp | 4 +- src/config.h.in | 5 +- src/dump_analysis.cpp | 19 +----- src/ir.cpp | 52 +++++++++------ src/ir_print.cpp | 6 +- src/ir_print.hpp | 2 + src/main.cpp | 75 ++++++++++++++------- src/memory_profiling.cpp | 139 +++++++++++++++++++++++++++++++++++++++ src/memory_profiling.hpp | 22 +++++++ src/util.cpp | 18 +++++ src/util.hpp | 35 ++++++++-- 12 files changed, 305 insertions(+), 74 deletions(-) create mode 100644 src/memory_profiling.cpp create mode 100644 src/memory_profiling.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fbc34f80511b580413579b84288996efe0c81605..3a34d034ddaaf8f5f106dc2062fd4e1715e067f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,7 @@ message("Configuring zig version ${ZIG_VERSION}") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries") set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL "Disable copying lib/ files to install prefix") +set(ZIG_ENABLE_MEM_PROFILE off CACHE BOOL "Activate memory usage instrumentation") if(ZIG_STATIC) set(ZIG_STATIC_LLVM "on") @@ -455,6 +456,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ir_print.cpp" "${CMAKE_SOURCE_DIR}/src/libc_installation.cpp" "${CMAKE_SOURCE_DIR}/src/link.cpp" + "${CMAKE_SOURCE_DIR}/src/memory_profiling.cpp" "${CMAKE_SOURCE_DIR}/src/os.cpp" "${CMAKE_SOURCE_DIR}/src/parser.cpp" "${CMAKE_SOURCE_DIR}/src/range_set.cpp" diff --git a/src/analyze.cpp b/src/analyze.cpp index 92b221a2ecdb3bc5370b85eee769f20b74ec9400..ed4547957bd424c5223b154fa54bc8374c663495 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -5783,8 +5783,8 @@ ConstExprValue *create_const_arg_tuple(CodeGen *g, size_t arg_index_start, size_ ConstExprValue *create_const_vals(size_t count) { - ConstGlobalRefs *global_refs = allocate(count); - ConstExprValue *vals = allocate(count); + ConstGlobalRefs *global_refs = allocate(count, "ConstGlobalRefs"); + ConstExprValue *vals = allocate(count, "ConstExprValue"); for (size_t i = 0; i < count; i += 1) { vals[i].global_refs = &global_refs[i]; } diff --git a/src/config.h.in b/src/config.h.in index a99aab0d7235cf238d10aedfa6b3d0dfb7068216..7a0ea3536a94df0dab99f7bfc634ac4bdc10ee2e 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -13,9 +13,6 @@ #define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@ #define ZIG_VERSION_STRING "@ZIG_VERSION@" -// Only used for running tests before installing. -#define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test" - // Used for communicating build information to self hosted build. #define ZIG_CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@" #define ZIG_CXX_COMPILER "@CMAKE_CXX_COMPILER@" @@ -24,4 +21,6 @@ #define ZIG_LLVM_CONFIG_EXE "@LLVM_CONFIG_EXE@" #define ZIG_DIA_GUIDS_LIB "@ZIG_DIA_GUIDS_LIB_ESCAPED@" +#cmakedefine ZIG_ENABLE_MEM_PROFILE + #endif diff --git a/src/dump_analysis.cpp b/src/dump_analysis.cpp index d5b109fbcb03db934eedb20bbacc1556d2a08aff..9703d3e57d107b157e5de956c048c24942fe473f 100644 --- a/src/dump_analysis.cpp +++ b/src/dump_analysis.cpp @@ -240,23 +240,6 @@ static void jw_string(JsonWriter *jw, const char *s) { static void tree_print(FILE *f, ZigType *ty, size_t indent); -static void pretty_print_bytes(FILE *f, double n) { - if (n > 1024.0 * 1024.0 * 1024.0) { - fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0); - return; - } - if (n > 1024.0 * 1024.0) { - fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0); - return; - } - if (n > 1024.0) { - fprintf(f, "%.02f KiB", n / 1024.0); - return; - } - fprintf(f, "%.02f bytes", n ); - return; -} - static int compare_type_abi_sizes_desc(const void *a, const void *b) { uint64_t size_a = (*(ZigType * const*)(a))->abi_size; uint64_t size_b = (*(ZigType * const*)(b))->abi_size; @@ -322,7 +305,7 @@ static void tree_print(FILE *f, ZigType *ty, size_t indent) { start_peer(f, indent); fprintf(f, "\"sizef\": \""); - pretty_print_bytes(f, ty->abi_size); + zig_pretty_print_bytes(f, ty->abi_size); fprintf(f, "\""); start_peer(f, indent); diff --git a/src/ir.cpp b/src/ir.cpp index 7dd141423c0c720a2009d90a5245b900bd50a2d1..b71a48dc2b7ba8121a4362b91d58f739d261c232 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -413,7 +413,7 @@ ZigType *ir_analyze_type_expr(IrAnalyze *ira, Scope *scope, AstNode *node) { } static IrBasicBlock *ir_create_basic_block(IrBuilder *irb, Scope *scope, const char *name_hint) { - IrBasicBlock *result = allocate(1); + IrBasicBlock *result = allocate(1, "IrBasicBlock"); result->scope = scope; result->name_hint = name_hint; result->debug_id = exec_next_debug_id(irb->exec); @@ -1085,13 +1085,18 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSpillEnd *) { template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { - T *special_instruction = allocate(1); + const char *name = nullptr; +#ifdef ZIG_ENABLE_MEM_PROFILE + T *dummy = nullptr; + name = ir_instruction_type_str(ir_instruction_id(dummy)); +#endif + T *special_instruction = allocate(1, name); special_instruction->base.id = ir_instruction_id(special_instruction); special_instruction->base.scope = scope; special_instruction->base.source_node = source_node; special_instruction->base.debug_id = exec_next_debug_id(irb->exec); special_instruction->base.owner_bb = irb->current_basic_block; - special_instruction->base.value.global_refs = allocate(1); + special_instruction->base.value.global_refs = allocate(1, "ConstGlobalRefs"); return special_instruction; } @@ -3569,7 +3574,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, switch (node->data.return_expr.kind) { case ReturnKindUnconditional: { - ResultLocReturn *result_loc_ret = allocate(1); + ResultLocReturn *result_loc_ret = allocate(1, "ResultLocReturn"); result_loc_ret->base.id = ResultLocIdReturn; ir_build_reset_result(irb, scope, node, &result_loc_ret->base); @@ -3664,7 +3669,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val, nullptr)); IrInstructionSpillBegin *spill_begin = ir_build_spill_begin(irb, scope, node, err_val, SpillIdRetErrCode); - ResultLocReturn *result_loc_ret = allocate(1); + ResultLocReturn *result_loc_ret = allocate(1, "ResultLocReturn"); result_loc_ret->base.id = ResultLocIdReturn; ir_build_reset_result(irb, scope, node, &result_loc_ret->base); ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base); @@ -3692,7 +3697,7 @@ static ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_s Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime, bool skip_name_check) { - ZigVar *variable_entry = allocate(1); + ZigVar *variable_entry = allocate(1, "ZigVar"); variable_entry->parent_scope = parent_scope; variable_entry->shadowable = is_shadowable; variable_entry->mem_slot_index = SIZE_MAX; @@ -3767,7 +3772,7 @@ static ZigVar *ir_create_var(IrBuilder *irb, AstNode *node, Scope *scope, Buf *n } static ResultLocPeer *create_peer_result(ResultLocPeerParent *peer_parent) { - ResultLocPeer *result = allocate(1); + ResultLocPeer *result = allocate(1, "ResultLocPeer"); result->base.id = ResultLocIdPeer; result->base.source_instruction = peer_parent->base.source_instruction; result->parent = peer_parent; @@ -3806,7 +3811,7 @@ static IrInstruction *ir_gen_block(IrBuilder *irb, Scope *parent_scope, AstNode scope_block->is_comptime = ir_build_const_bool(irb, parent_scope, block_node, ir_should_inline(irb->exec, parent_scope)); - scope_block->peer_parent = allocate(1); + scope_block->peer_parent = allocate(1, "ResultLocPeerParent"); scope_block->peer_parent->base.id = ResultLocIdPeerParent; scope_block->peer_parent->base.source_instruction = scope_block->is_comptime; scope_block->peer_parent->end_bb = scope_block->end_block; @@ -3933,7 +3938,7 @@ static IrInstruction *ir_gen_assign(IrBuilder *irb, Scope *scope, AstNode *node) if (lvalue == irb->codegen->invalid_instruction) return irb->codegen->invalid_instruction; - ResultLocInstruction *result_loc_inst = allocate(1); + ResultLocInstruction *result_loc_inst = allocate(1, "ResultLocInstruction"); result_loc_inst->base.id = ResultLocIdInstruction; result_loc_inst->base.source_instruction = lvalue; ir_ref_instruction(lvalue, irb->current_basic_block); @@ -4005,10 +4010,10 @@ static IrInstruction *ir_gen_bool_or(IrBuilder *irb, Scope *scope, AstNode *node ir_set_cursor_at_end_and_append_block(irb, true_block); - IrInstruction **incoming_values = allocate(2); + IrInstruction **incoming_values = allocate(2, "IrInstruction *"); incoming_values[0] = val1; incoming_values[1] = val2; - IrBasicBlock **incoming_blocks = allocate(2); + IrBasicBlock **incoming_blocks = allocate(2, "IrBasicBlock *"); incoming_blocks[0] = post_val1_block; incoming_blocks[1] = post_val2_block; @@ -8017,7 +8022,8 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A err_set_type->abi_size = irb->codegen->builtin_types.entry_global_error_set->abi_size; err_set_type->data.error_set.errors = allocate(err_count); - ErrorTableEntry **errors = allocate(irb->codegen->errors_by_index.length + err_count); + size_t errors_count = irb->codegen->errors_by_index.length + err_count; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0; i < err_count; i += 1) { AstNode *field_node = node->data.err_set_decl.decls.at(i); @@ -8048,7 +8054,7 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A } errors[err->value] = err; } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return ir_build_const_type(irb, parent_scope, node, err_set_type); } @@ -9574,7 +9580,8 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp if (type_is_global_error_set(set2)) { return set1; } - ErrorTableEntry **errors = allocate(ira->codegen->errors_by_index.length); + size_t errors_count = ira->codegen->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); populate_error_set_table(errors, set1); ZigList intersection_list = {}; @@ -9595,7 +9602,7 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp buf_appendf(&err_set_type->name, "%s%s", comma, buf_ptr(&existing_entry_with_docs->name)); } } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); err_set_type->data.error_set.err_count = intersection_list.length; err_set_type->data.error_set.errors = intersection_list.items; @@ -9792,7 +9799,8 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted return result; } - ErrorTableEntry **errors = allocate(g->errors_by_index.length); + size_t errors_count = g->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0; i < container_set->data.error_set.err_count; i += 1) { ErrorTableEntry *error_entry = container_set->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); @@ -9809,7 +9817,7 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted result.data.error_set_mismatch->missing_errors.append(contained_error_entry); } } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return result; } @@ -14686,14 +14694,15 @@ static IrInstruction *ir_analyze_instruction_merge_err_sets(IrAnalyze *ira, return ira->codegen->invalid_instruction; } - ErrorTableEntry **errors = allocate(ira->codegen->errors_by_index.length); + size_t errors_count = ira->codegen->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0, count = op1_type->data.error_set.err_count; i < count; i += 1) { ErrorTableEntry *error_entry = op1_type->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); errors[error_entry->value] = error_entry; } ZigType *result_type = get_error_set_union(ira->codegen, errors, op1_type, op2_type, instruction->type_name); - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return ir_const_type(ira, &instruction->base, result_type); } @@ -24034,7 +24043,8 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, return ira->codegen->invalid_instruction; } - AstNode **field_prev_uses = allocate(ira->codegen->errors_by_index.length); + size_t field_prev_uses_count = ira->codegen->errors_by_index.length; + AstNode **field_prev_uses = allocate(field_prev_uses_count, "AstNode *"); for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) { IrInstructionCheckSwitchProngsRange *range = &instruction->ranges[range_i]; @@ -24091,7 +24101,7 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, } } - free(field_prev_uses); + deallocate(field_prev_uses, field_prev_uses_count, "AstNode *"); } else if (switch_type->id == ZigTypeIdInt) { RangeSet rs = {0}; for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) { diff --git a/src/ir_print.cpp b/src/ir_print.cpp index ecd8248d69ee36c5dbb8d92047631a05dcbafd24..441808e834fd91210e78bd65bd2cfd075773ca15 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -38,8 +38,8 @@ struct IrPrint { static void ir_print_other_instruction(IrPrint *irp, IrInstruction *instruction); -static const char* ir_instruction_type_str(IrInstruction* instruction) { - switch (instruction->id) { +const char* ir_instruction_type_str(IrInstructionId id) { + switch (id) { case IrInstructionIdInvalid: return "Invalid"; case IrInstructionIdShuffleVector: @@ -387,7 +387,7 @@ static void ir_print_prefix(IrPrint *irp, IrInstruction *instruction, bool trail const char *ref_count = ir_has_side_effects(instruction) ? "-" : buf_ptr(buf_sprintf("%" ZIG_PRI_usize "", instruction->ref_count)); fprintf(irp->f, "%c%-3zu| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, - ir_instruction_type_str(instruction), type_name, ref_count); + ir_instruction_type_str(instruction->id), type_name, ref_count); } static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) { diff --git a/src/ir_print.hpp b/src/ir_print.hpp index 0960af4e6fe8c103cab6d42c3a7b8f003d0a7bdc..e3947077c8f018a98da79922ef00559237c2308f 100644 --- a/src/ir_print.hpp +++ b/src/ir_print.hpp @@ -15,4 +15,6 @@ void ir_print(CodeGen *codegen, FILE *f, IrExecutable *executable, int indent_size, IrPass pass); void ir_print_instruction(CodeGen *codegen, FILE *f, IrInstruction *instruction, int indent_size, IrPass pass); +const char* ir_instruction_type_str(IrInstructionId id); + #endif diff --git a/src/main.cpp b/src/main.cpp index b398eec9911229015ed7b442107f3c2353411561..4709035859e3f17b7d197faffffd730ae3e8a5eb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -64,6 +64,9 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " -fno-PIC disable Position Independent Code\n" " -ftime-report print timing diagnostics\n" " -fstack-report print stack size diagnostics\n" +#ifdef ZIG_ENABLE_MEM_PROFILE + " -fmem-report print memory usage diagnostics\n" +#endif " -fdump-analysis write analysis.json file with type information\n" " -femit-docs create a docs/ dir with html documentation\n" " -fno-emit-bin skip emitting machine code\n" @@ -306,9 +309,29 @@ static int zig_error_no_build_file(void) { extern "C" int ZigClang_main(int argc, char **argv); +#ifdef ZIG_ENABLE_MEM_PROFILE +bool mem_report = false; +#endif + +int main_exit(Stage2ProgressNode *root_progress_node, int exit_code) { + if (root_progress_node != nullptr) { + stage2_progress_end(root_progress_node); + } +#ifdef ZIG_ENABLE_MEM_PROFILE + if (mem_report) { + memprof_dump_stats(stderr); + } +#endif + return exit_code; +} + int main(int argc, char **argv) { stage2_attach_segfault_handler(); +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_init(); +#endif + char *arg0 = argv[0]; Error err; @@ -670,6 +693,13 @@ int main(int argc, char **argv) { timing_info = true; } else if (strcmp(arg, "-fstack-report") == 0) { stack_report = true; + } else if (strcmp(arg, "-fmem-report") == 0) { +#ifdef ZIG_ENABLE_MEM_PROFILE + mem_report = true; +#else + fprintf(stderr, "-fmem-report requires configuring with -DZIG_ENABLE_MEM_PROFILE=ON\n"); + return print_error_usage(arg0); +#endif } else if (strcmp(arg, "-fdump-analysis") == 0) { enable_dump_analysis = true; } else if (strcmp(arg, "-femit-docs") == 0) { @@ -1038,16 +1068,14 @@ int main(int argc, char **argv) { if (in_file) { ZigLibCInstallation libc; if ((err = zig_libc_parse(&libc, buf_create_from_str(in_file), &target, true))) - return EXIT_FAILURE; - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_FAILURE); + return main_exit(root_progress_node, EXIT_SUCCESS); } ZigLibCInstallation libc; if ((err = zig_libc_find_native(&libc, true))) - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); zig_libc_render(&libc, stdout); - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdBuiltin: { CodeGen *g = codegen_create(main_pkg_path, nullptr, &target, @@ -1065,10 +1093,9 @@ int main(int argc, char **argv) { Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdRun: case CmdBuild: @@ -1142,7 +1169,7 @@ int main(int argc, char **argv) { libc = allocate(1); if ((err = zig_libc_parse(libc, buf_create_from_str(libc_txt), &target, true))) { fprintf(stderr, "Unable to parse --libc text file: %s\n", err_str(err)); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } } Buf *cache_dir_buf; @@ -1219,7 +1246,7 @@ int main(int argc, char **argv) { codegen_set_rdynamic(g, rdynamic); if (mmacosx_version_min && mios_version_min) { fprintf(stderr, "-mmacosx-version-min and -mios-version-min options not allowed together\n"); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } if (mmacosx_version_min) { @@ -1259,6 +1286,11 @@ int main(int argc, char **argv) { zig_print_stack_report(g, stdout); if (cmd == CmdRun) { + stage2_progress_end(root_progress_node); +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dump_stats(stderr); +#endif + const char *exec_path = buf_ptr(&g->output_file_path); ZigList args = {0}; @@ -1282,10 +1314,9 @@ int main(int argc, char **argv) { buf_replace(&g->output_file_path, '/', '\\'); #endif if (printf("%s\n", buf_ptr(&g->output_file_path)) < 0) - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } else { zig_unreachable(); } @@ -1293,8 +1324,7 @@ int main(int argc, char **argv) { codegen_translate_c(g, in_file_buf, stdout, cmd == CmdTranslateCUserland); if (timing_info) codegen_print_timing_report(g, stderr); - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } else if (cmd == CmdTest) { codegen_set_emit_file_type(g, emit_file_type); @@ -1314,7 +1344,7 @@ int main(int argc, char **argv) { if (g->disable_bin_generation) { fprintf(stderr, "Semantic analysis complete. No binary produced due to -fno-emit-bin.\n"); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } Buf *test_exe_path_unresolved = &g->output_file_path; @@ -1324,7 +1354,7 @@ int main(int argc, char **argv) { if (emit_file_type != EmitFileTypeBinary) { fprintf(stderr, "Created %s but skipping execution because it is non executable.\n", buf_ptr(test_exe_path)); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } for (size_t i = 0; i < test_exec_args.length; i += 1) { @@ -1336,7 +1366,7 @@ int main(int argc, char **argv) { if (!target_can_exec(&native, &target) && test_exec_args.length == 0) { fprintf(stderr, "Created %s but skipping execution because it is non-native.\n", buf_ptr(test_exe_path)); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } Termination term; @@ -1348,21 +1378,20 @@ int main(int argc, char **argv) { fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n"); fprintf(stderr, "%s\n", buf_ptr(test_exe_path)); } - stage2_progress_end(root_progress_node); - return (term.how == TerminationIdClean) ? term.code : -1; + return main_exit(root_progress_node, (term.how == TerminationIdClean) ? term.code : -1); } else { zig_unreachable(); } } case CmdVersion: printf("%s\n", ZIG_VERSION_STRING); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); case CmdZen: { const char *ptr; size_t len; stage2_zen(&ptr, &len); fwrite(ptr, len, 1, stdout); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdTargets: return print_target_list(stdout); diff --git a/src/memory_profiling.cpp b/src/memory_profiling.cpp new file mode 100644 index 0000000000000000000000000000000000000000..494ffae117ac82776922d2e091d1f43404e0eb48 --- /dev/null +++ b/src/memory_profiling.cpp @@ -0,0 +1,139 @@ +#include "memory_profiling.hpp" +#include "hash_map.hpp" +#include "list.hpp" +#include "util.hpp" +#include + +#ifdef ZIG_ENABLE_MEM_PROFILE + +static bool str_eql_str(const char *a, const char *b) { + return strcmp(a, b) == 0; +} + +static uint32_t str_hash(const char *s) { + // FNV 32-bit hash + uint32_t h = 2166136261; + for (; *s; s += 1) { + h = h ^ *s; + h = h * 16777619; + } + return h; +} + +struct CountAndSize { + size_t item_count; + size_t type_size; +}; + +ZigList unknown_names = {}; +HashMap usage_table = {}; +bool table_active = false; + + +static const char *get_default_name(const char *name_or_null, size_t type_size) { + if (name_or_null != nullptr) return name_or_null; + if (type_size >= unknown_names.length) { + table_active = false; + unknown_names.resize(type_size + 1); + table_active = true; + } + if (unknown_names.at(type_size) == nullptr) { + char buf[100]; + sprintf(buf, "Unknown_%zu%c", type_size, 0); + unknown_names.at(type_size) = strdup(buf); + } + return unknown_names.at(type_size); +} + +void memprof_alloc(const char *name, size_t count, size_t type_size) { + if (!table_active) return; + if (count == 0) return; + // temporarily disable during table put + table_active = false; + name = get_default_name(name, type_size); + auto existing_entry = usage_table.put_unique(name, {count, type_size}); + if (existing_entry != nullptr) { + assert(existing_entry->value.type_size == type_size); // allocated name does not match type + existing_entry->value.item_count += count; + } + table_active = true; +} + +void memprof_dealloc(const char *name, size_t count, size_t type_size) { + if (!table_active) return; + if (count == 0) return; + name = get_default_name(name, type_size); + auto existing_entry = usage_table.maybe_get(name); + if (existing_entry == nullptr) { + zig_panic("deallocated more than allocated; compromised memory usage stats"); + } + if (existing_entry->value.type_size != type_size) { + zig_panic("deallocated name '%s' does not match expected type size %zu", name, type_size); + } + existing_entry->value.item_count -= count; +} + +void memprof_init(void) { + usage_table.init(1024); + table_active = true; +} + +struct MemItem { + const char *type_name; + CountAndSize count_and_size; +}; + +static size_t get_bytes(const MemItem *item) { + return item->count_and_size.item_count * item->count_and_size.type_size; +} + +static int compare_bytes_desc(const void *a, const void *b) { + size_t size_a = get_bytes((const MemItem *)(a)); + size_t size_b = get_bytes((const MemItem *)(b)); + if (size_a > size_b) + return -1; + if (size_a < size_b) + return 1; + return 0; +} + +void memprof_dump_stats(FILE *file) { + assert(table_active); + // disable modifications from this function + table_active = false; + + ZigList list = {}; + + auto it = usage_table.entry_iterator(); + for (;;) { + auto *entry = it.next(); + if (!entry) + break; + + list.append({entry->key, entry->value}); + } + + qsort(list.items, list.length, sizeof(MemItem), compare_bytes_desc); + + size_t total_bytes_used = 0; + + for (size_t i = 0; i < list.length; i += 1) { + const MemItem *item = &list.at(i); + fprintf(file, "%s: %zu items, %zu bytes each, total ", item->type_name, + item->count_and_size.item_count, item->count_and_size.type_size); + size_t bytes = get_bytes(item); + zig_pretty_print_bytes(file, bytes); + fprintf(file, "\n"); + + total_bytes_used += bytes; + } + + fprintf(stderr, "Total bytes used: "); + zig_pretty_print_bytes(file, total_bytes_used); + fprintf(file, "\n"); + + list.deinit(); + table_active = true; +} + +#endif diff --git a/src/memory_profiling.hpp b/src/memory_profiling.hpp new file mode 100644 index 0000000000000000000000000000000000000000..6d43d81e3ccaaa6e2a2394d6cdd9223307962433 --- /dev/null +++ b/src/memory_profiling.hpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_MEMORY_PROFILING_HPP +#define ZIG_MEMORY_PROFILING_HPP + +#include "config.h" + +#include +#include + +void memprof_init(void); + +void memprof_alloc(const char *name, size_t item_count, size_t type_size); +void memprof_dealloc(const char *name, size_t item_count, size_t type_size); + +void memprof_dump_stats(FILE *file); +#endif diff --git a/src/util.cpp b/src/util.cpp index 13bfbbcd4715ad74b038e3b431863f700d18ba7f..055d5720100077299efdc453a639132b9f46f8ff 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -119,3 +119,21 @@ Slice SplitIterator_rest(SplitIterator *self) { SplitIterator memSplit(Slice buffer, Slice split_bytes) { return SplitIterator{0, buffer, split_bytes}; } + +void zig_pretty_print_bytes(FILE *f, double n) { + if (n > 1024.0 * 1024.0 * 1024.0) { + fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0); + return; + } + if (n > 1024.0 * 1024.0) { + fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0); + return; + } + if (n > 1024.0) { + fprintf(f, "%.02f KiB", n / 1024.0); + return; + } + fprintf(f, "%.02f bytes", n ); + return; +} + diff --git a/src/util.hpp b/src/util.hpp index 8abcef32ceec78241148729f4cbed96c228c13a2..79bebd3355695ad81974157d691fdf211eeda110 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -8,6 +8,8 @@ #ifndef ZIG_UTIL_HPP #define ZIG_UTIL_HPP +#include "memory_profiling.hpp" + #include #include #include @@ -96,7 +98,10 @@ static inline int ctzll(unsigned long long mask) { template -ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) { +ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_alloc(name, count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (count == 0) @@ -109,7 +114,10 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) { } template -ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) { +ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_alloc(name, count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (count == 0) @@ -122,7 +130,7 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) { } template -static inline T *reallocate(T *old, size_t old_count, size_t new_count) { +static inline T *reallocate(T *old, size_t old_count, size_t new_count, const char *name = nullptr) { T *ptr = reallocate_nonzero(old, old_count, new_count); if (new_count > old_count) { memset(&ptr[old_count], 0, (new_count - old_count) * sizeof(T)); @@ -131,7 +139,11 @@ static inline T *reallocate(T *old, size_t old_count, size_t new_count) { } template -static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) { +static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dealloc(name, old_count, sizeof(T)); + memprof_alloc(name, new_count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (new_count == 0 && old == nullptr) @@ -143,6 +155,19 @@ static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) return ptr; } +template +static inline void deallocate(T *old, size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dealloc(name, count, sizeof(T)); +#endif + free(old); +} + +template +static inline void destroy(T *old, const char *name = nullptr) { + return deallocate(old, 1); +} + template constexpr size_t array_length(const T (&)[n]) { return n; @@ -225,6 +250,8 @@ static inline double zig_f16_to_double(float16_t x) { return z; } +void zig_pretty_print_bytes(FILE *f, double n); + template struct Optional { T value; -- 2.54.0