add -DZIG_ENABLE_MEM_PROFILE option and -fmem-report flag

This can be used to find what's taking up memory in the compiler.
Here's an example of how to use it:

```
./zig test ../lib/std/std.zig --cache off -fmem-report
```

And here's the output I get for this on x86_64-linux-gnu today:

```
Const: 6462833 items, 152 bytes each, total 936.84 MiB
ConstGlobalRefs: 17236534 items, 24 bytes each, total 394.51 MiB
ResetResult: 1698108 items, 160 bytes each, total 259.11 MiB
ConstExprValue: 3118299 items, 80 bytes each, total 237.91 MiB
EndExpr: 1345395 items, 168 bytes each, total 215.56 MiB
Unknown_8: 27370821 items, 8 bytes each, total 208.82 MiB
VarPtr: 1127866 items, 168 bytes each, total 180.70 MiB
IrBasicBlock: 794834 items, 120 bytes each, total 90.96 MiB
LoadPtr: 554024 items, 160 bytes each, total 84.54 MiB
Unknown_64: 1245715 items, 64 bytes each, total 76.03 MiB
Unknown_40: 1879218 items, 40 bytes each, total 71.69 MiB
Unknown_72: 989117 items, 72 bytes each, total 67.92 MiB
Return: 423783 items, 160 bytes each, total 64.66 MiB
Unknown_168: 332480 items, 168 bytes each, total 53.27 MiB
Unknown_152: 336890 items, 152 bytes each, total 48.84 MiB
AddImplicitReturnType: 230819 items, 168 bytes each, total 36.98 MiB
Br: 217835 items, 168 bytes each, total 34.90 MiB
Unknown_184: 179529 items, 184 bytes each, total 31.50 MiB
FieldPtr: 179388 items, 184 bytes each, total 31.48 MiB
BinOp: 171004 items, 176 bytes each, total 28.70 MiB
LoadPtrGen: 173287 items, 168 bytes each, total 27.76 MiB
CondBr: 137864 items, 192 bytes each, total 25.24 MiB
Unknown_720: 30918 items, 720 bytes each, total 21.23 MiB
CallSrc: 99320 items, 216 bytes each, total 20.46 MiB
Unknown_160: 129243 items, 160 bytes each, total 19.72 MiB
Unknown_1: 19339456 items, 1 bytes each, total 18.44 MiB
CheckStatementIsVoid: 119838 items, 160 bytes each, total 18.29 MiB
Unknown_48: 371178 items, 48 bytes each, total 16.99 MiB
TestComptime: 101443 items, 160 bytes each, total 15.48 MiB
DeclVarSrc: 72578 items, 184 bytes each, total 12.74 MiB
StorePtr: 72776 items, 176 bytes each, total 12.22 MiB
ZigVar: 79201 items, 160 bytes each, total 12.09 MiB
Unknown_16: 770643 items, 16 bytes each, total 11.76 MiB
Phi: 60482 items, 184 bytes each, total 10.61 MiB
TestErrSrc: 66177 items, 168 bytes each, total 10.60 MiB
Unknown_240: 45164 items, 240 bytes each, total 10.34 MiB
ElemPtr: 58232 items, 184 bytes each, total 10.22 MiB
AllocaSrc: 60053 items, 176 bytes each, total 10.08 MiB
CallGen: 44873 items, 224 bytes each, total 9.59 MiB
SaveErrRetAddr: 63787 items, 152 bytes each, total 9.25 MiB
Unknown_112: 82283 items, 112 bytes each, total 8.79 MiB
AllocaGen: 51909 items, 176 bytes each, total 8.71 MiB
Unknown_24: 373599 items, 24 bytes each, total 8.55 MiB
ResultLocPeer: 113683 items, 72 bytes each, total 7.81 MiB
DeclRef: 36343 items, 168 bytes each, total 5.82 MiB
UnwrapErrPayload: 34603 items, 168 bytes each, total 5.54 MiB
Ref: 33414 items, 168 bytes each, total 5.35 MiB
Unknown_104: 53882 items, 104 bytes each, total 5.34 MiB
DeclVarGen: 32540 items, 168 bytes each, total 5.21 MiB
StructFieldPtr: 30449 items, 176 bytes each, total 5.11 MiB
UnwrapErrCode: 31508 items, 168 bytes each, total 5.05 MiB
Unknown_56: 90256 items, 56 bytes each, total 4.82 MiB
SpillBegin: 28722 items, 168 bytes each, total 4.60 MiB
SpillEnd: 28722 items, 160 bytes each, total 4.38 MiB
ResultLocReturn: 64573 items, 48 bytes each, total 2.96 MiB
PtrType: 14702 items, 184 bytes each, total 2.58 MiB
SliceType: 15005 items, 176 bytes each, total 2.52 MiB
Unknown_176: 13326 items, 176 bytes each, total 2.24 MiB
RefGen: 12881 items, 168 bytes each, total 2.06 MiB
UnOp: 12102 items, 176 bytes each, total 2.03 MiB
SwitchBr: 9453 items, 200 bytes each, total 1.80 MiB
TestErrGen: 11143 items, 160 bytes each, total 1.70 MiB
Unknown_32: 52359 items, 32 bytes each, total 1.60 MiB
CheckSwitchProngs: 9094 items, 184 bytes each, total 1.60 MiB
TypeOf: 9259 items, 160 bytes each, total 1.41 MiB
IntCast: 8772 items, 168 bytes each, total 1.41 MiB
OptionalUnwrapPtr: 8755 items, 168 bytes each, total 1.40 MiB
SwitchTarget: 9094 items, 160 bytes each, total 1.39 MiB
Cast: 8198 items, 176 bytes each, total 1.38 MiB
WidenOrShorten: 8448 items, 160 bytes each, total 1.29 MiB
ErrorUnion: 7613 items, 176 bytes each, total 1.28 MiB
SliceSrc: 6249 items, 192 bytes each, total 1.14 MiB
ErrWrapCode: 7133 items, 168 bytes each, total 1.14 MiB
TypeName: 7328 items, 160 bytes each, total 1.12 MiB
ImplicitCast: 5480 items, 176 bytes each, total 941.88 KiB
ResolveResult: 5638 items, 168 bytes each, total 924.98 KiB
ResultLocInstruction: 22696 items, 40 bytes each, total 886.56 KiB
BitCastSrc: 4947 items, 168 bytes each, total 811.62 KiB
CompileErr: 5148 items, 160 bytes each, total 804.38 KiB
ReturnPtr: 5305 items, 152 bytes each, total 787.46 KiB
Unreachable: 5038 items, 152 bytes each, total 747.83 KiB
TestNonNull: 4716 items, 160 bytes each, total 736.88 KiB
BitCastGen: 4431 items, 160 bytes each, total 692.34 KiB
PtrToInt: 4289 items, 160 bytes each, total 670.16 KiB
SliceGen: 3573 items, 192 bytes each, total 669.94 KiB
ArrayType: 4081 items, 168 bytes each, total 669.54 KiB
IntType: 3868 items, 168 bytes each, total 634.59 KiB
Unknown_88: 7213 items, 88 bytes each, total 619.87 KiB
Truncate: 3771 items, 168 bytes each, total 618.68 KiB
TypeInfo: 3740 items, 160 bytes each, total 584.38 KiB
SwitchVar: 3385 items, 176 bytes each, total 581.80 KiB
ContainerInitFields: 3223 items, 184 bytes each, total 579.13 KiB
ContainerInitList: 2309 items, 192 bytes each, total 432.94 KiB
PtrCastGen: 2626 items, 168 bytes each, total 430.83 KiB
BoolNot: 2457 items, 160 bytes each, total 383.91 KiB
FnProto: 2054 items, 184 bytes each, total 369.08 KiB
MergeErrSets: 1927 items, 176 bytes each, total 331.20 KiB
Unknown_136: 2486 items, 136 bytes each, total 330.17 KiB
Unknown_80: 4059 items, 80 bytes each, total 317.11 KiB
Bswap: 1670 items, 168 bytes each, total 273.98 KiB
TypeId: 1680 items, 160 bytes each, total 262.50 KiB
PtrCastSrc: 1371 items, 176 bytes each, total 235.64 KiB
ErrName: 1193 items, 160 bytes each, total 186.41 KiB
UnionTag: 1120 items, 160 bytes each, total 175.00 KiB
TagName: 1050 items, 160 bytes each, total 164.06 KiB
SizeOf: 942 items, 160 bytes each, total 147.19 KiB
MemberName: 871 items, 168 bytes each, total 142.90 KiB
Import: 881 items, 160 bytes each, total 137.66 KiB
PtrOfArrayToSlice: 758 items, 168 bytes each, total 124.36 KiB
UnionFieldPtr: 710 items, 176 bytes each, total 122.03 KiB
EnumToInt: 778 items, 160 bytes each, total 121.56 KiB
CheckRuntimeScope: 700 items, 168 bytes each, total 114.84 KiB
FieldParentPtr: 632 items, 184 bytes each, total 113.56 KiB
BoolToInt: 719 items, 160 bytes each, total 112.34 KiB
ResultLocPeerParent: 904 items, 104 bytes each, total 91.81 KiB
IntToPtr: 537 items, 168 bytes each, total 88.10 KiB
AlignOf: 561 items, 160 bytes each, total 87.66 KiB
AtomicRmw: 356 items, 208 bytes each, total 72.31 KiB
MemberCount: 441 items, 160 bytes each, total 68.91 KiB
Memset: 342 items, 176 bytes each, total 58.78 KiB
PopCount: 321 items, 168 bytes each, total 52.66 KiB
AlignCast: 251 items, 168 bytes each, total 41.18 KiB
IrInstruction *: 5230 items, 8 bytes each, total 40.86 KiB
IrBasicBlock *: 5230 items, 8 bytes each, total 40.86 KiB
TagType: 261 items, 160 bytes each, total 40.78 KiB
HasDecl: 234 items, 168 bytes each, total 38.39 KiB
OverflowOp: 191 items, 200 bytes each, total 37.30 KiB
Export: 209 items, 176 bytes each, total 35.92 KiB
SetCold: 219 items, 160 bytes each, total 34.22 KiB
ReturnAddress: 216 items, 152 bytes each, total 32.06 KiB
FromBytes: 178 items, 176 bytes each, total 30.59 KiB
SetRuntimeSafety: 188 items, 160 bytes each, total 29.38 KiB
OptionalWrap: 151 items, 168 bytes each, total 24.77 KiB
Clz: 143 items, 168 bytes each, total 23.46 KiB
ResizeSlice: 135 items, 168 bytes each, total 22.15 KiB
UnionInitNamedField: 106 items, 184 bytes each, total 19.05 KiB
Panic: 102 items, 160 bytes each, total 15.94 KiB
SwitchElseVar: 93 items, 168 bytes each, total 15.26 KiB
ToBytes: 89 items, 168 bytes each, total 14.60 KiB
IntToFloat: 78 items, 168 bytes each, total 12.80 KiB
Unknown_4360: 3 items, 4360 bytes each, total 12.77 KiB
ErrWrapPayload: 72 items, 168 bytes each, total 11.81 KiB
FloatOp: 62 items, 176 bytes each, total 10.66 KiB
FloatToInt: 47 items, 168 bytes each, total 7.71 KiB
FloatCast: 46 items, 168 bytes each, total 7.55 KiB
ErrToInt: 47 items, 160 bytes each, total 7.34 KiB
Asm: 33 items, 216 bytes each, total 6.96 KiB
ErrSetCast: 40 items, 168 bytes each, total 6.56 KiB
Memcpy: 34 items, 176 bytes each, total 5.84 KiB
AtomicLoad: 17 items, 184 bytes each, total 3.05 KiB
AwaitSrc: 16 items, 168 bytes each, total 2.62 KiB
Resume: 14 items, 160 bytes each, total 2.19 KiB
AwaitGen: 12 items, 176 bytes each, total 2.06 KiB
ArgType: 12 items, 168 bytes each, total 1.97 KiB
AnyFrameType: 12 items, 160 bytes each, total 1.88 KiB
SuspendFinish: 10 items, 160 bytes each, total 1.56 KiB
SuspendBegin: 10 items, 160 bytes each, total 1.56 KiB
Ctz: 9 items, 168 bytes each, total 1.48 KiB
FrameHandle: 8 items, 152 bytes each, total 1.19 KiB
SetEvalBranchQuota: 7 items, 160 bytes each, total 1.09 KiB
AssertZero: 7 items, 160 bytes each, total 1.09 KiB
UndeclaredIdent: 7 items, 160 bytes each, total 1.09 KiB
CmpxchgSrc: 5 items, 216 bytes each, total 1.05 KiB
CmpxchgGen: 5 items, 200 bytes each, total 1000.00 bytes
IntToEnum: 4 items, 168 bytes each, total 672.00 bytes
VectorType: 4 items, 168 bytes each, total 672.00 bytes
ErrorReturnTrace: 2 items, 160 bytes each, total 320.00 bytes
Breakpoint: 2 items, 152 bytes each, total 304.00 bytes
FrameAddress: 2 items, 152 bytes each, total 304.00 bytes
Unknown_4: 61 items, 4 bytes each, total 244.00 bytes
VectorToArray: 1 items, 168 bytes each, total 168.00 bytes
SetAlignStack: 1 items, 160 bytes each, total 160.00 bytes
Unknown_12: 2 items, 12 bytes each, total 24.00 bytes
ErrorTableEntry *: 0 items, 8 bytes each, total 0.00 bytes
AstNode *: 0 items, 8 bytes each, total 0.00 bytes
Total bytes used: 3.51 GiB
```

You can see that most of the memory is taken up by IR instructions,
as well as comptime values. This points toward 2 changes which will
greatly reduce memory usage:

 * Rework semantic analysis so that IR instructions can be freed.
   Currently the comptime value struct (ConstExprValue) is embedded
   directly into the IrInstruction struct. If this is made to be
   separate, at the very least pass 1 IR instructions can be freed.
   This includes `Const` which is the largest usage of memory currently.

 * Rework the ConstExprValue struct to no longer be a tagged union.
   For example, there's no need for an integer comptime value to be
   80 bytes.

From this you can also see that this eliminates some things from being
the culprit. Before doing this analysis, I considered whether doing
string interning would help. From the above output, you can see that all
strings in the compiler account for only 18 MiB, so string interning
would have been a dead end.
master
Andrew Kelley 2019-10-19 03:23:23 -04:00
parent e42d86b657
commit fa9f1d2396
No known key found for this signature in database
GPG Key ID: 7C5F548F728501A9
12 changed files with 305 additions and 74 deletions

View File

@ -46,6 +46,7 @@ message("Configuring zig version ${ZIG_VERSION}")
set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)")
set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries")
set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL "Disable copying lib/ files to install prefix")
set(ZIG_ENABLE_MEM_PROFILE off CACHE BOOL "Activate memory usage instrumentation")
if(ZIG_STATIC)
set(ZIG_STATIC_LLVM "on")
@ -455,6 +456,7 @@ set(ZIG_SOURCES
"${CMAKE_SOURCE_DIR}/src/ir_print.cpp"
"${CMAKE_SOURCE_DIR}/src/libc_installation.cpp"
"${CMAKE_SOURCE_DIR}/src/link.cpp"
"${CMAKE_SOURCE_DIR}/src/memory_profiling.cpp"
"${CMAKE_SOURCE_DIR}/src/os.cpp"
"${CMAKE_SOURCE_DIR}/src/parser.cpp"
"${CMAKE_SOURCE_DIR}/src/range_set.cpp"

View File

@ -5783,8 +5783,8 @@ ConstExprValue *create_const_arg_tuple(CodeGen *g, size_t arg_index_start, size_
ConstExprValue *create_const_vals(size_t count) {
ConstGlobalRefs *global_refs = allocate<ConstGlobalRefs>(count);
ConstExprValue *vals = allocate<ConstExprValue>(count);
ConstGlobalRefs *global_refs = allocate<ConstGlobalRefs>(count, "ConstGlobalRefs");
ConstExprValue *vals = allocate<ConstExprValue>(count, "ConstExprValue");
for (size_t i = 0; i < count; i += 1) {
vals[i].global_refs = &global_refs[i];
}

View File

@ -13,9 +13,6 @@
#define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@
#define ZIG_VERSION_STRING "@ZIG_VERSION@"
// Only used for running tests before installing.
#define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test"
// Used for communicating build information to self hosted build.
#define ZIG_CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@"
#define ZIG_CXX_COMPILER "@CMAKE_CXX_COMPILER@"
@ -24,4 +21,6 @@
#define ZIG_LLVM_CONFIG_EXE "@LLVM_CONFIG_EXE@"
#define ZIG_DIA_GUIDS_LIB "@ZIG_DIA_GUIDS_LIB_ESCAPED@"
#cmakedefine ZIG_ENABLE_MEM_PROFILE
#endif

View File

@ -240,23 +240,6 @@ static void jw_string(JsonWriter *jw, const char *s) {
static void tree_print(FILE *f, ZigType *ty, size_t indent);
static void pretty_print_bytes(FILE *f, double n) {
if (n > 1024.0 * 1024.0 * 1024.0) {
fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0);
return;
}
if (n > 1024.0 * 1024.0) {
fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0);
return;
}
if (n > 1024.0) {
fprintf(f, "%.02f KiB", n / 1024.0);
return;
}
fprintf(f, "%.02f bytes", n );
return;
}
static int compare_type_abi_sizes_desc(const void *a, const void *b) {
uint64_t size_a = (*(ZigType * const*)(a))->abi_size;
uint64_t size_b = (*(ZigType * const*)(b))->abi_size;
@ -322,7 +305,7 @@ static void tree_print(FILE *f, ZigType *ty, size_t indent) {
start_peer(f, indent);
fprintf(f, "\"sizef\": \"");
pretty_print_bytes(f, ty->abi_size);
zig_pretty_print_bytes(f, ty->abi_size);
fprintf(f, "\"");
start_peer(f, indent);

View File

@ -413,7 +413,7 @@ ZigType *ir_analyze_type_expr(IrAnalyze *ira, Scope *scope, AstNode *node) {
}
static IrBasicBlock *ir_create_basic_block(IrBuilder *irb, Scope *scope, const char *name_hint) {
IrBasicBlock *result = allocate<IrBasicBlock>(1);
IrBasicBlock *result = allocate<IrBasicBlock>(1, "IrBasicBlock");
result->scope = scope;
result->name_hint = name_hint;
result->debug_id = exec_next_debug_id(irb->exec);
@ -1085,13 +1085,18 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSpillEnd *) {
template<typename T>
static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) {
T *special_instruction = allocate<T>(1);
const char *name = nullptr;
#ifdef ZIG_ENABLE_MEM_PROFILE
T *dummy = nullptr;
name = ir_instruction_type_str(ir_instruction_id(dummy));
#endif
T *special_instruction = allocate<T>(1, name);
special_instruction->base.id = ir_instruction_id(special_instruction);
special_instruction->base.scope = scope;
special_instruction->base.source_node = source_node;
special_instruction->base.debug_id = exec_next_debug_id(irb->exec);
special_instruction->base.owner_bb = irb->current_basic_block;
special_instruction->base.value.global_refs = allocate<ConstGlobalRefs>(1);
special_instruction->base.value.global_refs = allocate<ConstGlobalRefs>(1, "ConstGlobalRefs");
return special_instruction;
}
@ -3569,7 +3574,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
switch (node->data.return_expr.kind) {
case ReturnKindUnconditional:
{
ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1);
ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1, "ResultLocReturn");
result_loc_ret->base.id = ResultLocIdReturn;
ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
@ -3664,7 +3669,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val, nullptr));
IrInstructionSpillBegin *spill_begin = ir_build_spill_begin(irb, scope, node, err_val,
SpillIdRetErrCode);
ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1);
ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1, "ResultLocReturn");
result_loc_ret->base.id = ResultLocIdReturn;
ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base);
@ -3692,7 +3697,7 @@ static ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_s
Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime,
bool skip_name_check)
{
ZigVar *variable_entry = allocate<ZigVar>(1);
ZigVar *variable_entry = allocate<ZigVar>(1, "ZigVar");
variable_entry->parent_scope = parent_scope;
variable_entry->shadowable = is_shadowable;
variable_entry->mem_slot_index = SIZE_MAX;
@ -3767,7 +3772,7 @@ static ZigVar *ir_create_var(IrBuilder *irb, AstNode *node, Scope *scope, Buf *n
}
static ResultLocPeer *create_peer_result(ResultLocPeerParent *peer_parent) {
ResultLocPeer *result = allocate<ResultLocPeer>(1);
ResultLocPeer *result = allocate<ResultLocPeer>(1, "ResultLocPeer");
result->base.id = ResultLocIdPeer;
result->base.source_instruction = peer_parent->base.source_instruction;
result->parent = peer_parent;
@ -3806,7 +3811,7 @@ static IrInstruction *ir_gen_block(IrBuilder *irb, Scope *parent_scope, AstNode
scope_block->is_comptime = ir_build_const_bool(irb, parent_scope, block_node,
ir_should_inline(irb->exec, parent_scope));
scope_block->peer_parent = allocate<ResultLocPeerParent>(1);
scope_block->peer_parent = allocate<ResultLocPeerParent>(1, "ResultLocPeerParent");
scope_block->peer_parent->base.id = ResultLocIdPeerParent;
scope_block->peer_parent->base.source_instruction = scope_block->is_comptime;
scope_block->peer_parent->end_bb = scope_block->end_block;
@ -3933,7 +3938,7 @@ static IrInstruction *ir_gen_assign(IrBuilder *irb, Scope *scope, AstNode *node)
if (lvalue == irb->codegen->invalid_instruction)
return irb->codegen->invalid_instruction;
ResultLocInstruction *result_loc_inst = allocate<ResultLocInstruction>(1);
ResultLocInstruction *result_loc_inst = allocate<ResultLocInstruction>(1, "ResultLocInstruction");
result_loc_inst->base.id = ResultLocIdInstruction;
result_loc_inst->base.source_instruction = lvalue;
ir_ref_instruction(lvalue, irb->current_basic_block);
@ -4005,10 +4010,10 @@ static IrInstruction *ir_gen_bool_or(IrBuilder *irb, Scope *scope, AstNode *node
ir_set_cursor_at_end_and_append_block(irb, true_block);
IrInstruction **incoming_values = allocate<IrInstruction *>(2);
IrInstruction **incoming_values = allocate<IrInstruction *>(2, "IrInstruction *");
incoming_values[0] = val1;
incoming_values[1] = val2;
IrBasicBlock **incoming_blocks = allocate<IrBasicBlock *>(2);
IrBasicBlock **incoming_blocks = allocate<IrBasicBlock *>(2, "IrBasicBlock *");
incoming_blocks[0] = post_val1_block;
incoming_blocks[1] = post_val2_block;
@ -8017,7 +8022,8 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A
err_set_type->abi_size = irb->codegen->builtin_types.entry_global_error_set->abi_size;
err_set_type->data.error_set.errors = allocate<ErrorTableEntry *>(err_count);
ErrorTableEntry **errors = allocate<ErrorTableEntry *>(irb->codegen->errors_by_index.length + err_count);
size_t errors_count = irb->codegen->errors_by_index.length + err_count;
ErrorTableEntry **errors = allocate<ErrorTableEntry *>(errors_count, "ErrorTableEntry *");
for (uint32_t i = 0; i < err_count; i += 1) {
AstNode *field_node = node->data.err_set_decl.decls.at(i);
@ -8048,7 +8054,7 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A
}
errors[err->value] = err;
}
free(errors);
deallocate(errors, errors_count, "ErrorTableEntry *");
return ir_build_const_type(irb, parent_scope, node, err_set_type);
}
@ -9574,7 +9580,8 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp
if (type_is_global_error_set(set2)) {
return set1;
}
ErrorTableEntry **errors = allocate<ErrorTableEntry *>(ira->codegen->errors_by_index.length);
size_t errors_count = ira->codegen->errors_by_index.length;
ErrorTableEntry **errors = allocate<ErrorTableEntry *>(errors_count, "ErrorTableEntry *");
populate_error_set_table(errors, set1);
ZigList<ErrorTableEntry *> intersection_list = {};
@ -9595,7 +9602,7 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp
buf_appendf(&err_set_type->name, "%s%s", comma, buf_ptr(&existing_entry_with_docs->name));
}
}
free(errors);
deallocate(errors, errors_count, "ErrorTableEntry *");
err_set_type->data.error_set.err_count = intersection_list.length;
err_set_type->data.error_set.errors = intersection_list.items;
@ -9792,7 +9799,8 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted
return result;
}
ErrorTableEntry **errors = allocate<ErrorTableEntry *>(g->errors_by_index.length);
size_t errors_count = g->errors_by_index.length;
ErrorTableEntry **errors = allocate<ErrorTableEntry *>(errors_count, "ErrorTableEntry *");
for (uint32_t i = 0; i < container_set->data.error_set.err_count; i += 1) {
ErrorTableEntry *error_entry = container_set->data.error_set.errors[i];
assert(errors[error_entry->value] == nullptr);
@ -9809,7 +9817,7 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted
result.data.error_set_mismatch->missing_errors.append(contained_error_entry);
}
}
free(errors);
deallocate(errors, errors_count, "ErrorTableEntry *");
return result;
}
@ -14686,14 +14694,15 @@ static IrInstruction *ir_analyze_instruction_merge_err_sets(IrAnalyze *ira,
return ira->codegen->invalid_instruction;
}
ErrorTableEntry **errors = allocate<ErrorTableEntry *>(ira->codegen->errors_by_index.length);
size_t errors_count = ira->codegen->errors_by_index.length;
ErrorTableEntry **errors = allocate<ErrorTableEntry *>(errors_count, "ErrorTableEntry *");
for (uint32_t i = 0, count = op1_type->data.error_set.err_count; i < count; i += 1) {
ErrorTableEntry *error_entry = op1_type->data.error_set.errors[i];
assert(errors[error_entry->value] == nullptr);
errors[error_entry->value] = error_entry;
}
ZigType *result_type = get_error_set_union(ira->codegen, errors, op1_type, op2_type, instruction->type_name);
free(errors);
deallocate(errors, errors_count, "ErrorTableEntry *");
return ir_const_type(ira, &instruction->base, result_type);
}
@ -24034,7 +24043,8 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira,
return ira->codegen->invalid_instruction;
}
AstNode **field_prev_uses = allocate<AstNode *>(ira->codegen->errors_by_index.length);
size_t field_prev_uses_count = ira->codegen->errors_by_index.length;
AstNode **field_prev_uses = allocate<AstNode *>(field_prev_uses_count, "AstNode *");
for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) {
IrInstructionCheckSwitchProngsRange *range = &instruction->ranges[range_i];
@ -24091,7 +24101,7 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira,
}
}
free(field_prev_uses);
deallocate(field_prev_uses, field_prev_uses_count, "AstNode *");
} else if (switch_type->id == ZigTypeIdInt) {
RangeSet rs = {0};
for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) {

View File

@ -38,8 +38,8 @@ struct IrPrint {
static void ir_print_other_instruction(IrPrint *irp, IrInstruction *instruction);
static const char* ir_instruction_type_str(IrInstruction* instruction) {
switch (instruction->id) {
const char* ir_instruction_type_str(IrInstructionId id) {
switch (id) {
case IrInstructionIdInvalid:
return "Invalid";
case IrInstructionIdShuffleVector:
@ -387,7 +387,7 @@ static void ir_print_prefix(IrPrint *irp, IrInstruction *instruction, bool trail
const char *ref_count = ir_has_side_effects(instruction) ?
"-" : buf_ptr(buf_sprintf("%" ZIG_PRI_usize "", instruction->ref_count));
fprintf(irp->f, "%c%-3zu| %-22s| %-12s| %-2s| ", mark, instruction->debug_id,
ir_instruction_type_str(instruction), type_name, ref_count);
ir_instruction_type_str(instruction->id), type_name, ref_count);
}
static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) {

View File

@ -15,4 +15,6 @@
void ir_print(CodeGen *codegen, FILE *f, IrExecutable *executable, int indent_size, IrPass pass);
void ir_print_instruction(CodeGen *codegen, FILE *f, IrInstruction *instruction, int indent_size, IrPass pass);
const char* ir_instruction_type_str(IrInstructionId id);
#endif

View File

@ -64,6 +64,9 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) {
" -fno-PIC disable Position Independent Code\n"
" -ftime-report print timing diagnostics\n"
" -fstack-report print stack size diagnostics\n"
#ifdef ZIG_ENABLE_MEM_PROFILE
" -fmem-report print memory usage diagnostics\n"
#endif
" -fdump-analysis write analysis.json file with type information\n"
" -femit-docs create a docs/ dir with html documentation\n"
" -fno-emit-bin skip emitting machine code\n"
@ -306,9 +309,29 @@ static int zig_error_no_build_file(void) {
extern "C" int ZigClang_main(int argc, char **argv);
#ifdef ZIG_ENABLE_MEM_PROFILE
bool mem_report = false;
#endif
int main_exit(Stage2ProgressNode *root_progress_node, int exit_code) {
if (root_progress_node != nullptr) {
stage2_progress_end(root_progress_node);
}
#ifdef ZIG_ENABLE_MEM_PROFILE
if (mem_report) {
memprof_dump_stats(stderr);
}
#endif
return exit_code;
}
int main(int argc, char **argv) {
stage2_attach_segfault_handler();
#ifdef ZIG_ENABLE_MEM_PROFILE
memprof_init();
#endif
char *arg0 = argv[0];
Error err;
@ -670,6 +693,13 @@ int main(int argc, char **argv) {
timing_info = true;
} else if (strcmp(arg, "-fstack-report") == 0) {
stack_report = true;
} else if (strcmp(arg, "-fmem-report") == 0) {
#ifdef ZIG_ENABLE_MEM_PROFILE
mem_report = true;
#else
fprintf(stderr, "-fmem-report requires configuring with -DZIG_ENABLE_MEM_PROFILE=ON\n");
return print_error_usage(arg0);
#endif
} else if (strcmp(arg, "-fdump-analysis") == 0) {
enable_dump_analysis = true;
} else if (strcmp(arg, "-femit-docs") == 0) {
@ -1038,16 +1068,14 @@ int main(int argc, char **argv) {
if (in_file) {
ZigLibCInstallation libc;
if ((err = zig_libc_parse(&libc, buf_create_from_str(in_file), &target, true)))
return EXIT_FAILURE;
stage2_progress_end(root_progress_node);
return EXIT_SUCCESS;
return main_exit(root_progress_node, EXIT_FAILURE);
return main_exit(root_progress_node, EXIT_SUCCESS);
}
ZigLibCInstallation libc;
if ((err = zig_libc_find_native(&libc, true)))
return EXIT_FAILURE;
return main_exit(root_progress_node, EXIT_FAILURE);
zig_libc_render(&libc, stdout);
stage2_progress_end(root_progress_node);
return EXIT_SUCCESS;
return main_exit(root_progress_node, EXIT_SUCCESS);
}
case CmdBuiltin: {
CodeGen *g = codegen_create(main_pkg_path, nullptr, &target,
@ -1065,10 +1093,9 @@ int main(int argc, char **argv) {
Buf *builtin_source = codegen_generate_builtin_source(g);
if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) {
fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout)));
return EXIT_FAILURE;
return main_exit(root_progress_node, EXIT_FAILURE);
}
stage2_progress_end(root_progress_node);
return EXIT_SUCCESS;
return main_exit(root_progress_node, EXIT_SUCCESS);
}
case CmdRun:
case CmdBuild:
@ -1142,7 +1169,7 @@ int main(int argc, char **argv) {
libc = allocate<ZigLibCInstallation>(1);
if ((err = zig_libc_parse(libc, buf_create_from_str(libc_txt), &target, true))) {
fprintf(stderr, "Unable to parse --libc text file: %s\n", err_str(err));
return EXIT_FAILURE;
return main_exit(root_progress_node, EXIT_FAILURE);
}
}
Buf *cache_dir_buf;
@ -1219,7 +1246,7 @@ int main(int argc, char **argv) {
codegen_set_rdynamic(g, rdynamic);
if (mmacosx_version_min && mios_version_min) {
fprintf(stderr, "-mmacosx-version-min and -mios-version-min options not allowed together\n");
return EXIT_FAILURE;
return main_exit(root_progress_node, EXIT_FAILURE);
}
if (mmacosx_version_min) {
@ -1259,6 +1286,11 @@ int main(int argc, char **argv) {
zig_print_stack_report(g, stdout);
if (cmd == CmdRun) {
stage2_progress_end(root_progress_node);
#ifdef ZIG_ENABLE_MEM_PROFILE
memprof_dump_stats(stderr);
#endif
const char *exec_path = buf_ptr(&g->output_file_path);
ZigList<const char*> args = {0};
@ -1282,10 +1314,9 @@ int main(int argc, char **argv) {
buf_replace(&g->output_file_path, '/', '\\');
#endif
if (printf("%s\n", buf_ptr(&g->output_file_path)) < 0)
return EXIT_FAILURE;
return main_exit(root_progress_node, EXIT_FAILURE);
}
stage2_progress_end(root_progress_node);
return EXIT_SUCCESS;
return main_exit(root_progress_node, EXIT_SUCCESS);
} else {
zig_unreachable();
}
@ -1293,8 +1324,7 @@ int main(int argc, char **argv) {
codegen_translate_c(g, in_file_buf, stdout, cmd == CmdTranslateCUserland);
if (timing_info)
codegen_print_timing_report(g, stderr);
stage2_progress_end(root_progress_node);
return EXIT_SUCCESS;
return main_exit(root_progress_node, EXIT_SUCCESS);
} else if (cmd == CmdTest) {
codegen_set_emit_file_type(g, emit_file_type);
@ -1314,7 +1344,7 @@ int main(int argc, char **argv) {
if (g->disable_bin_generation) {
fprintf(stderr, "Semantic analysis complete. No binary produced due to -fno-emit-bin.\n");
return 0;
return main_exit(root_progress_node, EXIT_SUCCESS);
}
Buf *test_exe_path_unresolved = &g->output_file_path;
@ -1324,7 +1354,7 @@ int main(int argc, char **argv) {
if (emit_file_type != EmitFileTypeBinary) {
fprintf(stderr, "Created %s but skipping execution because it is non executable.\n",
buf_ptr(test_exe_path));
return 0;
return main_exit(root_progress_node, EXIT_SUCCESS);
}
for (size_t i = 0; i < test_exec_args.length; i += 1) {
@ -1336,7 +1366,7 @@ int main(int argc, char **argv) {
if (!target_can_exec(&native, &target) && test_exec_args.length == 0) {
fprintf(stderr, "Created %s but skipping execution because it is non-native.\n",
buf_ptr(test_exe_path));
return 0;
return main_exit(root_progress_node, EXIT_SUCCESS);
}
Termination term;
@ -1348,21 +1378,20 @@ int main(int argc, char **argv) {
fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n");
fprintf(stderr, "%s\n", buf_ptr(test_exe_path));
}
stage2_progress_end(root_progress_node);
return (term.how == TerminationIdClean) ? term.code : -1;
return main_exit(root_progress_node, (term.how == TerminationIdClean) ? term.code : -1);
} else {
zig_unreachable();
}
}
case CmdVersion:
printf("%s\n", ZIG_VERSION_STRING);
return EXIT_SUCCESS;
return main_exit(root_progress_node, EXIT_SUCCESS);
case CmdZen: {
const char *ptr;
size_t len;
stage2_zen(&ptr, &len);
fwrite(ptr, len, 1, stdout);
return EXIT_SUCCESS;
return main_exit(root_progress_node, EXIT_SUCCESS);
}
case CmdTargets:
return print_target_list(stdout);

139
src/memory_profiling.cpp Normal file
View File

@ -0,0 +1,139 @@
#include "memory_profiling.hpp"
#include "hash_map.hpp"
#include "list.hpp"
#include "util.hpp"
#include <string.h>
#ifdef ZIG_ENABLE_MEM_PROFILE
static bool str_eql_str(const char *a, const char *b) {
return strcmp(a, b) == 0;
}
static uint32_t str_hash(const char *s) {
// FNV 32-bit hash
uint32_t h = 2166136261;
for (; *s; s += 1) {
h = h ^ *s;
h = h * 16777619;
}
return h;
}
struct CountAndSize {
size_t item_count;
size_t type_size;
};
ZigList<const char *> unknown_names = {};
HashMap<const char *, CountAndSize, str_hash, str_eql_str> usage_table = {};
bool table_active = false;
static const char *get_default_name(const char *name_or_null, size_t type_size) {
if (name_or_null != nullptr) return name_or_null;
if (type_size >= unknown_names.length) {
table_active = false;
unknown_names.resize(type_size + 1);
table_active = true;
}
if (unknown_names.at(type_size) == nullptr) {
char buf[100];
sprintf(buf, "Unknown_%zu%c", type_size, 0);
unknown_names.at(type_size) = strdup(buf);
}
return unknown_names.at(type_size);
}
void memprof_alloc(const char *name, size_t count, size_t type_size) {
if (!table_active) return;
if (count == 0) return;
// temporarily disable during table put
table_active = false;
name = get_default_name(name, type_size);
auto existing_entry = usage_table.put_unique(name, {count, type_size});
if (existing_entry != nullptr) {
assert(existing_entry->value.type_size == type_size); // allocated name does not match type
existing_entry->value.item_count += count;
}
table_active = true;
}
void memprof_dealloc(const char *name, size_t count, size_t type_size) {
if (!table_active) return;
if (count == 0) return;
name = get_default_name(name, type_size);
auto existing_entry = usage_table.maybe_get(name);
if (existing_entry == nullptr) {
zig_panic("deallocated more than allocated; compromised memory usage stats");
}
if (existing_entry->value.type_size != type_size) {
zig_panic("deallocated name '%s' does not match expected type size %zu", name, type_size);
}
existing_entry->value.item_count -= count;
}
void memprof_init(void) {
usage_table.init(1024);
table_active = true;
}
struct MemItem {
const char *type_name;
CountAndSize count_and_size;
};
static size_t get_bytes(const MemItem *item) {
return item->count_and_size.item_count * item->count_and_size.type_size;
}
static int compare_bytes_desc(const void *a, const void *b) {
size_t size_a = get_bytes((const MemItem *)(a));
size_t size_b = get_bytes((const MemItem *)(b));
if (size_a > size_b)
return -1;
if (size_a < size_b)
return 1;
return 0;
}
void memprof_dump_stats(FILE *file) {
assert(table_active);
// disable modifications from this function
table_active = false;
ZigList<MemItem> list = {};
auto it = usage_table.entry_iterator();
for (;;) {
auto *entry = it.next();
if (!entry)
break;
list.append({entry->key, entry->value});
}
qsort(list.items, list.length, sizeof(MemItem), compare_bytes_desc);
size_t total_bytes_used = 0;
for (size_t i = 0; i < list.length; i += 1) {
const MemItem *item = &list.at(i);
fprintf(file, "%s: %zu items, %zu bytes each, total ", item->type_name,
item->count_and_size.item_count, item->count_and_size.type_size);
size_t bytes = get_bytes(item);
zig_pretty_print_bytes(file, bytes);
fprintf(file, "\n");
total_bytes_used += bytes;
}
fprintf(stderr, "Total bytes used: ");
zig_pretty_print_bytes(file, total_bytes_used);
fprintf(file, "\n");
list.deinit();
table_active = true;
}
#endif

22
src/memory_profiling.hpp Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright (c) 2019 Andrew Kelley
*
* This file is part of zig, which is MIT licensed.
* See http://opensource.org/licenses/MIT
*/
#ifndef ZIG_MEMORY_PROFILING_HPP
#define ZIG_MEMORY_PROFILING_HPP
#include "config.h"
#include <stddef.h>
#include <stdio.h>
void memprof_init(void);
void memprof_alloc(const char *name, size_t item_count, size_t type_size);
void memprof_dealloc(const char *name, size_t item_count, size_t type_size);
void memprof_dump_stats(FILE *file);
#endif

View File

@ -119,3 +119,21 @@ Slice<uint8_t> SplitIterator_rest(SplitIterator *self) {
SplitIterator memSplit(Slice<uint8_t> buffer, Slice<uint8_t> split_bytes) {
return SplitIterator{0, buffer, split_bytes};
}
void zig_pretty_print_bytes(FILE *f, double n) {
if (n > 1024.0 * 1024.0 * 1024.0) {
fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0);
return;
}
if (n > 1024.0 * 1024.0) {
fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0);
return;
}
if (n > 1024.0) {
fprintf(f, "%.02f KiB", n / 1024.0);
return;
}
fprintf(f, "%.02f bytes", n );
return;
}

View File

@ -8,6 +8,8 @@
#ifndef ZIG_UTIL_HPP
#define ZIG_UTIL_HPP
#include "memory_profiling.hpp"
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
@ -96,7 +98,10 @@ static inline int ctzll(unsigned long long mask) {
template<typename T>
ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) {
ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count, const char *name = nullptr) {
#ifdef ZIG_ENABLE_MEM_PROFILE
memprof_alloc(name, count, sizeof(T));
#endif
#ifndef NDEBUG
// make behavior when size == 0 portable
if (count == 0)
@ -109,7 +114,10 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) {
}
template<typename T>
ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) {
ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count, const char *name = nullptr) {
#ifdef ZIG_ENABLE_MEM_PROFILE
memprof_alloc(name, count, sizeof(T));
#endif
#ifndef NDEBUG
// make behavior when size == 0 portable
if (count == 0)
@ -122,7 +130,7 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) {
}
template<typename T>
static inline T *reallocate(T *old, size_t old_count, size_t new_count) {
static inline T *reallocate(T *old, size_t old_count, size_t new_count, const char *name = nullptr) {
T *ptr = reallocate_nonzero(old, old_count, new_count);
if (new_count > old_count) {
memset(&ptr[old_count], 0, (new_count - old_count) * sizeof(T));
@ -131,7 +139,11 @@ static inline T *reallocate(T *old, size_t old_count, size_t new_count) {
}
template<typename T>
static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) {
static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count, const char *name = nullptr) {
#ifdef ZIG_ENABLE_MEM_PROFILE
memprof_dealloc(name, old_count, sizeof(T));
memprof_alloc(name, new_count, sizeof(T));
#endif
#ifndef NDEBUG
// make behavior when size == 0 portable
if (new_count == 0 && old == nullptr)
@ -143,6 +155,19 @@ static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count)
return ptr;
}
template<typename T>
static inline void deallocate(T *old, size_t count, const char *name = nullptr) {
#ifdef ZIG_ENABLE_MEM_PROFILE
memprof_dealloc(name, count, sizeof(T));
#endif
free(old);
}
template<typename T>
static inline void destroy(T *old, const char *name = nullptr) {
return deallocate(old, 1);
}
template <typename T, size_t n>
constexpr size_t array_length(const T (&)[n]) {
return n;
@ -225,6 +250,8 @@ static inline double zig_f16_to_double(float16_t x) {
return z;
}
void zig_pretty_print_bytes(FILE *f, double n);
template<typename T>
struct Optional {
T value;