Zylann/godot_voxel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added basic stack tracking for memory pool allocations

Browse Source
This commit is contained in:
Marc Gilleron 2022-06-06 21:54:20 +01:00
parent 2098ca36d9
commit 177d7a99c9
14 changed files with 195 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
edition/voxel_mesh_sdf_gd.cpp
View File

@ -2,6 +2,7 @@
#include "../server/voxel_server.h"
#include "../server/voxel_server_updater.h"
#include "../storage/voxel_buffer_gd.h"
#include "../util/dstack.h"
#include "../util/godot/funcs.h"
#include "../util/math/color.h"
#include "../util/math/conv.h"
@ -92,6 +93,7 @@ Ref<Mesh> VoxelMeshSDF::get_mesh() const {
}
void VoxelMeshSDF::bake() {
ZN_DSTACK();
ZN_PROFILE_SCOPE();
Ref<Mesh> mesh = _mesh;
@ -187,6 +189,7 @@ void VoxelMeshSDF::bake_async(SceneTree *scene_tree) {
Ref<VoxelMeshSDF> obj_to_notify;
void run(ThreadedTaskContext ctx) override {
ZN_DSTACK();
ZN_PROFILE_SCOPE();
ZN_ASSERT(obj_to_notify.is_valid());
@ -399,6 +402,7 @@ Dictionary VoxelMeshSDF::_b_get_data() const {
}
void VoxelMeshSDF::_b_set_data(Dictionary d) {
ZN_DSTACK();
if (_is_baking) {
WARN_PRINT("Setting data while baking, that data will be overwritten when baking ends.");
}

2
editor/vox/vox_mesh_importer.cpp
View File

@ -4,6 +4,7 @@
#include "../../storage/voxel_buffer_internal.h"
#include "../../storage/voxel_memory_pool.h"
#include "../../streams/vox_data.h"
#include "../../util/dstack.h"
#include "../../util/macros.h"
#include "../../util/math/conv.h"
#include "../../util/memory.h"
@ -136,6 +137,7 @@ struct ModelInstance {
};
void extract_model_instances(const Data &vox_data, std::vector<ModelInstance> &out_instances) {
ZN_DSTACK();
// Gather all models and bake their rotations
for_each_model_instance(vox_data, [&out_instances](ForEachModelInstanceArgs args) {
ERR_FAIL_COND(args.model == nullptr);

6
register_types.cpp
View File

@ -268,12 +268,6 @@ void uninitialize_voxel_module(ModuleInitializationLevel p_level) {
VoxelServer::destroy_singleton();
// Do this last as VoxelServer might still be holding some refs to voxel blocks
const unsigned int used_blocks = VoxelMemoryPool::get_singleton().debug_get_used_blocks();
if (used_blocks > 0) {
ERR_PRINT(String("VoxelMemoryPool: "
"{0} memory blocks are still used when unregistering the module. Recycling leak?")
.format(varray(used_blocks)));
}
VoxelMemoryPool::destroy_singleton();
// TODO No remove?
}

2
server/load_block_data_task.cpp
View File

@ -1,5 +1,6 @@
#include "load_block_data_task.h"
#include "../storage/voxel_buffer_internal.h"
#include "../util/dstack.h"
#include "../util/godot/funcs.h"
#include "../util/log.h"
#include "../util/profiling.h"
@ -36,6 +37,7 @@ int LoadBlockDataTask::debug_get_running_count() {
}
void LoadBlockDataTask::run(zylann::ThreadedTaskContext ctx) {
ZN_DSTACK();
ZN_PROFILE_SCOPE();
CRASH_COND(_stream_dependency == nullptr);

3
server/mesh_block_task.cpp
View File

@ -1,4 +1,5 @@
#include "mesh_block_task.h"
#include "../util/dstack.h"
#include "../util/log.h"
#include "../util/profiling.h"
#include "voxel_server.h"
@ -11,6 +12,7 @@ namespace zylann::voxel {
static void copy_block_and_neighbors(Span<std::shared_ptr<VoxelBufferInternal>> blocks, VoxelBufferInternal &dst,
int min_padding, int max_padding, int channels_mask, Ref<VoxelGenerator> generator, int data_block_size,
uint8_t lod_index, Vector3i mesh_block_pos) {
ZN_DSTACK();
ZN_PROFILE_SCOPE();
// Extract wanted channels in a list
@ -161,6 +163,7 @@ int MeshBlockTask::debug_get_running_count() {
}
void MeshBlockTask::run(zylann::ThreadedTaskContext ctx) {
ZN_DSTACK();
ZN_PROFILE_SCOPE();
CRASH_COND(meshing_dependency == nullptr);

7
storage/voxel_buffer_gd.cpp
View File

@ -1,5 +1,6 @@
#include "voxel_buffer_gd.h"
#include "../edition/voxel_tool_buffer.h"
#include "../util/dstack.h"
#include "../util/math/color.h"
#include "../util/memory.h"
#include "voxel_metadata_variant.h"
@ -42,26 +43,31 @@ real_t VoxelBuffer::get_voxel_f(int x, int y, int z, unsigned int channel_index)
}
void VoxelBuffer::set_voxel_f(real_t value, int x, int y, int z, unsigned int channel_index) {
ZN_DSTACK();
return _buffer->set_voxel_f(value, x, y, z, channel_index);
}
void VoxelBuffer::copy_channel_from(Ref<VoxelBuffer> other, unsigned int channel) {
ZN_DSTACK();
ERR_FAIL_COND(other.is_null());
_buffer->copy_from(other->get_buffer(), channel);
}
void VoxelBuffer::copy_channel_from_area(
Ref<VoxelBuffer> other, Vector3i src_min, Vector3i src_max, Vector3i dst_min, unsigned int channel) {
ZN_DSTACK();
ERR_FAIL_COND(other.is_null());
_buffer->copy_from(other->get_buffer(), src_min, src_max, dst_min, channel);
}
void VoxelBuffer::fill(uint64_t defval, unsigned int channel_index) {
ZN_DSTACK();
ERR_FAIL_INDEX(channel_index, MAX_CHANNELS);
_buffer->fill(defval, channel_index);
}
void VoxelBuffer::fill_f(real_t value, unsigned int channel) {
ZN_DSTACK();
ERR_FAIL_INDEX(channel, MAX_CHANNELS);
_buffer->fill_f(value, channel);
}
@ -81,6 +87,7 @@ VoxelBuffer::Compression VoxelBuffer::get_channel_compression(unsigned int chann
}
void VoxelBuffer::downscale_to(Ref<VoxelBuffer> dst, Vector3i src_min, Vector3i src_max, Vector3i dst_min) const {
ZN_DSTACK();
ERR_FAIL_COND(dst.is_null());
_buffer->downscale_to(dst->get_buffer(), src_min, src_max, dst_min);
}

11
storage/voxel_buffer_internal.cpp
View File

@ -5,6 +5,7 @@
#endif
#include "../util/container_funcs.h"
#include "../util/dstack.h"
#include "../util/profiling.h"
#include "../util/string_funcs.h"
#include "voxel_buffer_internal.h"
@ -15,6 +16,7 @@
namespace zylann::voxel {
inline uint8_t *allocate_channel_data(size_t size) {
ZN_DSTACK();
#ifdef VOXEL_BUFFER_USE_MEMORY_POOL
return VoxelMemoryPool::get_singleton().allocate(size);
#else
@ -134,6 +136,7 @@ VoxelBufferInternal &VoxelBufferInternal::operator=(VoxelBufferInternal &&src) {
}
void VoxelBufferInternal::create(unsigned int sx, unsigned int sy, unsigned int sz) {
ZN_DSTACK();
ZN_ASSERT_RETURN(sx <= MAX_SIZE && sy <= MAX_SIZE && sz <= MAX_SIZE);
#ifdef TOOLS_ENABLED
if (sx == 0 || sy == 0 || sz == 0) {
@ -231,6 +234,7 @@ uint64_t VoxelBufferInternal::get_voxel(int x, int y, int z, unsigned int channe
}
void VoxelBufferInternal::set_voxel(uint64_t value, int x, int y, int z, unsigned int channel_index) {
ZN_DSTACK();
ZN_ASSERT_RETURN(channel_index < MAX_CHANNELS);
ZN_ASSERT_RETURN_MSG(is_position_valid(x, y, z), format("At position ({}, {}, {})", x, y, z));
@ -339,6 +343,7 @@ void VoxelBufferInternal::fill(uint64_t defval, unsigned int channel_index) {
}
void VoxelBufferInternal::fill_area(uint64_t defval, Vector3i min, Vector3i max, unsigned int channel_index) {
ZN_DSTACK();
ZN_ASSERT_RETURN(channel_index < MAX_CHANNELS);
Vector3iUtil::sort_min_max(min, max);
@ -482,6 +487,7 @@ void VoxelBufferInternal::compress_if_uniform(Channel &channel) {
}
void VoxelBufferInternal::decompress_channel(unsigned int channel_index) {
ZN_DSTACK();
ZN_ASSERT_RETURN(channel_index < MAX_CHANNELS);
Channel &channel = _channels[channel_index];
if (channel.data == nullptr) {
@ -512,6 +518,7 @@ void VoxelBufferInternal::copy_from(const VoxelBufferInternal &other) {
}
void VoxelBufferInternal::copy_from(const VoxelBufferInternal &other, unsigned int channel_index) {
ZN_DSTACK();
ZN_ASSERT_RETURN(channel_index < MAX_CHANNELS);
ZN_ASSERT_RETURN(other._size == _size);
@ -539,6 +546,7 @@ void VoxelBufferInternal::copy_from(const VoxelBufferInternal &other, unsigned i
void VoxelBufferInternal::copy_from(const VoxelBufferInternal &other, Vector3i src_min, Vector3i src_max,
Vector3i dst_min, unsigned int channel_index) {
//
ZN_DSTACK();
ZN_ASSERT_RETURN(channel_index < MAX_CHANNELS);
Channel &channel = _channels[channel_index];
@ -577,6 +585,7 @@ void VoxelBufferInternal::copy_from(const VoxelBufferInternal &other, Vector3i s
}
void VoxelBufferInternal::duplicate_to(VoxelBufferInternal &dst, bool include_metadata) const {
ZN_DSTACK();
dst.create(_size);
for (unsigned int i = 0; i < _channels.size(); ++i) {
dst.set_channel_depth(i, _channels[i].depth);
@ -619,6 +628,7 @@ bool VoxelBufferInternal::get_channel_raw(unsigned int channel_index, Span<uint8
}
bool VoxelBufferInternal::create_channel(int i, uint64_t defval) {
ZN_DSTACK();
if (!create_channel_noinit(i, _size)) {
return false;
}
@ -635,6 +645,7 @@ size_t VoxelBufferInternal::get_size_in_bytes_for_volume(Vector3i size, Depth de
}
bool VoxelBufferInternal::create_channel_noinit(int i, Vector3i size) {
ZN_DSTACK();
Channel &channel = _channels[i];
const size_t size_in_bytes = get_size_in_bytes_for_volume(size, channel.depth);
ZN_ASSERT_RETURN_V_MSG(size_in_bytes <= Channel::MAX_SIZE_IN_BYTES, false, "Buffer is too big");

52
storage/voxel_memory_pool.cpp
View File

@ -16,12 +16,63 @@ void VoxelMemoryPool::create_singleton() {
}
void VoxelMemoryPool::destroy_singleton() {
const unsigned int used_blocks = VoxelMemoryPool::get_singleton().debug_get_used_blocks();
if (used_blocks > 0) {
ZN_PRINT_ERROR(format("VoxelMemoryPool: "
"{} memory blocks are still used when unregistering the module. Recycling leak?",
used_blocks));
#ifdef DEBUG_ENABLED
VoxelMemoryPool::get_singleton().debug_print_used_blocks(10);
#endif
}
ZN_ASSERT(g_memory_pool != nullptr);
VoxelMemoryPool *pool = g_memory_pool;
g_memory_pool = nullptr;
memdelete(pool);
}
#ifdef DEBUG_ENABLED
void VoxelMemoryPool::debug_print_used_blocks(unsigned int max_count) {
struct L {
static void debug_print_used_blocks(const VoxelMemoryPool::DebugUsedBlocks &debug_used_blocks,
unsigned int &count, unsigned int max_count, size_t mem_size) {
if (count > max_count) {
count += debug_used_blocks.blocks.size();
return;
}
const unsigned int initial_count = count;
for (auto it = debug_used_blocks.blocks.begin(); it != debug_used_blocks.blocks.end(); ++it) {
if (count > max_count) {
break;
}
std::string s;
const dstack::Info &info = it->second;
info.to_string(s);
if (mem_size == 0) {
println(format("--- Alloc {}:", count));
} else {
println(format("--- Alloc {}, size {}:", count, mem_size));
}
println(s);
++count;
}
count = initial_count + debug_used_blocks.blocks.size();
}
};
unsigned int count = 0;
for (unsigned int pool_index = 0; pool_index < _pot_pools.size(); ++pool_index) {
const Pool &pool = _pot_pools[pool_index];
L::debug_print_used_blocks(pool.debug_used_blocks, count, max_count, get_size_from_pool_index(pool_index));
}
L::debug_print_used_blocks(_debug_nonpooled_used_blocks, count, max_count, 0);
if (count > 0 && count > max_count) {
println(format("[...] and {} more allocs.", max_count - count));
}
}
#endif
VoxelMemoryPool &VoxelMemoryPool::get_singleton() {
ZN_ASSERT(g_memory_pool != nullptr);
return *g_memory_pool;
@ -39,6 +90,7 @@ VoxelMemoryPool::~VoxelMemoryPool() {
}
uint8_t *VoxelMemoryPool::allocate(size_t size) {
ZN_DSTACK();
ZN_PROFILE_SCOPE();
ZN_ASSERT(size != 0);
uint8_t *block = nullptr;

17
storage/voxel_memory_pool.h
View File

@ -1,13 +1,14 @@
#ifndef VOXEL_MEMORY_POOL_H
#define VOXEL_MEMORY_POOL_H
#include "../util/dstack.h"
#include "../util/fixed_array.h"
#include "../util/math/funcs.h"
#include "../util/thread/mutex.h"
#include <atomic>
#include <limits>
#include <unordered_set>
#include <unordered_map>
#include <vector>
namespace zylann::voxel {
@ -22,14 +23,14 @@ private:
#ifdef DEBUG_ENABLED
struct DebugUsedBlocks {
Mutex mutex;
std::unordered_set<void *> blocks;
std::unordered_map<void *, dstack::Info> blocks;
void add(void *block) {
void add(void *mem) {
MutexLock lock(mutex);
auto it = blocks.find(block);
auto it = blocks.find(mem);
// Must not add twice
ZN_ASSERT(it == blocks.end());
blocks.insert(block);
blocks.insert({ mem, dstack::Info() });
}
void remove(void *block) {
@ -84,10 +85,14 @@ private:
return math::get_shift_from_power_of_two_32(math::get_next_power_of_two_32(size));
}
inline size_t get_size_from_pool_index(unsigned int i) const {
static inline size_t get_size_from_pool_index(unsigned int i) {
return size_t(1) << i;
}
#ifdef DEBUG_ENABLED
void debug_print_used_blocks(unsigned int max_amount);
#endif
// We handle allocations with up to 2^20 = 1,048,576 bytes.
// This is chosen based on practical needs.
// Each slot in this array corresponds to allocations

2
streams/vox_loader.cpp
View File

@ -1,11 +1,13 @@
#include "vox_loader.h"
#include "../meshers/cubes/voxel_color_palette.h"
#include "../storage/voxel_buffer_gd.h"
#include "../util/dstack.h"
#include "vox_data.h"
namespace zylann::voxel {
Error VoxelVoxLoader::load_from_file(String fpath, Ref<gd::VoxelBuffer> p_voxels, Ref<VoxelColorPalette> palette) {
ZN_DSTACK();
ERR_FAIL_COND_V(p_voxels.is_null(), ERR_INVALID_PARAMETER);
VoxelBufferInternal &voxels = p_voxels->get_buffer();

2
streams/voxel_block_serializer.cpp
View File

@ -1,6 +1,7 @@
#include "voxel_block_serializer.h"
#include "../storage/voxel_buffer_internal.h"
#include "../storage/voxel_memory_pool.h"
#include "../util/dstack.h"
#include "../util/macros.h"
#include "../util/math/vector3i.h"
#include "../util/profiling.h"
@ -548,6 +549,7 @@ bool migrate_v2_to_v3(Span<const uint8_t> p_data, std::vector<uint8_t> &dst) {
} // namespace legacy
bool deserialize(Span<const uint8_t> p_data, VoxelBufferInternal &out_voxel_buffer) {
ZN_DSTACK();
ZN_PROFILE_SCOPE();
std::vector<uint8_t> &metadata_tmp = tls_metadata_tmp;

2
terrain/variable_lod/voxel_lod_terrain_update_task.cpp
View File

@ -5,6 +5,7 @@
#include "../../server/save_block_data_task.h"
#include "../../server/voxel_server.h"
#include "../../util/container_funcs.h"
#include "../../util/dstack.h"
#include "../../util/math/conv.h"
#include "../../util/profiling.h"
#include "../../util/profiling_clock.h"
@ -14,6 +15,7 @@ namespace zylann::voxel {
void VoxelLodTerrainUpdateTask::flush_pending_lod_edits(VoxelLodTerrainUpdateData::State &state, VoxelDataLodMap &data,
Ref<VoxelGenerator> generator, bool full_load_mode, const int mesh_block_size) {
ZN_DSTACK();
ZN_PROFILE_SCOPE();
// Propagates edits performed so far to other LODs.
// These LODs must be currently in memory, otherwise terrain data will miss it.

47
util/dstack.cpp Normal file
View File

@ -0,0 +1,47 @@
#include "dstack.h"
#include "fixed_array.h"
#include "string_funcs.h"
#include <string>
namespace zylann {
namespace dstack {
struct Stack {
FixedArray<Frame, 64> frames;
unsigned int count = 0;
};
Stack &get_tls_stack() {
thread_local Stack tls_stack;
return tls_stack;
}
void push(const char *file, unsigned int line, const char *function) {
Stack &stack = get_tls_stack();
stack.frames[stack.count] = { file, function, line };
++stack.count;
}
void pop() {
Stack &stack = get_tls_stack();
--stack.count;
}
Info::Info() {
const Stack &stack = get_tls_stack();
_frames.resize(stack.count);
for (unsigned int i = 0; i < stack.count; ++i) {
_frames[i] = stack.frames[i];
}
}
void Info::to_string(FwdMutableStdString s) const {
for (unsigned int i = 0; i < _frames.size(); ++i) {
const Frame &frame = _frames[i];
s.s += format("{}: {} in {}\n", frame.function, frame.line, frame.file);
}
}
} // namespace dstack
} // namespace zylann

50
util/dstack.h Normal file
View File

@ -0,0 +1,50 @@
#ifndef ZN_DSTACK_H
#define ZN_DSTACK_H
#include "fwd_std_string.h"
#include <vector>
#ifdef DEBUG_ENABLED
#define ZN_DSTACK_ENABLED
#endif
#ifdef ZN_DSTACK_ENABLED
#define ZN_DSTACK() zylann::dstack::Scope dstack_scope_##__LINE__(__FILE__, __LINE__, __FUNCTION__)
#else
#define ZN_DSTACK()
#endif
namespace zylann {
namespace dstack {
void push(const char *file, unsigned int line, const char *fname);
void pop();
struct Scope {
Scope(const char *file, unsigned int line, const char *function) {
push(file, line, function);
}
~Scope() {
pop();
}
};
struct Frame {
const char *file = nullptr;
const char *function = nullptr;
unsigned int line = 0;
};
struct Info {
public:
Info();
void to_string(FwdMutableStdString s) const;
private:
std::vector<Frame> _frames;
};
} // namespace dstack
} // namespace zylann
#endif // ZN_DSTACK_H