Zylann/godot_voxel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Namespaced VoxelBlockSerializerInternal

Browse Source
This commit is contained in:
Marc Gilleron 2022-01-03 02:52:06 +00:00
parent 4a94d13565
commit 2fdf0bf6df
11 changed files with 47 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
streams/region/region_file.cpp
View File

@ -303,7 +303,7 @@ const RegionFormat &RegionFile::get_format() const {
} }
Error RegionFile::load_block( Error RegionFile::load_block(
Vector3i position, VoxelBufferInternal &out_block, VoxelBlockSerializerInternal &serializer) { Vector3i position, VoxelBufferInternal &out_block, zylann::voxel::BlockSerializer &serializer) {
// //
ERR_FAIL_COND_V(_file_access == nullptr, ERR_FILE_CANT_READ); ERR_FAIL_COND_V(_file_access == nullptr, ERR_FILE_CANT_READ);
FileAccess *f = _file_access; FileAccess *f = _file_access;
@ -336,7 +336,8 @@ Error RegionFile::load_block(
return OK; return OK;
} }
Error RegionFile::save_block(Vector3i position, VoxelBufferInternal &block, VoxelBlockSerializerInternal &serializer) { Error RegionFile::save_block(
Vector3i position, VoxelBufferInternal &block, zylann::voxel::BlockSerializer &serializer) {
// //
ERR_FAIL_COND_V(_header.format.verify_block(block) == false, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V(_header.format.verify_block(block) == false, ERR_INVALID_PARAMETER);
@ -361,7 +362,7 @@ Error RegionFile::save_block(Vector3i position, VoxelBufferInternal &block, Voxe
// Check position matches the sectors rule // Check position matches the sectors rule
CRASH_COND((block_offset - _blocks_begin_offset) % _header.format.sector_size != 0); CRASH_COND((block_offset - _blocks_begin_offset) % _header.format.sector_size != 0);
VoxelBlockSerializerInternal::SerializeResult res = serializer.serialize_and_compress(block); zylann::voxel::BlockSerializer::SerializeResult res = serializer.serialize_and_compress(block);
ERR_FAIL_COND_V(!res.success, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V(!res.success, ERR_INVALID_PARAMETER);
f->store_32(res.data.size()); f->store_32(res.data.size());
const unsigned int written_size = sizeof(int) + res.data.size(); const unsigned int written_size = sizeof(int) + res.data.size();
@ -389,7 +390,7 @@ Error RegionFile::save_block(Vector3i position, VoxelBufferInternal &block, Voxe
const int old_sector_count = block_info.get_sector_count(); const int old_sector_count = block_info.get_sector_count();
CRASH_COND(old_sector_count < 1); CRASH_COND(old_sector_count < 1);
VoxelBlockSerializerInternal::SerializeResult res = serializer.serialize_and_compress(block); zylann::voxel::BlockSerializer::SerializeResult res = serializer.serialize_and_compress(block);
ERR_FAIL_COND_V(!res.success, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V(!res.success, ERR_INVALID_PARAMETER);
const std::vector<uint8_t> &data = res.data; const std::vector<uint8_t> &data = res.data;
const int written_size = sizeof(int) + data.size(); const int written_size = sizeof(int) + data.size();

7
streams/region/region_file.h
View File

@ -8,10 +8,11 @@
#include <vector> #include <vector>
class FileAccess; class FileAccess;
class VoxelBlockSerializerInternal;
namespace zylann::voxel { namespace zylann::voxel {
class BlockSerializer;
struct RegionFormat { struct RegionFormat {
static const char *FILE_EXTENSION; static const char *FILE_EXTENSION;
static const uint32_t MAX_BLOCKS_ACROSS = 255; static const uint32_t MAX_BLOCKS_ACROSS = 255;
@ -83,8 +84,8 @@ public:
bool set_format(const RegionFormat &format); bool set_format(const RegionFormat &format);
const RegionFormat &get_format() const; const RegionFormat &get_format() const;
Error load_block(Vector3i position, VoxelBufferInternal &out_block, VoxelBlockSerializerInternal &serializer); Error load_block(Vector3i position, VoxelBufferInternal &out_block, zylann::voxel::BlockSerializer &serializer);
Error save_block(Vector3i position, VoxelBufferInternal &block, VoxelBlockSerializerInternal &serializer); Error save_block(Vector3i position, VoxelBufferInternal &block, zylann::voxel::BlockSerializer &serializer);
unsigned int get_header_block_count() const; unsigned int get_header_block_count() const;
bool has_block(Vector3i position) const; bool has_block(Vector3i position) const;

2
streams/region/voxel_stream_region_files.cpp
View File

@ -19,7 +19,7 @@ const char *META_FILE_NAME = "meta.vxrm";
} // namespace } // namespace
thread_local VoxelBlockSerializerInternal VoxelStreamRegionFiles::_block_serializer; thread_local zylann::voxel::BlockSerializer VoxelStreamRegionFiles::_block_serializer;
// Sorts a sequence without modifying it, returning a sorted list of pointers // Sorts a sequence without modifying it, returning a sorted list of pointers
template <typename T, typename Comparer_T> template <typename T, typename Comparer_T>

2
streams/region/voxel_stream_region_files.h
View File

@ -109,7 +109,7 @@ private:
} }
}; };
static thread_local VoxelBlockSerializerInternal _block_serializer; static thread_local zylann::voxel::BlockSerializer _block_serializer;
// TODO This is not thread-friendly. // TODO This is not thread-friendly.
// `VoxelRegionFile` is not thread-safe so we have to limit the usage to one thread at once, blocking the others. // `VoxelRegionFile` is not thread-safe so we have to limit the usage to one thread at once, blocking the others.

6
streams/sqlite/voxel_stream_sqlite.cpp
View File

@ -572,7 +572,7 @@ void VoxelStreamSQLiteInternal::save_meta(Meta meta) {
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
thread_local VoxelBlockSerializerInternal VoxelStreamSQLite::_voxel_block_serializer; thread_local zylann::voxel::BlockSerializer VoxelStreamSQLite::_voxel_block_serializer;
thread_local std::vector<uint8_t> VoxelStreamSQLite::_temp_block_data; thread_local std::vector<uint8_t> VoxelStreamSQLite::_temp_block_data;
thread_local std::vector<uint8_t> VoxelStreamSQLite::_temp_compressed_block_data; thread_local std::vector<uint8_t> VoxelStreamSQLite::_temp_compressed_block_data;
@ -885,7 +885,7 @@ void VoxelStreamSQLite::flush_cache(VoxelStreamSQLiteInternal *con) {
PRINT_VERBOSE(String("VoxelStreamSQLite: Flushing cache ({0} elements)") PRINT_VERBOSE(String("VoxelStreamSQLite: Flushing cache ({0} elements)")
.format(varray(_cache.get_indicative_block_count()))); .format(varray(_cache.get_indicative_block_count())));
VoxelBlockSerializerInternal &serializer = _voxel_block_serializer; zylann::voxel::BlockSerializer &serializer = _voxel_block_serializer;
ERR_FAIL_COND(con == nullptr); ERR_FAIL_COND(con == nullptr);
ERR_FAIL_COND(con->begin_transaction() == false); ERR_FAIL_COND(con->begin_transaction() == false);
@ -909,7 +909,7 @@ void VoxelStreamSQLite::flush_cache(VoxelStreamSQLiteInternal *con) {
const std::vector<uint8_t> empty; const std::vector<uint8_t> empty;
con->save_block(loc, empty, VoxelStreamSQLiteInternal::VOXELS); con->save_block(loc, empty, VoxelStreamSQLiteInternal::VOXELS);
} else { } else {
VoxelBlockSerializerInternal::SerializeResult res = serializer.serialize_and_compress(block.voxels); zylann::voxel::BlockSerializer::SerializeResult res = serializer.serialize_and_compress(block.voxels);
ERR_FAIL_COND(!res.success); ERR_FAIL_COND(!res.success);
con->save_block(loc, res.data, VoxelStreamSQLiteInternal::VOXELS); con->save_block(loc, res.data, VoxelStreamSQLiteInternal::VOXELS);
} }

9
streams/sqlite/voxel_stream_sqlite.h
View File

@ -28,11 +28,12 @@ public:
void immerge_blocks(Span<VoxelBlockRequest> p_blocks) override; void immerge_blocks(Span<VoxelBlockRequest> p_blocks) override;
bool supports_instance_blocks() const override; bool supports_instance_blocks() const override;
void load_instance_blocks( void load_instance_blocks(Span<VoxelStreamInstanceDataRequest> out_blocks, Span<Result> out_results) override;
Span<VoxelStreamInstanceDataRequest> out_blocks, Span<Result> out_results) override;
void save_instance_blocks(Span<VoxelStreamInstanceDataRequest> p_blocks) override; void save_instance_blocks(Span<VoxelStreamInstanceDataRequest> p_blocks) override;
bool supports_loading_all_blocks() const override { return true; } bool supports_loading_all_blocks() const override {
return true;
}
void load_all_blocks(FullLoadingResult &result) override; void load_all_blocks(FullLoadingResult &result) override;
int get_used_channels_mask() const override; int get_used_channels_mask() const override;
@ -66,7 +67,7 @@ private:
VoxelStreamCache _cache; VoxelStreamCache _cache;
// TODO I should consider specialized memory allocators // TODO I should consider specialized memory allocators
static thread_local VoxelBlockSerializerInternal _voxel_block_serializer; static thread_local zylann::voxel::BlockSerializer _voxel_block_serializer;
static thread_local std::vector<uint8_t> _temp_block_data; static thread_local std::vector<uint8_t> _temp_block_data;
static thread_local std::vector<uint8_t> _temp_compressed_block_data; static thread_local std::vector<uint8_t> _temp_compressed_block_data;
}; };

28
streams/voxel_block_serializer.cpp
View File

@ -12,6 +12,8 @@
#include <core/io/file_access.h> #include <core/io/file_access.h>
#include <limits> #include <limits>
namespace zylann::voxel {
namespace { namespace {
const uint8_t BLOCK_VERSION = 2; const uint8_t BLOCK_VERSION = 2;
const unsigned int BLOCK_TRAILING_MAGIC = 0x900df00d; const unsigned int BLOCK_TRAILING_MAGIC = 0x900df00d;
@ -182,8 +184,7 @@ size_t get_size_in_bytes(const VoxelBufferInternal &buffer, size_t &metadata_siz
return size + metadata_size_with_header + BLOCK_TRAILING_MAGIC_SIZE; return size + metadata_size_with_header + BLOCK_TRAILING_MAGIC_SIZE;
} }
VoxelBlockSerializerInternal::SerializeResult VoxelBlockSerializerInternal::serialize( BlockSerializer::SerializeResult BlockSerializer::serialize(const VoxelBufferInternal &voxel_buffer) {
const VoxelBufferInternal &voxel_buffer) {
// //
VOXEL_PROFILE_SCOPE(); VOXEL_PROFILE_SCOPE();
// Cannot serialize an empty block // Cannot serialize an empty block
@ -262,7 +263,7 @@ VoxelBlockSerializerInternal::SerializeResult VoxelBlockSerializerInternal::seri
return SerializeResult(_data, true); return SerializeResult(_data, true);
} }
bool VoxelBlockSerializerInternal::deserialize(Span<const uint8_t> p_data, VoxelBufferInternal &out_voxel_buffer) { bool BlockSerializer::deserialize(Span<const uint8_t> p_data, VoxelBufferInternal &out_voxel_buffer) {
// //
VOXEL_PROFILE_SCOPE(); VOXEL_PROFILE_SCOPE();
@ -364,32 +365,30 @@ bool VoxelBlockSerializerInternal::deserialize(Span<const uint8_t> p_data, Voxel
return true; return true;
} }
VoxelBlockSerializerInternal::SerializeResult VoxelBlockSerializerInternal::serialize_and_compress( BlockSerializer::SerializeResult BlockSerializer::serialize_and_compress(const VoxelBufferInternal &voxel_buffer) {
const VoxelBufferInternal &voxel_buffer) {
VOXEL_PROFILE_SCOPE(); VOXEL_PROFILE_SCOPE();
SerializeResult res = serialize(voxel_buffer); SerializeResult res = serialize(voxel_buffer);
ERR_FAIL_COND_V(!res.success, SerializeResult(_compressed_data, false)); ERR_FAIL_COND_V(!res.success, SerializeResult(_compressed_data, false));
const std::vector<uint8_t> &data = res.data; const std::vector<uint8_t> &data = res.data;
res.success = zylann::voxel::CompressedData::compress(Span<const uint8_t>(data.data(), 0, data.size()), res.success = CompressedData::compress(
_compressed_data, zylann::voxel::CompressedData::COMPRESSION_LZ4); Span<const uint8_t>(data.data(), 0, data.size()), _compressed_data, CompressedData::COMPRESSION_LZ4);
ERR_FAIL_COND_V(!res.success, SerializeResult(_compressed_data, false)); ERR_FAIL_COND_V(!res.success, SerializeResult(_compressed_data, false));
return SerializeResult(_compressed_data, true); return SerializeResult(_compressed_data, true);
} }
bool VoxelBlockSerializerInternal::decompress_and_deserialize( bool BlockSerializer::decompress_and_deserialize(Span<const uint8_t> p_data, VoxelBufferInternal &out_voxel_buffer) {
Span<const uint8_t> p_data, VoxelBufferInternal &out_voxel_buffer) {
VOXEL_PROFILE_SCOPE(); VOXEL_PROFILE_SCOPE();
const bool res = zylann::voxel::CompressedData::decompress(p_data, _data); const bool res = CompressedData::decompress(p_data, _data);
ERR_FAIL_COND_V(!res, false); ERR_FAIL_COND_V(!res, false);
return deserialize(to_span_const(_data), out_voxel_buffer); return deserialize(to_span_const(_data), out_voxel_buffer);
} }
bool VoxelBlockSerializerInternal::decompress_and_deserialize( bool BlockSerializer::decompress_and_deserialize(
FileAccess *f, unsigned int size_to_read, VoxelBufferInternal &out_voxel_buffer) { FileAccess *f, unsigned int size_to_read, VoxelBufferInternal &out_voxel_buffer) {
VOXEL_PROFILE_SCOPE(); VOXEL_PROFILE_SCOPE();
ERR_FAIL_COND_V(f == nullptr, false); ERR_FAIL_COND_V(f == nullptr, false);
@ -407,7 +406,7 @@ bool VoxelBlockSerializerInternal::decompress_and_deserialize(
return decompress_and_deserialize(to_span_const(_compressed_data), out_voxel_buffer); return decompress_and_deserialize(to_span_const(_compressed_data), out_voxel_buffer);
} }
int VoxelBlockSerializerInternal::serialize(Ref<StreamPeer> peer, VoxelBufferInternal &voxel_buffer, bool compress) { int BlockSerializer::serialize(Ref<StreamPeer> peer, VoxelBufferInternal &voxel_buffer, bool compress) {
if (compress) { if (compress) {
SerializeResult res = serialize_and_compress(voxel_buffer); SerializeResult res = serialize_and_compress(voxel_buffer);
ERR_FAIL_COND_V(!res.success, -1); ERR_FAIL_COND_V(!res.success, -1);
@ -422,8 +421,7 @@ int VoxelBlockSerializerInternal::serialize(Ref<StreamPeer> peer, VoxelBufferInt
} }
} }
void VoxelBlockSerializerInternal::deserialize( void BlockSerializer::deserialize(Ref<StreamPeer> peer, VoxelBufferInternal &voxel_buffer, int size, bool decompress) {
Ref<StreamPeer> peer, VoxelBufferInternal &voxel_buffer, int size, bool decompress) {
if (decompress) { if (decompress) {
_compressed_data.resize(size); _compressed_data.resize(size);
const Error err = peer->get_data(_compressed_data.data(), _compressed_data.size()); const Error err = peer->get_data(_compressed_data.data(), _compressed_data.size());
@ -439,6 +437,8 @@ void VoxelBlockSerializerInternal::deserialize(
} }
} }
} // namespace zylann::voxel
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int VoxelBlockSerializer::serialize(Ref<StreamPeer> peer, Ref<VoxelBuffer> voxel_buffer, bool compress) { int VoxelBlockSerializer::serialize(Ref<StreamPeer> peer, Ref<VoxelBuffer> voxel_buffer, bool compress) {

11
streams/voxel_block_serializer.h
View File

@ -10,15 +10,16 @@
class StreamPeer; class StreamPeer;
class VoxelBufferInternal; class VoxelBufferInternal;
class VoxelBlockSerializerInternal { namespace zylann::voxel {
class BlockSerializer {
// Had to be named differently to not conflict with the wrapper for Godot script API // Had to be named differently to not conflict with the wrapper for Godot script API
public: public:
struct SerializeResult { struct SerializeResult {
const std::vector<uint8_t> &data; const std::vector<uint8_t> &data;
bool success; bool success;
inline SerializeResult(const std::vector<uint8_t> &p_data, bool p_success) : inline SerializeResult(const std::vector<uint8_t> &p_data, bool p_success) : data(p_data), success(p_success) {}
data(p_data), success(p_success) {}
}; };
SerializeResult serialize(const VoxelBufferInternal &voxel_buffer); SerializeResult serialize(const VoxelBufferInternal &voxel_buffer);
@ -39,6 +40,8 @@ private:
FileAccessMemory _file_access_memory; FileAccessMemory _file_access_memory;
}; };
} // namespace zylann::voxel
class VoxelBuffer; class VoxelBuffer;
class VoxelBlockSerializer : public RefCounted { class VoxelBlockSerializer : public RefCounted {
@ -50,7 +53,7 @@ public:
private: private:
static void _bind_methods(); static void _bind_methods();
VoxelBlockSerializerInternal _serializer; zylann::voxel::BlockSerializer _serializer;
}; };
#endif // VOXEL_BLOCK_SERIALIZER_H #endif // VOXEL_BLOCK_SERIALIZER_H

4
streams/voxel_stream_block_files.cpp
View File

@ -12,7 +12,7 @@ const char *META_FILE_NAME = "meta.vxbm";
const char *BLOCK_FILE_EXTENSION = ".vxb"; const char *BLOCK_FILE_EXTENSION = ".vxb";
} // namespace } // namespace
thread_local VoxelBlockSerializerInternal VoxelStreamBlockFiles::_block_serializer; thread_local zylann::voxel::BlockSerializer VoxelStreamBlockFiles::_block_serializer;
VoxelStreamBlockFiles::VoxelStreamBlockFiles() { VoxelStreamBlockFiles::VoxelStreamBlockFiles() {
// Defaults // Defaults
@ -141,7 +141,7 @@ void VoxelStreamBlockFiles::immerge_block(VoxelBufferInternal &buffer, Vector3i
f->store_buffer((uint8_t *)FORMAT_BLOCK_MAGIC, 4); f->store_buffer((uint8_t *)FORMAT_BLOCK_MAGIC, 4);
f->store_8(FORMAT_VERSION); f->store_8(FORMAT_VERSION);
VoxelBlockSerializerInternal::SerializeResult res = _block_serializer.serialize_and_compress(buffer); zylann::voxel::BlockSerializer::SerializeResult res = _block_serializer.serialize_and_compress(buffer);
if (!res.success) { if (!res.success) {
memdelete(f); memdelete(f);
ERR_PRINT("Failed to save block"); ERR_PRINT("Failed to save block");

2
streams/voxel_stream_block_files.h
View File

@ -35,7 +35,7 @@ private:
String get_block_file_path(const Vector3i &block_pos, unsigned int lod) const; String get_block_file_path(const Vector3i &block_pos, unsigned int lod) const;
Vector3i get_block_position(const Vector3i &origin_in_voxels) const; Vector3i get_block_position(const Vector3i &origin_in_voxels) const;
static thread_local VoxelBlockSerializerInternal _block_serializer; static thread_local zylann::voxel::BlockSerializer _block_serializer;
String _directory_path; String _directory_path;

8
tests/tests.cpp
View File

@ -1034,8 +1034,8 @@ void test_block_serializer() {
voxel_buffer.fill_area(44, Vector3i(1, 2, 3), Vector3i(5, 5, 5), 1); voxel_buffer.fill_area(44, Vector3i(1, 2, 3), Vector3i(5, 5, 5), 1);
// Serialize // Serialize
VoxelBlockSerializerInternal serializer; BlockSerializer serializer;
VoxelBlockSerializerInternal::SerializeResult result = serializer.serialize_and_compress(voxel_buffer); BlockSerializer::SerializeResult result = serializer.serialize_and_compress(voxel_buffer);
ERR_FAIL_COND(!result.success); ERR_FAIL_COND(!result.success);
std::vector<uint8_t> data = result.data; std::vector<uint8_t> data = result.data;
@ -1077,7 +1077,7 @@ void test_region_file() {
ERR_FAIL_COND(open_error != OK); ERR_FAIL_COND(open_error != OK);
// Save block // Save block
VoxelBlockSerializerInternal serializer; BlockSerializer serializer;
const Error save_error = region_file.save_block(Vector3i(1, 2, 3), voxel_buffer, serializer); const Error save_error = region_file.save_block(Vector3i(1, 2, 3), voxel_buffer, serializer);
ERR_FAIL_COND(save_error != OK); ERR_FAIL_COND(save_error != OK);
@ -1099,7 +1099,7 @@ void test_region_file() {
// Read back // Read back
VoxelBufferInternal loaded_voxel_buffer; VoxelBufferInternal loaded_voxel_buffer;
VoxelBlockSerializerInternal serializer; BlockSerializer serializer;
const Error load_error = region_file.load_block(Vector3i(1, 2, 3), loaded_voxel_buffer, serializer); const Error load_error = region_file.load_block(Vector3i(1, 2, 3), loaded_voxel_buffer, serializer);
ERR_FAIL_COND(load_error != OK); ERR_FAIL_COND(load_error != OK);