Zylann/godot_voxel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Upgraded block serialized format, added some error handling

Browse Source
This commit is contained in:
Marc Gilleron 2021-01-30 01:36:37 +00:00
parent 158715922d
commit c6b169b978
7 changed files with 128 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
server/voxel_server.cpp
View File

@ -627,11 +627,19 @@ void VoxelServer::BlockDataRequest::run(VoxelTaskContext ctx) {
voxels->create(block_size, block_size, block_size);
// TODO No longer using batches? If that works ok, we should get rid of batch queries in files
const VoxelStream::Result result = stream->emerge_block(voxels, origin_in_voxels, lod);
if (result == VoxelStream::RESULT_ERROR) {
ERR_PRINT("Error loading block");
}
if (result == VoxelStream::RESULT_BLOCK_NOT_FOUND) {
Ref<VoxelGenerator> generator = stream_dependency->generator;
if (generator.is_valid()) {
VoxelServer::get_singleton()->request_block_generate_from_data_request(this);
type = TYPE_FALLBACK_ON_GENERATOR;
} else {
// If there is no generator... what do we do? What defines the format of that empty block?
// If the user leaves the defaults it's fine, but otherwise blocks of inconsistent format can
// end up in the volume and that can cause errors.
// TODO Define format on volume?
}
}
} break;

15
streams/region_file.cpp
View File

@ -363,10 +363,11 @@ Error VoxelRegionFile::save_block(Vector3i position, Ref<VoxelBuffer> block, Vox
// Check position matches the sectors rule
CRASH_COND((block_offset - _blocks_begin_offset) % _header.format.sector_size != 0);
const std::vector<uint8_t> &data = serializer.serialize_and_compress(**block);
f->store_32(data.size());
const unsigned int written_size = sizeof(int) + data.size();
f->store_buffer(data.data(), data.size());
VoxelBlockSerializerInternal::SerializeResult res = serializer.serialize_and_compress(**block);
ERR_FAIL_COND_V(!res.success, ERR_INVALID_PARAMETER);
f->store_32(res.data.size());
const unsigned int written_size = sizeof(int) + res.data.size();
f->store_buffer(res.data.data(), res.data.size());
const unsigned int end_pos = f->get_position();
CRASH_COND(written_size != (end_pos - block_offset));
@ -381,8 +382,6 @@ Error VoxelRegionFile::save_block(Vector3i position, Ref<VoxelBuffer> block, Vox
_header_modified = true;
//print_line(String("Block saved flen={0}").format(varray(f->get_len())));
} else {
// The block is already in the file
@ -392,7 +391,9 @@ Error VoxelRegionFile::save_block(Vector3i position, Ref<VoxelBuffer> block, Vox
const int old_sector_count = block_info.get_sector_count();
CRASH_COND(old_sector_count < 1);
const std::vector<uint8_t> &data = serializer.serialize_and_compress(**block);
VoxelBlockSerializerInternal::SerializeResult res = serializer.serialize_and_compress(**block);
ERR_FAIL_COND_V(!res.success, ERR_INVALID_PARAMETER);
const std::vector<uint8_t> &data = res.data;
const int written_size = sizeof(int) + data.size();
const int new_sector_count = get_sector_count_from_bytes(written_size);

5
streams/sqlite/voxel_stream_sqlite.cpp
View File

@ -620,8 +620,9 @@ void VoxelStreamSQLite::flush_cache(VoxelStreamSQLiteInternal *con) {
loc.lod = block.lod;
// TODO We might want to handle null blocks too, in case they intentionally get deleted
ERR_FAIL_COND(block.voxels.is_null());
const std::vector<uint8_t> &temp = serializer.serialize_and_compress(**block.voxels);
con->save_block(loc, temp);
VoxelBlockSerializerInternal::SerializeResult res = serializer.serialize_and_compress(**block.voxels);
ERR_FAIL_COND(!res.success);
con->save_block(loc, res.data);
});
ERR_FAIL_COND(con->end_transaction() == false);

118
streams/voxel_block_serializer.cpp
View File

@ -12,9 +12,7 @@
#include <core/os/file_access.h>
namespace {
// TODO Introduce versionning
//const uint16_t BLOCK_VERSION = 0;
//const unsigned int BLOCK_VERSION_HEADER_SIZE = sizeof(uint16_t);
const uint8_t BLOCK_VERSION = 2;
const unsigned int BLOCK_TRAILING_MAGIC = 0x900df00d;
const unsigned int BLOCK_TRAILING_MAGIC_SIZE = 4;
const unsigned int BLOCK_METADATA_HEADER_SIZE = sizeof(uint32_t);
@ -148,14 +146,16 @@ bool deserialize_metadata(uint8_t *p_src, VoxelBuffer &buffer, const size_t meta
}
size_t get_size_in_bytes(const VoxelBuffer &buffer, size_t &metadata_size) {
size_t size = 0;
// Version and size
size_t size = 1 * sizeof(uint8_t) + 3 * sizeof(uint16_t);
const Vector3i size_in_voxels = buffer.get_size();
for (unsigned int channel_index = 0; channel_index < VoxelBuffer::MAX_CHANNELS; ++channel_index) {
const VoxelBuffer::Compression compression = buffer.get_channel_compression(channel_index);
const VoxelBuffer::Depth depth = buffer.get_channel_depth(channel_index);
// For depth value
// For format value
size += 1;
switch (compression) {
@ -183,29 +183,45 @@ size_t get_size_in_bytes(const VoxelBuffer &buffer, size_t &metadata_size) {
return size + metadata_size_with_header + BLOCK_TRAILING_MAGIC_SIZE;
}
const std::vector<uint8_t> &VoxelBlockSerializerInternal::serialize(const VoxelBuffer &voxel_buffer) {
VoxelBlockSerializerInternal::SerializeResult VoxelBlockSerializerInternal::serialize(const VoxelBuffer &voxel_buffer) {
VOXEL_PROFILE_SCOPE();
size_t metadata_size = 0;
const size_t data_size = get_size_in_bytes(voxel_buffer, metadata_size);
_data.resize(data_size);
CRASH_COND(_file_access_memory.open_custom(_data.data(), _data.size()) != OK);
ERR_FAIL_COND_V(_file_access_memory.open_custom(_data.data(), _data.size()) != OK, SerializeResult(_data, false));
FileAccessMemory *f = &_file_access_memory;
f->store_8(BLOCK_VERSION);
ERR_FAIL_COND_V(voxel_buffer.get_size().x > std::numeric_limits<uint16_t>().max(), SerializeResult(_data, false));
f->store_16(voxel_buffer.get_size().x);
ERR_FAIL_COND_V(voxel_buffer.get_size().y > std::numeric_limits<uint16_t>().max(), SerializeResult(_data, false));
f->store_16(voxel_buffer.get_size().y);
ERR_FAIL_COND_V(voxel_buffer.get_size().z > std::numeric_limits<uint16_t>().max(), SerializeResult(_data, false));
f->store_16(voxel_buffer.get_size().z);
for (unsigned int channel_index = 0; channel_index < VoxelBuffer::MAX_CHANNELS; ++channel_index) {
VoxelBuffer::Compression compression = voxel_buffer.get_channel_compression(channel_index);
f->store_8(static_cast<uint8_t>(compression));
const VoxelBuffer::Compression compression = voxel_buffer.get_channel_compression(channel_index);
const VoxelBuffer::Depth depth = voxel_buffer.get_channel_depth(channel_index);
// Low nibble: compression (up to 16 values allowed)
// High nibble: depth (up to 16 values allowed)
const uint8_t fmt = static_cast<uint8_t>(compression) | (static_cast<uint8_t>(depth) << 4);
f->store_8(fmt);
switch (compression) {
case VoxelBuffer::COMPRESSION_NONE: {
ArraySlice<uint8_t> data;
CRASH_COND(!voxel_buffer.get_channel_raw(channel_index, data));
ERR_FAIL_COND_V(!voxel_buffer.get_channel_raw(channel_index, data), SerializeResult(_data, false));
f->store_buffer(data.data(), data.size());
} break;
case VoxelBuffer::COMPRESSION_UNIFORM: {
const uint64_t v = voxel_buffer.get_voxel(Vector3i(), channel_index);
switch (voxel_buffer.get_channel_depth(channel_index)) {
switch (depth) {
case VoxelBuffer::DEPTH_8_BIT:
f->store_8(v);
break;
@ -240,19 +256,54 @@ const std::vector<uint8_t> &VoxelBlockSerializerInternal::serialize(const VoxelB
f->store_32(BLOCK_TRAILING_MAGIC);
return _data;
return SerializeResult(_data, true);
}
bool VoxelBlockSerializerInternal::deserialize(const std::vector<uint8_t> &p_data, VoxelBuffer &out_voxel_buffer) {
VOXEL_PROFILE_SCOPE();
CRASH_COND(_file_access_memory.open_custom(p_data.data(), p_data.size()) != OK);
ERR_FAIL_COND_V(p_data.size() < sizeof(uint32_t), false);
const uint32_t magic = *reinterpret_cast<const uint32_t *>(&p_data[p_data.size() - sizeof(uint32_t)]);
ERR_FAIL_COND_V(magic != BLOCK_TRAILING_MAGIC, false);
ERR_FAIL_COND_V(_file_access_memory.open_custom(p_data.data(), p_data.size()) != OK, false);
FileAccessMemory *f = &_file_access_memory;
const uint8_t version = f->get_8();
if (version < 2) {
// In version 1, the first thing coming in block data is the compression value of the first channel.
// At the time, there was only 2 values this could take: 0 and 1.
// So we can recognize blocks using this old format and seek back.
// Formats before 2 also did not contain bit depth, they only had compression, leaving high nibble to 0.
// This means version 2 will read only 8-bit depth from the old block.
// "Fortunately", the old format also did not properly serialize formats using more than 8 bits.
// So we are kinda set to migrate without much changes, by assuming the block is already formatted properly.
f->seek(f->get_position() - 1);
WARN_PRINT("Reading block version < 2. Attempting to migrate.");
} else {
ERR_FAIL_COND_V(version != BLOCK_VERSION, false);
const unsigned int size_x = f->get_16();
const unsigned int size_y = f->get_16();
const unsigned int size_z = f->get_16();
out_voxel_buffer.create(Vector3i(size_x, size_y, size_z));
}
for (unsigned int channel_index = 0; channel_index < VoxelBuffer::MAX_CHANNELS; ++channel_index) {
uint8_t compression_value = f->get_8();
const uint8_t fmt = f->get_8();
const uint8_t compression_value = fmt & 0xf;
const uint8_t depth_value = (fmt >> 4) & 0xf;
ERR_FAIL_COND_V_MSG(compression_value >= VoxelBuffer::COMPRESSION_COUNT, false,
"At offset 0x" + String::num_int64(f->get_position() - 1, 16));
ERR_FAIL_COND_V_MSG(depth_value >= VoxelBuffer::DEPTH_COUNT, false,
"At offset 0x" + String::num_int64(f->get_position() - 1, 16));
VoxelBuffer::Compression compression = (VoxelBuffer::Compression)compression_value;
VoxelBuffer::Depth depth = (VoxelBuffer::Depth)depth_value;
out_voxel_buffer.set_channel_depth(channel_index, depth);
switch (compression) {
case VoxelBuffer::COMPRESSION_NONE: {
@ -261,7 +312,7 @@ bool VoxelBlockSerializerInternal::deserialize(const std::vector<uint8_t> &p_dat
ArraySlice<uint8_t> buffer;
CRASH_COND(!out_voxel_buffer.get_channel_raw(channel_index, buffer));
uint32_t read_len = f->get_buffer(buffer.data(), buffer.size());
const uint32_t read_len = f->get_buffer(buffer.data(), buffer.size());
if (read_len != buffer.size()) {
ERR_PRINT("Unexpected end of file");
return false;
@ -309,11 +360,16 @@ bool VoxelBlockSerializerInternal::deserialize(const std::vector<uint8_t> &p_dat
return true;
}
const std::vector<uint8_t> &VoxelBlockSerializerInternal::serialize_and_compress(const VoxelBuffer &voxel_buffer) {
VOXEL_PROFILE_SCOPE();
const std::vector<uint8_t> &data = serialize(voxel_buffer);
VoxelBlockSerializerInternal::SerializeResult VoxelBlockSerializerInternal::serialize_and_compress(
const VoxelBuffer &voxel_buffer) {
unsigned int header_size = sizeof(unsigned int);
VOXEL_PROFILE_SCOPE();
SerializeResult res = serialize(voxel_buffer);
ERR_FAIL_COND_V(!res.success, SerializeResult(_compressed_data, false));
const std::vector<uint8_t> &data = res.data;
const unsigned int header_size = sizeof(unsigned int);
_compressed_data.resize(header_size + LZ4_compressBound(data.size()));
// Write header
@ -331,21 +387,23 @@ const std::vector<uint8_t> &VoxelBlockSerializerInternal::serialize_and_compress
CRASH_COND(compressed_size == 0);
_compressed_data.resize(header_size + compressed_size);
return _compressed_data;
return SerializeResult(_compressed_data, true);
}
bool VoxelBlockSerializerInternal::decompress_and_deserialize(
const std::vector<uint8_t> &p_data, VoxelBuffer &out_voxel_buffer) {
VOXEL_PROFILE_SCOPE();
// Read header
unsigned int header_size = sizeof(unsigned int);
const unsigned int header_size = sizeof(unsigned int);
ERR_FAIL_COND_V(_file_access_memory.open_custom(p_data.data(), p_data.size()) != OK, false);
unsigned int decompressed_size = _file_access_memory.get_32();
const unsigned int decompressed_size = _file_access_memory.get_32();
_file_access_memory.close();
_data.resize(decompressed_size);
unsigned int actually_decompressed_size = LZ4_decompress_safe(
const unsigned int actually_decompressed_size = LZ4_decompress_safe(
(const char *)p_data.data() + header_size,
(char *)_data.data(),
p_data.size() - header_size,
@ -380,14 +438,16 @@ bool VoxelBlockSerializerInternal::decompress_and_deserialize(
int VoxelBlockSerializerInternal::serialize(Ref<StreamPeer> peer, Ref<VoxelBuffer> voxel_buffer, bool compress) {
if (compress) {
const std::vector<uint8_t> &data = serialize_and_compress(**voxel_buffer);
peer->put_data(data.data(), data.size());
return data.size();
SerializeResult res = serialize_and_compress(**voxel_buffer);
ERR_FAIL_COND_V(!res.success, -1);
peer->put_data(res.data.data(), res.data.size());
return res.data.size();
} else {
const std::vector<uint8_t> &data = serialize(**voxel_buffer);
peer->put_data(data.data(), data.size());
return data.size();
SerializeResult res = serialize(**voxel_buffer);
ERR_FAIL_COND_V(!res.success, -1);
peer->put_data(res.data.data(), res.data.size());
return res.data.size();
}
}

13
streams/voxel_block_serializer.h
View File

@ -11,10 +11,18 @@ class StreamPeer;
class VoxelBlockSerializerInternal {
// Had to be named differently to not conflict with the wrapper for Godot script API
public:
const std::vector<uint8_t> &serialize(const VoxelBuffer &voxel_buffer);
struct SerializeResult {
const std::vector<uint8_t> &data;
bool success;
inline SerializeResult(const std::vector<uint8_t> &p_data, bool p_success) :
data(p_data), success(p_success) {}
};
SerializeResult serialize(const VoxelBuffer &voxel_buffer);
bool deserialize(const std::vector<uint8_t> &p_data, VoxelBuffer &out_voxel_buffer);
const std::vector<uint8_t> &serialize_and_compress(const VoxelBuffer &voxel_buffer);
SerializeResult serialize_and_compress(const VoxelBuffer &voxel_buffer);
bool decompress_and_deserialize(const std::vector<uint8_t> &p_data, VoxelBuffer &out_voxel_buffer);
bool decompress_and_deserialize(FileAccess *f, unsigned int size_to_read, VoxelBuffer &out_voxel_buffer);
@ -22,6 +30,7 @@ public:
void deserialize(Ref<StreamPeer> peer, Ref<VoxelBuffer> voxel_buffer, int size, bool decompress);
private:
// Make thread-locals?
std::vector<uint8_t> _data;
std::vector<uint8_t> _compressed_data;
std::vector<uint8_t> _metadata_tmp;

13
streams/voxel_stream_block_files.cpp
View File

@ -124,8 +124,6 @@ void VoxelStreamBlockFiles::immerge_block(Ref<VoxelBuffer> buffer, Vector3i orig
Vector3i block_pos = get_block_position(origin_in_voxels) >> lod;
String file_path = get_block_file_path(block_pos, lod);
//print_line(String("Saving VXB {0}").format(varray(block_pos.to_vec3())));
{
const Error err = check_directory_created(file_path.get_base_dir());
ERR_FAIL_COND(err != OK);
@ -144,9 +142,14 @@ void VoxelStreamBlockFiles::immerge_block(Ref<VoxelBuffer> buffer, Vector3i orig
f->store_buffer((uint8_t *)FORMAT_BLOCK_MAGIC, 4);
f->store_8(FORMAT_VERSION);
const std::vector<uint8_t> &data = _block_serializer.serialize_and_compress(**buffer);
f->store_32(data.size());
f->store_buffer(data.data(), data.size());
VoxelBlockSerializerInternal::SerializeResult res = _block_serializer.serialize_and_compress(**buffer);
if (!res.success) {
memdelete(f);
ERR_PRINT("Failed to save block");
return;
}
f->store_32(res.data.size());
f->store_buffer(res.data.data(), res.data.size());
f->close();
memdelete(f);

3
streams/voxel_stream_region_files.cpp
View File

@ -209,7 +209,6 @@ void VoxelStreamRegionFiles::_immerge_block(Ref<VoxelBuffer> voxel_buffer, Vecto
Vector3i block_pos = get_block_position_from_voxels(origin_in_voxels) >> lod;
Vector3i region_pos = get_region_position_from_blocks(block_pos);
Vector3i block_rpos = block_pos.wrap(region_size);
//print_line(String("Immerging block {0} r {1}").format(varray(block_pos.to_vec3(), region_pos.to_vec3())));
CachedRegion *cache = open_region(region_pos, lod, true);
ERR_FAIL_COND_MSG(cache == nullptr, "Could not save region file data");
@ -738,7 +737,7 @@ void VoxelStreamRegionFiles::_convert_files(Meta new_meta) {
close_all_regions();
print_line("Done converting region files");
PRINT_VERBOSE("Done converting region files");
}
Vector3i VoxelStreamRegionFiles::get_region_size() const {