From e9445a9ccb1f7acafc97e421836b634b185a9b46 Mon Sep 17 00:00:00 2001 From: Marc Gilleron Date: Sun, 17 Jul 2022 18:49:48 +0100 Subject: [PATCH] Removed VoxelStreamBlockFiles --- config.py | 1 - doc/classes/VoxelStreamBlockFiles.xml | 20 -- doc/source/api/VoxelStreamBlockFiles.md | 26 --- doc/source/changelog.md | 1 + misc/fuzzer_config.txt | 1 - register_types.cpp | 2 - streams/voxel_stream_block_files.cpp | 293 ------------------------ streams/voxel_stream_block_files.h | 56 ----- 8 files changed, 1 insertion(+), 399 deletions(-) delete mode 100644 doc/classes/VoxelStreamBlockFiles.xml delete mode 100644 doc/source/api/VoxelStreamBlockFiles.md delete mode 100644 streams/voxel_stream_block_files.cpp delete mode 100644 streams/voxel_stream_block_files.h diff --git a/config.py b/config.py index 18514a89..cb01310b 100644 --- a/config.py +++ b/config.py @@ -44,7 +44,6 @@ def get_doc_classes(): "VoxelStream", "VoxelStreamFile", - "VoxelStreamBlockFiles", "VoxelStreamRegionFiles", "VoxelStreamSQLite", "VoxelStreamScript", diff --git a/doc/classes/VoxelStreamBlockFiles.xml b/doc/classes/VoxelStreamBlockFiles.xml deleted file mode 100644 index 33572f86..00000000 --- a/doc/classes/VoxelStreamBlockFiles.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - Loads and saves blocks as individual files under a directory. - - - Loads and saves blocks to the filesystem, under a directory. Each block gets its own file, which may produce a lot of them. This is a naive implementation and may be very slow in practice. At the very least it serves as proof of concept, but will probably be removed in the future. - - - - - - - - Directory under which the data is saved. - - - - - diff --git a/doc/source/api/VoxelStreamBlockFiles.md b/doc/source/api/VoxelStreamBlockFiles.md deleted file mode 100644 index 6b2e241a..00000000 --- a/doc/source/api/VoxelStreamBlockFiles.md +++ /dev/null @@ -1,26 +0,0 @@ -# VoxelStreamBlockFiles - -Inherits: [VoxelStream](VoxelStream.md) - - -Loads and saves blocks as individual files under a directory. - -## Description: - -Loads and saves blocks to the filesystem, under a directory. Each block gets its own file, which may produce a lot of them. This is a naive implementation and may be very slow in practice. At the very least it serves as proof of concept, but will probably be removed in the future. - -## Properties: - - -Type | Name | Default ---------- | -------------------------- | -------- -`String` | [directory](#i_directory) | "" -

- -## Property Descriptions - -- [String](https://docs.godotengine.org/en/stable/classes/class_string.html) **directory** = "" - -Directory under which the data is saved. - -_Generated on Nov 06, 2021_ diff --git a/doc/source/changelog.md b/doc/source/changelog.md index 1cf9f2e8..ee3be6c7 100644 --- a/doc/source/changelog.md +++ b/doc/source/changelog.md @@ -102,6 +102,7 @@ Godot 4 is required from this version. - `VoxelStreamScript`: renamed `_immerge_block` => `_save_voxel_block` - `VoxelGeneratorGraph`: the `Select` node's `threshold` port is now a parameter instead. - `FastNoiseLite` was renamed `ZN_FastNoiseLite`, as now Godot 4 comes with its own implementation, with a few differences. + - Removed `VoxelStreamBlockFiles` - Known issues - Some nodes and resources no longer start with predefined properties due to a warning introduced in Godot4 when properties are resources. diff --git a/misc/fuzzer_config.txt b/misc/fuzzer_config.txt index dc2f3745..8bd3bb78 100644 --- a/misc/fuzzer_config.txt +++ b/misc/fuzzer_config.txt @@ -21,7 +21,6 @@ VoxelViewer # Cause freeze when using with VoxelTerrain, due iterating each cell VoxelInstancer VoxelStream VoxelStreamFile -VoxelStreamBlockFiles VoxelStreamRegionFiles VoxelStreamSQLite VoxelStreamScript diff --git a/register_types.cpp b/register_types.cpp index 7e6ab14b..d2053b11 100644 --- a/register_types.cpp +++ b/register_types.cpp @@ -28,7 +28,6 @@ #include "streams/sqlite/voxel_stream_sqlite.h" #include "streams/vox_loader.h" #include "streams/voxel_block_serializer_gd.h" -#include "streams/voxel_stream_block_files.h" #include "streams/voxel_stream_script.h" #include "terrain/fixed_lod/voxel_box_mover.h" #include "terrain/fixed_lod/voxel_terrain.h" @@ -176,7 +175,6 @@ void initialize_voxel_module(ModuleInitializationLevel p_level) { // Streams ClassDB::register_abstract_class(); - ClassDB::register_class(); ClassDB::register_class(); ClassDB::register_class(); ClassDB::register_class(); diff --git a/streams/voxel_stream_block_files.cpp b/streams/voxel_stream_block_files.cpp deleted file mode 100644 index 6c7593d1..00000000 --- a/streams/voxel_stream_block_files.cpp +++ /dev/null @@ -1,293 +0,0 @@ -#include "voxel_stream_block_files.h" -#include "../engine/voxel_engine.h" -#include "voxel_block_serializer.h" - -#include -#include - -using namespace zylann; -using namespace voxel; - -namespace { -const uint8_t FORMAT_VERSION = 1; -const char *FORMAT_META_MAGIC = "VXBM"; -const char *FORMAT_BLOCK_MAGIC = "VXB_"; -const char *META_FILE_NAME = "meta.vxbm"; -const char *BLOCK_FILE_EXTENSION = ".vxb"; -} // namespace - -VoxelStreamBlockFiles::VoxelStreamBlockFiles() { - // Defaults - _meta.block_size_po2 = 4; - _meta.lod_count = 1; - _meta.version = FORMAT_VERSION; - fill(_meta.channel_depths, VoxelBufferInternal::DEFAULT_CHANNEL_DEPTH); -} - -// TODO Have configurable block size - -void VoxelStreamBlockFiles::load_voxel_block(VoxelStream::VoxelQueryData &q) { - // - if (_directory_path.is_empty()) { - q.result = RESULT_BLOCK_NOT_FOUND; - return; - } - - q.result = RESULT_ERROR; - - if (!_meta_loaded) { - if (load_meta() != FILE_OK) { - return; - } - } - - CRASH_COND(!_meta_loaded); - - const Vector3i block_size = Vector3iUtil::create(1 << _meta.block_size_po2); - - ERR_FAIL_COND(q.lod >= _meta.lod_count); - ERR_FAIL_COND(block_size != q.voxel_buffer.get_size()); - - const Vector3i block_pos = get_block_position(q.origin_in_voxels) >> q.lod; - const String file_path = get_block_file_path(block_pos, q.lod); - const CharString file_path_utf8 = file_path.utf8(); - - Ref f; - VoxelFileLockerRead file_rlock(file_path_utf8.get_data()); - { - Error err; - f = FileAccess::open(file_path, FileAccess::READ, &err); - // Had to add ERR_FILE_CANT_OPEN because that's what Godot actually returns when the file doesn't exist... - if (f == nullptr && (err == ERR_FILE_NOT_FOUND || err == ERR_FILE_CANT_OPEN)) { - q.result = RESULT_BLOCK_NOT_FOUND; - return; - } - } - - ERR_FAIL_COND(f == nullptr); - - { - { - uint8_t version; - const FileResult err = check_magic_and_version(**f, FORMAT_VERSION, FORMAT_BLOCK_MAGIC, version); - if (err != FILE_OK) { - ERR_PRINT(String("Invalid file header: ") + ::to_string(err)); - return; - } - } - - // Configure depths, as they currently are only specified in the meta file. - // Files are expected to contain such depths, and use those in the buffer to know how much data to read. - for (unsigned int channel_index = 0; channel_index < _meta.channel_depths.size(); ++channel_index) { - q.voxel_buffer.set_channel_depth(channel_index, _meta.channel_depths[channel_index]); - } - - uint32_t size_to_read = f->get_32(); - if (!BlockSerializer::decompress_and_deserialize(**f, size_to_read, q.voxel_buffer)) { - ERR_PRINT("Failed to decompress and deserialize"); - } - } - - q.result = RESULT_BLOCK_FOUND; -} - -void VoxelStreamBlockFiles::save_voxel_block(VoxelStream::VoxelQueryData &q) { - ERR_FAIL_COND(_directory_path.is_empty()); - - if (!_meta_loaded) { - // If it's not loaded, always try to load meta file first if it exists already, - // because we could want to save blocks without reading any - const FileResult res = load_meta(); - if (res != FILE_OK && res != FILE_CANT_OPEN) { - // The file is present but there is a problem with it - String meta_path = _directory_path.plus_file(META_FILE_NAME); - ERR_PRINT(String("Could not read {0}: {1}").format(varray(meta_path, ::to_string(res)))); - return; - } - } - - if (!_meta_saved) { - // First time we save the meta file, initialize it from the first block format - for (unsigned int i = 0; i < _meta.channel_depths.size(); ++i) { - _meta.channel_depths[i] = q.voxel_buffer.get_channel_depth(i); - } - const FileResult res = save_meta(); - ERR_FAIL_COND(res != FILE_OK); - } - - // Check format - const Vector3i block_size = Vector3iUtil::create(1 << _meta.block_size_po2); - ERR_FAIL_COND(q.voxel_buffer.get_size() != block_size); - for (unsigned int channel_index = 0; channel_index < _meta.channel_depths.size(); ++channel_index) { - ERR_FAIL_COND(q.voxel_buffer.get_channel_depth(channel_index) != _meta.channel_depths[channel_index]); - } - - const Vector3i block_pos = get_block_position(q.origin_in_voxels) >> q.lod; - const String file_path = get_block_file_path(block_pos, q.lod); - - { - const CharString file_path_base_dir = file_path.get_base_dir().utf8(); - const Error err = check_directory_created_using_file_locker(file_path_base_dir.get_data()); - ERR_FAIL_COND(err != OK); - } - - { - Ref f; - const CharString file_path_utf8 = file_path.utf8(); - VoxelFileLockerWrite file_wlock(file_path_utf8.get_data()); - { - Error err; - // Create file if not exists, always truncate - f = FileAccess::open(file_path, FileAccess::WRITE, &err); - } - ERR_FAIL_COND(f == nullptr); - - f->store_buffer((uint8_t *)FORMAT_BLOCK_MAGIC, 4); - f->store_8(FORMAT_VERSION); - - BlockSerializer::SerializeResult res = BlockSerializer::serialize_and_compress(q.voxel_buffer); - if (!res.success) { - ERR_PRINT("Failed to save block"); - return; - } - f->store_32(res.data.size()); - f->store_buffer(res.data.data(), res.data.size()); - } -} - -int VoxelStreamBlockFiles::get_used_channels_mask() const { - // Assuming all, since that stream can store anything. - return VoxelBufferInternal::ALL_CHANNELS_MASK; -} - -String VoxelStreamBlockFiles::get_directory() const { - return _directory_path; -} - -void VoxelStreamBlockFiles::set_directory(String dirpath) { - if (_directory_path != dirpath) { - _directory_path = dirpath; - _meta_loaded = false; - } -} - -int VoxelStreamBlockFiles::get_block_size_po2() const { - return _meta.block_size_po2; -} - -FileResult VoxelStreamBlockFiles::save_meta() { - CRASH_COND(_directory_path.is_empty()); - - // Make sure the directory exists - { - const CharString directory_path_utf8 = _directory_path.utf8(); - const Error err = check_directory_created_using_file_locker(directory_path_utf8.get_data()); - if (err != OK) { - ERR_PRINT("Could not save meta"); - return FILE_CANT_OPEN; - } - } - - const String meta_path = _directory_path.plus_file(META_FILE_NAME); - - { - Error err; - const CharString meta_path_utf8 = meta_path.utf8(); - VoxelFileLockerWrite file_wlock(meta_path_utf8.get_data()); - Ref f = FileAccess::open(meta_path, FileAccess::WRITE, &err); - ERR_FAIL_COND_V(f == nullptr, FILE_CANT_OPEN); - - f->store_buffer((uint8_t *)FORMAT_META_MAGIC, 4); - f->store_8(FORMAT_VERSION); - - f->store_8(_meta.lod_count); - f->store_8(_meta.block_size_po2); - - for (unsigned int i = 0; i < _meta.channel_depths.size(); ++i) { - f->store_8(_meta.channel_depths[i]); - } - } - - _meta_loaded = true; - _meta_saved = true; - return FILE_OK; -} - -FileResult VoxelStreamBlockFiles::load_or_create_meta() { - const FileResult res = load_meta(); - if (res == FILE_DOES_NOT_EXIST) { - const FileResult save_result = save_meta(); - ERR_FAIL_COND_V(save_result != FILE_OK, save_result); - return FILE_OK; - } - return res; -} - -FileResult VoxelStreamBlockFiles::load_meta() { - CRASH_COND(_directory_path.is_empty()); - - const String meta_path = _directory_path.plus_file(META_FILE_NAME); - - Meta meta; - { - Error open_result; - const CharString meta_path_utf8 = meta_path.utf8(); - VoxelFileLockerRead file_rlock(meta_path_utf8.get_data()); - Ref f = FileAccess::open(meta_path, FileAccess::READ, &open_result); - // Had to add ERR_FILE_CANT_OPEN because that's what Godot actually returns when the file doesn't exist... - if (!_meta_saved && (open_result == ERR_FILE_NOT_FOUND || open_result == ERR_FILE_CANT_OPEN)) { - // This is a new terrain, save the meta we have and consider it current - return FILE_DOES_NOT_EXIST; - } - ERR_FAIL_COND_V(f.is_null(), FILE_CANT_OPEN); - - FileResult check_result = check_magic_and_version(**f, FORMAT_VERSION, FORMAT_META_MAGIC, meta.version); - if (check_result != FILE_OK) { - return check_result; - } - - meta.lod_count = f->get_8(); - meta.block_size_po2 = f->get_8(); - - for (unsigned int i = 0; i < meta.channel_depths.size(); ++i) { - uint8_t depth = f->get_8(); - ERR_FAIL_COND_V(depth >= VoxelBufferInternal::DEPTH_COUNT, FILE_INVALID_DATA); - meta.channel_depths[i] = (VoxelBufferInternal::Depth)depth; - } - - ERR_FAIL_COND_V(meta.lod_count < 1 || meta.lod_count > 32, FILE_INVALID_DATA); - ERR_FAIL_COND_V(meta.block_size_po2 < 1 || meta.block_size_po2 > 8, FILE_INVALID_DATA); - } - - _meta_loaded = true; - _meta = meta; - return FILE_OK; -} - -String VoxelStreamBlockFiles::get_block_file_path(const Vector3i &block_pos, unsigned int lod) const { - // TODO This is probably extremely inefficient, also given the nature of Godot strings - - // Save under a folder, because there could be other kinds of data to store in this terrain - String path = "blocks/lod"; - path += String::num_uint64(lod); - path += '/'; - for (unsigned int i = 0; i < 3; ++i) { - if (block_pos[i] >= 0) { - path += '+'; - } - path += String::num_int64(block_pos[i]); - } - path += BLOCK_FILE_EXTENSION; - return _directory_path.plus_file(path); -} - -Vector3i VoxelStreamBlockFiles::get_block_position(const Vector3i &origin_in_voxels) const { - return origin_in_voxels >> _meta.block_size_po2; -} - -void VoxelStreamBlockFiles::_bind_methods() { - ClassDB::bind_method(D_METHOD("set_directory", "directory"), &VoxelStreamBlockFiles::set_directory); - ClassDB::bind_method(D_METHOD("get_directory"), &VoxelStreamBlockFiles::get_directory); - - ADD_PROPERTY(PropertyInfo(Variant::STRING, "directory", PROPERTY_HINT_DIR), "set_directory", "get_directory"); -} diff --git a/streams/voxel_stream_block_files.h b/streams/voxel_stream_block_files.h deleted file mode 100644 index bd254899..00000000 --- a/streams/voxel_stream_block_files.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef VOXEL_STREAM_BLOCK_FILES_H -#define VOXEL_STREAM_BLOCK_FILES_H - -#include "../storage/voxel_buffer_internal.h" -#include "file_utils.h" -#include "voxel_stream.h" - -class FileAccess; - -namespace zylann::voxel { - -// Loads and saves blocks to the filesystem, under a directory. -// Each block gets its own file, which may produce a lot of them, but it makes it simple to implement. -// This is a naive implementation and may be very slow in practice, so maybe it will be removed in the future. -class VoxelStreamBlockFiles : public VoxelStream { - GDCLASS(VoxelStreamBlockFiles, VoxelStream) -public: - VoxelStreamBlockFiles(); - - void load_voxel_block(VoxelStream::VoxelQueryData &q) override; - void save_voxel_block(VoxelStream::VoxelQueryData &q) override; - - int get_used_channels_mask() const override; - - String get_directory() const; - void set_directory(String dirpath); - - int get_block_size_po2() const override; - -protected: - static void _bind_methods(); - -private: - FileResult save_meta(); - FileResult load_meta(); - FileResult load_or_create_meta(); - String get_block_file_path(const Vector3i &block_pos, unsigned int lod) const; - Vector3i get_block_position(const Vector3i &origin_in_voxels) const; - - String _directory_path; - - struct Meta { - uint8_t version = -1; - uint8_t lod_count = 0; - uint8_t block_size_po2 = 0; // How many voxels in a block - FixedArray channel_depths; - }; - - Meta _meta; - bool _meta_loaded = false; - bool _meta_saved = false; -}; - -} // namespace zylann::voxel - -#endif // VOXEL_STREAM_BLOCK_FILES_H