Zylann
/
godot_voxel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Removed VoxelStreamBlockFiles

master
Marc Gilleron 2022-07-17 18:49:48 +01:00
parent 7254dcb7fc
commit e9445a9ccb
8 changed files with 1 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config.py
View File

@ -44,7 +44,6 @@ def get_doc_classes():
"VoxelStream",
"VoxelStreamFile",
"VoxelStreamBlockFiles",
"VoxelStreamRegionFiles",
"VoxelStreamSQLite",
"VoxelStreamScript",

20
doc/classes/VoxelStreamBlockFiles.xml
View File

@ -1,20 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<class name="VoxelStreamBlockFiles" inherits="VoxelStream" version="3.5">
<brief_description>
Loads and saves blocks as individual files under a directory.
</brief_description>
<description>
Loads and saves blocks to the filesystem, under a directory. Each block gets its own file, which may produce a lot of them. This is a naive implementation and may be very slow in practice. At the very least it serves as proof of concept, but will probably be removed in the future.
</description>
<tutorials>
</tutorials>
<methods>
</methods>
<members>
<member name="directory" type="String" setter="set_directory" getter="get_directory" default="&quot;&quot;">
Directory under which the data is saved.
</member>
</members>
<constants>
</constants>
</class>

26
doc/source/api/VoxelStreamBlockFiles.md
View File

@ -1,26 +0,0 @@
# VoxelStreamBlockFiles
Inherits: [VoxelStream](VoxelStream.md)
Loads and saves blocks as individual files under a directory.
## Description:
Loads and saves blocks to the filesystem, under a directory. Each block gets its own file, which may produce a lot of them. This is a naive implementation and may be very slow in practice. At the very least it serves as proof of concept, but will probably be removed in the future.
## Properties:
Type | Name | Default
--------- | -------------------------- | --------
`String` | [directory](#i_directory) | ""
<p></p>
## Property Descriptions
- [String](https://docs.godotengine.org/en/stable/classes/class_string.html)<span id="i_directory"></span> **directory** = ""
Directory under which the data is saved.
_Generated on Nov 06, 2021_

1
doc/source/changelog.md
View File

@ -102,6 +102,7 @@ Godot 4 is required from this version.
- `VoxelStreamScript`: renamed `_immerge_block` => `_save_voxel_block`
- `VoxelGeneratorGraph`: the `Select` node's `threshold` port is now a parameter instead.
- `FastNoiseLite` was renamed `ZN_FastNoiseLite`, as now Godot 4 comes with its own implementation, with a few differences.
- Removed `VoxelStreamBlockFiles`
- Known issues
- Some nodes and resources no longer start with predefined properties due to a warning introduced in Godot4 when properties are resources.

1
misc/fuzzer_config.txt
View File

@ -21,7 +21,6 @@ VoxelViewer # Cause freeze when using with VoxelTerrain, due iterating each cell
VoxelInstancer
VoxelStream
VoxelStreamFile
VoxelStreamBlockFiles
VoxelStreamRegionFiles
VoxelStreamSQLite
VoxelStreamScript

2
register_types.cpp
View File

@ -28,7 +28,6 @@
#include "streams/sqlite/voxel_stream_sqlite.h"
#include "streams/vox_loader.h"
#include "streams/voxel_block_serializer_gd.h"
#include "streams/voxel_stream_block_files.h"
#include "streams/voxel_stream_script.h"
#include "terrain/fixed_lod/voxel_box_mover.h"
#include "terrain/fixed_lod/voxel_terrain.h"
@ -176,7 +175,6 @@ void initialize_voxel_module(ModuleInitializationLevel p_level) {
// Streams
ClassDB::register_abstract_class<VoxelStream>();
ClassDB::register_class<VoxelStreamBlockFiles>();
ClassDB::register_class<VoxelStreamRegionFiles>();
ClassDB::register_class<VoxelStreamScript>();
ClassDB::register_class<VoxelStreamSQLite>();

293
streams/voxel_stream_block_files.cpp
View File

@ -1,293 +0,0 @@
#include "voxel_stream_block_files.h"
#include "../engine/voxel_engine.h"
#include "voxel_block_serializer.h"
#include <core/io/dir_access.h>
#include <core/io/file_access.h>
using namespace zylann;
using namespace voxel;
namespace {
const uint8_t FORMAT_VERSION = 1;
const char *FORMAT_META_MAGIC = "VXBM";
const char *FORMAT_BLOCK_MAGIC = "VXB_";
const char *META_FILE_NAME = "meta.vxbm";
const char *BLOCK_FILE_EXTENSION = ".vxb";
} // namespace
VoxelStreamBlockFiles::VoxelStreamBlockFiles() {
// Defaults
_meta.block_size_po2 = 4;
_meta.lod_count = 1;
_meta.version = FORMAT_VERSION;
fill(_meta.channel_depths, VoxelBufferInternal::DEFAULT_CHANNEL_DEPTH);
}
// TODO Have configurable block size
void VoxelStreamBlockFiles::load_voxel_block(VoxelStream::VoxelQueryData &q) {
//
if (_directory_path.is_empty()) {
q.result = RESULT_BLOCK_NOT_FOUND;
return;
}
q.result = RESULT_ERROR;
if (!_meta_loaded) {
if (load_meta() != FILE_OK) {
return;
}
}
CRASH_COND(!_meta_loaded);
const Vector3i block_size = Vector3iUtil::create(1 << _meta.block_size_po2);
ERR_FAIL_COND(q.lod >= _meta.lod_count);
ERR_FAIL_COND(block_size != q.voxel_buffer.get_size());
const Vector3i block_pos = get_block_position(q.origin_in_voxels) >> q.lod;
const String file_path = get_block_file_path(block_pos, q.lod);
const CharString file_path_utf8 = file_path.utf8();
Ref<FileAccess> f;
VoxelFileLockerRead file_rlock(file_path_utf8.get_data());
{
Error err;
f = FileAccess::open(file_path, FileAccess::READ, &err);
// Had to add ERR_FILE_CANT_OPEN because that's what Godot actually returns when the file doesn't exist...
if (f == nullptr && (err == ERR_FILE_NOT_FOUND || err == ERR_FILE_CANT_OPEN)) {
q.result = RESULT_BLOCK_NOT_FOUND;
return;
}
}
ERR_FAIL_COND(f == nullptr);
{
{
uint8_t version;
const FileResult err = check_magic_and_version(**f, FORMAT_VERSION, FORMAT_BLOCK_MAGIC, version);
if (err != FILE_OK) {
ERR_PRINT(String("Invalid file header: ") + ::to_string(err));
return;
}
}
// Configure depths, as they currently are only specified in the meta file.
// Files are expected to contain such depths, and use those in the buffer to know how much data to read.
for (unsigned int channel_index = 0; channel_index < _meta.channel_depths.size(); ++channel_index) {
q.voxel_buffer.set_channel_depth(channel_index, _meta.channel_depths[channel_index]);
}
uint32_t size_to_read = f->get_32();
if (!BlockSerializer::decompress_and_deserialize(**f, size_to_read, q.voxel_buffer)) {
ERR_PRINT("Failed to decompress and deserialize");
}
}
q.result = RESULT_BLOCK_FOUND;
}
void VoxelStreamBlockFiles::save_voxel_block(VoxelStream::VoxelQueryData &q) {
ERR_FAIL_COND(_directory_path.is_empty());
if (!_meta_loaded) {
// If it's not loaded, always try to load meta file first if it exists already,
// because we could want to save blocks without reading any
const FileResult res = load_meta();
if (res != FILE_OK && res != FILE_CANT_OPEN) {
// The file is present but there is a problem with it
String meta_path = _directory_path.plus_file(META_FILE_NAME);
ERR_PRINT(String("Could not read {0}: {1}").format(varray(meta_path, ::to_string(res))));
return;
}
}
if (!_meta_saved) {
// First time we save the meta file, initialize it from the first block format
for (unsigned int i = 0; i < _meta.channel_depths.size(); ++i) {
_meta.channel_depths[i] = q.voxel_buffer.get_channel_depth(i);
}
const FileResult res = save_meta();
ERR_FAIL_COND(res != FILE_OK);
}
// Check format
const Vector3i block_size = Vector3iUtil::create(1 << _meta.block_size_po2);
ERR_FAIL_COND(q.voxel_buffer.get_size() != block_size);
for (unsigned int channel_index = 0; channel_index < _meta.channel_depths.size(); ++channel_index) {
ERR_FAIL_COND(q.voxel_buffer.get_channel_depth(channel_index) != _meta.channel_depths[channel_index]);
}
const Vector3i block_pos = get_block_position(q.origin_in_voxels) >> q.lod;
const String file_path = get_block_file_path(block_pos, q.lod);
{
const CharString file_path_base_dir = file_path.get_base_dir().utf8();
const Error err = check_directory_created_using_file_locker(file_path_base_dir.get_data());
ERR_FAIL_COND(err != OK);
}
{
Ref<FileAccess> f;
const CharString file_path_utf8 = file_path.utf8();
VoxelFileLockerWrite file_wlock(file_path_utf8.get_data());
{
Error err;
// Create file if not exists, always truncate
f = FileAccess::open(file_path, FileAccess::WRITE, &err);
}
ERR_FAIL_COND(f == nullptr);
f->store_buffer((uint8_t *)FORMAT_BLOCK_MAGIC, 4);
f->store_8(FORMAT_VERSION);
BlockSerializer::SerializeResult res = BlockSerializer::serialize_and_compress(q.voxel_buffer);
if (!res.success) {
ERR_PRINT("Failed to save block");
return;
}
f->store_32(res.data.size());
f->store_buffer(res.data.data(), res.data.size());
}
}
int VoxelStreamBlockFiles::get_used_channels_mask() const {
// Assuming all, since that stream can store anything.
return VoxelBufferInternal::ALL_CHANNELS_MASK;
}
String VoxelStreamBlockFiles::get_directory() const {
return _directory_path;
}
void VoxelStreamBlockFiles::set_directory(String dirpath) {
if (_directory_path != dirpath) {
_directory_path = dirpath;
_meta_loaded = false;
}
}
int VoxelStreamBlockFiles::get_block_size_po2() const {
return _meta.block_size_po2;
}
FileResult VoxelStreamBlockFiles::save_meta() {
CRASH_COND(_directory_path.is_empty());
// Make sure the directory exists
{
const CharString directory_path_utf8 = _directory_path.utf8();
const Error err = check_directory_created_using_file_locker(directory_path_utf8.get_data());
if (err != OK) {
ERR_PRINT("Could not save meta");
return FILE_CANT_OPEN;
}
}
const String meta_path = _directory_path.plus_file(META_FILE_NAME);
{
Error err;
const CharString meta_path_utf8 = meta_path.utf8();
VoxelFileLockerWrite file_wlock(meta_path_utf8.get_data());
Ref<FileAccess> f = FileAccess::open(meta_path, FileAccess::WRITE, &err);
ERR_FAIL_COND_V(f == nullptr, FILE_CANT_OPEN);
f->store_buffer((uint8_t *)FORMAT_META_MAGIC, 4);
f->store_8(FORMAT_VERSION);
f->store_8(_meta.lod_count);
f->store_8(_meta.block_size_po2);
for (unsigned int i = 0; i < _meta.channel_depths.size(); ++i) {
f->store_8(_meta.channel_depths[i]);
}
}
_meta_loaded = true;
_meta_saved = true;
return FILE_OK;
}
FileResult VoxelStreamBlockFiles::load_or_create_meta() {
const FileResult res = load_meta();
if (res == FILE_DOES_NOT_EXIST) {
const FileResult save_result = save_meta();
ERR_FAIL_COND_V(save_result != FILE_OK, save_result);
return FILE_OK;
}
return res;
}
FileResult VoxelStreamBlockFiles::load_meta() {
CRASH_COND(_directory_path.is_empty());
const String meta_path = _directory_path.plus_file(META_FILE_NAME);
Meta meta;
{
Error open_result;
const CharString meta_path_utf8 = meta_path.utf8();
VoxelFileLockerRead file_rlock(meta_path_utf8.get_data());
Ref<FileAccess> f = FileAccess::open(meta_path, FileAccess::READ, &open_result);
// Had to add ERR_FILE_CANT_OPEN because that's what Godot actually returns when the file doesn't exist...
if (!_meta_saved && (open_result == ERR_FILE_NOT_FOUND || open_result == ERR_FILE_CANT_OPEN)) {
// This is a new terrain, save the meta we have and consider it current
return FILE_DOES_NOT_EXIST;
}
ERR_FAIL_COND_V(f.is_null(), FILE_CANT_OPEN);
FileResult check_result = check_magic_and_version(**f, FORMAT_VERSION, FORMAT_META_MAGIC, meta.version);
if (check_result != FILE_OK) {
return check_result;
}
meta.lod_count = f->get_8();
meta.block_size_po2 = f->get_8();
for (unsigned int i = 0; i < meta.channel_depths.size(); ++i) {
uint8_t depth = f->get_8();
ERR_FAIL_COND_V(depth >= VoxelBufferInternal::DEPTH_COUNT, FILE_INVALID_DATA);
meta.channel_depths[i] = (VoxelBufferInternal::Depth)depth;
}
ERR_FAIL_COND_V(meta.lod_count < 1 || meta.lod_count > 32, FILE_INVALID_DATA);
ERR_FAIL_COND_V(meta.block_size_po2 < 1 || meta.block_size_po2 > 8, FILE_INVALID_DATA);
}
_meta_loaded = true;
_meta = meta;
return FILE_OK;
}
String VoxelStreamBlockFiles::get_block_file_path(const Vector3i &block_pos, unsigned int lod) const {
// TODO This is probably extremely inefficient, also given the nature of Godot strings
// Save under a folder, because there could be other kinds of data to store in this terrain
String path = "blocks/lod";
path += String::num_uint64(lod);
path += '/';
for (unsigned int i = 0; i < 3; ++i) {
if (block_pos[i] >= 0) {
path += '+';
}
path += String::num_int64(block_pos[i]);
}
path += BLOCK_FILE_EXTENSION;
return _directory_path.plus_file(path);
}
Vector3i VoxelStreamBlockFiles::get_block_position(const Vector3i &origin_in_voxels) const {
return origin_in_voxels >> _meta.block_size_po2;
}
void VoxelStreamBlockFiles::_bind_methods() {
ClassDB::bind_method(D_METHOD("set_directory", "directory"), &VoxelStreamBlockFiles::set_directory);
ClassDB::bind_method(D_METHOD("get_directory"), &VoxelStreamBlockFiles::get_directory);
ADD_PROPERTY(PropertyInfo(Variant::STRING, "directory", PROPERTY_HINT_DIR), "set_directory", "get_directory");
}

56
streams/voxel_stream_block_files.h
View File

@ -1,56 +0,0 @@
#ifndef VOXEL_STREAM_BLOCK_FILES_H
#define VOXEL_STREAM_BLOCK_FILES_H
#include "../storage/voxel_buffer_internal.h"
#include "file_utils.h"
#include "voxel_stream.h"
class FileAccess;
namespace zylann::voxel {
// Loads and saves blocks to the filesystem, under a directory.
// Each block gets its own file, which may produce a lot of them, but it makes it simple to implement.
// This is a naive implementation and may be very slow in practice, so maybe it will be removed in the future.
class VoxelStreamBlockFiles : public VoxelStream {
GDCLASS(VoxelStreamBlockFiles, VoxelStream)
public:
VoxelStreamBlockFiles();
void load_voxel_block(VoxelStream::VoxelQueryData &q) override;
void save_voxel_block(VoxelStream::VoxelQueryData &q) override;
int get_used_channels_mask() const override;
String get_directory() const;
void set_directory(String dirpath);
int get_block_size_po2() const override;
protected:
static void _bind_methods();
private:
FileResult save_meta();
FileResult load_meta();
FileResult load_or_create_meta();
String get_block_file_path(const Vector3i &block_pos, unsigned int lod) const;
Vector3i get_block_position(const Vector3i &origin_in_voxels) const;
String _directory_path;
struct Meta {
uint8_t version = -1;
uint8_t lod_count = 0;
uint8_t block_size_po2 = 0; // How many voxels in a block
FixedArray<VoxelBufferInternal::Depth, VoxelBufferInternal::MAX_CHANNELS> channel_depths;
};
Meta _meta;
bool _meta_loaded = false;
bool _meta_saved = false;
};
} // namespace zylann::voxel
#endif // VOXEL_STREAM_BLOCK_FILES_H