Zylann/godot_voxel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Made file streams thread-safe (but still won't be used in more than one thread)

Browse Source 
- Added a global file locker so threads can synchronize when accessing files
- Added mutexes to serialize access to file streams (i.e to call in series)
- VoxelStream API now expects implementations to be thread-safe
- Did some logic/data separation in RegionFile in case we need an
  in-memory implementation, however I might just use SQLite in the
  future instead
This commit is contained in:
Marc Gilleron 2021-01-16 13:34:45 +00:00
parent 66141be0b3
commit 89750a20f8
15 changed files with 526 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
math/vector3i.h
View File

@ -211,6 +211,10 @@ _FORCE_INLINE_ Vector3i operator>>(const Vector3i &a, int b) {
return Vector3i(a.x >> b, a.y >> b, a.z >> b);
}
_FORCE_INLINE_ Vector3i operator&(const Vector3i &a, int b) {
return Vector3i(a.x & b, a.y & b, a.z & b);
}
inline Vector3i operator%(const Vector3i &a, const Vector3i &b) {
return Vector3i(a.x % b.x, a.y % b.y, a.z % b.z);
}

33
server/voxel_server.h
View File

@ -3,6 +3,7 @@
#include "../meshers/blocky/voxel_mesher_blocky.h"
#include "../streams/voxel_stream.h"
#include "../util/file_locker.h"
#include "struct_db.h"
#include "voxel_thread_pool.h"
#include <scene/main/node.h>
@ -114,6 +115,10 @@ public:
void process();
void wait_and_clear_all_tasks(bool warn);
inline VoxelFileLocker &get_file_locker() {
return _file_locker;
}
static inline int get_octree_lod_block_region_extent(float split_scale) {
// This is a bounding radius of blocks around a viewer within which we may load them.
// It depends on the LOD split scale, which tells how close to a block we need to be for it to subdivide.
@ -263,6 +268,8 @@ private:
VoxelThreadPool _streaming_thread_pool;
VoxelThreadPool _meshing_thread_pool;
VoxelFileLocker _file_locker;
};
// TODO Hack to make VoxelServer update... need ways to integrate callbacks from main loop!
@ -279,4 +286,30 @@ private:
VoxelServerUpdater();
};
struct VoxelFileLockerRead {
VoxelFileLockerRead(String path) :
_path(path) {
VoxelServer::get_singleton()->get_file_locker().lock_read(path);
}
~VoxelFileLockerRead() {
VoxelServer::get_singleton()->get_file_locker().unlock(_path);
}
String _path;
};
struct VoxelFileLockerWrite {
VoxelFileLockerWrite(String path) :
_path(path) {
VoxelServer::get_singleton()->get_file_locker().lock_write(path);
}
~VoxelFileLockerWrite() {
VoxelServer::get_singleton()->get_file_locker().unlock(_path);
}
String _path;
};
#endif // VOXEL_SERVER_H

4
streams/file_utils.cpp
View File

@ -1,4 +1,5 @@
#include "file_utils.h"
#include "../server/voxel_server.h"
const char *to_string(VoxelFileResult res) {
switch (res) {
@ -6,6 +7,8 @@ const char *to_string(VoxelFileResult res) {
return "OK";
case VOXEL_FILE_CANT_OPEN:
return "Can't open";
case VOXEL_FILE_DOES_NOT_EXIST:
return "Doesn't exist";
case VOXEL_FILE_UNEXPECTED_EOF:
return "Unexpected end of file";
case VOXEL_FILE_INVALID_MAGIC:
@ -41,6 +44,7 @@ VoxelFileResult check_magic_and_version(FileAccess *f, uint8_t expected_version,
}
Error check_directory_created(const String &directory_path) {
VoxelFileLockerWrite file_wlock(directory_path);
DirAccess *d = DirAccess::create_for_path(directory_path);
if (d == nullptr) {

1
streams/file_utils.h
View File

@ -36,6 +36,7 @@ inline void store_vec3u32(FileAccess *f, const Vector3i v) {
enum VoxelFileResult {
VOXEL_FILE_OK = 0,
VOXEL_FILE_CANT_OPEN,
VOXEL_FILE_DOES_NOT_EXIST,
VOXEL_FILE_UNEXPECTED_EOF,
VOXEL_FILE_INVALID_MAGIC,
VOXEL_FILE_INVALID_VERSION,

301
streams/region_file.cpp
View File

@ -16,11 +16,159 @@ const uint8_t FORMAT_VERSION_LEGACY_1 = 1;
const char *FORMAT_REGION_MAGIC = "VXR_";
const uint32_t MAGIC_AND_VERSION_SIZE = 4 + 1;
const uint32_t FIXED_HEADER_DATA_SIZE = 7 + VoxelRegionFile::CHANNEL_COUNT;
const uint32_t FIXED_HEADER_DATA_SIZE = 7 + VoxelRegionFormat::CHANNEL_COUNT;
const uint32_t PALETTE_SIZE_IN_BYTES = 256 * 4;
} // namespace
const char *VoxelRegionFile::FILE_EXTENSION = "vxr";
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const char *VoxelRegionFormat::FILE_EXTENSION = "vxr";
bool VoxelRegionFormat::validate() const {
ERR_FAIL_COND_V(region_size.x < 0 || region_size.x >= static_cast<int>(MAX_BLOCKS_ACROSS), false);
ERR_FAIL_COND_V(region_size.y < 0 || region_size.y >= static_cast<int>(MAX_BLOCKS_ACROSS), false);
ERR_FAIL_COND_V(region_size.z < 0 || region_size.z >= static_cast<int>(MAX_BLOCKS_ACROSS), false);
ERR_FAIL_COND_V(block_size_po2 <= 0, false);
// Test worst case limits (this does not include arbitrary metadata, so it can't be 100% accurrate...)
size_t bytes_per_block = 0;
for (unsigned int i = 0; i < channel_depths.size(); ++i) {
bytes_per_block += VoxelBuffer::get_depth_bit_count(channel_depths[i]) / 8;
}
bytes_per_block *= Vector3i(1 << block_size_po2).volume();
const size_t sectors_per_block = (bytes_per_block - 1) / sector_size + 1;
ERR_FAIL_COND_V(sectors_per_block > VoxelRegionBlockInfo::MAX_SECTOR_COUNT, false);
const size_t max_potential_sectors = region_size.volume() * sectors_per_block;
ERR_FAIL_COND_V(max_potential_sectors > VoxelRegionBlockInfo::MAX_SECTOR_INDEX, false);
return true;
}
bool VoxelRegionFormat::verify_block(const VoxelBuffer &block) const {
ERR_FAIL_COND_V(block.get_size() != Vector3i(1 << block_size_po2), false);
for (unsigned int i = 0; i < VoxelBuffer::MAX_CHANNELS; ++i) {
ERR_FAIL_COND_V(block.get_channel_depth(i) != channel_depths[i], false);
}
return true;
}
static uint32_t get_header_size_v3(const VoxelRegionFormat &format) {
// Which file offset blocks data is starting
// magic + version + blockinfos
return MAGIC_AND_VERSION_SIZE + FIXED_HEADER_DATA_SIZE +
(format.has_palette ? PALETTE_SIZE_IN_BYTES : 0) +
format.region_size.volume() * sizeof(VoxelRegionBlockInfo);
}
static bool save_header(FileAccess *f, uint8_t version, const VoxelRegionFormat &format,
const std::vector<VoxelRegionBlockInfo> &block_infos) {
ERR_FAIL_COND_V(f == nullptr, ERR_INVALID_PARAMETER);
f->seek(0);
f->store_buffer(reinterpret_cast<const uint8_t *>(FORMAT_REGION_MAGIC), 4);
f->store_8(version);
f->store_8(format.block_size_po2);
f->store_8(format.region_size.x);
f->store_8(format.region_size.y);
f->store_8(format.region_size.z);
for (unsigned int i = 0; i < format.channel_depths.size(); ++i) {
f->store_8(format.channel_depths[i]);
}
f->store_16(format.sector_size);
if (format.has_palette) {
f->store_8(0xff);
for (unsigned int i = 0; i < format.palette.size(); ++i) {
const Color8 c = format.palette[i];
f->store_8(c.r);
f->store_8(c.g);
f->store_8(c.b);
f->store_8(c.a);
}
} else {
f->store_8(0x00);
}
// TODO Deal with endianess
f->store_buffer(reinterpret_cast<const uint8_t *>(block_infos.data()),
block_infos.size() * sizeof(VoxelRegionBlockInfo));
size_t blocks_begin_offset = f->get_position();
#ifdef DEBUG_ENABLED
CRASH_COND(blocks_begin_offset != get_header_size_v3(format));
#endif
return true;
}
static bool load_header(FileAccess *f, uint8_t &out_version, VoxelRegionFormat &out_format,
std::vector<VoxelRegionBlockInfo> &out_block_infos) {
ERR_FAIL_COND_V(f == nullptr, ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(f->get_position() != 0, ERR_PARSE_ERROR);
ERR_FAIL_COND_V(f->get_len() < MAGIC_AND_VERSION_SIZE, ERR_PARSE_ERROR);
FixedArray<char, 5> magic(0);
ERR_FAIL_COND_V(f->get_buffer(reinterpret_cast<uint8_t *>(magic.data()), 4) != 4, ERR_PARSE_ERROR);
ERR_FAIL_COND_V(strcmp(magic.data(), FORMAT_REGION_MAGIC) != 0, ERR_PARSE_ERROR);
const uint8_t version = f->get_8();
if (version == FORMAT_VERSION) {
out_format.block_size_po2 = f->get_8();
out_format.region_size.x = f->get_8();
out_format.region_size.y = f->get_8();
out_format.region_size.z = f->get_8();
for (unsigned int i = 0; i < out_format.channel_depths.size(); ++i) {
const uint8_t d = f->get_8();
ERR_FAIL_COND_V(d >= VoxelBuffer::DEPTH_COUNT, ERR_PARSE_ERROR);
out_format.channel_depths[i] = static_cast<VoxelBuffer::Depth>(d);
}
out_format.sector_size = f->get_16();
const uint8_t palette_size = f->get_8();
if (palette_size == 0xff) {
out_format.has_palette = true;
for (unsigned int i = 0; i < out_format.palette.size(); ++i) {
Color8 c;
c.r = f->get_8();
c.g = f->get_8();
c.b = f->get_8();
c.a = f->get_8();
out_format.palette[i] = c;
}
} else if (palette_size == 0x00) {
out_format.has_palette = false;
} else {
ERR_PRINT(String("Unexpected palette value: {0}").format(varray(palette_size)));
return ERR_PARSE_ERROR;
}
}
out_version = version;
out_block_infos.resize(out_format.region_size.volume());
// TODO Deal with endianess
const size_t blocks_len = out_block_infos.size() * sizeof(VoxelRegionBlockInfo);
const size_t read_size = f->get_buffer((uint8_t *)out_block_infos.data(), blocks_len);
ERR_FAIL_COND_V(read_size != blocks_len, ERR_PARSE_ERROR);
return true;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
VoxelRegionFile::VoxelRegionFile() {
// Defaults
@ -80,14 +228,14 @@ Error VoxelRegionFile::open(const String &fpath, bool create_if_not_found) {
// This will be useful to know when sectors get moved on insertion and removal
struct BlockInfoAndIndex {
BlockInfo b;
VoxelRegionBlockInfo b;
unsigned int i;
};
// Filter only present blocks and keep the index around because it represents the 3D position of the block
std::vector<BlockInfoAndIndex> blocks_sorted_by_offset;
for (unsigned int i = 0; i < _header.blocks.size(); ++i) {
const BlockInfo b = _header.blocks[i];
const VoxelRegionBlockInfo b = _header.blocks[i];
if (b.data != 0) {
BlockInfoAndIndex p;
p.b = b;
@ -136,26 +284,9 @@ bool VoxelRegionFile::is_open() const {
return _file_access != nullptr;
}
bool VoxelRegionFile::set_format(const VoxelRegionFile::Format &format) {
bool VoxelRegionFile::set_format(const VoxelRegionFormat &format) {
ERR_FAIL_COND_V_MSG(_file_access != nullptr, false, "Can't set format when the file already exists");
ERR_FAIL_COND_V(format.region_size.x < 0 || format.region_size.x >= static_cast<int>(MAX_BLOCKS_ACROSS), false);
ERR_FAIL_COND_V(format.region_size.y < 0 || format.region_size.y >= static_cast<int>(MAX_BLOCKS_ACROSS), false);
ERR_FAIL_COND_V(format.region_size.z < 0 || format.region_size.z >= static_cast<int>(MAX_BLOCKS_ACROSS), false);
ERR_FAIL_COND_V(format.block_size_po2 <= 0, false);
// Test worst case limits (this does not include arbitrary metadata, so it can't be 100% accurrate...)
{
size_t bytes_per_block = 0;
for (unsigned int i = 0; i < format.channel_depths.size(); ++i) {
bytes_per_block += VoxelBuffer::get_depth_bit_count(format.channel_depths[i]) / 8;
}
bytes_per_block *= Vector3i(1 << format.block_size_po2).volume();
const size_t sectors_per_block = (bytes_per_block - 1) / format.sector_size + 1;
ERR_FAIL_COND_V(sectors_per_block > BlockInfo::MAX_SECTOR_COUNT, false);
const size_t max_potential_sectors = format.region_size.volume() * sectors_per_block;
ERR_FAIL_COND_V(max_potential_sectors > BlockInfo::MAX_SECTOR_INDEX, false);
}
ERR_FAIL_COND_V(!format.validate(), false);
// This will be the format used to create the next file if not found on open()
_header.format = format;
@ -164,7 +295,7 @@ bool VoxelRegionFile::set_format(const VoxelRegionFile::Format &format) {
return true;
}
const VoxelRegionFile::Format &VoxelRegionFile::get_format() const {
const VoxelRegionFormat &VoxelRegionFile::get_format() const {
return _header.format;
}
@ -177,7 +308,7 @@ Error VoxelRegionFile::load_block(
const unsigned int lut_index = get_block_index_in_header(position);
ERR_FAIL_COND_V(lut_index >= _header.blocks.size(), ERR_INVALID_PARAMETER);
const BlockInfo &block_info = _header.blocks[lut_index];
const VoxelRegionBlockInfo &block_info = _header.blocks[lut_index];
if (block_info.data == 0) {
return ERR_DOES_NOT_EXIST;
@ -202,17 +333,9 @@ Error VoxelRegionFile::load_block(
return OK;
}
bool VoxelRegionFile::verify_format(VoxelBuffer &block) {
ERR_FAIL_COND_V(block.get_size() != Vector3i(1 << _header.format.block_size_po2), false);
for (unsigned int i = 0; i < VoxelBuffer::MAX_CHANNELS; ++i) {
ERR_FAIL_COND_V(block.get_channel_depth(i) != _header.format.channel_depths[i], false);
}
return true;
}
Error VoxelRegionFile::save_block(Vector3i position, Ref<VoxelBuffer> block, VoxelBlockSerializerInternal &serializer) {
ERR_FAIL_COND_V(block.is_null(), ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(verify_format(**block) == false, ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(_header.format.verify_block(**block) == false, ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(_file_access == nullptr, ERR_FILE_CANT_WRITE);
FileAccess *f = _file_access;
@ -224,7 +347,7 @@ Error VoxelRegionFile::save_block(Vector3i position, Ref<VoxelBuffer> block, Vox
const unsigned int lut_index = get_block_index_in_header(position);
ERR_FAIL_COND_V(lut_index >= _header.blocks.size(), ERR_INVALID_PARAMETER);
BlockInfo &block_info = _header.blocks[lut_index];
VoxelRegionBlockInfo &block_info = _header.blocks[lut_index];
if (block_info.data == 0) {
// The block isn't in the file yet, append at the end
@ -349,7 +472,7 @@ void VoxelRegionFile::remove_sectors_from_block(Vector3i block_pos, unsigned int
const unsigned int block_index = get_block_index_in_header(block_pos);
CRASH_COND(block_index >= _header.blocks.size());
BlockInfo &block_info = _header.blocks[block_index];
VoxelRegionBlockInfo &block_info = _header.blocks[block_index];
unsigned int src_offset = _blocks_begin_offset +
(block_info.get_sector_index() + block_info.get_sector_count()) * sector_size;
@ -401,7 +524,7 @@ void VoxelRegionFile::remove_sectors_from_block(Vector3i block_pos, unsigned int
// Shift sector index of following blocks
if (old_sector_index < _sectors.size()) {
for (unsigned int i = 0; i < _header.blocks.size(); ++i) {
BlockInfo &b = _header.blocks[i];
VoxelRegionBlockInfo &b = _header.blocks[i];
if (b.data != 0 && b.get_sector_index() > old_sector_index) {
b.set_sector_index(b.get_sector_index() - p_sector_count);
}
@ -414,51 +537,13 @@ bool VoxelRegionFile::save_header(FileAccess *f) {
if (_header.version != FORMAT_VERSION) {
ERR_FAIL_COND_V(migrate_to_latest(f) == false, false);
}
f->seek(0);
f->store_buffer(reinterpret_cast<const uint8_t *>(FORMAT_REGION_MAGIC), 4);
f->store_8(_header.version);
f->store_8(_header.format.block_size_po2);
f->store_8(_header.format.region_size.x);
f->store_8(_header.format.region_size.y);
f->store_8(_header.format.region_size.z);
for (unsigned int i = 0; i < _header.format.channel_depths.size(); ++i) {
f->store_8(_header.format.channel_depths[i]);
}
f->store_16(_header.format.sector_size);
if (_header.format.has_palette) {
f->store_8(0xff);
for (unsigned int i = 0; i < _header.format.palette.size(); ++i) {
const Color8 c = _header.format.palette[i];
f->store_8(c.r);
f->store_8(c.g);
f->store_8(c.b);
f->store_8(c.a);
}
} else {
f->store_8(0x00);
}
// TODO Deal with endianess
f->store_buffer(reinterpret_cast<const uint8_t *>(_header.blocks.data()),
_header.blocks.size() * sizeof(BlockInfo));
ERR_FAIL_COND_V(!::save_header(f, _header.version, _header.format, _header.blocks), false);
_blocks_begin_offset = f->get_position();
#ifdef DEBUG_ENABLED
CRASH_COND(_blocks_begin_offset != get_header_size_v3(_header.format));
#endif
_header_modified = false;
return true;
}
bool VoxelRegionFile::migrate_from_v2_to_v3(FileAccess *f, VoxelRegionFile::Format &format) {
bool VoxelRegionFile::migrate_from_v2_to_v3(FileAccess *f, VoxelRegionFormat &format) {
PRINT_VERBOSE(String("Migrating region file {0} from v2 to v3").format(varray(_file_path)));
// We can migrate if we know in advance what format the file should contain.
@ -512,60 +597,8 @@ bool VoxelRegionFile::migrate_to_latest(FileAccess *f) {
}
Error VoxelRegionFile::load_header(FileAccess *f) {
ERR_FAIL_COND_V(f->get_position() != 0, ERR_PARSE_ERROR);
ERR_FAIL_COND_V(f->get_len() < MAGIC_AND_VERSION_SIZE, ERR_PARSE_ERROR);
FixedArray<char, 5> magic(0);
ERR_FAIL_COND_V(f->get_buffer(reinterpret_cast<uint8_t *>(magic.data()), 4) != 4, ERR_PARSE_ERROR);
ERR_FAIL_COND_V(strcmp(magic.data(), FORMAT_REGION_MAGIC) != 0, ERR_PARSE_ERROR);
const uint8_t version = f->get_8();
if (version == FORMAT_VERSION) {
_header.format.block_size_po2 = f->get_8();
_header.format.region_size.x = f->get_8();
_header.format.region_size.y = f->get_8();
_header.format.region_size.z = f->get_8();
for (unsigned int i = 0; i < _header.format.channel_depths.size(); ++i) {
const uint8_t d = f->get_8();
ERR_FAIL_COND_V(d >= VoxelBuffer::DEPTH_COUNT, ERR_PARSE_ERROR);
_header.format.channel_depths[i] = static_cast<VoxelBuffer::Depth>(d);
}
_header.format.sector_size = f->get_16();
const uint8_t palette_size = f->get_8();
if (palette_size == 0xff) {
_header.format.has_palette = true;
for (unsigned int i = 0; i < _header.format.palette.size(); ++i) {
Color8 c;
c.r = f->get_8();
c.g = f->get_8();
c.b = f->get_8();
c.a = f->get_8();
_header.format.palette[i] = c;
}
} else if (palette_size == 0x00) {
_header.format.has_palette = false;
} else {
ERR_PRINT(String("Unexpected palette value: {0}").format(varray(palette_size)));
return ERR_PARSE_ERROR;
}
}
_header.version = version;
_header.blocks.resize(_header.format.region_size.volume());
// TODO Deal with endianess
const size_t blocks_len = _header.blocks.size() * sizeof(BlockInfo);
const size_t read_size = f->get_buffer((uint8_t *)_header.blocks.data(), blocks_len);
ERR_FAIL_COND_V(read_size != blocks_len, ERR_PARSE_ERROR);
ERR_FAIL_COND_V(!::load_header(f, _header.version, _header.format, _header.blocks), ERR_PARSE_ERROR);
_blocks_begin_offset = f->get_position();
return OK;
}
@ -581,14 +614,6 @@ uint32_t VoxelRegionFile::get_sector_count_from_bytes(uint32_t size_in_bytes) co
return (size_in_bytes - 1) / _header.format.sector_size + 1;
}
uint32_t VoxelRegionFile::get_header_size_v3(const Format &format) {
// Which file offset blocks data is starting
// magic + version + blockinfos
return MAGIC_AND_VERSION_SIZE + FIXED_HEADER_DATA_SIZE +
(format.has_palette ? PALETTE_SIZE_IN_BYTES : 0) +
format.region_size.volume() * sizeof(BlockInfo);
}
unsigned int VoxelRegionFile::get_header_block_count() const {
ERR_FAIL_COND_V(!is_open(), 0);
return _header.blocks.size();

111
streams/region_file.h
View File

@ -10,29 +10,65 @@
class FileAccess;
class VoxelBlockSerializerInternal;
// Archive file storing voxels in a fixed sparse grid data structure.
// The format is designed to be easily writable in chunks so it can be used for partial in-game loading and saving.
// Inspired by https://www.seedofandromeda.com/blogs/1-creating-a-region-file-system-for-a-voxel-game
class VoxelRegionFile {
public:
struct VoxelRegionFormat {
static const char *FILE_EXTENSION;
static const uint32_t MAX_BLOCKS_ACROSS = 255;
static const uint32_t CHANNEL_COUNT = 8;
static_assert(CHANNEL_COUNT == VoxelBuffer::MAX_CHANNELS, "This format doesn't support variable channel count");
struct Format {
// How many voxels in a cubic block, as power of two
uint8_t block_size_po2 = 0;
// How many blocks across all dimensions (stored as 3 bytes)
Vector3i region_size;
FixedArray<VoxelBuffer::Depth, CHANNEL_COUNT> channel_depths;
// Blocks are stored at offsets multiple of that size
uint32_t sector_size = 0;
FixedArray<Color8, 256> palette;
bool has_palette = false;
};
// How many voxels in a cubic block, as power of two
uint8_t block_size_po2 = 0;
// How many blocks across all dimensions (stored as 3 bytes)
Vector3i region_size;
FixedArray<VoxelBuffer::Depth, CHANNEL_COUNT> channel_depths;
// Blocks are stored at offsets multiple of that size
uint32_t sector_size = 0;
FixedArray<Color8, 256> palette;
bool has_palette = false;
bool validate() const;
bool verify_block(const VoxelBuffer &block) const;
};
struct VoxelRegionBlockInfo {
static const unsigned int MAX_SECTOR_INDEX = 0xffffff;
static const unsigned int MAX_SECTOR_COUNT = 0xff;
// AAAB
// A: 3 bytes for sector index
// B: 1 byte for size of the block, in sectors
uint32_t data = 0;
inline uint32_t get_sector_index() const {
return data >> 8;
}
inline void set_sector_index(uint32_t i) {
CRASH_COND(i > MAX_SECTOR_INDEX);
data = (i << 8) | (data & 0xff);
}
inline uint32_t get_sector_count() const {
return data & 0xff;
}
inline void set_sector_count(uint32_t c) {
CRASH_COND(c > 0xff);
data = (c & 0xff) | (data & 0xffffff00);
}
};
// Archive file storing voxels in a fixed sparse grid data structure.
// The format is designed to be easily writable in chunks so it can be used for partial in-game loading and saving.
// Inspired by https://www.seedofandromeda.com/blogs/1-creating-a-region-file-system-for-a-voxel-game
//
// This is a stream implementation, where the file handle remains in use for read and write and only keeps a fraction
// of data in memory.
// It isn't thread-safe.
//
class VoxelRegionFile {
public:
VoxelRegionFile();
~VoxelRegionFile();
@ -40,8 +76,8 @@ public:
Error close();
bool is_open() const;
bool set_format(const Format &format);
const Format &get_format() const;
bool set_format(const VoxelRegionFormat &format);
const VoxelRegionFormat &get_format() const;
Error load_block(Vector3i position, Ref<VoxelBuffer> out_block, VoxelBlockSerializerInternal &serializer);
Error save_block(Vector3i position, Ref<VoxelBuffer> block, VoxelBlockSerializerInternal &serializer);
@ -62,46 +98,15 @@ private:
void remove_sectors_from_block(Vector3i block_pos, unsigned int p_sector_count);
bool migrate_to_latest(FileAccess *f);
static uint32_t get_header_size_v3(const Format &format);
bool migrate_from_v2_to_v3(FileAccess *f, VoxelRegionFile::Format &format);
bool verify_format(VoxelBuffer &block);
struct BlockInfo {
static const unsigned int MAX_SECTOR_INDEX = 0xffffff;
static const unsigned int MAX_SECTOR_COUNT = 0xff;
// AAAB
// A: 3 bytes for sector index
// B: 1 byte for size of the block, in sectors
uint32_t data = 0;
inline uint32_t get_sector_index() const {
return data >> 8;
}
inline void set_sector_index(uint32_t i) {
CRASH_COND(i > MAX_SECTOR_INDEX);
data = (i << 8) | (data & 0xff);
}
inline uint32_t get_sector_count() const {
return data & 0xff;
}
inline void set_sector_count(uint32_t c) {
CRASH_COND(c > 0xff);
data = (c & 0xff) | (data & 0xffffff00);
}
};
bool migrate_from_v2_to_v3(FileAccess *f, VoxelRegionFormat &format);
struct Header {
uint8_t version = -1;
Format format;
VoxelRegionFormat format;
// Location and size of blocks, indexed by flat position.
// This table always has the same size,
// and the same index always corresponds to the same 3D position.
std::vector<BlockInfo> blocks;
std::vector<VoxelRegionBlockInfo> blocks;
};
FileAccess *_file_access = nullptr;
@ -123,7 +128,7 @@ private:
// and which position their block is. The same block can span multiple sectors.
// This is essentially a reverse table of `Header::blocks`.
std::vector<Vector3u16> _sectors;
size_t _blocks_begin_offset;
uint32_t _blocks_begin_offset;
String _file_path;
};

12
streams/voxel_stream.cpp
View File

@ -28,14 +28,6 @@ void VoxelStream::immerge_blocks(Vector<VoxelBlockRequest> &p_blocks) {
}
}
bool VoxelStream::is_thread_safe() const {
return false;
}
bool VoxelStream::is_cloneable() const {
return false;
}
void VoxelStream::_emerge_block(Ref<VoxelBuffer> out_buffer, Vector3 origin_in_voxels, int lod) {
ERR_FAIL_COND(lod < 0);
emerge_block(out_buffer, Vector3i(origin_in_voxels), lod);
@ -54,10 +46,6 @@ int VoxelStream::_get_used_channels_mask() const {
return get_used_channels_mask();
}
VoxelStream::Stats VoxelStream::get_statistics() const {
return _stats;
}
bool VoxelStream::has_script() const {
Ref<Script> s = get_script();
return s.is_valid();

16
streams/voxel_stream.h
View File

@ -10,11 +10,6 @@
class VoxelStream : public Resource {
GDCLASS(VoxelStream, Resource)
public:
struct Stats {
int file_openings = 0;
int time_spent_opening_files = 0;
};
VoxelStream();
// TODO Rename load_block()
@ -37,15 +32,6 @@ public:
// Declares the format expected from this stream
virtual int get_used_channels_mask() const;
// If the stream is thread-safe, the same instance will be used by the streamer across all threads.
// If the stream is not thread-safe:
// - If it is cloneable, the streamer will duplicate the instance for each thread.
// - If it isn't, the streamer will be limited to a single thread.
virtual bool is_thread_safe() const;
virtual bool is_cloneable() const;
Stats get_statistics() const;
virtual bool has_script() const;
protected:
@ -54,8 +40,6 @@ protected:
void _emerge_block(Ref<VoxelBuffer> out_buffer, Vector3 origin_in_voxels, int lod);
void _immerge_block(Ref<VoxelBuffer> buffer, Vector3 origin_in_voxels, int lod);
int _get_used_channels_mask() const;
Stats _stats;
};
#endif // VOXEL_STREAM_H

33
streams/voxel_stream_block_files.cpp
View File

@ -1,4 +1,5 @@
#include "voxel_stream_block_files.h"
#include "../server/voxel_server.h"
#include "../util/utility.h"
#include <core/os/dir_access.h>
#include <core/os/file_access.h>
@ -46,6 +47,7 @@ void VoxelStreamBlockFiles::emerge_block(Ref<VoxelBuffer> out_buffer, Vector3i o
String file_path = get_block_file_path(block_pos, lod);
FileAccess *f = nullptr;
VoxelFileLockerRead file_rlock(file_path);
{
Error err;
f = open_file(file_path, FileAccess::READ, &err);
@ -62,7 +64,11 @@ void VoxelStreamBlockFiles::emerge_block(Ref<VoxelBuffer> out_buffer, Vector3i o
{
uint8_t version;
VoxelFileResult err = check_magic_and_version(f, FORMAT_VERSION, FORMAT_BLOCK_MAGIC, version);
ERR_FAIL_COND_MSG(err != VOXEL_FILE_OK, ::to_string(err));
if (err != VOXEL_FILE_OK) {
memdelete(f);
ERR_PRINT(String("Invalid file header: ") + ::to_string(err));
return;
}
}
// Configure depths, as they currently are only specified in the meta file.
@ -72,7 +78,9 @@ void VoxelStreamBlockFiles::emerge_block(Ref<VoxelBuffer> out_buffer, Vector3i o
}
uint32_t size_to_read = f->get_32();
ERR_FAIL_COND(!_block_serializer.decompress_and_deserialize(f, size_to_read, **out_buffer));
if (!_block_serializer.decompress_and_deserialize(f, size_to_read, **out_buffer)) {
ERR_PRINT("Failed to decompress and deserialize");
}
}
f->close();
@ -80,7 +88,6 @@ void VoxelStreamBlockFiles::emerge_block(Ref<VoxelBuffer> out_buffer, Vector3i o
}
void VoxelStreamBlockFiles::immerge_block(Ref<VoxelBuffer> buffer, Vector3i origin_in_voxels, int lod) {
ERR_FAIL_COND(_directory_path.empty());
ERR_FAIL_COND(buffer.is_null());
@ -118,12 +125,13 @@ void VoxelStreamBlockFiles::immerge_block(Ref<VoxelBuffer> buffer, Vector3i orig
//print_line(String("Saving VXB {0}").format(varray(block_pos.to_vec3())));
{
Error err = check_directory_created(file_path.get_base_dir());
const Error err = check_directory_created(file_path.get_base_dir());
ERR_FAIL_COND(err != OK);
}
{
FileAccess *f = nullptr;
VoxelFileLockerWrite file_wlock(file_path);
{
Error err;
// Create file if not exists, always truncate
@ -159,7 +167,6 @@ int VoxelStreamBlockFiles::get_block_size_po2() const {
}
VoxelFileResult VoxelStreamBlockFiles::save_meta() {
CRASH_COND(_directory_path.empty());
// Make sure the directory exists
@ -175,6 +182,7 @@ VoxelFileResult VoxelStreamBlockFiles::save_meta() {
{
Error err;
VoxelFileLockerWrite file_wlock(meta_path);
FileAccess *f = open_file(meta_path, FileAccess::WRITE, &err);
ERR_FAIL_COND_V(f == nullptr, VOXEL_FILE_CANT_OPEN);
@ -196,6 +204,16 @@ VoxelFileResult VoxelStreamBlockFiles::save_meta() {
return VOXEL_FILE_OK;
}
VoxelFileResult VoxelStreamBlockFiles::load_or_create_meta() {
VoxelFileResult res = load_meta();
if (res == VOXEL_FILE_DOES_NOT_EXIST) {
VoxelFileResult save_result = save_meta();
ERR_FAIL_COND_V(save_result != VOXEL_FILE_OK, save_result);
return VOXEL_FILE_OK;
}
return res;
}
VoxelFileResult VoxelStreamBlockFiles::load_meta() {
CRASH_COND(_directory_path.empty());
@ -204,13 +222,12 @@ VoxelFileResult VoxelStreamBlockFiles::load_meta() {
Meta meta;
{
Error open_result;
VoxelFileLockerRead file_rlock(meta_path);
FileAccessRef f = open_file(meta_path, FileAccess::READ, &open_result);
// Had to add ERR_FILE_CANT_OPEN because that's what Godot actually returns when the file doesn't exist...
if (!_meta_saved && (open_result == ERR_FILE_NOT_FOUND || open_result == ERR_FILE_CANT_OPEN)) {
// This is a new terrain, save the meta we have and consider it current
VoxelFileResult save_result = save_meta();
ERR_FAIL_COND_V(save_result != VOXEL_FILE_OK, save_result);
return VOXEL_FILE_OK;
return VOXEL_FILE_DOES_NOT_EXIST;
}
ERR_FAIL_COND_V(!f, VOXEL_FILE_CANT_OPEN);

2
streams/voxel_stream_block_files.h
View File

@ -8,6 +8,7 @@ class FileAccess;
// Loads and saves blocks to the filesystem, under a directory.
// Each block gets its own file, which may produce a lot of them, but it makes it simple to implement.
// This is a naive implementation and may be very slow in practice, so maybe it will be removed in the future.
class VoxelStreamBlockFiles : public VoxelStreamFile {
GDCLASS(VoxelStreamBlockFiles, VoxelStreamFile)
public:
@ -27,6 +28,7 @@ protected:
private:
VoxelFileResult save_meta();
VoxelFileResult load_meta();
VoxelFileResult load_or_create_meta();
String get_block_file_path(const Vector3i &block_pos, unsigned int lod) const;
Vector3i get_block_position(const Vector3i &origin_in_voxels) const;

64
streams/voxel_stream_file.cpp
View File

@ -1,23 +1,36 @@
#include "voxel_stream_file.h"
#include "../server/voxel_server.h"
#include "../util/profiling.h"
#include <core/os/file_access.h>
#include <core/os/os.h>
VoxelStreamFile::VoxelStreamFile() {
_parameters_lock = RWLock::create();
}
VoxelStreamFile::~VoxelStreamFile() {
memdelete(_parameters_lock);
}
void VoxelStreamFile::set_save_fallback_output(bool enabled) {
_save_fallback_output = enabled;
RWLockWrite wlock(_parameters_lock);
_parameters.save_fallback_output = enabled;
}
bool VoxelStreamFile::get_save_fallback_output() const {
return _save_fallback_output;
RWLockRead rlock(_parameters_lock);
return _parameters.save_fallback_output;
}
Ref<VoxelStream> VoxelStreamFile::get_fallback_stream() const {
return _fallback_stream;
RWLockRead rlock(_parameters_lock);
return _parameters.fallback_stream;
}
void VoxelStreamFile::set_fallback_stream(Ref<VoxelStream> stream) {
ERR_FAIL_COND(*stream == this);
_fallback_stream = stream;
RWLockWrite wlock(_parameters_lock);
_parameters.fallback_stream = stream;
}
void VoxelStreamFile::emerge_block_fallback(Ref<VoxelBuffer> out_buffer, Vector3i origin_in_voxels, int lod) {
@ -37,32 +50,46 @@ void VoxelStreamFile::emerge_block_fallback(Ref<VoxelBuffer> out_buffer, Vector3
void VoxelStreamFile::emerge_blocks_fallback(Vector<VoxelBlockRequest> &requests) {
VOXEL_PROFILE_SCOPE();
if (_fallback_stream.is_valid()) {
_fallback_stream->emerge_blocks(requests);
Parameters params;
{
RWLockRead rlock(_parameters_lock);
params = _parameters;
}
if (_save_fallback_output) {
if (params.fallback_stream.is_valid()) {
params.fallback_stream->emerge_blocks(requests);
if (params.save_fallback_output) {
immerge_blocks(requests);
}
}
}
int VoxelStreamFile::get_used_channels_mask() const {
if (_fallback_stream.is_valid()) {
return _fallback_stream->get_used_channels_mask();
RWLockRead rlock(_parameters_lock);
if (_parameters.fallback_stream.is_valid()) {
return _parameters.fallback_stream->get_used_channels_mask();
}
return VoxelStream::get_used_channels_mask();
}
FileAccess *VoxelStreamFile::open_file(const String &fpath, int mode_flags, Error *err) {
VOXEL_PROFILE_SCOPE();
uint64_t time_before = OS::get_singleton()->get_ticks_usec();
//uint64_t time_before = OS::get_singleton()->get_ticks_usec();
FileAccess *f = FileAccess::open(fpath, mode_flags, err);
uint64_t time_spent = OS::get_singleton()->get_ticks_usec() - time_before;
_stats.time_spent_opening_files += time_spent;
++_stats.file_openings;
//uint64_t time_spent = OS::get_singleton()->get_ticks_usec() - time_before;
//_stats.time_spent_opening_files += time_spent;
//++_stats.file_openings;
return f;
}
void VoxelStreamFile::close_file(FileAccess *f) {
ERR_FAIL_COND(f == nullptr);
f->close();
memdelete(f);
VoxelServer::get_singleton()->get_file_locker().unlock(f->get_path());
}
int VoxelStreamFile::get_block_size_po2() const {
return 4;
}
@ -76,8 +103,9 @@ Vector3 VoxelStreamFile::_get_block_size() const {
}
bool VoxelStreamFile::has_script() const {
if (_fallback_stream.is_valid()) {
return _fallback_stream->has_script();
RWLockRead rlock(_parameters_lock);
if (_parameters.fallback_stream.is_valid()) {
return _parameters.fallback_stream->has_script();
}
return VoxelStream::has_script();
}
@ -91,6 +119,8 @@ void VoxelStreamFile::_bind_methods() {
ClassDB::bind_method(D_METHOD("get_block_size"), &VoxelStreamFile::_get_block_size);
ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "fallback_stream", PROPERTY_HINT_RESOURCE_TYPE, "VoxelStream"), "set_fallback_stream", "get_fallback_stream");
ADD_PROPERTY(PropertyInfo(Variant::BOOL, "save_fallback_output"), "set_save_fallback_output", "get_save_fallback_output");
ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "fallback_stream", PROPERTY_HINT_RESOURCE_TYPE, "VoxelStream"),
"set_fallback_stream", "get_fallback_stream");
ADD_PROPERTY(PropertyInfo(Variant::BOOL, "save_fallback_output"),
"set_save_fallback_output", "get_save_fallback_output");
}

13
streams/voxel_stream_file.h
View File

@ -13,6 +13,9 @@ class FileAccess;
class VoxelStreamFile : public VoxelStream {
GDCLASS(VoxelStreamFile, VoxelStream)
public:
VoxelStreamFile();
~VoxelStreamFile();
void set_save_fallback_output(bool enabled);
bool get_save_fallback_output() const;
@ -35,14 +38,20 @@ protected:
void emerge_blocks_fallback(Vector<VoxelBlockRequest> &requests);
FileAccess *open_file(const String &fpath, int mode_flags, Error *err);
void close_file(FileAccess *f);
VoxelBlockSerializerInternal _block_serializer;
private:
Vector3 _get_block_size() const;
Ref<VoxelStream> _fallback_stream;
bool _save_fallback_output = true;
struct Parameters {
Ref<VoxelStream> fallback_stream;
bool save_fallback_output = true;
};
Parameters _parameters;
RWLock *_parameters_lock = nullptr;
};
#endif // VOXEL_STREAM_FILE_H

112
streams/voxel_stream_region_files.cpp
View File

@ -1,5 +1,6 @@
#include "voxel_stream_region_files.h"
#include "../math/rect3i.h"
#include "../server/voxel_server.h"
#include "../util/macros.h"
#include "../util/profiling.h"
#include "../util/utility.h"
@ -24,10 +25,12 @@ VoxelStreamRegionFiles::VoxelStreamRegionFiles() {
_meta.sector_size = 512; // next_power_of_2(_meta.block_size.volume() / 10) // based on compression ratios
_meta.lod_count = 1;
_meta.channel_depths.fill(VoxelBuffer::DEFAULT_CHANNEL_DEPTH);
_mutex = Mutex::create();
}
VoxelStreamRegionFiles::~VoxelStreamRegionFiles() {
close_all_regions();
memdelete(_mutex);
}
void VoxelStreamRegionFiles::emerge_block(Ref<VoxelBuffer> out_buffer, Vector3i origin_in_voxels, int lod) {
@ -99,6 +102,8 @@ VoxelStreamRegionFiles::EmergeResult VoxelStreamRegionFiles::_emerge_block(
VOXEL_PROFILE_SCOPE();
ERR_FAIL_COND_V(out_buffer.is_null(), EMERGE_FAILED);
MutexLock lock(_mutex);
if (_directory_path.empty()) {
return EMERGE_OK_FALLBACK;
}
@ -156,6 +161,8 @@ VoxelStreamRegionFiles::EmergeResult VoxelStreamRegionFiles::_emerge_block(
void VoxelStreamRegionFiles::_immerge_block(Ref<VoxelBuffer> voxel_buffer, Vector3i origin_in_voxels, int lod) {
VOXEL_PROFILE_SCOPE();
MutexLock lock(_mutex);
ERR_FAIL_COND(_directory_path.empty());
ERR_FAIL_COND(voxel_buffer.is_null());
@ -199,11 +206,14 @@ void VoxelStreamRegionFiles::_immerge_block(Ref<VoxelBuffer> voxel_buffer, Vecto
}
String VoxelStreamRegionFiles::get_directory() const {
MutexLock lock(_mutex);
return _directory_path;
}
void VoxelStreamRegionFiles::set_directory(String dirpath) {
MutexLock lock(_mutex);
if (_directory_path != dirpath) {
close_all_regions();
_directory_path = dirpath.strip_edges();
_meta_loaded = false;
_meta_saved = false;
@ -266,6 +276,7 @@ VoxelFileResult VoxelStreamRegionFiles::save_meta() {
String meta_path = _directory_path.plus_file(META_FILE_NAME);
Error err;
VoxelFileLockerWrite file_wlock(meta_path);
FileAccessRef f = open_file(meta_path, FileAccess::WRITE, &err);
if (!f) {
ERR_PRINT(String("Could not save {0}").format(varray(meta_path)));
@ -312,6 +323,7 @@ VoxelFileResult VoxelStreamRegionFiles::load_meta() {
{
Error err;
VoxelFileLockerRead file_rlock(meta_path);
FileAccessRef f = open_file(meta_path, FileAccess::READ, &err);
if (!f) {
return VOXEL_FILE_CANT_OPEN;
@ -391,7 +403,7 @@ String VoxelStreamRegionFiles::get_region_file_path(const Vector3i &region_pos,
a[1] = region_pos.x;
a[2] = region_pos.y;
a[3] = region_pos.z;
a[4] = VoxelRegionFile::FILE_EXTENSION;
a[4] = VoxelRegionFormat::FILE_EXTENSION;
return _directory_path.plus_file(String("regions/lod{0}/r.{1}.{2}.{3}.{4}").format(a));
}
@ -430,7 +442,7 @@ VoxelStreamRegionFiles::CachedRegion *VoxelStreamRegionFiles::open_region(
// Configure format because we might have to create the file, and some old file versions don't embed format
{
VoxelRegionFile::Format format;
VoxelRegionFormat format;
format.block_size_po2 = _meta.block_size_po2;
format.channel_depths = _meta.channel_depths;
// TODO Palette support
@ -464,7 +476,7 @@ VoxelStreamRegionFiles::CachedRegion *VoxelStreamRegionFiles::open_region(
// Make sure it has correct format
{
const VoxelRegionFile::Format &format = cached_region->region.get_format();
const VoxelRegionFormat &format = cached_region->region.get_format();
if (format.block_size_po2 != _meta.block_size_po2 ||
format.channel_depths != _meta.channel_depths ||
format.region_size != Vector3i(1 << _meta.region_size_po2) ||
@ -587,7 +599,7 @@ void VoxelStreamRegionFiles::_convert_files(Meta new_meta) {
const String lod_folder =
old_stream->_directory_path.plus_file("regions").plus_file("lod") + String::num_int64(lod);
const String ext = String(".") + VoxelRegionFile::FILE_EXTENSION;
const String ext = String(".") + VoxelRegionFormat::FILE_EXTENSION;
DirAccessRef da = DirAccess::open(lod_folder);
if (!da) {
@ -719,6 +731,7 @@ void VoxelStreamRegionFiles::_convert_files(Meta new_meta) {
}
Vector3i VoxelStreamRegionFiles::get_region_size() const {
MutexLock lock(_mutex);
return Vector3i(1 << _meta.region_size_po2);
}
@ -727,18 +740,22 @@ Vector3 VoxelStreamRegionFiles::get_region_size_v() const {
}
int VoxelStreamRegionFiles::get_region_size_po2() const {
MutexLock lock(_mutex);
return _meta.region_size_po2;
}
int VoxelStreamRegionFiles::get_block_size_po2() const {
MutexLock lock(_mutex);
return _meta.block_size_po2;
}
int VoxelStreamRegionFiles::get_lod_count() const {
MutexLock lock(_mutex);
return _meta.lod_count;
}
int VoxelStreamRegionFiles::get_sector_size() const {
MutexLock lock(_mutex);
return _meta.sector_size;
}
@ -748,46 +765,61 @@ int VoxelStreamRegionFiles::get_sector_size() const {
// This can be made easier by adding a button to the inspector to convert existing files just in case
void VoxelStreamRegionFiles::set_region_size_po2(int p_region_size_po2) {
if (_meta.region_size_po2 == p_region_size_po2) {
return;
{
MutexLock lock(_mutex);
if (_meta.region_size_po2 == p_region_size_po2) {
return;
}
ERR_FAIL_COND_MSG(_meta_loaded, "Can't change existing region size without heavy conversion. Use convert_files().");
ERR_FAIL_COND(p_region_size_po2 < 1);
ERR_FAIL_COND(p_region_size_po2 > 8);
_meta.region_size_po2 = p_region_size_po2;
}
ERR_FAIL_COND_MSG(_meta_loaded, "Can't change existing region size without heavy conversion. Use convert_files().");
ERR_FAIL_COND(p_region_size_po2 < 1);
ERR_FAIL_COND(p_region_size_po2 > 8);
_meta.region_size_po2 = p_region_size_po2;
emit_changed();
}
void VoxelStreamRegionFiles::set_block_size_po2(int p_block_size_po2) {
if (_meta.block_size_po2 == p_block_size_po2) {
return;
{
MutexLock lock(_mutex);
if (_meta.block_size_po2 == p_block_size_po2) {
return;
}
ERR_FAIL_COND_MSG(_meta_loaded,
"Can't change existing block size without heavy conversion. Use convert_files().");
ERR_FAIL_COND(p_block_size_po2 < 1);
ERR_FAIL_COND(p_block_size_po2 > 8);
_meta.block_size_po2 = p_block_size_po2;
}
ERR_FAIL_COND_MSG(_meta_loaded, "Can't change existing block size without heavy conversion. Use convert_files().");
ERR_FAIL_COND(p_block_size_po2 < 1);
ERR_FAIL_COND(p_block_size_po2 > 8);
_meta.block_size_po2 = p_block_size_po2;
emit_changed();
}
void VoxelStreamRegionFiles::set_sector_size(int p_sector_size) {
if (static_cast<int>(_meta.sector_size) == p_sector_size) {
return;
{
MutexLock lock(_mutex);
if (static_cast<int>(_meta.sector_size) == p_sector_size) {
return;
}
ERR_FAIL_COND_MSG(_meta_loaded,
"Can't change existing sector size without heavy conversion. Use convert_files().");
ERR_FAIL_COND(p_sector_size < 256);
ERR_FAIL_COND(p_sector_size > 65536);
_meta.sector_size = p_sector_size;
}
ERR_FAIL_COND_MSG(_meta_loaded, "Can't change existing sector size without heavy conversion. Use convert_files().");
ERR_FAIL_COND(p_sector_size < 256);
ERR_FAIL_COND(p_sector_size > 65536);
_meta.sector_size = p_sector_size;
emit_changed();
}
void VoxelStreamRegionFiles::set_lod_count(int p_lod_count) {
if (_meta.lod_count == p_lod_count) {
return;
{
MutexLock lock(_mutex);
if (_meta.lod_count == p_lod_count) {
return;
}
ERR_FAIL_COND_MSG(_meta_loaded,
"Can't change existing LOD count without heavy conversion. Use convert_files().");
ERR_FAIL_COND(p_lod_count < 1);
ERR_FAIL_COND(p_lod_count > 32);
_meta.lod_count = p_lod_count;
}
ERR_FAIL_COND_MSG(_meta_loaded, "Can't change existing LOD count without heavy conversion. Use convert_files().");
ERR_FAIL_COND(p_lod_count < 1);
ERR_FAIL_COND(p_lod_count > 32);
_meta.lod_count = p_lod_count;
emit_changed();
}
@ -799,22 +831,26 @@ void VoxelStreamRegionFiles::convert_files(Dictionary d) {
meta.sector_size = int(d["sector_size"]);
meta.lod_count = int(d["lod_count"]);
ERR_FAIL_COND_MSG(!check_meta(meta), "Invalid setting");
{
MutexLock lock(_mutex);
if (!_meta_loaded) {
ERR_FAIL_COND_MSG(!check_meta(meta), "Invalid setting");
if (load_meta() != VOXEL_FILE_OK) {
// New stream, nothing to convert
_meta = meta;
if (!_meta_loaded) {
if (load_meta() != VOXEL_FILE_OK) {
// New stream, nothing to convert
_meta = meta;
} else {
// Just opened existing stream
_convert_files(meta);
}
} else {
// Just opened existing stream
// That stream was previously used
_convert_files(meta);
}
} else {
// That stream was previously used
_convert_files(meta);
}
emit_changed();

8
streams/voxel_stream_region_files.h
View File

@ -15,6 +15,8 @@ class FileAccess;
// because it allows to keep using the same file handles and avoid switching.
// Inspired by https://www.seedofandromeda.com/blogs/1-creating-a-region-file-system-for-a-voxel-game
//
// Region files are not thread-safe. Because of this, internal mutexing may often constrain the use by one thread only.
//
class VoxelStreamRegionFiles : public VoxelStreamFile {
GDCLASS(VoxelStreamRegionFiles, VoxelStreamFile)
public:
@ -105,6 +107,10 @@ private:
}
};
// TODO This is not thread-friendly.
// `VoxelRegionFile` is not thread-safe so we have to limit the usage to one thread at once, blocking the others.
// A refactoring should be done to allow better threading.
struct CachedRegion {
Vector3i position;
int lod = 0;
@ -121,6 +127,8 @@ private:
std::vector<CachedRegion *> _region_cache;
// TODO Add memory caches to increase capacity.
unsigned int _max_open_regions = MIN(8, FOPEN_MAX);
Mutex *_mutex;
};
#endif // VOXEL_STREAM_REGION_H

96
util/file_locker.h Normal file
View File

@ -0,0 +1,96 @@
#ifndef VOXEL_FILE_LOCKER_H
#define VOXEL_FILE_LOCKER_H
#include <core/hash_map.h>
#include <core/os/file_access.h>
#include <core/os/rw_lock.h>
// Performs software locking on paths,
// so that multiple threads (controlled by this module) wanting to access the same file will lock a shared mutex.
// Note: has nothing to do with voxels, it's just prefixed.
class VoxelFileLocker {
public:
VoxelFileLocker() {
_files_mutex = Mutex::create();
}
~VoxelFileLocker() {
{
MutexLock lock(_files_mutex);
const String *key = nullptr;
while ((key = _files.next(key))) {
File *fp = _files.getptr(*key);
memdelete(fp->lock);
}
}
memdelete(_files_mutex);
}
void lock_read(String fpath) {
lock(fpath, true);
}
void lock_write(String fpath) {
lock(fpath, false);
}
void unlock(String fpath) {
unlock_internal(fpath);
}
private:
struct File {
RWLock *lock;
bool read_only;
};
void lock(String fpath, bool read_only) {
File *fp = nullptr;
{
MutexLock lock(_files_mutex);
fp = _files.getptr(fpath);
if (fp == nullptr) {
File f;
f.lock = RWLock::create();
_files.set(fpath, f);
fp = _files.getptr(fpath);
}
}
if (read_only) {
fp->lock->read_lock();
// The read lock was acquired. It means nobody is writing.
fp->read_only = true;
} else {
fp->lock->write_lock();
// The write lock was acquired. It means only one thread is writing.
fp->read_only = false;
}
}
void unlock_internal(String fpath) {
File *fp = nullptr;
// I assume `get_path` returns the same string that was used to open it
{
MutexLock lock(_files_mutex);
fp = _files.getptr(fpath);
}
ERR_FAIL_COND(fp == nullptr);
// TODO FileAccess::reopen can have been called, nullifying my efforts to enforce thread sync :|
// So for now please don't do that
if (fp->read_only) {
fp->lock->read_unlock();
} else {
fp->lock->write_unlock();
}
}
private:
Mutex *_files_mutex;
HashMap<String, File> _files;
};
#endif // VOXEL_FILE_LOCKER_H