Merge branch 'master' into modifiers
Marc Gilleron
commit
19f94e9912
|
|
@ -80,9 +80,9 @@ jobs:
|
|||
# Download and extract zip archive with project, folder is renamed to be able to easy change used project
|
||||
- name: Download test project
|
||||
run: |
|
||||
wget https://github.com/qarmin/RegressionTestProject/archive/master.zip
|
||||
unzip master.zip
|
||||
mv "RegressionTestProject-master" "test_project"
|
||||
wget https://github.com/godotengine/regression-test-project/archive/refs/heads/4.0.zip
|
||||
unzip 4.0.zip
|
||||
mv "regression-test-project-4.0" "test_project"
|
||||
|
||||
# Editor is quite complicated piece of software, so it is easy to introduce bug here
|
||||
- name: Open and close editor
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
*.obj
|
||||
*.pyc
|
||||
*.o
|
||||
*.autosave
|
||||
*.bc
|
||||
*.d
|
||||
*.log
|
||||
*.o
|
||||
*.obj
|
||||
*.pyc
|
||||
*.tmp
|
||||
|
|
|
|||
|
|
@ -26,25 +26,27 @@ Short recipes
|
|||
The following sections contain basic quick start instructions to get voxel terrains running.
|
||||
For more in-depth information, see the rest of the documentation.
|
||||
|
||||
Before each example, you may do the following:
|
||||
|
||||
- Create a new project and a new 3D scene. Give it some light by adding a `DirectionalLight` node, and orientate it so it shines approximately downwards. You can enable `Shadows` too.
|
||||
|
||||
### Blocky heightmap terrain using `VoxelTerrain`
|
||||
|
||||
1) Create a new project and a new 3D scene. Give it some light by adding a `DirectionalLight` node, and orientate it so it shines approximately downwards. You can enable `Shadows` too.
|
||||
|
||||
2) Add a `VoxelTerrain` node, and adjust the following settings in the inspector:
|
||||
1) Add a `VoxelTerrain` node, and adjust the following settings in the inspector:
|
||||
|
||||
2.1) Under the `materials` section, create a new `SpatialMaterial` in the first slot. Then click on it and enable the `vertex_color_as_albedo` option. This will give the blocks better shading.
|
||||
1.1) Under the `materials` section, create a new `SpatialMaterial` in the first slot. Then click on it and enable the `vertex_color_as_albedo` option. This will give the blocks better shading.
|
||||
|
||||
2.1) `Generator`: create a new `VoxelGeneratorNoise2D`. Then click on it and set its `Channel` parameter to `TYPE`. Also make sure the `noise` property is assigned to a noise resource.
|
||||
1.1) `Generator`: create a new `VoxelGeneratorNoise2D`. Then click on it and set its `Channel` parameter to `TYPE`. Also make sure the `noise` property is assigned to a noise resource.
|
||||
|
||||
2.2) `Mesher`: create a new `VoxelMesherBlocky`. In the `library` property, create a new `VoxelBlockyLibrary`. Set `voxel_count` to `2`. A list of two voxels types appear. In the first slot, create a new `VoxelBlockyModel` (this will be "air"). In the second slot, create another `VoxelBlockyModel`. In that second one, set its `geometry_type` to `Cube` (this will be "solid").
|
||||
1.2) `Mesher`: create a new `VoxelMesherBlocky`. In the `library` property, create a new `VoxelBlockyLibrary`. Set `voxel_count` to `2`. A list of two voxels types appear. In the first slot, create a new `VoxelBlockyModel` (this will be "air"). In the second slot, create another `VoxelBlockyModel`. In that second one, set its `geometry_type` to `Cube` (this will be "solid").
|
||||
|
||||
2.3) Select the terrain node again, and in the `Terrain` menu on top of the viewport, click `Re-generate`. At this point you should start to see a terrain made of cubes appear in the editor viewport.
|
||||
1.3) Select the terrain node again, and in the `Terrain` menu on top of the viewport, click `Re-generate`. At this point you should start to see a terrain made of cubes appear in the editor viewport.
|
||||
|
||||
3) The terrain is not setup to appear in-game yet, so there is some extra setup needed. Add a `Camera` node, and elevate it so it's above the terrain. You may also want to angle it a bit downward to see more of the landscape.
|
||||
2) The terrain is not setup to appear in-game yet, so there is some extra setup needed. Add a `Camera` node, and elevate it so it's above the terrain. You may also want to angle it a bit downward to see more of the landscape.
|
||||
|
||||
4) Add a `VoxelViewer` node under the camera. When the game runs, this node will tell the voxel engine where to generate voxels, as the camera moves around.
|
||||
3) Add a `VoxelViewer` node under the camera. When the game runs, this node will tell the voxel engine where to generate voxels, as the camera moves around.
|
||||
|
||||
5) Play the scene: you should see the terrain appear!
|
||||
4) Play the scene: you should see the terrain appear!
|
||||
|
||||
|
||||
|
||||
|
|
@ -53,23 +55,21 @@ You can modify the shape of the terrain by changing noise parameters under the g
|
|||
`VoxelMesherBlocky` allows to specify way more than just white cubes: you can define multiple models, with varying textures, materials and shapes, in order to compose a world like you would see in Minecraft for example.
|
||||
|
||||
|
||||
### Large smooth world using `VoxelLODTerrain`
|
||||
### Large smooth heightmap with overhangs using `VoxelLODTerrain`
|
||||
|
||||
1) Create a new project and a new 3D scene. Give it some light by adding a `DirectionalLight` node, and orientate it so it shines approximately downwards. You can enable `Shadows` too.
|
||||
1) Add a `VoxelLODTerrain` node, and adjust the following settings in the inspector:
|
||||
|
||||
2) Add a `VoxelLODTerrain` node, and adjust the following settings in the inspector:
|
||||
1.1) `Generator`: create a new `VoxelGeneratorNoise`. Then click on it and set its `Channel` parameter to `SDF`. Also make sure the `noise` property is assigned to a noise resource.
|
||||
|
||||
2.1) `Generator`: create a new `VoxelGeneratorNoise`. Then click on it and set its `Channel` parameter to `SDF`. Also make sure the `noise` property is assigned to a noise resource.
|
||||
1.2) `Mesher`: create a new `VoxelMesherTransvoxel`.
|
||||
|
||||
2.2) `Mesher`: create a new `VoxelMesherTransvoxel`.
|
||||
2) At this point you should start to see a smooth, spongy terrain appear in the editor viewport. If you can't see anything, you can force a reload by reopening the scene, or using the menu `Terrain -> Re-generate`.
|
||||
|
||||
3) At this point you should start to see a smooth, spongy terrain appear in the editor viewport. If you can't see anything, you can force a reload by reopening the scene, or using the menu `Terrain -> Re-generate`.
|
||||
3) The terrain is not setup to appear in-game yet, so there is some extra setup needed. Add a `Camera` node, and elevate it so it's above the terrain. You may also want to angle it a bit downward to see more of the landscape.
|
||||
|
||||
4) The terrain is not setup to appear in-game yet, so there is some extra setup needed. Add a `Camera` node, and elevate it so it's above the terrain. You may also want to angle it a bit downward to see more of the landscape.
|
||||
4) Add a `VoxelViewer` node under the camera. When the game runs, this node will tell the voxel engine where to generate voxels, as the camera moves around.
|
||||
|
||||
5) Add a `VoxelViewer` node under the camera. When the game runs, this node will tell the voxel engine where to generate voxels, as the camera moves around.
|
||||
|
||||
6) Play the scene: you should see the terrain appear!
|
||||
5) Play the scene: you should see the terrain appear!
|
||||
|
||||
|
||||
|
||||
|
|
@ -91,4 +91,4 @@ It's easy to think a project needs voxels, but they are less needed than it soun
|
|||
|
||||
- "I need to make a planet": you can make more efficient planets by stitching 6 spherified heightmaps together. Take a cube where each face is a heightmap, then puff that cube to turn it into a sphere.
|
||||
|
||||
- "GridMap sucks": how large do you want your grid to be? How complex are your models? This module is geared towards very large grids with simple geometry, so it has its own restrictions.
|
||||
- "GridMap sucks": how large do you want your grid to be? How complex are your models? This module's blocky mesher is geared towards very large grids with simple geometry, so it has its own restrictions.
|
||||
|
|
|
|||
|
|
@ -389,6 +389,11 @@ For information about LOD behavior in the editor, see [Camera options in editor]
|
|||
|
||||
Currently, the size of voxels is fixed to 1 space unit. It might be possible in a future version to change it. For now, a workaround is to scale down the node. However, make sure it is a uniform scale, and careful not to scale too low otherwise it might blow up.
|
||||
|
||||
`scale` from Node3D must not be confused with the concept of *size*. If you change `scale`, *it will also scale the voxel grid*, view distances, all the dimensions you might have set in generators, and of course it will apply to child nodes as well. The result will *look the same*, just bigger, no more details. So if you want something larger *with more details as a result*, it is recommended to change these sizes instead of scaling everything.
|
||||
For example, if your generator contains a sphere and Perlin noise, you may change the radius of the sphere and the frequency/period of the noise instead of scaling the node. Doing it this way preserve the size of voxels and so it preserves accuracy.
|
||||
|
||||
Godot also allows you to scale non-uniformly, but it's not recommended (might cause collision issues too).
|
||||
|
||||
|
||||
### Full load mode
|
||||
|
||||
|
|
|
|||
|
|
@ -117,8 +117,7 @@ void VoxelBlockyModel::_get_property_list(List<PropertyInfo> *p_list) const {
|
|||
Variant::NIL, "Mesh collision", PROPERTY_HINT_NONE, "collision_enabled_", PROPERTY_USAGE_GROUP));
|
||||
|
||||
for (unsigned int i = 0; i < _surface_count; ++i) {
|
||||
p_list->push_back(PropertyInfo(Variant::OBJECT, String("collision_enabled_{0}").format(varray(i)),
|
||||
PROPERTY_HINT_RESOURCE_TYPE, Material::get_class_static()));
|
||||
p_list->push_back(PropertyInfo(Variant::BOOL, String("collision_enabled_{0}").format(varray(i))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
#include "voxel_stream_sqlite.h"
|
||||
#include "../../thirdparty/sqlite/sqlite3.h"
|
||||
#include "../../util/errors.h"
|
||||
#include "../../util/godot/funcs.h"
|
||||
#include "../../util/log.h"
|
||||
#include "../../util/math/conv.h"
|
||||
#include "../../util/profiling.h"
|
||||
#include "../../util/string_funcs.h"
|
||||
#include "../compressed_data.h"
|
||||
|
||||
#include <limits>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
|
||||
namespace zylann::voxel {
|
||||
|
||||
|
|
@ -18,17 +21,14 @@ struct BlockLocation {
|
|||
uint8_t lod;
|
||||
|
||||
static bool validate(const Vector3i pos, uint8_t lod) {
|
||||
ERR_FAIL_COND_V(pos.x < std::numeric_limits<int16_t>::min(), false);
|
||||
ERR_FAIL_COND_V(pos.y < std::numeric_limits<int16_t>::min(), false);
|
||||
ERR_FAIL_COND_V(pos.z < std::numeric_limits<int16_t>::min(), false);
|
||||
ERR_FAIL_COND_V(pos.x > std::numeric_limits<int16_t>::max(), false);
|
||||
ERR_FAIL_COND_V(pos.y > std::numeric_limits<int16_t>::max(), false);
|
||||
ERR_FAIL_COND_V(pos.z > std::numeric_limits<int16_t>::max(), false);
|
||||
ZN_ASSERT_RETURN_V(can_convert_to_i16(pos), false);
|
||||
ZN_ASSERT_RETURN_V(lod < constants::MAX_LOD, false);
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t encode() const {
|
||||
// 0l xx yy zz
|
||||
// TODO Is this valid with negative numbers?
|
||||
return ((static_cast<uint64_t>(lod) & 0xffff) << 48) | ((static_cast<uint64_t>(x) & 0xffff) << 32) |
|
||||
((static_cast<uint64_t>(y) & 0xffff) << 16) | (static_cast<uint64_t>(z) & 0xffff);
|
||||
}
|
||||
|
|
@ -96,6 +96,9 @@ public:
|
|||
void (*process_block_func)(void *callback_data, BlockLocation location, Span<const uint8_t> voxel_data,
|
||||
Span<const uint8_t> instances_data));
|
||||
|
||||
bool load_all_block_keys(
|
||||
void *callback_data, void (*process_block_func)(void *callback_data, BlockLocation location));
|
||||
|
||||
Meta load_meta();
|
||||
void save_meta(Meta meta);
|
||||
|
||||
|
|
@ -139,6 +142,7 @@ private:
|
|||
sqlite3_stmt *_load_channels_statement = nullptr;
|
||||
sqlite3_stmt *_save_channel_statement = nullptr;
|
||||
sqlite3_stmt *_load_all_blocks_statement = nullptr;
|
||||
sqlite3_stmt *_load_all_block_keys_statement = nullptr;
|
||||
};
|
||||
|
||||
VoxelStreamSQLiteInternal::VoxelStreamSQLiteInternal() {}
|
||||
|
|
@ -215,6 +219,9 @@ bool VoxelStreamSQLiteInternal::open(const char *fpath) {
|
|||
if (!prepare(db, &_load_all_blocks_statement, "SELECT * FROM blocks")) {
|
||||
return false;
|
||||
}
|
||||
if (!prepare(db, &_load_all_block_keys_statement, "SELECT loc FROM blocks")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Is the database setup?
|
||||
Meta meta = load_meta();
|
||||
|
|
@ -250,6 +257,7 @@ void VoxelStreamSQLiteInternal::close() {
|
|||
finalize(_load_channels_statement);
|
||||
finalize(_save_channel_statement);
|
||||
finalize(_load_all_blocks_statement);
|
||||
finalize(_load_all_block_keys_statement);
|
||||
sqlite3_close(_db);
|
||||
_db = nullptr;
|
||||
_opened_path.clear();
|
||||
|
|
@ -449,6 +457,47 @@ bool VoxelStreamSQLiteInternal::load_all_blocks(void *callback_data,
|
|||
return true;
|
||||
}
|
||||
|
||||
bool VoxelStreamSQLiteInternal::load_all_block_keys(
|
||||
void *callback_data, void (*process_block_func)(void *callback_data, BlockLocation location)) {
|
||||
ZN_PROFILE_SCOPE();
|
||||
ZN_ASSERT(process_block_func != nullptr);
|
||||
|
||||
sqlite3 *db = _db;
|
||||
sqlite3_stmt *load_all_block_keys_statement = _load_all_block_keys_statement;
|
||||
|
||||
int rc;
|
||||
|
||||
rc = sqlite3_reset(load_all_block_keys_statement);
|
||||
if (rc != SQLITE_OK) {
|
||||
ERR_PRINT(sqlite3_errmsg(db));
|
||||
return false;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
rc = sqlite3_step(load_all_block_keys_statement);
|
||||
|
||||
if (rc == SQLITE_ROW) {
|
||||
ZN_PROFILE_SCOPE_NAMED("Row");
|
||||
|
||||
const uint64_t eloc = sqlite3_column_int64(load_all_block_keys_statement, 0);
|
||||
const BlockLocation loc = BlockLocation::decode(eloc);
|
||||
|
||||
// Using a function pointer because returning a big list of a copy of all the blobs can
|
||||
// waste a lot of temporary memory
|
||||
process_block_func(callback_data, loc);
|
||||
|
||||
} else if (rc == SQLITE_DONE) {
|
||||
break;
|
||||
|
||||
} else {
|
||||
ERR_PRINT(String("Unexpected SQLite return code: {0}; errmsg: {1}").format(rc, sqlite3_errmsg(db)));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
VoxelStreamSQLiteInternal::Meta VoxelStreamSQLiteInternal::load_meta() {
|
||||
sqlite3 *db = _db;
|
||||
sqlite3_stmt *load_meta_statement = _load_meta_statement;
|
||||
|
|
@ -578,8 +627,10 @@ void VoxelStreamSQLiteInternal::save_meta(Meta meta) {
|
|||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
thread_local std::vector<uint8_t> VoxelStreamSQLite::_temp_block_data;
|
||||
thread_local std::vector<uint8_t> VoxelStreamSQLite::_temp_compressed_block_data;
|
||||
namespace {
|
||||
thread_local std::vector<uint8_t> tls_temp_block_data;
|
||||
thread_local std::vector<uint8_t> tls_temp_compressed_block_data;
|
||||
} // namespace
|
||||
|
||||
VoxelStreamSQLite::VoxelStreamSQLite() {}
|
||||
|
||||
|
|
@ -617,6 +668,7 @@ void VoxelStreamSQLite::set_database_path(String path) {
|
|||
for (auto it = _connection_pool.begin(); it != _connection_pool.end(); ++it) {
|
||||
delete *it;
|
||||
}
|
||||
_block_keys_cache.clear();
|
||||
_connection_pool.clear();
|
||||
_connection_path = path;
|
||||
// Don't actually open anything here. We'll do it only when necessary
|
||||
|
|
@ -647,6 +699,13 @@ void VoxelStreamSQLite::load_voxel_blocks(Span<VoxelStream::VoxelQueryData> p_bl
|
|||
VoxelStream::VoxelQueryData &q = p_blocks[i];
|
||||
const Vector3i pos = q.origin_in_voxels >> (bs_po2 + q.lod);
|
||||
|
||||
ZN_ASSERT_CONTINUE(can_convert_to_i16(pos));
|
||||
|
||||
if (_block_keys_cache_enabled && !_block_keys_cache.contains(to_vec3i16(pos), q.lod)) {
|
||||
q.result = RESULT_BLOCK_NOT_FOUND;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_cache.load_voxel_block(pos, q.lod, q.voxel_buffer)) {
|
||||
q.result = RESULT_BLOCK_FOUND;
|
||||
|
||||
|
|
@ -678,11 +737,11 @@ void VoxelStreamSQLite::load_voxel_blocks(Span<VoxelStream::VoxelQueryData> p_bl
|
|||
loc.z = q.origin_in_voxels.z >> po2;
|
||||
loc.lod = q.lod;
|
||||
|
||||
const ResultCode res = con->load_block(loc, _temp_block_data, VoxelStreamSQLiteInternal::VOXELS);
|
||||
const ResultCode res = con->load_block(loc, tls_temp_block_data, VoxelStreamSQLiteInternal::VOXELS);
|
||||
|
||||
if (res == RESULT_BLOCK_FOUND) {
|
||||
// TODO Not sure if we should actually expect non-null. There can be legit not found blocks.
|
||||
BlockSerializer::decompress_and_deserialize(to_span_const(_temp_block_data), q.voxel_buffer);
|
||||
BlockSerializer::decompress_and_deserialize(to_span_const(tls_temp_block_data), q.voxel_buffer);
|
||||
}
|
||||
|
||||
q.result = res;
|
||||
|
|
@ -708,6 +767,9 @@ void VoxelStreamSQLite::save_voxel_blocks(Span<VoxelStream::VoxelQueryData> p_bl
|
|||
}
|
||||
|
||||
_cache.save_voxel_block(pos, q.lod, q.voxel_buffer);
|
||||
if (_block_keys_cache_enabled) {
|
||||
_block_keys_cache.add(to_vec3i16(pos), q.lod);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO We should consider using a serialized cache, and measure the threshold in bytes
|
||||
|
|
@ -761,16 +823,17 @@ void VoxelStreamSQLite::load_instance_blocks(Span<VoxelStream::InstancesQueryDat
|
|||
loc.z = q.position.z;
|
||||
loc.lod = q.lod;
|
||||
|
||||
const ResultCode res = con->load_block(loc, _temp_compressed_block_data, VoxelStreamSQLiteInternal::INSTANCES);
|
||||
const ResultCode res =
|
||||
con->load_block(loc, tls_temp_compressed_block_data, VoxelStreamSQLiteInternal::INSTANCES);
|
||||
|
||||
if (res == RESULT_BLOCK_FOUND) {
|
||||
if (!CompressedData::decompress(to_span_const(_temp_compressed_block_data), _temp_block_data)) {
|
||||
if (!CompressedData::decompress(to_span_const(tls_temp_compressed_block_data), tls_temp_block_data)) {
|
||||
ERR_PRINT("Failed to decompress instance block");
|
||||
q.result = RESULT_ERROR;
|
||||
continue;
|
||||
}
|
||||
q.data = make_unique_instance<InstanceBlockData>();
|
||||
if (!deserialize_instance_block_data(*q.data, to_span_const(_temp_block_data))) {
|
||||
if (!deserialize_instance_block_data(*q.data, to_span_const(tls_temp_block_data))) {
|
||||
ERR_PRINT("Failed to deserialize instance block");
|
||||
q.result = RESULT_ERROR;
|
||||
continue;
|
||||
|
|
@ -792,7 +855,16 @@ void VoxelStreamSQLite::save_instance_blocks(Span<VoxelStream::InstancesQueryDat
|
|||
// First put in cache
|
||||
for (size_t i = 0; i < p_blocks.size(); ++i) {
|
||||
VoxelStream::InstancesQueryData &q = p_blocks[i];
|
||||
|
||||
if (!BlockLocation::validate(q.position, q.lod)) {
|
||||
ZN_PRINT_ERROR(format("Instance block position {} is outside of supported range", q.position));
|
||||
continue;
|
||||
}
|
||||
|
||||
_cache.save_instance_block(q.position, q.lod, std::move(q.data));
|
||||
if (_block_keys_cache_enabled) {
|
||||
_block_keys_cache.add(to_vec3i16(q.position), q.lod);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO Optimization: we should consider using a serialized cache, and measure the threshold in bytes
|
||||
|
|
@ -808,7 +880,6 @@ void VoxelStreamSQLite::load_all_blocks(FullLoadingResult &result) {
|
|||
ERR_FAIL_COND(con == nullptr);
|
||||
|
||||
struct Context {
|
||||
VoxelStreamSQLite &stream;
|
||||
FullLoadingResult &result;
|
||||
};
|
||||
|
||||
|
|
@ -837,7 +908,7 @@ void VoxelStreamSQLite::load_all_blocks(FullLoadingResult &result) {
|
|||
}
|
||||
|
||||
if (instances_data.size() > 0) {
|
||||
std::vector<uint8_t> &temp_block_data = ctx->stream._temp_block_data;
|
||||
std::vector<uint8_t> &temp_block_data = tls_temp_block_data;
|
||||
if (!CompressedData::decompress(instances_data, temp_block_data)) {
|
||||
ERR_PRINT("Failed to decompress instance block");
|
||||
return;
|
||||
|
|
@ -855,7 +926,7 @@ void VoxelStreamSQLite::load_all_blocks(FullLoadingResult &result) {
|
|||
|
||||
// Had to suffix `_outer`,
|
||||
// because otherwise GCC thinks it shadows a variable inside the local function/captureless lambda
|
||||
Context ctx_outer{ *this, result };
|
||||
Context ctx_outer{ result };
|
||||
const bool request_result = con->load_all_blocks(&ctx_outer, L::process_block_func);
|
||||
ERR_FAIL_COND(request_result == false);
|
||||
}
|
||||
|
|
@ -880,8 +951,8 @@ void VoxelStreamSQLite::flush_cache(VoxelStreamSQLiteInternal *con) {
|
|||
ERR_FAIL_COND(con == nullptr);
|
||||
ERR_FAIL_COND(con->begin_transaction() == false);
|
||||
|
||||
std::vector<uint8_t> &temp_data = _temp_block_data;
|
||||
std::vector<uint8_t> &temp_compressed_data = _temp_compressed_block_data;
|
||||
std::vector<uint8_t> &temp_data = tls_temp_block_data;
|
||||
std::vector<uint8_t> &temp_compressed_data = tls_temp_compressed_block_data;
|
||||
|
||||
// TODO Needs better error rollback handling
|
||||
_cache.flush([con, &temp_data, &temp_compressed_data](VoxelStreamCache::Block &block) {
|
||||
|
|
@ -935,6 +1006,8 @@ VoxelStreamSQLiteInternal *VoxelStreamSQLite::get_connection() {
|
|||
_connection_mutex.unlock();
|
||||
return s;
|
||||
}
|
||||
// First connection we get since we set the database path
|
||||
|
||||
String fpath = _connection_path;
|
||||
_connection_mutex.unlock();
|
||||
|
||||
|
|
@ -947,6 +1020,13 @@ VoxelStreamSQLiteInternal *VoxelStreamSQLite::get_connection() {
|
|||
delete con;
|
||||
con = nullptr;
|
||||
}
|
||||
if (_block_keys_cache_enabled) {
|
||||
RWLockWrite wlock(_block_keys_cache.rw_lock);
|
||||
con->load_all_block_keys(&_block_keys_cache, [](void *ctx, BlockLocation loc) {
|
||||
BlockKeysCache *cache = static_cast<BlockKeysCache *>(ctx);
|
||||
cache->add_no_lock({ loc.x, loc.y, loc.z }, loc.lod);
|
||||
});
|
||||
}
|
||||
return con;
|
||||
}
|
||||
|
||||
|
|
@ -963,10 +1043,21 @@ void VoxelStreamSQLite::recycle_connection(VoxelStreamSQLiteInternal *con) {
|
|||
}
|
||||
}
|
||||
|
||||
void VoxelStreamSQLite::set_key_cache_enabled(bool enable) {
|
||||
_block_keys_cache_enabled = enable;
|
||||
}
|
||||
|
||||
bool VoxelStreamSQLite::is_key_cache_enabled() const {
|
||||
return _block_keys_cache_enabled;
|
||||
}
|
||||
|
||||
void VoxelStreamSQLite::_bind_methods() {
|
||||
ClassDB::bind_method(D_METHOD("set_database_path", "path"), &VoxelStreamSQLite::set_database_path);
|
||||
ClassDB::bind_method(D_METHOD("get_database_path"), &VoxelStreamSQLite::get_database_path);
|
||||
|
||||
ClassDB::bind_method(D_METHOD("set_key_cache_enabled", "enabled"), &VoxelStreamSQLite::set_key_cache_enabled);
|
||||
ClassDB::bind_method(D_METHOD("is_key_cache_enabled"), &VoxelStreamSQLite::is_key_cache_enabled);
|
||||
|
||||
ADD_PROPERTY(PropertyInfo(Variant::STRING, "database_path", PROPERTY_HINT_FILE), "set_database_path",
|
||||
"get_database_path");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
#ifndef VOXEL_STREAM_SQLITE_H
|
||||
#define VOXEL_STREAM_SQLITE_H
|
||||
|
||||
#include "../../util/math/vector3i16.h"
|
||||
#include "../../util/thread/mutex.h"
|
||||
#include "../voxel_block_serializer.h"
|
||||
#include "../voxel_stream.h"
|
||||
#include "../voxel_stream_cache.h"
|
||||
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace zylann::voxel {
|
||||
|
|
@ -20,6 +23,10 @@ public:
|
|||
VoxelStreamSQLite();
|
||||
~VoxelStreamSQLite();
|
||||
|
||||
// Warning: changing this path from a valid one to another is not always safe in a multithreaded context.
|
||||
// If threads were about to write into database A but it gets changed to database B,
|
||||
// that remaining data will get written in database B.
|
||||
// The nominal use case is to set this path when the game starts and not change it until the end of the session.
|
||||
void set_database_path(String path);
|
||||
String get_database_path() const;
|
||||
|
||||
|
|
@ -42,7 +49,49 @@ public:
|
|||
|
||||
void flush_cache();
|
||||
|
||||
// Might improve query performance if saved data is very sparse (like when only edited blocks are saved).
|
||||
void set_key_cache_enabled(bool enable);
|
||||
bool is_key_cache_enabled() const;
|
||||
|
||||
private:
|
||||
void rebuild_key_cache();
|
||||
|
||||
struct BlockKeysCache {
|
||||
FixedArray<std::unordered_set<Vector3i16>, constants::MAX_LOD> lods;
|
||||
RWLock rw_lock;
|
||||
|
||||
inline bool contains(Vector3i16 bpos, unsigned int lod_index) const {
|
||||
const std::unordered_set<Vector3i16> &keys = lods[lod_index];
|
||||
RWLockRead rlock(rw_lock);
|
||||
return keys.find(bpos) != keys.end();
|
||||
}
|
||||
|
||||
inline void add_no_lock(Vector3i16 bpos, unsigned int lod_index) {
|
||||
lods[lod_index].insert(bpos);
|
||||
}
|
||||
|
||||
inline void add(Vector3i16 bpos, unsigned int lod_index) {
|
||||
RWLockWrite wlock(rw_lock);
|
||||
add_no_lock(bpos, lod_index);
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
RWLockWrite wlock(rw_lock);
|
||||
for (unsigned int i = 0; i < lods.size(); ++i) {
|
||||
lods[i].clear();
|
||||
}
|
||||
}
|
||||
|
||||
// inline size_t get_memory_usage() const {
|
||||
// size_t mem = 0;
|
||||
// for (unsigned int i = 0; i < lods.size(); ++i) {
|
||||
// const std::unordered_set<Vector3i> &keys = lods[i];
|
||||
// mem += sizeof(Vector3i) * keys.size();
|
||||
// }
|
||||
// return mem;
|
||||
// }
|
||||
};
|
||||
|
||||
// An SQlite3 database is safe to use with multiple threads in serialized mode,
|
||||
// but after having a look at the implementation while stepping with a debugger, here are what actually happens:
|
||||
//
|
||||
|
|
@ -66,11 +115,18 @@ private:
|
|||
String _connection_path;
|
||||
std::vector<VoxelStreamSQLiteInternal *> _connection_pool;
|
||||
Mutex _connection_mutex;
|
||||
// This cache stores blocks in memory, and gets flushed to the database when big enough.
|
||||
// This is because save queries are more expensive.
|
||||
// It also speeds up queries of blocks that were recently saved.
|
||||
VoxelStreamCache _cache;
|
||||
|
||||
// TODO I should consider specialized memory allocators
|
||||
static thread_local std::vector<uint8_t> _temp_block_data;
|
||||
static thread_local std::vector<uint8_t> _temp_compressed_block_data;
|
||||
// The current way we stream data is by querying every block location near each player, to know if there is data.
|
||||
// Therefore testing if a block is present is the beginning of the most frequently executed code path.
|
||||
// In configurations where only edited blocks get saved, very few blocks even get stored in the database,
|
||||
// so it makes sense to cache keys to make this query fast and concurrent.
|
||||
// Note: in the long term, on a game that systematically saves everything it generates instead of just edits,
|
||||
// such a cache can become quite large. In this case we could either allow turning it off, or use an octree.
|
||||
BlockKeysCache _block_keys_cache;
|
||||
bool _block_keys_cache_enabled = false;
|
||||
};
|
||||
|
||||
} // namespace zylann::voxel
|
||||
|
|
|
|||
|
|
@ -4,12 +4,13 @@ namespace zylann::voxel {
|
|||
|
||||
bool VoxelStreamCache::load_voxel_block(Vector3i position, uint8_t lod_index, VoxelBufferInternal &out_voxels) {
|
||||
const Lod &lod = _cache[lod_index];
|
||||
lod.rw_lock.read_lock();
|
||||
|
||||
RWLockRead rlock(lod.rw_lock);
|
||||
|
||||
auto it = lod.blocks.find(position);
|
||||
|
||||
if (it == lod.blocks.end()) {
|
||||
// Not in cache, will have to query
|
||||
lod.rw_lock.read_unlock();
|
||||
return false;
|
||||
|
||||
} else {
|
||||
|
|
@ -21,7 +22,6 @@ bool VoxelStreamCache::load_voxel_block(Vector3i position, uint8_t lod_index, Vo
|
|||
// and the requests wants us to populate the buffer it provides
|
||||
vb.duplicate_to(out_voxels, true);
|
||||
|
||||
lod.rw_lock.read_unlock();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@
|
|||
#include "vector3d.h"
|
||||
#include "vector3f.h"
|
||||
#include "vector3i.h"
|
||||
#include "vector3i16.h"
|
||||
#include <limits>
|
||||
|
||||
namespace zylann {
|
||||
|
||||
|
|
@ -40,6 +42,16 @@ inline Vector3d to_vec3d(const Vector3f v) {
|
|||
return Vector3d(v.x, v.y, v.z);
|
||||
}
|
||||
|
||||
inline Vector3i16 to_vec3i16(const Vector3i v) {
|
||||
return Vector3i16(v.x, v.y, v.z);
|
||||
}
|
||||
|
||||
inline bool can_convert_to_i16(Vector3i p) {
|
||||
return p.x >= std::numeric_limits<int16_t>::min() && p.x <= std::numeric_limits<int16_t>::max() &&
|
||||
p.y >= std::numeric_limits<int16_t>::min() && p.y <= std::numeric_limits<int16_t>::max() &&
|
||||
p.z >= std::numeric_limits<int16_t>::min() && p.z <= std::numeric_limits<int16_t>::max();
|
||||
}
|
||||
|
||||
namespace math {
|
||||
|
||||
inline Vector3i floor_to_int(const Vector3 &f) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
#ifndef ZN_VECTOR3I16_H
|
||||
#define ZN_VECTOR3I16_H
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
|
||||
namespace zylann {
|
||||
|
||||
struct Vector3i16 {
|
||||
int16_t x;
|
||||
int16_t y;
|
||||
int16_t z;
|
||||
|
||||
Vector3i16() : x(0), y(0), z(0) {}
|
||||
Vector3i16(int16_t p_x, int16_t p_y, int16_t p_z) : x(p_x), y(p_y), z(p_z) {}
|
||||
|
||||
inline bool operator==(const Vector3i16 p_v) const {
|
||||
return x == p_v.x && y == p_v.y && z == p_v.z;
|
||||
}
|
||||
|
||||
inline bool operator!=(const Vector3i16 p_v) const {
|
||||
return x != p_v.x || y != p_v.y || z != p_v.z;
|
||||
}
|
||||
};
|
||||
|
||||
inline size_t get_hash_st(const zylann::Vector3i16 &v) {
|
||||
// TODO Optimization: benchmark this hash, I just wanted one that works
|
||||
uint64_t m = 0;
|
||||
*(zylann::Vector3i16 *)m = v;
|
||||
return std::hash<uint64_t>{}(m);
|
||||
}
|
||||
|
||||
} // namespace zylann
|
||||
|
||||
// For STL
|
||||
namespace std {
|
||||
template <>
|
||||
struct hash<zylann::Vector3i16> {
|
||||
size_t operator()(const zylann::Vector3i16 &v) const {
|
||||
return zylann::get_hash_st(v);
|
||||
}
|
||||
};
|
||||
} // namespace std
|
||||
|
||||
#endif // ZN_VECTOR3I16_H
|
||||
|
|
@ -45,4 +45,14 @@ unsigned int Thread::get_hardware_concurrency() {
|
|||
return std::thread::hardware_concurrency();
|
||||
}
|
||||
|
||||
static uint64_t get_hash(const std::thread::id &p_t) {
|
||||
static std::hash<std::thread::id> hasher;
|
||||
return hasher(p_t);
|
||||
}
|
||||
|
||||
Thread::ID Thread::get_caller_id() {
|
||||
static thread_local ID caller_id = get_hash(std::this_thread::get_id());
|
||||
return caller_id;
|
||||
}
|
||||
|
||||
} // namespace zylann
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ public:
|
|||
PRIORITY_HIGH
|
||||
};
|
||||
|
||||
typedef uint64_t ID;
|
||||
|
||||
typedef void (*Callback)(void *p_userdata);
|
||||
|
||||
Thread();
|
||||
|
|
@ -31,6 +33,9 @@ public:
|
|||
static void set_name(const char *name);
|
||||
static void sleep_usec(uint32_t microseconds);
|
||||
|
||||
// Get ID of the current thread
|
||||
static ID get_caller_id();
|
||||
|
||||
private:
|
||||
ThreadImpl *_impl = nullptr;
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue