From ef6fa6ca1a3f3cd6716be69cce8856427f976f91 Mon Sep 17 00:00:00 2001 From: random-geek <35757396+random-geek@users.noreply.github.com> Date: Fri, 5 Feb 2021 21:31:52 -0800 Subject: [PATCH] MapBlock Caching part 1 --- src/cmd_line.rs | 5 ++- src/commands/clone.rs | 42 +++++++++++++++------- src/map_block/compression.rs | 2 +- src/map_block/map_block.rs | 3 +- src/map_block/metadata.rs | 4 ++- src/map_block/name_id_map.rs | 2 +- src/map_block/node_data.rs | 2 +- src/map_block/node_timer.rs | 2 +- src/map_block/static_object.rs | 2 +- src/map_database.rs | 2 +- src/time_keeper.rs | 11 +++--- src/utils.rs | 66 +++++++++++++++++++++++++++++++++- 12 files changed, 117 insertions(+), 26 deletions(-) diff --git a/src/cmd_line.rs b/src/cmd_line.rs index 64be8b5..5e7f2be 100644 --- a/src/cmd_line.rs +++ b/src/cmd_line.rs @@ -247,7 +247,10 @@ fn print_editing_status(done: usize, total: usize, real_start: Instant, fn print_log(log_type: LogType, msg: String) { - eprintln!("{}: {}", log_type, msg) + let prefix = format!("{}: ", log_type); + let indented = msg.lines().collect::>() + .join(&format!( "\n{}", " ".repeat(prefix.len()) )); + eprintln!("{}{}", prefix, indented); } diff --git a/src/commands/clone.rs b/src/commands/clone.rs index 2e34159..201f58d 100644 --- a/src/commands/clone.rs +++ b/src/commands/clone.rs @@ -1,11 +1,12 @@ use super::Command; +use crate::unwrap_or; use crate::spatial::{Vec3, area_rel_block_overlap, area_abs_block_overlap}; -use crate::map_block::{MapBlock, NodeMetadataList}; +use crate::map_block::{MapBlock, is_valid_generated, NodeMetadataList}; use crate::block_utils::{merge_blocks, merge_metadata, clean_name_id_map}; use crate::instance::{ArgType, InstBundle}; -use crate::utils::query_keys; +use crate::utils::{CacheMap, CachedMapDatabase, query_keys}; use crate::time_keeper::TimeKeeper; @@ -29,19 +30,23 @@ fn clone(inst: &mut InstBundle) { (Vec3::from_block_key(*k) * sort_dir + sort_offset).to_block_key() }); - inst.status.begin_editing(); - + // let mut db = CachedMapDatabase::new(&mut inst.db, 256); + let mut block_cache = CacheMap::::with_capacity(256); let mut tk = TimeKeeper::new(); - for key in keys { + + inst.status.begin_editing(); + for dst_key in keys { inst.status.inc_done(); - let dst_data = inst.db.get_block(key).unwrap(); - // TODO: is_valid_generated + let dst_data = inst.db.get_block(dst_key).unwrap(); + if !is_valid_generated(&dst_data) { + continue; + } let mut dst_block = MapBlock::deserialize(&dst_data).unwrap(); let mut dst_meta = NodeMetadataList::deserialize( dst_block.metadata.get_ref()).unwrap(); - let dst_pos = Vec3::from_block_key(key); + let dst_pos = Vec3::from_block_key(dst_key); let dst_part_abs = area_abs_block_overlap(&dst_area, dst_pos) .unwrap(); let src_part_abs = dst_part_abs - offset; @@ -51,8 +56,21 @@ fn clone(inst: &mut InstBundle) { if !src_pos.is_valid_block_pos() { continue; } - let src_data = inst.db.get_block(src_pos.to_block_key()).unwrap(); - let src_block = MapBlock::deserialize(&src_data).unwrap(); + let src_key = src_pos.to_block_key(); + let src_block = if let Some(block) = block_cache.get(&src_key) { + let _t = tk.get_timer("get_block (cached)"); + block.clone() + } else { + let _t = tk.get_timer("get_block (database)"); + let src_data = unwrap_or!(inst.db.get_block(src_key), + continue); + if !is_valid_generated(&src_data) { + continue; + } + let src_block = MapBlock::deserialize(&src_data).unwrap(); + block_cache.insert(src_key, src_block.clone()); + src_block + }; let src_meta = NodeMetadataList::deserialize( &src_block.metadata.get_ref()).unwrap(); @@ -80,11 +98,11 @@ fn clone(inst: &mut InstBundle) { } *dst_block.metadata.get_mut() = dst_meta.serialize(dst_block.version); - inst.db.set_block(key, &dst_block.serialize()).unwrap(); + inst.db.set_block(dst_key, &dst_block.serialize()).unwrap(); } - // tk.print(); inst.status.end_editing(); + tk.print(&inst.status); } diff --git a/src/map_block/compression.rs b/src/map_block/compression.rs index 7be1b9e..77f1859 100644 --- a/src/map_block/compression.rs +++ b/src/map_block/compression.rs @@ -34,7 +34,7 @@ impl Compress for Vec { } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct ZlibContainer { compressed: Option>, data: T diff --git a/src/map_block/map_block.rs b/src/map_block/map_block.rs index 221afdb..8f2a3b5 100644 --- a/src/map_block/map_block.rs +++ b/src/map_block/map_block.rs @@ -12,7 +12,7 @@ pub fn is_valid_generated(data: &[u8]) -> bool { } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct MapBlock { pub version: u8, pub flags: u8, @@ -81,6 +81,7 @@ impl MapBlock { } pub fn serialize(&self) -> Vec { + // TODO: Retain compression level used by Minetest? // TODO: Use a bigger buffer (unsafe?) to reduce heap allocations. let mut buf = Vec::with_capacity(BLOCK_BUF_SIZE); let mut data = Cursor::new(buf); diff --git a/src/map_block/metadata.rs b/src/map_block/metadata.rs index 3944abe..fc8d429 100644 --- a/src/map_block/metadata.rs +++ b/src/map_block/metadata.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::cmp::min; use memmem::{Searcher, TwoWaySearcher}; @@ -16,7 +17,8 @@ impl NodeMetadata { -> Result { let var_count = data.read_u32::()?; - let mut vars = HashMap::with_capacity(var_count as usize); + // Avoid memory allocation errors with corrupt data. + let mut vars = HashMap::with_capacity(min(var_count as usize, 0xFFFF)); for _ in 0..var_count { let name = read_string16(data)?; diff --git a/src/map_block/name_id_map.rs b/src/map_block/name_id_map.rs index b4be105..a4e4b02 100644 --- a/src/map_block/name_id_map.rs +++ b/src/map_block/name_id_map.rs @@ -5,7 +5,7 @@ use super::*; /// Maps 16-bit node IDs to actual node names. /// Relevant Minetest source file: /src/nameidmapping.cpp -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct NameIdMap { // Use a BTreeMap instead of a HashMap to preserve the order of IDs. pub map: BTreeMap>, diff --git a/src/map_block/node_data.rs b/src/map_block/node_data.rs index e2dd047..0cd6dd2 100644 --- a/src/map_block/node_data.rs +++ b/src/map_block/node_data.rs @@ -9,7 +9,7 @@ const BLOCK_SIZE: usize = 16; const NODE_COUNT: usize = BLOCK_SIZE * BLOCK_SIZE * BLOCK_SIZE; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct NodeData { pub nodes: Vec, pub param1: Vec, diff --git a/src/map_block/node_timer.rs b/src/map_block/node_timer.rs index e57bef9..56e5c21 100644 --- a/src/map_block/node_timer.rs +++ b/src/map_block/node_timer.rs @@ -1,7 +1,7 @@ use super::*; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct NodeTimer { pub pos: u16, pub timeout: u32, diff --git a/src/map_block/static_object.rs b/src/map_block/static_object.rs index aaa71c4..c3be5f6 100644 --- a/src/map_block/static_object.rs +++ b/src/map_block/static_object.rs @@ -2,7 +2,7 @@ use super::*; use crate::spatial::Vec3; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct StaticObject { pub obj_type: u8, pub f_pos: Vec3, diff --git a/src/map_database.rs b/src/map_database.rs index 1e7dccc..e9b4242 100644 --- a/src/map_database.rs +++ b/src/map_database.rs @@ -100,7 +100,7 @@ impl<'a> MapDatabase<'a> { Ok(()) } - pub fn iter_rows(&mut self) -> MapDatabaseRows { + pub fn iter_rows(&self) -> MapDatabaseRows { self.begin_if_needed().unwrap(); let stmt = self.conn.prepare("SELECT pos, data FROM blocks").unwrap(); MapDatabaseRows {stmt_get: stmt} diff --git a/src/time_keeper.rs b/src/time_keeper.rs index 65ac272..e85162f 100644 --- a/src/time_keeper.rs +++ b/src/time_keeper.rs @@ -1,6 +1,8 @@ use std::collections::HashMap; use std::time::{Instant, Duration}; +use crate::instance::StatusServer; + pub struct Timer<'a> { parent: &'a mut TimeKeeper, @@ -38,11 +40,12 @@ impl TimeKeeper { Timer {parent: self, name: name.to_string(), start: Instant::now()} } - /*pub fn print(&mut self) { - println!(""); + pub fn print(&mut self, status: &StatusServer) { + let mut msg = String::new(); for (name, (duration, count)) in &self.times { - println!("{}: {} x {:?} each; {:?} total", + msg += &format!("{}: {} x {:?} each; {:?} total\n", name, count, *duration / *count, duration); } - }*/ + status.log_info(msg); + } } diff --git a/src/utils.rs b/src/utils.rs index 43c3f64..18b966b 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,10 +1,12 @@ use std::time::Duration; +use std::collections::{HashMap, VecDeque}; use memmem::{Searcher, TwoWaySearcher}; use byteorder::{WriteBytesExt, BigEndian}; use crate::instance::{InstState, StatusServer}; -use crate::map_database::MapDatabase; +use crate::map_block::MapBlock; +use crate::map_database::{MapDatabase, DBError}; use crate::spatial::{Area, Vec3}; @@ -67,6 +69,68 @@ pub fn query_keys( } +pub struct CacheMap { + key_queue: VecDeque, + map: HashMap, + cap: usize, +} + +impl CacheMap { + pub fn with_capacity(cap: usize) -> Self { + Self { + key_queue: VecDeque::with_capacity(cap), + map: HashMap::with_capacity(cap), + cap + } + } + + pub fn insert(&mut self, key: K, value: V) { + if self.key_queue.len() >= self.cap { + if let Some(oldest_key) = self.key_queue.pop_front() { + self.map.remove(&oldest_key); + } + } + self.key_queue.push_back(key.clone()); + self.map.insert(key, value); + } + + #[inline] + pub fn get(&self, key: &K) -> Option<&V> { + self.map.get(key) + } +} + + +pub struct CachedMapDatabase<'a, 'b> { + db: &'a mut MapDatabase<'b>, + cache: CacheMap> +} + +impl<'a, 'b> CachedMapDatabase<'a, 'b> { + pub fn new(db: &'a mut MapDatabase<'b>, cap: usize) -> Self { + Self { db, cache: CacheMap::with_capacity(cap) } + } + + pub fn get_block(&mut self, key: i64) -> Option { + if let Some(block) = self.cache.get(&key) { + block.clone() + } else { + let data = self.db.get_block(key).ok(); + let block = match data { + Some(d) => MapBlock::deserialize(&d).ok(), + None => None + }; + self.cache.insert(key, block.clone()); + block + } + } + + pub fn set_block(&mut self, key: i64, data: &[u8]) -> Result<(), DBError> { + self.db.set_block(key, data) + } +} + + pub fn to_bytes(s: &String) -> Vec { s.as_bytes().to_vec() }