MapBlock Caching part 1

src/cmd_line.rs

@@ -247,7 +247,10 @@ fn print_editing_status(done: usize, total: usize, real_start: Instant,
 
 
 fn print_log(log_type: LogType, msg: String) {
-	eprintln!("{}: {}", log_type, msg)
+	let prefix = format!("{}: ", log_type);
+	let indented = msg.lines().collect::<Vec<_>>()
+		.join(&format!( "\n{}", " ".repeat(prefix.len()) ));
+	eprintln!("{}{}", prefix, indented);
 }
 
 

src/commands/clone.rs

@@ -1,11 +1,12 @@
 use super::Command;
 
+use crate::unwrap_or;
 use crate::spatial::{Vec3, area_rel_block_overlap,
 	area_abs_block_overlap};
-use crate::map_block::{MapBlock, NodeMetadataList};
+use crate::map_block::{MapBlock, is_valid_generated, NodeMetadataList};
 use crate::block_utils::{merge_blocks, merge_metadata, clean_name_id_map};
 use crate::instance::{ArgType, InstBundle};
-use crate::utils::query_keys;
+use crate::utils::{CacheMap, CachedMapDatabase, query_keys};
 use crate::time_keeper::TimeKeeper;
 
 
@@ -29,19 +30,23 @@ fn clone(inst: &mut InstBundle) {
 		(Vec3::from_block_key(*k) * sort_dir + sort_offset).to_block_key()
 	});
 
-	inst.status.begin_editing();
-
+	// let mut db = CachedMapDatabase::new(&mut inst.db, 256);
+	let mut block_cache = CacheMap::<i64, MapBlock>::with_capacity(256);
 	let mut tk = TimeKeeper::new();
-	for key in keys {
+
+	inst.status.begin_editing();
+	for dst_key in keys {
 		inst.status.inc_done();
 
-		let dst_data = inst.db.get_block(key).unwrap();
-		// TODO: is_valid_generated
+		let dst_data = inst.db.get_block(dst_key).unwrap();
+		if !is_valid_generated(&dst_data) {
+			continue;
+		}
 		let mut dst_block = MapBlock::deserialize(&dst_data).unwrap();
 		let mut dst_meta = NodeMetadataList::deserialize(
 			dst_block.metadata.get_ref()).unwrap();
 
-		let dst_pos = Vec3::from_block_key(key);
+		let dst_pos = Vec3::from_block_key(dst_key);
 		let dst_part_abs = area_abs_block_overlap(&dst_area, dst_pos)
 			.unwrap();
 		let src_part_abs = dst_part_abs - offset;
@@ -51,8 +56,21 @@ fn clone(inst: &mut InstBundle) {
 			if !src_pos.is_valid_block_pos() {
 				continue;
 			}
-			let src_data = inst.db.get_block(src_pos.to_block_key()).unwrap();
-			let src_block = MapBlock::deserialize(&src_data).unwrap();
+			let src_key = src_pos.to_block_key();
+			let src_block = if let Some(block) = block_cache.get(&src_key) {
+				let _t = tk.get_timer("get_block (cached)");
+				block.clone()
+			} else {
+				let _t = tk.get_timer("get_block (database)");
+				let src_data = unwrap_or!(inst.db.get_block(src_key),
+					continue);
+				if !is_valid_generated(&src_data) {
+					continue;
+				}
+				let src_block = MapBlock::deserialize(&src_data).unwrap();
+				block_cache.insert(src_key, src_block.clone());
+				src_block
+			};
 			let src_meta = NodeMetadataList::deserialize(
 				&src_block.metadata.get_ref()).unwrap();
 
@@ -80,11 +98,11 @@ fn clone(inst: &mut InstBundle) {
 		}
 
 		*dst_block.metadata.get_mut() = dst_meta.serialize(dst_block.version);
-		inst.db.set_block(key, &dst_block.serialize()).unwrap();
+		inst.db.set_block(dst_key, &dst_block.serialize()).unwrap();
 	}
 
-	// tk.print();
 	inst.status.end_editing();
+	tk.print(&inst.status);
 }
 
 

src/map_block/compression.rs

@@ -34,7 +34,7 @@ impl Compress for Vec<u8> {
 }
 
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct ZlibContainer<T: Compress> {
 	compressed: Option<Vec<u8>>,
 	data: T

src/map_block/map_block.rs

@@ -12,7 +12,7 @@ pub fn is_valid_generated(data: &[u8]) -> bool {
 }
 
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct MapBlock {
 	pub version: u8,
 	pub flags: u8,
@@ -81,6 +81,7 @@ impl MapBlock {
 	}
 
 	pub fn serialize(&self) -> Vec<u8> {
+		// TODO: Retain compression level used by Minetest?
 		// TODO: Use a bigger buffer (unsafe?) to reduce heap allocations.
 		let mut buf = Vec::with_capacity(BLOCK_BUF_SIZE);
 		let mut data = Cursor::new(buf);

src/map_block/metadata.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::cmp::min;
 
 use memmem::{Searcher, TwoWaySearcher};
 
@@ -16,7 +17,8 @@ impl NodeMetadata {
 		-> Result<Self, MapBlockError>
 	{
 		let var_count = data.read_u32::<BigEndian>()?;
-		let mut vars = HashMap::with_capacity(var_count as usize);
+		// Avoid memory allocation errors with corrupt data.
+		let mut vars = HashMap::with_capacity(min(var_count as usize, 0xFFFF));
 
 		for _ in 0..var_count {
 			let name = read_string16(data)?;

src/map_block/name_id_map.rs

@@ -5,7 +5,7 @@ use super::*;
 
 /// Maps 16-bit node IDs to actual node names.
 /// Relevant Minetest source file: /src/nameidmapping.cpp
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct NameIdMap {
 	// Use a BTreeMap instead of a HashMap to preserve the order of IDs.
 	pub map: BTreeMap<u16, Vec<u8>>,

src/map_block/node_data.rs

@@ -9,7 +9,7 @@ const BLOCK_SIZE: usize = 16;
 const NODE_COUNT: usize = BLOCK_SIZE * BLOCK_SIZE * BLOCK_SIZE;
 
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct NodeData {
 	pub nodes: Vec<u16>,
 	pub param1: Vec<u8>,

src/map_block/node_timer.rs

@@ -1,7 +1,7 @@
 use super::*;
 
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct NodeTimer {
 	pub pos: u16,
 	pub timeout: u32,

src/map_block/static_object.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::spatial::Vec3;
 
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct StaticObject {
 	pub obj_type: u8,
 	pub f_pos: Vec3,

src/map_database.rs

@@ -100,7 +100,7 @@ impl<'a> MapDatabase<'a> {
 		Ok(())
 	}
 
-	pub fn iter_rows(&mut self) -> MapDatabaseRows {
+	pub fn iter_rows(&self) -> MapDatabaseRows {
 		self.begin_if_needed().unwrap();
 		let stmt = self.conn.prepare("SELECT pos, data FROM blocks").unwrap();
 		MapDatabaseRows {stmt_get: stmt}

src/time_keeper.rs

@@ -1,6 +1,8 @@
 use std::collections::HashMap;
 use std::time::{Instant, Duration};
 
+use crate::instance::StatusServer;
+
 
 pub struct Timer<'a> {
 	parent: &'a mut TimeKeeper,
@@ -38,11 +40,12 @@ impl TimeKeeper {
 		Timer {parent: self, name: name.to_string(), start: Instant::now()}
 	}
 
-	/*pub fn print(&mut self) {
-		println!("");
+	pub fn print(&mut self, status: &StatusServer) {
+		let mut msg = String::new();
 		for (name, (duration, count)) in &self.times {
-			println!("{}: {} x {:?} each; {:?} total",
+			msg += &format!("{}: {} x {:?} each; {:?} total\n",
 				name, count, *duration / *count, duration);
 		}
-	}*/
+		status.log_info(msg);
+	}
 }

src/utils.rs

@@ -1,10 +1,12 @@
 use std::time::Duration;
+use std::collections::{HashMap, VecDeque};
 
 use memmem::{Searcher, TwoWaySearcher};
 use byteorder::{WriteBytesExt, BigEndian};
 
 use crate::instance::{InstState, StatusServer};
-use crate::map_database::MapDatabase;
+use crate::map_block::MapBlock;
+use crate::map_database::{MapDatabase, DBError};
 use crate::spatial::{Area, Vec3};
 
 
@@ -67,6 +69,68 @@ pub fn query_keys(
 }
 
 
+pub struct CacheMap<K, V> {
+	key_queue: VecDeque<K>,
+	map: HashMap<K, V>,
+	cap: usize,
+}
+
+impl<K: Eq + std::hash::Hash + Clone, V> CacheMap<K, V> {
+	pub fn with_capacity(cap: usize) -> Self {
+		Self {
+			key_queue: VecDeque::with_capacity(cap),
+			map: HashMap::with_capacity(cap),
+			cap
+		}
+	}
+
+	pub fn insert(&mut self, key: K, value: V) {
+		if self.key_queue.len() >= self.cap {
+			if let Some(oldest_key) = self.key_queue.pop_front() {
+				self.map.remove(&oldest_key);
+			}
+		}
+		self.key_queue.push_back(key.clone());
+		self.map.insert(key, value);
+	}
+
+	#[inline]
+	pub fn get(&self, key: &K) -> Option<&V> {
+		self.map.get(key)
+	}
+}
+
+
+pub struct CachedMapDatabase<'a, 'b> {
+	db: &'a mut MapDatabase<'b>,
+	cache: CacheMap<i64, Option<MapBlock>>
+}
+
+impl<'a, 'b> CachedMapDatabase<'a, 'b> {
+	pub fn new(db: &'a mut MapDatabase<'b>, cap: usize) -> Self {
+		Self { db, cache: CacheMap::with_capacity(cap) }
+	}
+
+	pub fn get_block(&mut self, key: i64) -> Option<MapBlock> {
+		if let Some(block) = self.cache.get(&key) {
+			block.clone()
+		} else {
+			let data = self.db.get_block(key).ok();
+			let block = match data {
+				Some(d) => MapBlock::deserialize(&d).ok(),
+				None => None
+			};
+			self.cache.insert(key, block.clone());
+			block
+		}
+	}
+
+	pub fn set_block(&mut self, key: i64, data: &[u8]) -> Result<(), DBError> {
+		self.db.set_block(key, data)
+	}
+}
+
+
 pub fn to_bytes(s: &String) -> Vec<u8> {
 	s.as_bytes().to_vec()
 }