Texture cache on client (mostly made by sapier) (breaks network compatibility)

src/client.cpp

@@ -36,6 +36,22 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "tooldef.h"
 #include "craftitemdef.h"
 #include <IFileSystem.h>
+#include "sha1.h"
+#include "base64.h"
+
+static std::string getTextureCacheDir()
+{
+	return porting::path_userdata + DIR_DELIM + "cache" + DIR_DELIM + "texture";
+}
+
+struct TextureRequest
+{
+	std::string name;
+
+	TextureRequest(const std::string &name_=""):
+		name(name_)
+	{}
+};
 
 /*
 	QueuedMeshUpdate
@@ -1232,6 +1248,157 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
 		event.deathscreen.camera_point_target_z = camera_point_target.Z;
 		m_client_event_queue.push_back(event);
 	}
+	else if(command == TOCLIENT_ANNOUNCE_TEXTURES)
+	{
+		io::IFileSystem *irrfs = m_device->getFileSystem();
+		video::IVideoDriver *vdrv = m_device->getVideoDriver();
+
+		std::string datastring((char*)&data[2], datasize-2);
+		std::istringstream is(datastring, std::ios_base::binary);
+
+
+		// Stop threads while updating content definitions
+		m_mesh_update_thread.setRun(false);
+		// Process the remaining TextureSource queue to let MeshUpdateThread
+		// get it's remaining textures and thus let it stop
+		while(m_mesh_update_thread.IsRunning()){
+			m_tsrc->processQueue();
+		}
+
+		int num_textures = readU16(is);
+
+		core::list<TextureRequest> texture_requests;
+
+		for(int i=0; i<num_textures; i++){
+
+			bool texture_found = false;
+
+			//read texture from cache
+			std::string name = deSerializeString(is);
+			std::string sha1_texture = deSerializeString(is);
+
+			std::string tpath = getTextureCacheDir() + DIR_DELIM + name;
+			// Read data
+			std::ifstream fis(tpath.c_str(), std::ios_base::binary);
+
+
+			if(fis.good() == false){
+				infostream<<"Client::Texture not found in cache: "
+						<<name << " expected it at: "<<tpath<<std::endl;
+			}
+			else
+			{
+				std::ostringstream tmp_os(std::ios_base::binary);
+				bool bad = false;
+				for(;;){
+					char buf[1024];
+					fis.read(buf, 1024);
+					std::streamsize len = fis.gcount();
+					tmp_os.write(buf, len);
+					if(fis.eof())
+						break;
+					if(!fis.good()){
+						bad = true;
+						break;
+					}
+				}
+				if(bad){
+					infostream<<"Client: Failed to read texture from cache\""
+							<<name<<"\""<<std::endl;
+				}
+				else {
+
+					SHA1 sha1;
+					sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
+
+					unsigned char *digest = sha1.getDigest();
+
+					std::string digest_string = base64_encode(digest, 20);
+
+					if (digest_string == sha1_texture) {
+						// Silly irrlicht's const-incorrectness
+						Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size());
+
+						// Create an irrlicht memory file
+						io::IReadFile *rfile = irrfs->createMemoryReadFile(
+								*data_rw,  tmp_os.str().size(), "_tempreadfile");
+						assert(rfile);
+						// Read image
+						video::IImage *img = vdrv->createImageFromFile(rfile);
+						if(!img){
+							infostream<<"Client: Cannot create image from data of "
+									<<"received texture \""<<name<<"\""<<std::endl;
+							rfile->drop();
+						}
+						else {
+							m_tsrc->insertSourceImage(name, img);
+							img->drop();
+							rfile->drop();
+
+							texture_found = true;
+						}
+					}
+					else {
+						infostream<<"Client::Texture cached sha1 hash not matching server hash: "
+								<<name << ": server ->"<<sha1_texture <<" client -> "<<digest_string<<std::endl;
+					}
+
+					delete(digest);
+				}
+			}
+
+			//add texture request
+			if (!texture_found) {
+				infostream<<"Client: Adding texture to request list: \""
+						<<name<<"\""<<std::endl;
+				texture_requests.push_back(TextureRequest(name));
+			}
+
+		}
+		// Resume threads
+		m_mesh_update_thread.setRun(true);
+		m_mesh_update_thread.Start();
+
+		ClientEvent event;
+		event.type = CE_TEXTURES_UPDATED;
+		m_client_event_queue.push_back(event);
+
+
+		//send Texture request
+		/*
+				u16 command
+				u16 number of textures requested
+				for each texture {
+					u16 length of name
+					string name
+					u16 length of path
+					string path
+				}
+		 */
+		std::ostringstream os(std::ios_base::binary);
+		u8 buf[12];
+
+
+		// Write command
+		writeU16(buf, TOSERVER_REQUEST_TEXTURES);
+		os.write((char*)buf, 2);
+
+		writeU16(buf,texture_requests.size());
+		os.write((char*)buf, 2);
+
+
+		for(core::list<TextureRequest>::Iterator i = texture_requests.begin();
+				i != texture_requests.end(); i++) {
+			os<<serializeString(i->name);
+		}
+
+		// Make data buffer
+		std::string s = os.str();
+		SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+		// Send as reliable
+		Send(0, data, true);
+		infostream<<"Client: Sending request list to server " <<std::endl;
+	}
 	else if(command == TOCLIENT_TEXTURES)
 	{
 		io::IFileSystem *irrfs = m_device->getFileSystem();
@@ -1286,6 +1453,20 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
 				rfile->drop();
 				continue;
 			}
+
+			fs::CreateAllDirs(getTextureCacheDir());
+
+			std::string filename = getTextureCacheDir() + DIR_DELIM + name;
+			std::ofstream outfile(filename.c_str(), std::ios_base::binary | std::ios_base::trunc);
+
+			if (outfile.good()) {
+				outfile.write(data.c_str(),data.length());
+				outfile.close();
+			}
+			else {
+				errorstream<<"Client: Unable to open cached texture file "<< filename <<std::endl;
+			}
+
 			m_tsrc->insertSourceImage(name, img);
 			img->drop();
 			rfile->drop();

src/client.h

@@ -31,6 +31,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "utility.h" // For IntervalLimiter
 #include "gamedef.h"
 #include "inventorymanager.h"
+#include "filesys.h"
 
 struct MeshMakeData;
 class IGameDef;

src/clientserver.h

@@ -37,9 +37,11 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 		Obsolete TOSERVER_GROUND_ACTION
 	PROTOCOL_VERSION 5:
 		Make players to be handled mostly as ActiveObjects
+	PROTOCOL_VERSION 6:
+		Only non-cached textures are sent
 */
 
-#define PROTOCOL_VERSION 5
+#define PROTOCOL_VERSION 6
 
 #define PROTOCOL_ID 0x4f457403
 
@@ -235,6 +237,19 @@ enum ToClientCommand
 		u32 length of the next item
 		serialized CraftiItemDefManager
 	*/
+
+	TOCLIENT_ANNOUNCE_TEXTURES = 0x3c,
+
+	/*
+		u16 command
+		u32 number of textures
+		for each texture {
+			u16 length of name
+			string name
+			u16 length of sha1_digest
+			string sha1_digest
+		}
+	*/
 };
 
 enum ToServerCommand
@@ -408,6 +423,17 @@ enum ToServerCommand
 		(Obsoletes TOSERVER_GROUND_ACTION and TOSERVER_CLICK_ACTIVEOBJECT.)
 	*/
 	
+	TOSERVER_REQUEST_TEXTURES = 0x40,
+
+	/*
+			u16 command
+			u16 number of textures requested
+			for each texture {
+				u16 length of name
+				string name
+			}
+	 */
+
 };
 
 inline SharedBuffer<u8> makePacket_TOCLIENT_TIME_OF_DAY(u16 time)

src/server.cpp

@@ -48,6 +48,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "mapgen.h"
 #include "content_abm.h"
 #include "mods.h"
+#include "sha1.h"
+#include "base64.h"
 
 #define PP(x) "("<<(x).X<<","<<(x).Y<<","<<(x).Z<<")"
 
@@ -929,6 +931,9 @@ Server::Server(
 		}
 	}
 	
+	// Read Textures and calculate sha1 sums
+	PrepareTextures();
+
 	// Initialize Environment
 	
 	m_env = new ServerEnvironment(new ServerMap(mapsavedir, this), m_lua,
@@ -2110,8 +2115,8 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
 		// Send CraftItem definitions
 		SendCraftItemDef(m_con, peer_id, m_craftitemdef);
 		
-		// Send textures
+		// Send texture announcement
-		SendTextures(peer_id);
+		SendTextureAnnouncement(peer_id);
 		
 		// Send player info to all players
 		//SendPlayerInfos();
@@ -2825,6 +2830,26 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
 		// ActiveObject is added to environment in AsyncRunStep after
 		// the previous addition has been succesfully removed
 	}
+	else if(command == TOSERVER_REQUEST_TEXTURES) {
+		std::string datastring((char*)&data[2], datasize-2);
+		std::istringstream is(datastring, std::ios_base::binary);
+
+		infostream<<"TOSERVER_REQUEST_TEXTURES: "<<std::endl;
+
+		core::list<TextureRequest> tosend;
+
+		u16 numtextures = readU16(is);
+
+		for(int i = 0; i < numtextures; i++) {
+
+			std::string name = deSerializeString(is);
+
+			tosend.push_back(TextureRequest(name));
+			infostream<<"TOSERVER_REQUEST_TEXTURES: requested texture " << name <<std::endl;
+		}
+
+		SendTexturesRequested(peer_id, tosend);
+	}
 	else if(command == TOSERVER_INTERACT)
 	{
 		std::string datastring((char*)&data[2], datasize-2);
@@ -4233,6 +4258,124 @@ void Server::SendBlocks(float dtime)
 	}
 }
 
+void Server::PrepareTextures() {
+	DSTACK(__FUNCTION_NAME);
+
+	infostream<<"Server::PrepareTextures(): Calculate sha1 sums of textures"<<std::endl;
+
+	for(core::list<ModSpec>::Iterator i = m_mods.begin();
+					i != m_mods.end(); i++){
+			const ModSpec &mod = *i;
+			std::string texturepath = mod.path + DIR_DELIM + "textures";
+			std::vector<fs::DirListNode> dirlist = fs::GetDirListing(texturepath);
+			for(u32 j=0; j<dirlist.size(); j++){
+				if(dirlist[j].dir) // Ignode dirs
+					continue;
+				std::string tname = dirlist[j].name;
+				std::string tpath = texturepath + DIR_DELIM + tname;
+				// Read data
+				std::ifstream fis(tpath.c_str(), std::ios_base::binary);
+				if(fis.good() == false){
+					errorstream<<"Server::PrepareTextures(): Could not open \""
+							<<tname<<"\" for reading"<<std::endl;
+					continue;
+				}
+				std::ostringstream tmp_os(std::ios_base::binary);
+				bool bad = false;
+				for(;;){
+					char buf[1024];
+					fis.read(buf, 1024);
+					std::streamsize len = fis.gcount();
+					tmp_os.write(buf, len);
+					if(fis.eof())
+						break;
+					if(!fis.good()){
+						bad = true;
+						break;
+					}
+				}
+				if(bad){
+					errorstream<<"Server::PrepareTextures(): Failed to read \""
+							<<tname<<"\""<<std::endl;
+					continue;
+				}
+
+				SHA1 sha1;
+				sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
+
+				unsigned char *digest = sha1.getDigest();
+				std::string digest_string = base64_encode(digest, 20);
+
+				delete(digest);
+
+				// Put in list
+				this->m_Textures[tname] = TextureInformation(tpath,digest_string);
+				infostream<<"Server::PrepareTextures(): added sha1 for "<< tname <<std::endl;
+			}
+	}
+}
+
+struct SendableTextureAnnouncement
+	{
+		std::string name;
+		std::string sha1_digest;
+
+		SendableTextureAnnouncement(const std::string name_="",
+				const std::string sha1_digest_=""):
+			name(name_),
+			sha1_digest(sha1_digest_)
+		{
+		}
+	};
+
+void Server::SendTextureAnnouncement(u16 peer_id){
+	DSTACK(__FUNCTION_NAME);
+
+	infostream<<"Server::SendTextureAnnouncement(): Calculate sha1 sums of textures and send to client"<<std::endl;
+
+	core::list<SendableTextureAnnouncement> texture_announcements;
+
+	for (std::map<std::string,TextureInformation>::iterator i = m_Textures.begin();i != m_Textures.end(); i++ ) {
+
+		// Put in list
+		texture_announcements.push_back(
+				SendableTextureAnnouncement(i->first, i->second.sha1_digest));
+	}
+
+	//send announcements
+
+	/*
+		u16 command
+		u32 number of textures
+		for each texture {
+			u16 length of name
+			string name
+			u16 length of digest string
+			string sha1_digest
+		}
+	*/
+	std::ostringstream os(std::ios_base::binary);
+
+	writeU16(os, 	TOCLIENT_ANNOUNCE_TEXTURES);
+	writeU16(os, texture_announcements.size());
+
+	for(core::list<SendableTextureAnnouncement>::Iterator
+			j = texture_announcements.begin();
+			j != texture_announcements.end(); j++){
+		os<<serializeString(j->name);
+		os<<serializeString(j->sha1_digest);
+	}
+
+	// Make data buffer
+	std::string s = os.str();
+	infostream<<"Server::SendTextureAnnouncement(): Send to client"<<std::endl;
+	SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+
+	// Send as reliable
+	m_con.Send(peer_id, 0, data, true);
+
+}
+
 struct SendableTexture
 {
 	std::string name;
@@ -4247,112 +4390,109 @@ struct SendableTexture
 	{}
 };
 
-void Server::SendTextures(u16 peer_id)
+void Server::SendTexturesRequested(u16 peer_id,core::list<TextureRequest> tosend) {
-{
 	DSTACK(__FUNCTION_NAME);
 
-	infostream<<"Server::SendTextures(): Sending textures to client"<<std::endl;
+	infostream<<"Server::SendTexturesRequested(): Sending textures to client"<<std::endl;
-	
+
 	/* Read textures */
-	
+
 	// Put 5kB in one bunch (this is not accurate)
 	u32 bytes_per_bunch = 5000;
-	
+
 	core::array< core::list<SendableTexture> > texture_bunches;
 	texture_bunches.push_back(core::list<SendableTexture>());
-	
+
 	u32 texture_size_bunch_total = 0;
-	for(core::list<ModSpec>::Iterator i = m_mods.begin();
+
-			i != m_mods.end(); i++){
+	for(core::list<TextureRequest>::Iterator i = tosend.begin(); i != tosend.end(); i++) {
-		const ModSpec &mod = *i;
+
-		std::string texturepath = mod.path + DIR_DELIM + "textures";
+		//TODO get path + name
-		std::vector<fs::DirListNode> dirlist = fs::GetDirListing(texturepath);
+		std::string tpath = m_Textures[(*i).name].path;
-		for(u32 j=0; j<dirlist.size(); j++){
+
-			if(dirlist[j].dir) // Ignode dirs
+		// Read data
-				continue;
+		std::ifstream fis(tpath.c_str(), std::ios_base::binary);
-			std::string tname = dirlist[j].name;
+		if(fis.good() == false){
-			std::string tpath = texturepath + DIR_DELIM + tname;
+			errorstream<<"Server::SendTexturesRequested(): Could not open \""
-			// Read data
+					<<tpath<<"\" for reading"<<std::endl;
-			std::ifstream fis(tpath.c_str(), std::ios_base::binary);
+			continue;
-			if(fis.good() == false){
+		}
-				errorstream<<"Server::SendTextures(): Could not open \""
+		std::ostringstream tmp_os(std::ios_base::binary);
-						<<tname<<"\" for reading"<<std::endl;
+		bool bad = false;
-				continue;
+		for(;;){
-			}
+			char buf[1024];
-			std::ostringstream tmp_os(std::ios_base::binary);
+			fis.read(buf, 1024);
-			bool bad = false;
+			std::streamsize len = fis.gcount();
-			for(;;){
+			tmp_os.write(buf, len);
-				char buf[1024];
+			texture_size_bunch_total += len;
-				fis.read(buf, 1024);
+			if(fis.eof())
-				std::streamsize len = fis.gcount();
+				break;
-				tmp_os.write(buf, len);
+			if(!fis.good()){
-				texture_size_bunch_total += len;
+				bad = true;
-				if(fis.eof())
+				break;
-					break;
-				if(!fis.good()){
-					bad = true;
-					break;
-				}
-			}
-			if(bad){
-				errorstream<<"Server::SendTextures(): Failed to read \""
-						<<tname<<"\""<<std::endl;
-				continue;
-			}
-			/*infostream<<"Server::SendTextures(): Loaded \""
-					<<tname<<"\""<<std::endl;*/
-			// Put in list
-			texture_bunches[texture_bunches.size()-1].push_back(
-					SendableTexture(tname, tpath, tmp_os.str()));
-			
-			// Start next bunch if got enough data
-			if(texture_size_bunch_total >= bytes_per_bunch){
-				texture_bunches.push_back(core::list<SendableTexture>());
-				texture_size_bunch_total = 0;
 			}
 		}
+		if(bad){
+			errorstream<<"Server::SendTexturesRequested(): Failed to read \""
+					<<(*i).name<<"\""<<std::endl;
+			continue;
+		}
+		/*infostream<<"Server::SendTexturesRequested(): Loaded \""
+				<<tname<<"\""<<std::endl;*/
+		// Put in list
+		texture_bunches[texture_bunches.size()-1].push_back(
+				SendableTexture((*i).name, tpath, tmp_os.str()));
+
+		// Start next bunch if got enough data
+		if(texture_size_bunch_total >= bytes_per_bunch){
+			texture_bunches.push_back(core::list<SendableTexture>());
+			texture_size_bunch_total = 0;
+		}
+
 	}
 
 	/* Create and send packets */
-	
-	u32 num_bunches = texture_bunches.size();
-	for(u32 i=0; i<num_bunches; i++)
-	{
-		/*
-			u16 command
-			u16 total number of texture bunches
-			u16 index of this bunch
-			u32 number of textures in this bunch
-			for each texture {
-				u16 length of name
-				string name
-				u32 length of data
-				data
-			}
-		*/
-		std::ostringstream os(std::ios_base::binary);
 
-		writeU16(os, TOCLIENT_TEXTURES);
+		u32 num_bunches = texture_bunches.size();
-		writeU16(os, num_bunches);
+		for(u32 i=0; i<num_bunches; i++)
-		writeU16(os, i);
+		{
-		writeU32(os, texture_bunches[i].size());
+			/*
-		
+				u16 command
-		for(core::list<SendableTexture>::Iterator
+				u16 total number of texture bunches
-				j = texture_bunches[i].begin();
+				u16 index of this bunch
-				j != texture_bunches[i].end(); j++){
+				u32 number of textures in this bunch
-			os<<serializeString(j->name);
+				for each texture {
-			os<<serializeLongString(j->data);
+					u16 length of name
+					string name
+					u32 length of data
+					data
+				}
+			*/
+			std::ostringstream os(std::ios_base::binary);
+
+			writeU16(os, TOCLIENT_TEXTURES);
+			writeU16(os, num_bunches);
+			writeU16(os, i);
+			writeU32(os, texture_bunches[i].size());
+
+			for(core::list<SendableTexture>::Iterator
+					j = texture_bunches[i].begin();
+					j != texture_bunches[i].end(); j++){
+				os<<serializeString(j->name);
+				os<<serializeLongString(j->data);
+			}
+
+			// Make data buffer
+			std::string s = os.str();
+			infostream<<"Server::SendTexturesRequested(): bunch "<<i<<"/"<<num_bunches
+					<<" textures="<<texture_bunches[i].size()
+					<<" size=" <<s.size()<<std::endl;
+			SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+			// Send as reliable
+			m_con.Send(peer_id, 0, data, true);
 		}
-		
+
-		// Make data buffer
+
-		std::string s = os.str();
-		infostream<<"Server::SendTextures(): bunch "<<i<<"/"<<num_bunches
-				<<" textures="<<texture_bunches[i].size()
-				<<" size=" <<s.size()<<std::endl;
-		SharedBuffer<u8> data((u8*)s.c_str(), s.size());
-		// Send as reliable
-		m_con.Send(peer_id, 0, data, true);
-	}
 }
 
 /*

src/server.h

@@ -236,6 +236,28 @@ struct PrioritySortedBlockTransfer
 	u16 peer_id;
 };
 
+struct TextureRequest
+{
+	std::string name;
+
+	TextureRequest(const std::string &name_=""):
+		name(name_)
+	{}
+};
+
+struct TextureInformation
+{
+	std::string path;
+	std::string sha1_digest;
+
+	TextureInformation(const std::string path_="",
+			const std::string sha1_digest_=""):
+		path(path_),
+		sha1_digest(sha1_digest_)
+	{
+	}
+};
+
 class RemoteClient
 {
 public:
@@ -564,7 +586,11 @@ private:
 	// Sends blocks to clients (locks env and con on its own)
 	void SendBlocks(float dtime);
 	
-	void SendTextures(u16 peer_id);
+	void PrepareTextures();
+
+	void SendTextureAnnouncement(u16 peer_id);
+
+	void SendTexturesRequested(u16 peer_id,core::list<TextureRequest> tosend);
 
 	/*
 		Something random
@@ -744,6 +770,8 @@ private:
 
 	friend class EmergeThread;
 	friend class RemoteClient;
+
+	std::map<std::string,TextureInformation> m_Textures;
 };
 
 /*