client, builtin/client_file: Transferring of file content into cache

CMakeLists.txt

@@ -58,6 +58,7 @@ set(CLIENT_SRCS
 	src/client/state.cpp
 	src/client/app.cpp
 	src/core/log.cpp
+	src/impl/fs.cpp
 	src/impl/tcpsocket.cpp
 	src/impl/sha1.cpp
 	src/impl/packet_stream.cpp

share/builtin/client_file/client_file.cpp

@@ -2,7 +2,6 @@
 #include "interface/module.h"
 #include "interface/server.h"
 #include "interface/event.h"
-#include "interface/fs.h"
 #include "interface/sha1.h"
 #include "client_file/include/api.h"
 #include "network/include/api.h"
@@ -45,12 +44,16 @@ struct Module: public interface::Module, public client_file::Interface
 		log_v(MODULE, "client_file init");
 		m_server->sub_event(this, Event::t("core:start"));
 		m_server->sub_event(this, Event::t("network:new_client"));
+		m_server->sub_event(this,
+				Event::t("network:packet_received/core:request_file"));
 	}
 
 	void event(const Event::Type &type, const Event::Private *p)
 	{
 		EVENT_VOIDN("core:start", on_start)
 		EVENT_TYPEN("network:new_client", on_new_client, network::NewClient)
+		EVENT_TYPEN("network:packet_received/core:request_file", on_request_file,
+				network::Packet)
 	}
 
 	void on_start()
@@ -116,6 +119,43 @@ struct Module: public interface::Module, public client_file::Interface
 #endif
 	}
 
+	void on_request_file(const network::Packet &packet)
+	{
+		ss_ file_name;
+		ss_ file_hash;
+		std::istringstream is(packet.data, std::ios::binary);
+		{
+			cereal::BinaryInputArchive ar(is);
+			ar(file_name);
+			ar(file_hash);
+		}
+		log_v(MODULE, "File request: %s %s", cs(file_name),
+				cs(interface::sha1::hex(file_hash)));
+		auto it = m_files.find(file_name);
+		if(it == m_files.end()){
+			log_w(MODULE, "Requested file does not exist: \"%s\"", cs(file_name));
+			return;
+		}
+		const FileInfo &info = *it->second.get();
+		if(info.hash != file_hash){
+			log_w(MODULE, "Requested file differs in hash: \"%s\": "
+					"requested %s, actual %s", cs(file_name),
+					cs(interface::sha1::hex(file_hash)),
+					cs(interface::sha1::hex(info.hash)));
+			return;
+		}
+		std::ostringstream os(std::ios::binary);
+		{
+			cereal::BinaryOutputArchive ar(os);
+			ar(info.name);
+			ar(info.hash);
+			ar(info.content);
+		}
+		network::access(m_server, [&](network::Interface * inetwork){
+			inetwork->send(packet.sender, "core:file_content", os.str());
+		});
+	}
+
 	// Interface
 
 	void add_file_content(const ss_ &name, const ss_ &content)

share/builtin/network/include/api.h

@@ -15,9 +15,11 @@ namespace network
 	struct Packet: public interface::Event::Private
 	{
 		typedef size_t Type;
+		PeerInfo::Id sender = 0;
 		ss_ name;
 		ss_ data;
-		Packet(const ss_ &name, const ss_ &data): name(name), data(data){}
+		Packet(PeerInfo::Id sender, const ss_ &name, const ss_ &data):
+			sender(sender), name(name), data(data){}
 	};
 
 	struct NewClient: public interface::Event::Private

share/builtin/network/network.cpp

@@ -141,8 +141,8 @@ struct Module: public interface::Module, public network::Interface
 		peer.packet_stream.input(peer.socket_buffer,
 		[&](const ss_ & name, const ss_ & data){
 			// Emit event
-			m_server->emit_event(ss_()+"network:packet_received:"+name,
+			m_server->emit_event(ss_()+"network:packet_received/"+name,
-					new Packet(name, data));
+					new Packet(peer.id, name, data));
 		});
 	}
 

src/client/app.cpp

@@ -277,6 +277,7 @@ struct CApp: public Polycode::EventHandler, public App
 	lua_setglobal(L, "__buildat_" #name);\
 }
 		DEF_BUILDAT_FUNC(send_packet);
+		DEF_BUILDAT_FUNC(get_file_content)
 
 		ss_ init_lua_path = g_client_config.share_path+"/client/init.lua";
 		int error = luaL_dofile(L, init_lua_path.c_str());
@@ -343,6 +344,22 @@ struct CApp: public Polycode::EventHandler, public App
 
 		return 0;
 	}
+
+	// get_file_content(name: string)
+	static int l_get_file_content(lua_State *L)
+	{
+		size_t name_len = 0;
+		const char *name_c = lua_tolstring(L, 1, &name_len);
+		ss_ name(name_c, name_len);
+
+		lua_getfield(L, LUA_REGISTRYINDEX, "__buildat_app");
+		CApp *self = (CApp*)lua_touserdata(L, -1);
+		lua_pop(L, 1);
+
+		ss_ content = self->m_state->get_file_content(name);
+		lua_pushlstring(L, content.c_str(), content.size());
+		return 1;
+	}
 };
 
 App* createApp(Polycode::PolycodeView *view)

src/client/state.cpp

@@ -4,6 +4,8 @@
 #include "client/config.h"
 #include "interface/tcpsocket.h"
 #include "interface/packet_stream.h"
+#include "interface/sha1.h"
+#include "interface/fs.h"
 #include <cereal/archives/binary.hpp>
 #include <cereal/types/string.hpp>
 #include <cstring>
@@ -22,11 +24,25 @@ struct CState: public State
 	std::deque<char> m_socket_buffer;
 	interface::PacketStream m_packet_stream;
 	sp_<app::App> m_app;
+	ss_ m_cache_path;
 
 	CState(sp_<app::App> app):
 		m_socket(interface::createTCPSocket()),
-		m_app(app)
+		m_app(app),
-	{}
+		m_cache_path(g_client_config.cache_path+"/remote")
+	{
+		// Create directory for cached files
+		auto *fs = interface::getGlobalFilesystem();
+		fs->create_directories(m_cache_path);
+	}
+
+	void update()
+	{
+		if(m_socket->wait_data(0)){
+			read_socket();
+			handle_socket_buffer();
+		}
+	}
 
 	bool connect(const ss_ &address, const ss_ &port)
 	{
@@ -47,12 +63,10 @@ struct CState: public State
 		});
 	}
 
-	void update()
+	ss_ get_file_content(const ss_ &name)
 	{
-		if(m_socket->wait_data(0)){
+		// TODO
-			read_socket();
+		return "Rullatortilla";
-			handle_socket_buffer();
-		}
 	}
 
 	void read_socket()
@@ -99,30 +113,53 @@ struct CState: public State
 				ar(file_name);
 				ar(file_hash);
 			}
-			// TODO: Check if we already have this file
+			ss_ file_hash_hex = interface::sha1::hex(file_hash);
-			ss_ path = g_client_config.cache_path+"/remote/"+file_hash;
+			// Check if we already have this file
+			ss_ path = m_cache_path+"/"+file_hash_hex;
 			std::ifstream ifs(path, std::ios::binary);
 			if(ifs.good()){
 				// We have it; no need to ask this file
+				log_i(MODULE, "%s %s: cached",
+						cs(file_hash_hex), cs(file_name));
 			} else {
 				// We don't have it; request this file
+				log_i(MODULE, "%s %s: requesting",
+						cs(file_hash_hex), cs(file_name));
+				std::ostringstream os(std::ios::binary);
+				{
+					cereal::BinaryOutputArchive ar(os);
+					ar(file_name);
+					ar(file_hash);
 				}
+				send_packet("core:request_file", os.str());
 			}
-		if(packet_name == "core:transfer_file"){
+			return;
+		}
+		if(packet_name == "core:file_content"){
 			ss_ file_name;
+			ss_ file_hash;
 			ss_ file_content;
 			std::istringstream is(data, std::ios::binary);
 			{
 				cereal::BinaryInputArchive ar(is);
 				ar(file_name);
+				ar(file_hash);
 				ar(file_content);
 			}
-			// TODO: Check filename for malicious characters "/.\"\\"
+			ss_ file_hash2 = interface::sha1::calculate(file_content);
-			// TODO: Never use a filename in the filesystem that was supplied by
+			if(file_hash != file_hash2){
-			//       server
+				log_w(MODULE, "Requested file differs in hash: \"%s\": "
-			ss_ path = g_client_config.cache_path+"/remote/"+file_name;
+						"requested %s, actual %s", cs(file_name),
+						cs(interface::sha1::hex(file_hash)),
+						cs(interface::sha1::hex(file_hash2)));
+				return;
+			}
+			ss_ file_hash_hex = interface::sha1::hex(file_hash);
+			ss_ path = g_client_config.cache_path+"/remote/"+file_hash_hex;
+			log_i(MODULE, "Saving %s to %s", cs(file_name), cs(path));
 			std::ofstream of(path, std::ios::binary);
 			of<<file_content;
+			return;
 		}
 	}
 };

src/client/state.h

@@ -15,9 +15,10 @@ namespace client
 	struct State
 	{
 		virtual ~State(){}
+		virtual void update() = 0;
 		virtual bool connect(const ss_ &address, const ss_ &port) = 0;
 		virtual void send_packet(const ss_ &name, const ss_ &data) = 0;
-		virtual void update() = 0;
+		virtual ss_ get_file_content(const ss_ &name) = 0;
 	};
 
 	State* createState(sp_<app::App> app);

src/impl/fs.cpp

@@ -17,6 +17,10 @@ struct CFilesystem : public Filesystem
 		}
 		return result;
 	}
+	bool create_directories(const ss_ &path)
+	{
+		return c55fs::CreateAllDirs(path);
+	}
 };
 
 Filesystem* getGlobalFilesystem()

src/impl/packet_stream.cpp

@@ -11,13 +11,13 @@ void PacketStream::input(std::deque<char> &socket_buffer,
 		if(socket_buffer.size() < 6)
 			return;
 		size_t type =
-				socket_buffer[0]<<0 |
+				(socket_buffer[0] & 0xff)<<0 |
-				socket_buffer[1]<<8;
+				(socket_buffer[1] & 0xff)<<8;
 		size_t size =
-				socket_buffer[2]<<0 |
+				(socket_buffer[2] & 0xff)<<0 |
-				socket_buffer[3]<<8 |
+				(socket_buffer[3] & 0xff)<<8 |
-				socket_buffer[4]<<16 |
+				(socket_buffer[4] & 0xff)<<16 |
-				socket_buffer[5]<<24;
+				(socket_buffer[5] & 0xff)<<24;
 		log_d(MODULE, "size=%zu", size);
 		if(socket_buffer.size() < 6 + size)
 			return;

src/interface/fs.h

@@ -13,6 +13,8 @@ namespace interface
 		};
 
 		virtual sv_<Node> list_directory(const ss_ &path) = 0;
+
+		virtual bool create_directories(const ss_ &path) = 0;
 	};
 
 	Filesystem* getGlobalFilesystem();

test/testmodules/test1/test1.cpp

@@ -82,9 +82,7 @@ struct Module: public interface::Module
 		log_v(MODULE, "on_files_sent(): recipient=%zu", event.recipient);
 		network::access(m_server, [&](network::Interface * inetwork){
 			inetwork->send(event.recipient, "core:run_script",
-					"print(\"TODO: Run init.lua\")");
+					"__buildat_run_script_file(\"test1/init.lua\")");
-			inetwork->send(event.recipient, "core:run_script",
-					"buildat:send_packet(\"foo\", \"bar\")");
 		});
 	}