From 60c0a790bb4b7c63a477b32641385f5205390c69 Mon Sep 17 00:00:00 2001 From: Perttu Ahola Date: Fri, 19 Sep 2014 02:22:53 +0300 Subject: [PATCH] client, builtin/client_file: Transferring of file content into cache --- CMakeLists.txt | 1 + share/builtin/client_file/client_file.cpp | 42 ++++++++++++++- share/builtin/network/include/api.h | 4 +- share/builtin/network/network.cpp | 4 +- src/client/app.cpp | 19 ++++++- src/client/state.cpp | 65 ++++++++++++++++++----- src/client/state.h | 3 +- src/impl/fs.cpp | 4 ++ src/impl/packet_stream.cpp | 12 ++--- src/interface/fs.h | 2 + test/testmodules/test1/test1.cpp | 4 +- 11 files changed, 131 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a1ec657..1b6c5ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,7 @@ set(CLIENT_SRCS src/client/state.cpp src/client/app.cpp src/core/log.cpp + src/impl/fs.cpp src/impl/tcpsocket.cpp src/impl/sha1.cpp src/impl/packet_stream.cpp diff --git a/share/builtin/client_file/client_file.cpp b/share/builtin/client_file/client_file.cpp index d3b0803..7345af8 100644 --- a/share/builtin/client_file/client_file.cpp +++ b/share/builtin/client_file/client_file.cpp @@ -2,7 +2,6 @@ #include "interface/module.h" #include "interface/server.h" #include "interface/event.h" -#include "interface/fs.h" #include "interface/sha1.h" #include "client_file/include/api.h" #include "network/include/api.h" @@ -45,12 +44,16 @@ struct Module: public interface::Module, public client_file::Interface log_v(MODULE, "client_file init"); m_server->sub_event(this, Event::t("core:start")); m_server->sub_event(this, Event::t("network:new_client")); + m_server->sub_event(this, + Event::t("network:packet_received/core:request_file")); } void event(const Event::Type &type, const Event::Private *p) { EVENT_VOIDN("core:start", on_start) EVENT_TYPEN("network:new_client", on_new_client, network::NewClient) + EVENT_TYPEN("network:packet_received/core:request_file", on_request_file, + network::Packet) } void on_start() @@ -116,6 +119,43 @@ struct Module: public interface::Module, public client_file::Interface #endif } + void on_request_file(const network::Packet &packet) + { + ss_ file_name; + ss_ file_hash; + std::istringstream is(packet.data, std::ios::binary); + { + cereal::BinaryInputArchive ar(is); + ar(file_name); + ar(file_hash); + } + log_v(MODULE, "File request: %s %s", cs(file_name), + cs(interface::sha1::hex(file_hash))); + auto it = m_files.find(file_name); + if(it == m_files.end()){ + log_w(MODULE, "Requested file does not exist: \"%s\"", cs(file_name)); + return; + } + const FileInfo &info = *it->second.get(); + if(info.hash != file_hash){ + log_w(MODULE, "Requested file differs in hash: \"%s\": " + "requested %s, actual %s", cs(file_name), + cs(interface::sha1::hex(file_hash)), + cs(interface::sha1::hex(info.hash))); + return; + } + std::ostringstream os(std::ios::binary); + { + cereal::BinaryOutputArchive ar(os); + ar(info.name); + ar(info.hash); + ar(info.content); + } + network::access(m_server, [&](network::Interface * inetwork){ + inetwork->send(packet.sender, "core:file_content", os.str()); + }); + } + // Interface void add_file_content(const ss_ &name, const ss_ &content) diff --git a/share/builtin/network/include/api.h b/share/builtin/network/include/api.h index 98bb994..4ff2036 100644 --- a/share/builtin/network/include/api.h +++ b/share/builtin/network/include/api.h @@ -15,9 +15,11 @@ namespace network struct Packet: public interface::Event::Private { typedef size_t Type; + PeerInfo::Id sender = 0; ss_ name; ss_ data; - Packet(const ss_ &name, const ss_ &data): name(name), data(data){} + Packet(PeerInfo::Id sender, const ss_ &name, const ss_ &data): + sender(sender), name(name), data(data){} }; struct NewClient: public interface::Event::Private diff --git a/share/builtin/network/network.cpp b/share/builtin/network/network.cpp index c81b124..b592c81 100644 --- a/share/builtin/network/network.cpp +++ b/share/builtin/network/network.cpp @@ -141,8 +141,8 @@ struct Module: public interface::Module, public network::Interface peer.packet_stream.input(peer.socket_buffer, [&](const ss_ & name, const ss_ & data){ // Emit event - m_server->emit_event(ss_()+"network:packet_received:"+name, - new Packet(name, data)); + m_server->emit_event(ss_()+"network:packet_received/"+name, + new Packet(peer.id, name, data)); }); } diff --git a/src/client/app.cpp b/src/client/app.cpp index 227cd32..17ca2b0 100644 --- a/src/client/app.cpp +++ b/src/client/app.cpp @@ -272,11 +272,12 @@ struct CApp: public Polycode::EventHandler, public App lua_pushlightuserdata(L, (void*)this); lua_setfield(L, LUA_REGISTRYINDEX, "__buildat_app"); -#define DEF_BUILDAT_FUNC(name) {\ +#define DEF_BUILDAT_FUNC(name){\ lua_pushcfunction(L, l_##name);\ lua_setglobal(L, "__buildat_" #name);\ } DEF_BUILDAT_FUNC(send_packet); + DEF_BUILDAT_FUNC(get_file_content) ss_ init_lua_path = g_client_config.share_path+"/client/init.lua"; int error = luaL_dofile(L, init_lua_path.c_str()); @@ -343,6 +344,22 @@ struct CApp: public Polycode::EventHandler, public App return 0; } + + // get_file_content(name: string) + static int l_get_file_content(lua_State *L) + { + size_t name_len = 0; + const char *name_c = lua_tolstring(L, 1, &name_len); + ss_ name(name_c, name_len); + + lua_getfield(L, LUA_REGISTRYINDEX, "__buildat_app"); + CApp *self = (CApp*)lua_touserdata(L, -1); + lua_pop(L, 1); + + ss_ content = self->m_state->get_file_content(name); + lua_pushlstring(L, content.c_str(), content.size()); + return 1; + } }; App* createApp(Polycode::PolycodeView *view) diff --git a/src/client/state.cpp b/src/client/state.cpp index 9295192..5b55efb 100644 --- a/src/client/state.cpp +++ b/src/client/state.cpp @@ -4,6 +4,8 @@ #include "client/config.h" #include "interface/tcpsocket.h" #include "interface/packet_stream.h" +#include "interface/sha1.h" +#include "interface/fs.h" #include #include #include @@ -22,11 +24,25 @@ struct CState: public State std::deque m_socket_buffer; interface::PacketStream m_packet_stream; sp_ m_app; + ss_ m_cache_path; CState(sp_ app): m_socket(interface::createTCPSocket()), - m_app(app) - {} + m_app(app), + m_cache_path(g_client_config.cache_path+"/remote") + { + // Create directory for cached files + auto *fs = interface::getGlobalFilesystem(); + fs->create_directories(m_cache_path); + } + + void update() + { + if(m_socket->wait_data(0)){ + read_socket(); + handle_socket_buffer(); + } + } bool connect(const ss_ &address, const ss_ &port) { @@ -47,12 +63,10 @@ struct CState: public State }); } - void update() + ss_ get_file_content(const ss_ &name) { - if(m_socket->wait_data(0)){ - read_socket(); - handle_socket_buffer(); - } + // TODO + return "Rullatortilla"; } void read_socket() @@ -99,30 +113,53 @@ struct CState: public State ar(file_name); ar(file_hash); } - // TODO: Check if we already have this file - ss_ path = g_client_config.cache_path+"/remote/"+file_hash; + ss_ file_hash_hex = interface::sha1::hex(file_hash); + // Check if we already have this file + ss_ path = m_cache_path+"/"+file_hash_hex; std::ifstream ifs(path, std::ios::binary); if(ifs.good()){ // We have it; no need to ask this file + log_i(MODULE, "%s %s: cached", + cs(file_hash_hex), cs(file_name)); } else { // We don't have it; request this file + log_i(MODULE, "%s %s: requesting", + cs(file_hash_hex), cs(file_name)); + std::ostringstream os(std::ios::binary); + { + cereal::BinaryOutputArchive ar(os); + ar(file_name); + ar(file_hash); + } + send_packet("core:request_file", os.str()); } + return; } - if(packet_name == "core:transfer_file"){ + if(packet_name == "core:file_content"){ ss_ file_name; + ss_ file_hash; ss_ file_content; std::istringstream is(data, std::ios::binary); { cereal::BinaryInputArchive ar(is); ar(file_name); + ar(file_hash); ar(file_content); } - // TODO: Check filename for malicious characters "/.\"\\" - // TODO: Never use a filename in the filesystem that was supplied by - // server - ss_ path = g_client_config.cache_path+"/remote/"+file_name; + ss_ file_hash2 = interface::sha1::calculate(file_content); + if(file_hash != file_hash2){ + log_w(MODULE, "Requested file differs in hash: \"%s\": " + "requested %s, actual %s", cs(file_name), + cs(interface::sha1::hex(file_hash)), + cs(interface::sha1::hex(file_hash2))); + return; + } + ss_ file_hash_hex = interface::sha1::hex(file_hash); + ss_ path = g_client_config.cache_path+"/remote/"+file_hash_hex; + log_i(MODULE, "Saving %s to %s", cs(file_name), cs(path)); std::ofstream of(path, std::ios::binary); of< app); diff --git a/src/impl/fs.cpp b/src/impl/fs.cpp index 5fcec96..d4626da 100644 --- a/src/impl/fs.cpp +++ b/src/impl/fs.cpp @@ -17,6 +17,10 @@ struct CFilesystem : public Filesystem } return result; } + bool create_directories(const ss_ &path) + { + return c55fs::CreateAllDirs(path); + } }; Filesystem* getGlobalFilesystem() diff --git a/src/impl/packet_stream.cpp b/src/impl/packet_stream.cpp index 5083117..af32c4d 100644 --- a/src/impl/packet_stream.cpp +++ b/src/impl/packet_stream.cpp @@ -11,13 +11,13 @@ void PacketStream::input(std::deque &socket_buffer, if(socket_buffer.size() < 6) return; size_t type = - socket_buffer[0]<<0 | - socket_buffer[1]<<8; + (socket_buffer[0] & 0xff)<<0 | + (socket_buffer[1] & 0xff)<<8; size_t size = - socket_buffer[2]<<0 | - socket_buffer[3]<<8 | - socket_buffer[4]<<16 | - socket_buffer[5]<<24; + (socket_buffer[2] & 0xff)<<0 | + (socket_buffer[3] & 0xff)<<8 | + (socket_buffer[4] & 0xff)<<16 | + (socket_buffer[5] & 0xff)<<24; log_d(MODULE, "size=%zu", size); if(socket_buffer.size() < 6 + size) return; diff --git a/src/interface/fs.h b/src/interface/fs.h index 81fdf5d..bcd4c98 100644 --- a/src/interface/fs.h +++ b/src/interface/fs.h @@ -13,6 +13,8 @@ namespace interface }; virtual sv_ list_directory(const ss_ &path) = 0; + + virtual bool create_directories(const ss_ &path) = 0; }; Filesystem* getGlobalFilesystem(); diff --git a/test/testmodules/test1/test1.cpp b/test/testmodules/test1/test1.cpp index d416e81..45a6921 100644 --- a/test/testmodules/test1/test1.cpp +++ b/test/testmodules/test1/test1.cpp @@ -82,9 +82,7 @@ struct Module: public interface::Module log_v(MODULE, "on_files_sent(): recipient=%zu", event.recipient); network::access(m_server, [&](network::Interface * inetwork){ inetwork->send(event.recipient, "core:run_script", - "print(\"TODO: Run init.lua\")"); - inetwork->send(event.recipient, "core:run_script", - "buildat:send_packet(\"foo\", \"bar\")"); + "__buildat_run_script_file(\"test1/init.lua\")"); }); }