diff --git a/minetest.conf.example b/minetest.conf.example index 3dbbd773..59d8b32f 100644 --- a/minetest.conf.example +++ b/minetest.conf.example @@ -191,9 +191,16 @@ # right clicks when holding the right mouse button #repeat_rightclick_time = 0.25 -# will only work for servers which use remote_media setting -# and only for clients compiled with cURL -#media_fetch_threads = 8 +# Default timeout for cURL, in milliseconds +# Only has an effect if compiled with cURL +#curl_timeout = 5000 +# Limits number of parallel HTTP requests. Affects: +# - Media fetch if server uses remote_media setting +# - Serverlist download and server announcement +# - Downloads performed by main menu (e.g. mod manager) +# - Downloads performed by mods (minetest.httpfetch) +# Only has an effect if compiled with cURL +#curl_parallel_limit = 8 # Url to the server list displayed in the Multiplayer Tab #serverlist_url = servers.minetest.net diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 104e5640..90f58678 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -360,6 +360,7 @@ set(minetest_SRCS guiDeathScreen.cpp guiChatConsole.cpp client.cpp + clientmedia.cpp filecache.cpp tile.cpp shader.cpp diff --git a/src/client.cpp b/src/client.cpp index 8b80e3ec..ee63cf7c 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -19,6 +19,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "client.h" #include +#include #include "clientserver.h" #include "jthread/jmutexautolock.h" #include "main.h" @@ -37,17 +38,17 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "itemdef.h" #include "shader.h" #include -#include "sha1.h" #include "base64.h" #include "clientmap.h" -#include "filecache.h" +#include "clientmedia.h" #include "sound.h" #include "util/string.h" -#include "hex.h" #include "IMeshCache.h" +#include "serialization.h" #include "util/serialize.h" #include "config.h" #include "util/directiontables.h" +#include "util/pointedthing.h" #include "version.h" #if USE_CURL @@ -222,46 +223,9 @@ void * MeshUpdateThread::Thread() return NULL; } -void * MediaFetchThread::Thread() -{ - ThreadStarted(); - - log_register_thread("MediaFetchThread"); - - DSTACK(__FUNCTION_NAME); - - BEGIN_DEBUG_EXCEPTION_HANDLER - - #if USE_CURL - CURL *curl; - CURLcode res; - for (std::list::iterator i = m_file_requests.begin(); - i != m_file_requests.end(); ++i) { - curl = curl_easy_init(); - assert(curl); - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_URL, (m_remote_url + i->name).c_str()); - curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); - std::ostringstream stream; - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_data); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream); - curl_easy_setopt(curl, CURLOPT_USERAGENT, (std::string("Minetest ")+minetest_version_hash).c_str()); - res = curl_easy_perform(curl); - if (res == CURLE_OK) { - std::string data = stream.str(); - m_file_data.push_back(make_pair(i->name, data)); - } else { - m_failed.push_back(*i); - infostream << "cURL request failed for " << i->name << " (" << curl_easy_strerror(res) << ")"<< std::endl; - } - curl_easy_cleanup(curl); - } - #endif - - END_DEBUG_EXCEPTION_HANDLER(errorstream) - - return NULL; -} +/* + Client +*/ Client::Client( IrrlichtDevice *device, @@ -303,12 +267,9 @@ Client::Client( m_map_seed(0), m_password(password), m_access_denied(false), - m_media_cache(getMediaCacheDir()), - m_media_receive_started(false), - m_media_count(0), - m_media_received_count(0), m_itemdef_received(false), m_nodedef_received(false), + m_media_downloader(new ClientMediaDownloader()), m_time_of_day_set(false), m_last_time_of_day_f(-1), m_time_of_day_update_timer(0), @@ -332,9 +293,6 @@ Client::Client( m_env.addPlayer(player); } - - for (size_t i = 0; i < g_settings->getU16("media_fetch_threads"); ++i) - m_media_fetch_threads.push_back(new MediaFetchThread(this)); } Client::~Client() @@ -364,10 +322,6 @@ Client::~Client() } } - for (std::list::iterator i = m_media_fetch_threads.begin(); - i != m_media_fetch_threads.end(); ++i) - delete *i; - // cleanup 3d model meshes on client shutdown while (m_device->getSceneManager()->getMeshCache()->getMeshCount() != 0) { scene::IAnimatedMesh * mesh = @@ -797,57 +751,11 @@ void Client::step(float dtime) /* Load fetched media */ - if (m_media_receive_started) { - bool all_stopped = true; - for (std::list::iterator thread = m_media_fetch_threads.begin(); - thread != m_media_fetch_threads.end(); ++thread) { - all_stopped &= !(*thread)->IsRunning(); - while (!(*thread)->m_file_data.empty()) { - std::pair out = (*thread)->m_file_data.pop_front(); - if(m_media_received_count < m_media_count) - m_media_received_count++; - - bool success = loadMedia(out.second, out.first); - if(success){ - verbosestream<<"Client: Loaded received media: " - <<"\""<::iterator n; - n = m_media_name_sha1_map.find(out.first); - if(n == m_media_name_sha1_map.end()) - errorstream<<"The server sent a file that has not " - <<"been announced."< fetch_failed; - for (std::list::iterator thread = m_media_fetch_threads.begin(); - thread != m_media_fetch_threads.end(); ++thread) { - for (std::list::iterator request = (*thread)->m_failed.begin(); - request != (*thread)->m_failed.end(); ++request) - fetch_failed.push_back(*request); - (*thread)->m_failed.clear(); - } - if (fetch_failed.size() > 0) { - infostream << "Failed to remote-fetch " << fetch_failed.size() << " files. " - << "Requesting them the usual way." << std::endl; - request_media(fetch_failed); - } + if (m_media_downloader && m_media_downloader->isStarted()) { + m_media_downloader->step(this); + if (m_media_downloader->isDone()) { + delete m_media_downloader; + m_media_downloader = NULL; } } @@ -1048,15 +956,15 @@ void Client::deletingPeer(con::Peer *peer, bool timeout) string name } */ -void Client::request_media(const std::list &file_requests) +void Client::request_media(const std::list &file_requests) { std::ostringstream os(std::ios_base::binary); writeU16(os, TOSERVER_REQUEST_MEDIA); writeU16(os, file_requests.size()); - for(std::list::const_iterator i = file_requests.begin(); + for(std::list::const_iterator i = file_requests.begin(); i != file_requests.end(); ++i) { - os<name); + os< &file_requests) < data((u8*)s.c_str(), s.size()); + // Send as reliable + Send(0, data, true); + infostream<<"Client: Notifying server that we received all media" + < file_requests; + if (m_media_downloader == NULL || + m_media_downloader->isStarted()) { + const char *problem = m_media_downloader ? + "we already saw another announcement" : + "all media has been received already"; + errorstream<<"Client: Received media announcement but " + <addFile(name, sha1_raw); } - std::string remote_media = ""; + std::vector remote_media; try { - remote_media = deSerializeString(is); + Strfnd sf(deSerializeString(is)); + while(!sf.atend()) { + std::string baseurl = trim(sf.next(",")); + if(baseurl != "") + m_media_downloader->addRemoteServer(baseurl); + } } catch(SerializationError) { // not supported by server or turned off } - m_media_count = file_requests.size(); - m_media_receive_started = true; - - if (remote_media == "" || !USE_CURL) { - request_media(file_requests); - } else { - #if USE_CURL - std::list::iterator cur = m_media_fetch_threads.begin(); - for(std::list::iterator i = file_requests.begin(); - i != file_requests.end(); ++i) { - (*cur)->m_file_requests.push_back(*i); - cur++; - if (cur == m_media_fetch_threads.end()) - cur = m_media_fetch_threads.begin(); - } - for (std::list::iterator i = m_media_fetch_threads.begin(); - i != m_media_fetch_threads.end(); ++i) { - (*i)->m_remote_url = remote_media; - (*i)->Start(); - } - #endif - - // notify server we received everything - std::ostringstream os(std::ios_base::binary); - writeU16(os, TOSERVER_RECEIVED_MEDIA); - std::string s = os.str(); - SharedBuffer data((u8*)s.c_str(), s.size()); - // Send as reliable - Send(0, data, true); + m_media_downloader->step(this); + if (m_media_downloader->isDone()) { + // might be done already if all media is in the cache + delete m_media_downloader; + m_media_downloader = NULL; } - ClientEvent event; - event.type = CE_TEXTURES_UPDATED; - m_client_event_queue.push_back(event); } else if(command == TOCLIENT_MEDIA) { @@ -1775,67 +1654,37 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id) <::iterator n; - n = m_media_name_sha1_map.find(name); - if(n == m_media_name_sha1_map.end()) - errorstream<<"The server sent a file that has not " - <<"been announced."<conventionalTransferDone( + name, data, this); } - ClientEvent event; - event.type = CE_TEXTURES_UPDATED; - m_client_event_queue.push_back(event); + if (m_media_downloader->isDone()) { + delete m_media_downloader; + m_media_downloader = NULL; + } } else if(command == TOCLIENT_TOOLDEF) { @@ -2885,6 +2734,14 @@ ClientEvent Client::getClientEvent() return m_client_event_queue.pop_front(); } +float Client::mediaReceiveProgress() +{ + if (m_media_downloader) + return m_media_downloader->getProgress(); + else + return 1.0; // downloader only exists when not yet done +} + void draw_load_screen(const std::wstring &text, IrrlichtDevice* device, gui::IGUIFont* font, float dtime=0 ,int percent=0, bool clouds=true); @@ -2893,12 +2750,8 @@ void Client::afterContentReceived(IrrlichtDevice *device, gui::IGUIFont* font) infostream<<"Client::afterContentReceived() started"<rebuildImagesAndTextures(); diff --git a/src/client.h b/src/client.h index eb0f225a..5969adc8 100644 --- a/src/client.h +++ b/src/client.h @@ -31,32 +31,21 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "clientobject.h" #include "gamedef.h" #include "inventorymanager.h" -#include "filecache.h" #include "localplayer.h" -#include "server.h" +#include "hud.h" #include "particles.h" -#include "util/pointedthing.h" -#include struct MeshMakeData; class MapBlockMesh; -class IGameDef; class IWritableTextureSource; class IWritableShaderSource; class IWritableItemDefManager; class IWritableNodeDefManager; //class IWritableCraftDefManager; -class ClientEnvironment; +class ClientMediaDownloader; struct MapDrawControl; class MtEventManager; - -class ClientNotReadyException : public BaseException -{ -public: - ClientNotReadyException(const char *s): - BaseException(s) - {} -}; +struct PointedThing; struct QueuedMeshUpdate { @@ -132,31 +121,12 @@ public: IGameDef *m_gamedef; }; -class MediaFetchThread : public SimpleThread -{ -public: - - MediaFetchThread(IGameDef *gamedef): - m_gamedef(gamedef) - { - } - - void * Thread(); - - std::list m_file_requests; - MutexedQueue > m_file_data; - std::list m_failed; - std::string m_remote_url; - IGameDef *m_gamedef; -}; - enum ClientEventType { CE_NONE, CE_PLAYER_DAMAGE, CE_PLAYER_FORCE_MOVE, CE_DEATHSCREEN, - CE_TEXTURES_UPDATED, CE_SHOW_FORMSPEC, CE_SPAWN_PARTICLE, CE_ADD_PARTICLESPAWNER, @@ -426,19 +396,15 @@ public: std::wstring accessDeniedReason() { return m_access_denied_reason; } - float mediaReceiveProgress() - { - if (!m_media_receive_started) return 0; - return 1.0 * m_media_received_count / m_media_count; - } - - bool texturesReceived() - { return m_media_receive_started && m_media_received_count == m_media_count; } bool itemdefReceived() { return m_itemdef_received; } bool nodedefReceived() { return m_nodedef_received; } - + bool mediaReceived() + { return m_media_downloader == NULL; } + + float mediaReceiveProgress(); + void afterContentReceived(IrrlichtDevice *device, gui::IGUIFont* font); float getRTT(void); @@ -455,12 +421,15 @@ public: virtual bool checkLocalPrivilege(const std::string &priv) { return checkPrivilege(priv); } -private: - + // The following set of functions is used by ClientMediaDownloader // Insert a media file appropriately into the appropriate manager bool loadMedia(const std::string &data, const std::string &filename); + // Send a request for conventional media transfer + void request_media(const std::list &file_requests); + // Send a notification that no conventional media transfer is needed + void received_media(); - void request_media(const std::list &file_requests); +private: // Virtual methods from con::PeerHandler void peerAdded(con::Peer *peer); @@ -488,7 +457,6 @@ private: MtEventManager *m_event; MeshUpdateThread m_mesh_update_thread; - std::list m_media_fetch_threads; ClientEnvironment m_env; con::Connection m_con; IrrlichtDevice *m_device; @@ -514,14 +482,9 @@ private: bool m_access_denied; std::wstring m_access_denied_reason; Queue m_client_event_queue; - FileCache m_media_cache; - // Mapping from media file name to SHA1 checksum - std::map m_media_name_sha1_map; - bool m_media_receive_started; - u32 m_media_count; - u32 m_media_received_count; bool m_itemdef_received; bool m_nodedef_received; + ClientMediaDownloader *m_media_downloader; // time_of_day speed approximation for old protocol bool m_time_of_day_set; diff --git a/src/clientmedia.cpp b/src/clientmedia.cpp new file mode 100644 index 00000000..8260d5f5 --- /dev/null +++ b/src/clientmedia.cpp @@ -0,0 +1,656 @@ +/* +Minetest +Copyright (C) 2013 celeron55, Perttu Ahola + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#include "clientmedia.h" +#include "httpfetch.h" +#include "client.h" +#include "clientserver.h" +#include "filecache.h" +#include "hex.h" +#include "sha1.h" +#include "debug.h" +#include "log.h" +#include "porting.h" +#include "settings.h" +#include "main.h" +#include "util/serialize.h" +#include "util/string.h" + +static std::string getMediaCacheDir() +{ + return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media"; +} + +/* + ClientMediaDownloader +*/ + +ClientMediaDownloader::ClientMediaDownloader(): + m_media_cache(getMediaCacheDir()) +{ + m_initial_step_done = false; + m_name_bound = ""; // works because "" is an invalid file name + m_uncached_count = 0; + m_uncached_received_count = 0; + m_httpfetch_caller = HTTPFETCH_DISCARD; + m_httpfetch_active = 0; + m_httpfetch_active_limit = 0; + m_httpfetch_next_id = 0; + m_httpfetch_timeout = 0; + m_outstanding_hash_sets = 0; +} + +ClientMediaDownloader::~ClientMediaDownloader() +{ + if (m_httpfetch_caller != HTTPFETCH_DISCARD) + httpfetch_caller_free(m_httpfetch_caller); + + for (std::map::iterator it = m_files.begin(); + it != m_files.end(); ++it) + delete it->second; + + for (u32 i = 0; i < m_remotes.size(); ++i) + delete m_remotes[i]; +} + +void ClientMediaDownloader::addFile(std::string name, std::string sha1) +{ + assert(!m_initial_step_done); + + // if name was already announced, ignore the new announcement + if (m_files.count(name) != 0) { + errorstream << "Client: ignoring duplicate media announcement " + << "sent by server: \"" << name << "\"" + << std::endl; + return; + } + + // if name is empty or contains illegal characters, ignore the file + if (name.empty() || !string_allowed(name, TEXTURENAME_ALLOWED_CHARS)) { + errorstream << "Client: ignoring illegal file name " + << "sent by server: \"" << name << "\"" + << std::endl; + return; + } + + // length of sha1 must be exactly 20 (160 bits), else ignore the file + if (sha1.size() != 20) { + errorstream << "Client: ignoring illegal SHA1 sent by server: " + << hex_encode(sha1) << " \"" << name << "\"" + << std::endl; + return; + } + + FileStatus *filestatus = new FileStatus; + filestatus->received = false; + filestatus->sha1 = sha1; + filestatus->current_remote = -1; + m_files.insert(std::make_pair(name, filestatus)); +} + +void ClientMediaDownloader::addRemoteServer(std::string baseurl) +{ + assert(!m_initial_step_done); + + #ifdef USE_CURL + + infostream << "Client: Adding remote server \"" + << baseurl << "\" for media download" << std::endl; + + RemoteServerStatus *remote = new RemoteServerStatus; + remote->baseurl = baseurl; + remote->active_count = 0; + remote->request_by_filename = false; + m_remotes.push_back(remote); + + #else + + infostream << "Client: Ignoring remote server \"" + << baseurl << "\" because cURL support is not compiled in" + << std::endl; + + #endif +} + +void ClientMediaDownloader::step(Client *client) +{ + if (!m_initial_step_done) { + initialStep(client); + m_initial_step_done = true; + } + + // Remote media: check for completion of fetches + if (m_httpfetch_active) { + bool fetched_something = false; + HTTPFetchResult fetchresult; + + while (httpfetch_async_get(m_httpfetch_caller, fetchresult)) { + m_httpfetch_active--; + fetched_something = true; + + // Is this a hashset (index.mth) or a media file? + if (fetchresult.request_id < m_remotes.size()) + remoteHashSetReceived(fetchresult); + else + remoteMediaReceived(fetchresult, client); + } + + if (fetched_something) + startRemoteMediaTransfers(); + + // Did all remote transfers end and no new ones can be started? + // If so, request still missing files from the minetest server + // (Or report that we have all files.) + if (m_httpfetch_active == 0) { + if (m_uncached_received_count < m_uncached_count) { + infostream << "Client: Failed to remote-fetch " + << (m_uncached_count-m_uncached_received_count) + << " files. Requesting them" + << " the usual way." << std::endl; + } + startConventionalTransfers(client); + } + } +} + +void ClientMediaDownloader::initialStep(Client *client) +{ + // Check media cache + m_uncached_count = m_files.size(); + for (std::map::iterator + it = m_files.begin(); + it != m_files.end(); ++it) { + std::string name = it->first; + FileStatus *filestatus = it->second; + const std::string &sha1 = filestatus->sha1; + + std::ostringstream tmp_os(std::ios_base::binary); + bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os); + + // If found in cache, try to load it from there + if (found_in_cache) { + bool success = checkAndLoad(name, sha1, + tmp_os.str(), true, client); + if (success) { + filestatus->received = true; + m_uncached_count--; + } + } + } + + assert(m_uncached_received_count == 0); + + // Create the media cache dir if we are likely to write to it + if (m_uncached_count != 0) { + bool did = fs::CreateAllDirs(getMediaCacheDir()); + if (!did) { + errorstream << "Client: " + << "Could not create media cache directory: " + << getMediaCacheDir() + << std::endl; + } + } + + // If we found all files in the cache, report this fact to the server. + // If the server reported no remote servers, immediately start + // conventional transfers. Note: if cURL support is not compiled in, + // m_remotes is always empty, so "!USE_CURL" is redundant but may + // reduce the size of the compiled code + if (!USE_CURL || m_uncached_count == 0 || m_remotes.empty()) { + startConventionalTransfers(client); + } + else { + // Otherwise start off by requesting each server's sha1 set + + // This is the first time we use httpfetch, so alloc a caller ID + m_httpfetch_caller = httpfetch_caller_alloc(); + m_httpfetch_timeout = g_settings->getS32("curl_timeout"); + + // Set the active fetch limit to curl_parallel_limit or 84, + // whichever is greater. This gives us some leeway so that + // inefficiencies in communicating with the httpfetch thread + // don't slow down fetches too much. (We still want some limit + // so that when the first remote server returns its hash set, + // not all files are requested from that server immediately.) + // One such inefficiency is that ClientMediaDownloader::step() + // is only called a couple times per second, while httpfetch + // might return responses much faster than that. + // Note that httpfetch strictly enforces curl_parallel_limit + // but at no inter-thread communication cost. This however + // doesn't help with the aforementioned inefficiencies. + // The signifance of 84 is that it is 2*6*9 in base 13. + m_httpfetch_active_limit = g_settings->getS32("curl_parallel_limit"); + m_httpfetch_active_limit = MYMAX(m_httpfetch_active_limit, 84); + + // Write a list of hashes that we need. This will be POSTed + // to the server using Content-Type: application/octet-stream + std::string required_hash_set = serializeRequiredHashSet(); + + // minor fixme: this loop ignores m_httpfetch_active_limit + + // another minor fixme, unlikely to matter in normal usage: + // these index.mth fetches do (however) count against + // m_httpfetch_active_limit when starting actual media file + // requests, so if there are lots of remote servers that are + // not responding, those will stall new media file transfers. + + for (u32 i = 0; i < m_remotes.size(); ++i) { + assert(m_httpfetch_next_id == i); + + RemoteServerStatus *remote = m_remotes[i]; + actionstream << "Client: Contacting remote server \"" + << remote->baseurl << "\"" << std::endl; + + HTTPFetchRequest fetchrequest; + fetchrequest.url = + remote->baseurl + MTHASHSET_FILE_NAME; + fetchrequest.caller = m_httpfetch_caller; + fetchrequest.request_id = m_httpfetch_next_id; // == i + fetchrequest.timeout = m_httpfetch_timeout; + fetchrequest.connect_timeout = m_httpfetch_timeout; + fetchrequest.post_fields = required_hash_set; + fetchrequest.extra_headers.push_back( + "Content-Type: application/octet-stream"); + httpfetch_async(fetchrequest); + + m_httpfetch_active++; + m_httpfetch_next_id++; + m_outstanding_hash_sets++; + } + } +} + +void ClientMediaDownloader::remoteHashSetReceived( + const HTTPFetchResult &fetchresult) +{ + u32 remote_id = fetchresult.request_id; + assert(remote_id < m_remotes.size()); + RemoteServerStatus *remote = m_remotes[remote_id]; + + m_outstanding_hash_sets--; + + if (fetchresult.succeeded) { + try { + // Server sent a list of file hashes that are + // available on it, try to parse the list + + std::set sha1_set; + deSerializeHashSet(fetchresult.data, sha1_set); + + // Parsing succeeded: For every file that is + // available on this server, add this server + // to the available_remotes array + + for(std::map::iterator + it = m_files.upper_bound(m_name_bound); + it != m_files.end(); ++it) { + FileStatus *f = it->second; + if (!f->received && sha1_set.count(f->sha1)) + f->available_remotes.push_back(remote_id); + } + } + catch (SerializationError &e) { + infostream << "Client: Remote server \"" + << remote->baseurl << "\" sent invalid hash set: " + << e.what() << std::endl; + } + } + + // For compatibility: If index.mth is not found, assume that the + // server contains files named like the original files (not their sha1) + + if (!fetchresult.succeeded && !fetchresult.timeout && + fetchresult.response_code == 404) { + infostream << "Client: Enabling compatibility mode for remote " + << "server \"" << remote->baseurl << "\"" << std::endl; + remote->request_by_filename = true; + + // Assume every file is available on this server + + for(std::map::iterator + it = m_files.upper_bound(m_name_bound); + it != m_files.end(); ++it) { + FileStatus *f = it->second; + if (!f->received) + f->available_remotes.push_back(remote_id); + } + } +} + +void ClientMediaDownloader::remoteMediaReceived( + const HTTPFetchResult &fetchresult, + Client *client) +{ + // Some remote server sent us a file. + // -> decrement number of active fetches + // -> mark file as received if fetch succeeded + // -> try to load media + + std::string name; + { + std::map::iterator it = + m_remote_file_transfers.find(fetchresult.request_id); + assert(it != m_remote_file_transfers.end()); + name = it->second; + m_remote_file_transfers.erase(it); + } + + assert(m_files.count(name) != 0); + + FileStatus *filestatus = m_files[name]; + assert(!filestatus->received); + assert(filestatus->current_remote >= 0); + + RemoteServerStatus *remote = m_remotes[filestatus->current_remote]; + + filestatus->current_remote = -1; + remote->active_count--; + + // If fetch succeeded, try to load media file + + if (fetchresult.succeeded) { + bool success = checkAndLoad(name, filestatus->sha1, + fetchresult.data, false, client); + if (success) { + filestatus->received = true; + assert(m_uncached_received_count < m_uncached_count); + m_uncached_received_count++; + } + } +} + +s32 ClientMediaDownloader::selectRemoteServer(FileStatus *filestatus) +{ + assert(filestatus != NULL); + assert(!filestatus->received); + assert(filestatus->current_remote < 0); + + if (filestatus->available_remotes.empty()) + return -1; + else { + // Of all servers that claim to provide the file (and haven't + // been unsuccessfully tried before), find the one with the + // smallest number of currently active transfers + + s32 best = 0; + s32 best_remote_id = filestatus->available_remotes[best]; + s32 best_active_count = m_remotes[best_remote_id]->active_count; + + for (u32 i = 1; i < filestatus->available_remotes.size(); ++i) { + s32 remote_id = filestatus->available_remotes[i]; + s32 active_count = m_remotes[remote_id]->active_count; + if (active_count < best_active_count) { + best = i; + best_remote_id = remote_id; + best_active_count = active_count; + } + } + + filestatus->available_remotes.erase( + filestatus->available_remotes.begin() + best); + + return best_remote_id; + } +} + +void ClientMediaDownloader::startRemoteMediaTransfers() +{ + bool changing_name_bound = true; + + for (std::map::iterator + files_iter = m_files.upper_bound(m_name_bound); + files_iter != m_files.end(); ++files_iter) { + + // Abort if active fetch limit is exceeded + if (m_httpfetch_active >= m_httpfetch_active_limit) + break; + + const std::string &name = files_iter->first; + FileStatus *filestatus = files_iter->second; + + if (!filestatus->received && filestatus->current_remote < 0) { + // File has not been received yet and is not currently + // being transferred. Choose a server for it. + s32 remote_id = selectRemoteServer(filestatus); + if (remote_id >= 0) { + // Found a server, so start fetching + RemoteServerStatus *remote = + m_remotes[remote_id]; + + std::string url = remote->baseurl + + (remote->request_by_filename ? name : + hex_encode(filestatus->sha1)); + verbosestream << "Client: " + << "Requesting remote media file " + << "\"" << name << "\" " + << "\"" << url << "\"" << std::endl; + + HTTPFetchRequest fetchrequest; + fetchrequest.url = url; + fetchrequest.caller = m_httpfetch_caller; + fetchrequest.request_id = m_httpfetch_next_id; + fetchrequest.timeout = 0; // no data timeout! + fetchrequest.connect_timeout = + m_httpfetch_timeout; + httpfetch_async(fetchrequest); + + m_remote_file_transfers.insert(std::make_pair( + m_httpfetch_next_id, + name)); + + filestatus->current_remote = remote_id; + remote->active_count++; + m_httpfetch_active++; + m_httpfetch_next_id++; + } + } + + if (filestatus->received || + (filestatus->current_remote < 0 && + !m_outstanding_hash_sets)) { + // If we arrive here, we conclusively know that we + // won't fetch this file from a remote server in the + // future. So update the name bound if possible. + if (changing_name_bound) + m_name_bound = name; + } + else + changing_name_bound = false; + } + +} + +void ClientMediaDownloader::startConventionalTransfers(Client *client) +{ + assert(m_httpfetch_active == 0); + + if (m_uncached_received_count == m_uncached_count) { + // In this case all media was found in the cache or + // has been downloaded from some remote server; + // report this fact to the server + client->received_media(); + } + else { + // Some media files have not been received yet, use the + // conventional slow method (minetest protocol) to get them + std::list file_requests; + for (std::map::iterator + it = m_files.begin(); + it != m_files.end(); ++it) { + if (!it->second->received) + file_requests.push_back(it->first); + } + assert((s32) file_requests.size() == + m_uncached_count - m_uncached_received_count); + client->request_media(file_requests); + } +} + +void ClientMediaDownloader::conventionalTransferDone( + const std::string &name, + const std::string &data, + Client *client) +{ + // Check that file was announced + std::map::iterator + file_iter = m_files.find(name); + if (file_iter == m_files.end()) { + errorstream << "Client: server sent media file that was" + << "not announced, ignoring it: \"" << name << "\"" + << std::endl; + return; + } + FileStatus *filestatus = file_iter->second; + assert(filestatus != NULL); + + // Check that file hasn't already been received + if (filestatus->received) { + errorstream << "Client: server sent media file that we already" + << "received, ignoring it: \"" << name << "\"" + << std::endl; + return; + } + + // Mark file as received, regardless of whether loading it works and + // whether the checksum matches (because at this point there is no + // other server that could send a replacement) + filestatus->received = true; + assert(m_uncached_received_count < m_uncached_count); + m_uncached_received_count++; + + // Check that received file matches announced checksum + // If so, load it + checkAndLoad(name, filestatus->sha1, data, false, client); +} + +bool ClientMediaDownloader::checkAndLoad( + const std::string &name, const std::string &sha1, + const std::string &data, bool is_from_cache, Client *client) +{ + const char *cached_or_received = is_from_cache ? "cached" : "received"; + const char *cached_or_received_uc = is_from_cache ? "Cached" : "Received"; + std::string sha1_hex = hex_encode(sha1); + + // Compute actual checksum of data + std::string data_sha1; + { + SHA1 data_sha1_calculator; + data_sha1_calculator.addBytes(data.c_str(), data.size()); + unsigned char *data_tmpdigest = data_sha1_calculator.getDigest(); + data_sha1.assign((char*) data_tmpdigest, 20); + free(data_tmpdigest); + } + + // Check that received file matches announced checksum + if (data_sha1 != sha1) { + std::string data_sha1_hex = hex_encode(data_sha1); + infostream << "Client: " + << cached_or_received_uc << " media file " + << sha1_hex << " \"" << name << "\" " + << "mismatches actual checksum " << data_sha1_hex + << std::endl; + return false; + } + + // Checksum is ok, try loading the file + bool success = client->loadMedia(data, name); + if (!success) { + infostream << "Client: " + << "Failed to load " << cached_or_received << " media: " + << sha1_hex << " \"" << name << "\"" + << std::endl; + return false; + } + + verbosestream << "Client: " + << "Loaded " << cached_or_received << " media: " + << sha1_hex << " \"" << name << "\"" + << std::endl; + + // Update cache (unless we just loaded the file from the cache) + if (!is_from_cache) + m_media_cache.update(sha1_hex, data); + + return true; +} + + +/* + Minetest Hashset File Format + + All values are stored in big-endian byte order. + [u32] signature: 'MTHS' + [u16] version: 1 + For each hash in set: + [u8*20] SHA1 hash + + Version changes: + 1 - Initial version +*/ + +std::string ClientMediaDownloader::serializeRequiredHashSet() +{ + std::ostringstream os(std::ios::binary); + + writeU32(os, MTHASHSET_FILE_SIGNATURE); // signature + writeU16(os, 1); // version + + // Write list of hashes of files that have not been + // received (found in cache) yet + for (std::map::iterator + it = m_files.begin(); + it != m_files.end(); ++it) { + if (!it->second->received) { + assert(it->second->sha1.size() == 20); + os << it->second->sha1; + } + } + + return os.str(); +} + +void ClientMediaDownloader::deSerializeHashSet(const std::string &data, + std::set &result) +{ + if (data.size() < 6 || data.size() % 20 != 6) { + throw SerializationError( + "ClientMediaDownloader::deSerializeHashSet: " + "invalid hash set file size"); + } + + const u8 *data_cstr = (const u8*) data.c_str(); + + u32 signature = readU32(&data_cstr[0]); + if (signature != MTHASHSET_FILE_SIGNATURE) { + throw SerializationError( + "ClientMediaDownloader::deSerializeHashSet: " + "invalid hash set file signature"); + } + + u16 version = readU16(&data_cstr[4]); + if (version != 1) { + throw SerializationError( + "ClientMediaDownloader::deSerializeHashSet: " + "unsupported hash set file version"); + } + + for (u32 pos = 6; pos < data.size(); pos += 20) { + result.insert(data.substr(pos, 20)); + } +} diff --git a/src/clientmedia.h b/src/clientmedia.h new file mode 100644 index 00000000..54004423 --- /dev/null +++ b/src/clientmedia.h @@ -0,0 +1,150 @@ +/* +Minetest +Copyright (C) 2013 celeron55, Perttu Ahola + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#ifndef CLIENTMEDIA_HEADER +#define CLIENTMEDIA_HEADER + +#include "irrlichttypes.h" +#include "filecache.h" +#include +#include +#include +#include + +class Client; +struct HTTPFetchResult; + +#define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS' +#define MTHASHSET_FILE_NAME "index.mth" + +class ClientMediaDownloader +{ +public: + ClientMediaDownloader(); + ~ClientMediaDownloader(); + + float getProgress() const { + if (m_uncached_count >= 1) + return 1.0 * m_uncached_received_count / + m_uncached_count; + else + return 0.0; + } + + bool isStarted() const { + return m_initial_step_done; + } + + // If this returns true, the downloader is done and can be deleted + bool isDone() const { + return m_initial_step_done && + m_uncached_received_count == m_uncached_count; + } + + // Add a file to the list of required file (but don't fetch it yet) + void addFile(std::string name, std::string sha1); + + // Add a remote server to the list; ignored if not built with cURL + void addRemoteServer(std::string baseurl); + + // Steps the media downloader: + // - May load media into client by calling client->loadMedia() + // - May check media cache for files + // - May add files to media cache + // - May start remote transfers by calling httpfetch_async + // - May check for completion of current remote transfers + // - May start conventional transfers by calling client->request_media() + // - May inform server that all media has been loaded + // by calling client->received_media() + // After step has been called once, don't call addFile/addRemoteServer. + void step(Client *client); + + // Must be called for each file received through TOCLIENT_MEDIA + void conventionalTransferDone( + const std::string &name, + const std::string &data, + Client *client); + +private: + struct FileStatus { + bool received; + std::string sha1; + s32 current_remote; + std::vector available_remotes; + }; + + struct RemoteServerStatus { + std::string baseurl; + s32 active_count; + bool request_by_filename; + }; + + void initialStep(Client *client); + void remoteHashSetReceived(const HTTPFetchResult &fetchresult); + void remoteMediaReceived(const HTTPFetchResult &fetchresult, + Client *client); + s32 selectRemoteServer(FileStatus *filestatus); + void startRemoteMediaTransfers(); + void startConventionalTransfers(Client *client); + + bool checkAndLoad(const std::string &name, const std::string &sha1, + const std::string &data, bool is_from_cache, + Client *client); + + std::string serializeRequiredHashSet(); + static void deSerializeHashSet(const std::string &data, + std::set &result); + + // Maps filename to file status + std::map m_files; + + // Array of remote media servers + std::vector m_remotes; + + // Filesystem-based media cache + FileCache m_media_cache; + + // Has an attempt been made to load media files from the file cache? + // Have hash sets been requested from remote servers? + bool m_initial_step_done; + + // Total number of media files to load + s32 m_uncached_count; + + // Number of media files that have been received + s32 m_uncached_received_count; + + // Status of remote transfers + unsigned long m_httpfetch_caller; + unsigned long m_httpfetch_next_id; + long m_httpfetch_timeout; + s32 m_httpfetch_active; + s32 m_httpfetch_active_limit; + s32 m_outstanding_hash_sets; + std::map m_remote_file_transfers; + + // All files up to this name have either been received from a + // remote server or failed on all remote servers, so those files + // don't need to be looked at again + // (use m_files.upper_bound(m_name_bound) to get an iterator) + std::string m_name_bound; + +}; + +#endif // !CLIENTMEDIA_HEADER diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index 662717c8..a217fda4 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -139,7 +139,7 @@ void set_default_settings(Settings *settings) settings->setDefault("repeat_rightclick_time", "0.25"); settings->setDefault("enable_particles", "true"); - settings->setDefault("media_fetch_threads", "8"); + settings->setDefault("curl_timeout", "5000"); settings->setDefault("curl_parallel_limit", "8"); settings->setDefault("serverlist_url", "servers.minetest.net"); @@ -284,7 +284,6 @@ void set_default_settings(Settings *settings) settings->setDefault("mgmath_generator", "mandelbox"); - settings->setDefault("curl_timeout", "5000"); // IPv6 settings->setDefault("enable_ipv6", "true"); diff --git a/src/filecache.cpp b/src/filecache.cpp index c4de6cf8..33677cc8 100644 --- a/src/filecache.cpp +++ b/src/filecache.cpp @@ -23,12 +23,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "clientserver.h" #include "log.h" #include "filesys.h" -#include "hex.h" -#include "sha1.h" #include #include #include -#include #include bool FileCache::loadByPath(const std::string &path, std::ostream &os) @@ -85,36 +82,8 @@ bool FileCache::update(const std::string &name, const std::string &data) std::string path = m_dir + DIR_DELIM + name; return updateByPath(path, data); } -bool FileCache::update_sha1(const std::string &data) -{ - SHA1 sha1; - sha1.addBytes(data.c_str(), data.size()); - unsigned char *digest = sha1.getDigest(); - std::string sha1_raw((char*)digest, 20); - free(digest); - std::string sha1_hex = hex_encode(sha1_raw); - return update(sha1_hex, data); -} bool FileCache::load(const std::string &name, std::ostream &os) { std::string path = m_dir + DIR_DELIM + name; return loadByPath(path, os); } -bool FileCache::load_sha1(const std::string &sha1_raw, std::ostream &os) -{ - std::ostringstream tmp_os(std::ios_base::binary); - if(!load(hex_encode(sha1_raw), tmp_os)) - return false; - SHA1 sha1; - sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length()); - unsigned char *digest = sha1.getDigest(); - std::string sha1_real_raw((char*)digest, 20); - free(digest); - if(sha1_real_raw != sha1_raw){ - verbosestream<<"FileCache["< #include #include "util/directiontables.h" +#include "util/pointedthing.h" /* Text input system @@ -1207,7 +1208,7 @@ void the_game( server->step(dtime); // End condition - if(client.texturesReceived() && + if(client.mediaReceived() && client.itemdefReceived() && client.nodedefReceived()){ got_content = true; @@ -1422,7 +1423,7 @@ void the_game( bool invert_mouse = g_settings->getBool("invert_mouse"); bool respawn_menu_active = false; - bool update_wielded_item_trigger = false; + bool update_wielded_item_trigger = true; bool show_hud = true; bool show_chat = true; @@ -2290,10 +2291,6 @@ void the_game( delete(event.show_formspec.formspec); delete(event.show_formspec.formname); } - else if(event.type == CE_TEXTURES_UPDATED) - { - update_wielded_item_trigger = true; - } else if(event.type == CE_SPAWN_PARTICLE) { LocalPlayer* player = client.getEnv().getLocalPlayer(); diff --git a/src/server.cpp b/src/server.cpp index 2c38c66d..9d51b794 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2745,7 +2745,7 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id) std::string datastring((char*)&data[2], datasize-2); std::istringstream is(datastring, std::ios_base::binary); - std::list tosend; + std::list tosend; u16 numfiles = readU16(is); infostream<<"Sending "< &tosend) + const std::list &tosend) { DSTACK(__FUNCTION_NAME); @@ -4470,17 +4470,19 @@ void Server::sendRequestedMedia(u16 peer_id, u32 file_size_bunch_total = 0; - for(std::list::const_iterator i = tosend.begin(); + for(std::list::const_iterator i = tosend.begin(); i != tosend.end(); ++i) { - if(m_media.find(i->name) == m_media.end()){ + const std::string &name = *i; + + if(m_media.find(name) == m_media.end()){ errorstream<<"Server::sendRequestedMedia(): Client asked for " - <<"unknown file \""<<(i->name)<<"\""<= bytes_per_bunch){ diff --git a/src/server.h b/src/server.h index 87a60353..e71a811e 100644 --- a/src/server.h +++ b/src/server.h @@ -153,15 +153,6 @@ struct PrioritySortedBlockTransfer u16 peer_id; }; -struct MediaRequest -{ - std::string name; - - MediaRequest(const std::string &name_=""): - name(name_) - {} -}; - struct MediaInfo { std::string path; @@ -569,7 +560,7 @@ private: void fillMediaCache(); void sendMediaAnnouncement(u16 peer_id); void sendRequestedMedia(u16 peer_id, - const std::list &tosend); + const std::list &tosend); void sendDetachedInventory(const std::string &name, u16 peer_id); void sendDetachedInventoryToAll(const std::string &name);