builtin/client_file, interface/file_watch: Watch changes in automatically transferred files and reload content for new clients

builtin/client_file/api.h

@@ -14,7 +14,9 @@ namespace client_file
 
 	struct Interface
 	{
+		virtual void update_file_content(const ss_ &name, const ss_ &content) = 0;
 		virtual void add_file_content(const ss_ &name, const ss_ &content) = 0;
+		virtual void add_file_path(const ss_ &name, const ss_ &path) = 0;
 	};
 
 	inline bool access(interface::Server *server,

builtin/client_file/client_file.cpp

@@ -3,6 +3,7 @@
 #include "interface/server.h"
 #include "interface/event.h"
 #include "interface/sha1.h"
+#include "interface/file_watch.h"
 #include "client_file/api.h"
 #include "network/api.h"
 #include <cereal/archives/binary.hpp>
@@ -26,10 +27,12 @@ struct Module: public interface::Module, public client_file::Interface
 {
 	interface::Server *m_server;
 	sm_<ss_, sp_<FileInfo>> m_files;
+	sp_<interface::MultiFileWatch> m_watch;
 
 	Module(interface::Server *server):
+		interface::Module("client_file"),
 		m_server(server),
-		interface::Module("client_file")
+		m_watch(interface::createMultiFileWatch())
 	{
 		log_v(MODULE, "client_file construct");
 	}
@@ -37,6 +40,8 @@ struct Module: public interface::Module, public client_file::Interface
 	~Module()
 	{
 		log_v(MODULE, "client_file destruct");
+		for(int fd : m_watch->get_fds())
+			m_server->remove_socket_event(fd);
 	}
 
 	void init()
@@ -48,6 +53,10 @@ struct Module: public interface::Module, public client_file::Interface
 				Event::t("network:packet_received/core:request_file"));
 		m_server->sub_event(this,
 				Event::t("network:packet_received/core:all_files_transferred"));
+		m_server->sub_event(this, Event::t("watch_file:watch_fd_event"));
+
+		for(int fd : m_watch->get_fds())
+			m_server->add_socket_event(fd, Event::t("watch_file:watch_fd_event"));
 	}
 
 	void event(const Event::Type &type, const Event::Private *p)
@@ -58,6 +67,8 @@ struct Module: public interface::Module, public client_file::Interface
 				network::Packet)
 		EVENT_TYPEN("network:packet_received/core:all_files_transferred",
 				on_all_files_transferred, network::Packet)
+		EVENT_TYPEN("watch_file:watch_fd_event", on_watch_fd_event,
+				interface::SocketEvent);
 	}
 
 	void on_start()
@@ -130,14 +141,57 @@ struct Module: public interface::Module, public client_file::Interface
 				new FilesTransmitted(packet.sender));
 	}
 
+	void on_watch_fd_event(const interface::SocketEvent &event)
+	{
+		log_v(MODULE, "on_watch_fd_event()");
+		m_watch->report_fd(event.fd);
+	}
+
 	// Interface
 
+	void update_file_content(const ss_ &name, const ss_ &content)
+	{
+		ss_ hash = interface::sha1::calculate(content);
+
+		auto it = m_files.find(name);
+		if(it != m_files.end()){
+			// File already added; ignore if content wasn't modified
+			sp_<FileInfo> old_info = it->second;
+			if(old_info->hash == hash){
+				log_v(MODULE, "File stayed the same: %s: %s", cs(name),
+						cs(interface::sha1::hex(hash)));
+				return;
+			}
+		}
+
+		log_v(MODULE, "File updated: %s: %s", cs(name),
+				cs(interface::sha1::hex(hash)));
+		m_files[name] = sp_<FileInfo>(new FileInfo(name, content, hash));
+
+		// Note: Clients have to reconnect to see the new file content
+	}
+
 	void add_file_content(const ss_ &name, const ss_ &content)
 	{
+		update_file_content(name, content);
+	}
+
+	void add_file_path(const ss_ &name, const ss_ &path)
+	{
+		std::ifstream f(path);
+		std::string content((std::istreambuf_iterator<char>(f)),
+				std::istreambuf_iterator<char>());
 		ss_ hash = interface::sha1::calculate(content);
-		log_v(MODULE, "File: %s: %s", cs(name),
+		log_v(MODULE, "File added: %s: %s (%s)", cs(name),
-				cs(interface::sha1::hex(hash)));
+				cs(interface::sha1::hex(hash)), cs(path));
-		m_files[name] = up_<FileInfo>(new FileInfo(name, content, hash));
+		m_files[name] = sp_<FileInfo>(new FileInfo(name, content, hash));
+		m_watch->add(path, [this, name, path](const ss_ & path_){
+			log_v(MODULE, "File modified: %s (%s)", cs(name), cs(path));
+			std::ifstream f(path);
+			std::string content((std::istreambuf_iterator<char>(f)),
+					std::istreambuf_iterator<char>());
+			update_file_content(name, content);
+		});
 	}
 
 	void* get_interface()

builtin/client_lua/client_lua.cpp

@@ -73,14 +73,10 @@ struct Module: public interface::Module
 		for(const interface::Filesystem::Node &n : list){
 			if(n.is_directory)
 				continue;
-			const ss_ &file_name = n.name;
+			const ss_ &file_path = client_lua_path+"/"+n.name;
-			std::ifstream f(client_lua_path+"/"+file_name);
-			std::string file_content((std::istreambuf_iterator<char>(f)),
-					std::istreambuf_iterator<char>());
-
 			const ss_ &public_file_name = module_name+"/"+n.name;
 			client_file::access(m_server, [&](client_file::Interface * i){
-				i->add_file_content(public_file_name, file_content);
+				i->add_file_path(public_file_name, file_path);
 			});
 		}
 	}

client/sandbox.lua

@@ -66,6 +66,6 @@ function buildat:run_script_file(name)
 		log:error("Failed to load script file: "+name)
 		return false
 	end
-	log:info("buildat:run_script_file("..name.."): #code="..#code)
+	log:info("buildat:run_script_file("..name.."): code length: "..#code)
 	return __buildat_run_in_sandbox(code)
 end

src/impl/linux/file_watch.cpp

@@ -112,4 +112,121 @@ FileWatch* createFileWatch(const sv_<ss_> &paths, std::function<void()> cb)
 	return new CFileWatch(paths, cb);
 }
 
+struct CMultiFileWatch: MultiFileWatch
+{
+	struct WatchThing {
+		ss_ path;
+		std::function<void(const ss_ &path)> cb;
+		WatchThing(const ss_ &path, std::function<void(const ss_ &path)> cb):
+			path(path), cb(cb){}
+	};
+
+	int m_fd = -1;
+	sm_<int, sp_<WatchThing>> m_watch;
+
+	CMultiFileWatch()
+	{
+		m_fd = inotify_init1(IN_NONBLOCK);
+		if(m_fd == -1){
+			throw Exception(ss_()+"inotify_init() failed: "+strerror(errno));
+		}
+	}
+
+	~CMultiFileWatch()
+	{
+		close(m_fd);
+	}
+
+	void add(const ss_ &path, std::function<void(const ss_ &path)> cb)
+	{
+		int r = inotify_add_watch(m_fd, path.c_str(), IN_CLOSE_WRITE |
+				IN_MOVED_TO | IN_CREATE | IN_MOVED_FROM | IN_DELETE |
+				IN_MODIFY | IN_ATTRIB);
+		if(r == -1){
+			throw Exception(ss_()+"inotify_add_watch() failed: "+
+					strerror(errno)+" (path="+path+")");
+		}
+		log_v(MODULE, "Watching path \"%s\" (inotify fd=%i)", cs(path), m_fd);
+		m_watch[r] = sp_<WatchThing>(new WatchThing(path, cb));
+	}
+
+	// Used on Linux; no-op on Windows
+	sv_<int> get_fds()
+	{
+		if(m_fd == -1)
+			return {};
+		return {m_fd};
+	}
+	void report_fd(int fd)
+	{
+		if(fd != m_fd)
+			return;
+		struct inotify_event in_event;
+		for(;;){
+			int r = read(fd, &in_event, sizeof(struct inotify_event));
+			if(r != sizeof(struct inotify_event))
+				break;
+			sv_<char> name_buf(in_event.len);
+			read(fd, &name_buf[0], in_event.len);
+			ss_ name(name_buf.begin(), name_buf.end());
+			ss_ mask_s;
+			if(in_event.mask & IN_ACCESS) mask_s += "IN_ACCESS | ";
+			if(in_event.mask & IN_ATTRIB) mask_s += "IN_ATTRIB | ";
+			if(in_event.mask & IN_CLOSE_WRITE) mask_s += "IN_CLOSE_WRITE | ";
+			if(in_event.mask & IN_CLOSE_NOWRITE) mask_s += "IN_CLOSE_NOWRITE | ";
+			if(in_event.mask & IN_CREATE) mask_s += "IN_CREATE | ";
+			if(in_event.mask & IN_DELETE) mask_s += "IN_DELETE | ";
+			if(in_event.mask & IN_DELETE_SELF) mask_s += "IN_DELETE_SELF | ";
+			if(in_event.mask & IN_MODIFY) mask_s += "IN_MODIFY | ";
+			if(in_event.mask & IN_MOVE_SELF) mask_s += "IN_MOVE_SELF | ";
+			if(in_event.mask & IN_MOVED_FROM) mask_s += "IN_MOVED_FROM | ";
+			if(in_event.mask & IN_MOVED_TO) mask_s += "IN_MOVED_TO | ";
+			if(in_event.mask & IN_OPEN) mask_s += "IN_OPEN | ";
+			if(in_event.mask & IN_IGNORED) mask_s += "IN_IGNORED | ";
+			if(in_event.mask & IN_ISDIR) mask_s += "IN_ISDIR | ";
+			if(in_event.mask & IN_Q_OVERFLOW) mask_s += "IN_Q_OVERFLOW | ";
+			if(in_event.mask & IN_UNMOUNT) mask_s += "IN_UNMOUNT | ";
+
+			mask_s = mask_s.substr(0, mask_s.size()-3);
+
+			log_d(MODULE, "in_event.wd=%i, mask=%s, name=%s",
+					in_event.wd, cs(mask_s), cs(name));
+
+			auto it = m_watch.find(in_event.wd);
+			if(it == m_watch.end())
+				throw Exception("inotify returned unknown wd");
+			sp_<WatchThing> thing = it->second;
+			thing->cb(thing->path);
+
+			if(in_event.mask & IN_IGNORED){
+				// Inotify removed path from watch
+				const ss_ &path = thing->path;
+				m_watch.erase(in_event.wd);
+				int r = inotify_add_watch(m_fd, path.c_str(), IN_CLOSE_WRITE |
+						IN_MOVED_TO | IN_CREATE | IN_MOVED_FROM | IN_DELETE |
+						IN_MODIFY | IN_ATTRIB);
+				if(r == -1){
+					log_w(MODULE, "inotify_add_watch() failed: %s (while trying "
+							"to re-watch ignored path \"%s\")",
+							strerror(errno), cs(path));
+				} else {
+					log_v(MODULE, "Re-watching auto-ignored path \"%s\" (inotify fd=%i)",
+							cs(path), m_fd);
+					m_watch[r] = thing;
+				}
+			}
+		}
+	}
+
+	// Used on Windows; no-op on Linux
+	void update()
+	{
+	}
+};
+
+MultiFileWatch* createMultiFileWatch()
+{
+	return new CMultiFileWatch();
+}
+
 }

src/interface/file_watch.h

@@ -19,4 +19,22 @@ namespace interface
 	// cb is called at either report_fd() or update().
 	FileWatch* createFileWatch(
 			const sv_<ss_> &paths, std::function<void()> cb);
+
+	struct MultiFileWatch
+	{
+		virtual ~MultiFileWatch(){}
+
+		virtual void add(const ss_ &path,
+				std::function<void(const ss_ &path)> cb) = 0;
+
+		// Used on Linux; no-op on Windows
+		virtual sv_<int> get_fds() = 0;
+		virtual void report_fd(int fd) = 0;
+
+		// Used on Windows; no-op on Linux
+		virtual void update() = 0;
+	};
+
+	// cb is called at either report_fd() or update().
+	MultiFileWatch* createMultiFileWatch();
 }

todo.txt

@@ -2,15 +2,10 @@ Buildat TODO
 ============
 - Cache a hash of each compiled module in order not to rebuild them
   unnecessarily
-- Modules should be unloadable if possible. Figure out if so, and how to handle
-  references to modules when unloading them.
-- There should be some kind of an easy synchronous calling mechanism between
-  modules.
 - Modules should be run in threads.
 - Network should leave socket fds in m_server when destructing, and pick them up
   on core:continue.
-- Implement file modification tracking in client_lua or client_file, or somehow
+- builtin/client_file should somehow not forget its file list at reload
-  otherwise make newly connected clients get new file contents
 
 Buildat TODO later
 ==================