From 87ae4b1d7a8eb7eba43c0be6d85a3826f9cd822e Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Wed, 19 Apr 2017 11:28:43 -0700 Subject: [PATCH] Add media scanning and collection. This removes the need entirely for any other script. At startup the program will scan all folders recursively and hardlink or copy the content over to the webroot, and then serve the index.mth. --- main.go | 105 ++++++++++++++++++++++++++++++++++++++++++++++-- mtmediasrv.yaml | 31 ++++++++++++++ readme.md | 19 +++++++++ 3 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 mtmediasrv.yaml diff --git a/main.go b/main.go index 03340b8..a163430 100644 --- a/main.go +++ b/main.go @@ -32,7 +32,9 @@ import ( "net/http" "net/http/fcgi" "os" + "path/filepath" "strings" + "github.com/spf13/viper" ) @@ -40,6 +42,8 @@ var ( Version string Build string + newmedia int + arr []string ) @@ -134,13 +138,108 @@ func parseMedia(path string) { } } +func collectMedia(l bool, c bool, e map[string]bool, w string) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Print(err) + return nil + } + if info.IsDir() { + return nil + } + ext := filepath.Ext(path) + if e[ext] { + sha, err := getHash(path) + if err != nil { + return err + } + + of := strings.Join([]string{w, sha}, "/") + + if l { + err := os.Link(path, of) + if err != nil { + return err + } + newmedia++ + } else if c { + in, err := os.Open(path) + if err != nil { + return err + } + defer in.Close() + os.Remove(of) + out, err := os.Create(of) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, in) + closeErr := out.Close() + if err != nil { + return err + } + newmedia++ + return closeErr + } + } + return nil + } +} func main() { - p := "/var/www/media" - parseMedia(p) + // config stuff + viper.SetConfigName("mtmediasrv") + viper.SetConfigType("yaml") + + viper.AddConfigPath("/usr/share/defaults/etc") + viper.AddConfigPath("/etc") + viper.AddConfigPath("$HOME/.config") + + viper.SetDefault("socket", "/run/mtmediasrv/sock") + viper.SetDefault("webroot", "/var/www/media") + viper.SetDefault("mediapath", []string{}) + viper.SetDefault("mediascan", "true") + viper.SetDefault("medialink", "true") + viper.SetDefault("mediacopy", "false") + viper.SetDefault("extensions", []string{ ".png", ".jpg", ".jpeg", ".ogg", ".x", ".b3d", ".obj"}) + + err := viper.ReadInConfig() + if err != nil { + log.Fatal("Error in confog file: ", err) + } + + // step 1, collect media files + w := viper.GetString("webroot") + ext := viper.GetStringSlice("extensions") + extmap := make(map[string]bool) + for i := 0; i < len(ext); i++ { + extmap[ext[i]] = true + } + if viper.GetBool("mediascan") { + l := viper.GetBool("medialink") + c := viper.GetBool("mediacopy") + if (!(l || c)) { + log.Fatal("mediascan enabled but both medialink and mediacopy are disabled!") + } + if len(viper.GetStringSlice("mediapath")) == 0 { + log.Fatal("empty mediapath list, but mediascan was enabled!") + } + for _, v := range viper.GetStringSlice("mediapath") { + log.Print("Scaning mediapath: ", v) + err := filepath.Walk(v, collectMedia(l, c, extmap, w)) + if err != nil { + log.Fatal(err) + } + } + log.Print("mediascan linked/copied files: ", newmedia) + } + + // step 2, fill our hash table `arr` + parseMedia(w) log.Print("mtmediasrv: Number of media files: ", len(arr)) - s := "/run/mtmediasrv/sock" + s := viper.GetString("socket") os.Remove(s) listener, err := net.Listen("unix", s) diff --git a/mtmediasrv.yaml b/mtmediasrv.yaml new file mode 100644 index 0000000..342cdc0 --- /dev/null +++ b/mtmediasrv.yaml @@ -0,0 +1,31 @@ +# +# mtmediasrv.yaml - example config file +# + +# Specifies the path to the HTTP content that will be served +# webroot: /var/www/media + +# Socket location/name. Folder must be writable by mtmediasrv +# socket: /run/mtmediasrv/sock + +# Enable media scanning and copying? +# mediascan: true + +# Extensions of files that will be copied/linked if found in the scan: +# extensions: [ .png, .jpg, .jpeg, .ogg, .x, .b3d, .obj ] + +# Enable hard linking of media into webroot +# medialink: true + +# Enable copying of media into webroot (not used if medialink is enabled) +# mediacopy: false + +# List of paths where media should be copied from, recursively +# mediapath: +# (default: empty) +# +# Example of multiple paths: +# mediapath: +# - /home/minetest/git/minetest +# - /home/minetest/git/minetest_game + diff --git a/readme.md b/readme.md index 58975a9..c2b4085 100644 --- a/readme.md +++ b/readme.md @@ -3,6 +3,13 @@ A Minetest Media server implementation as fcgi server. +This program is useful to distribute minetest media (textures, models, +sounds) to minetest clients for multiplayer server owners that wish to +have their media hosted on a `remote media server` URL. Doing this as +a separate download removes some of the download bandwidth from the +actual game server and offloads it to a different HTTP server. This +will work for clients that have cURL support enabled. + ### Requirements @@ -10,6 +17,7 @@ A Minetest Media server implementation as fcgi server. - webserver must be able to access file sockets in /run/mtmediasrv - systemd for controlling the service startup - go to build the service +- mod assets to serve This program is intended to run as fcgi process and handle POST requests for the `/index.mth` URI. It listens on a local unix domain @@ -17,6 +25,11 @@ socket, and needs to read the media files in the media folder to create sha1 hashes. It creates a hash list of files it has available and keeps this in memory. +At startup, the program can optionally scan mods and subgames to find +and copy or hardlink (the default) all the assets into the webroot. +The hardlink method is better for space, but may fail if the media +is not on the same filesystem as the webroot. + When a client connects, the client POSTS their list of known sha1 hashes of files they need. @@ -30,6 +43,8 @@ need to have your web server serve that content as static files. ### Building +mtmediasrv uses `viper` to read configuration files. You must +`go get https://github.com/spf13/viper` before building. Run `go build` in this folder to create the binary `mtschemsrv`. Please note the currently hardcoded values in the binary and @@ -49,6 +64,10 @@ automatically. If the content changes, you need to restart the program. You should not have a file called `index.mth` in the media folder, although this will not break anything, it will probably be confusing. +Copy and edit the `mtmediasrv.yaml` file and point it at the proper +webroot, socket path, and mediapath entries. Place it in /etc/ or +~/.config/. + ### logging