Add media scanning and collection.

This removes the need entirely for any other script. At startup
the program will scan all folders recursively and hardlink or copy
the content over to the webroot, and then serve the index.mth.

main.go

@@ -32,7 +32,9 @@ import (
 	"net/http"
 	"net/http/fcgi"
 	"os"
+	"path/filepath"
 	"strings"
+	"github.com/spf13/viper"
 )
 
 
@@ -40,6 +42,8 @@ var (
 	Version string
 	Build string
 
+	newmedia int
+
 	arr []string
 )
 
@@ -134,13 +138,108 @@ func parseMedia(path string) {
 	}
 }
 
+func collectMedia(l bool, c bool, e map[string]bool, w string) filepath.WalkFunc {
+	return func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			log.Print(err)
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+		ext := filepath.Ext(path)
+		if e[ext] {
+			sha, err := getHash(path)
+			if err != nil {
+				return err
+			}
+
+			of := strings.Join([]string{w, sha}, "/")
+
+			if l {
+				err := os.Link(path, of)
+				if err != nil {
+					return err
+				}
+				newmedia++
+			} else if c {
+				in, err := os.Open(path)
+				if err != nil {
+					return err
+				}
+				defer in.Close()
+				os.Remove(of)
+				out, err := os.Create(of)
+				if err != nil {
+					return err
+				}
+				defer out.Close()
+				_, err = io.Copy(out, in)
+				closeErr := out.Close()
+				if err != nil {
+					return err
+				}
+				newmedia++
+				return closeErr
+			}
+		}
+		return nil
+	}
+}
 
 func main() {
-	p := "/var/www/media"
-	parseMedia(p)
+	// config stuff
+	viper.SetConfigName("mtmediasrv")
+	viper.SetConfigType("yaml")
+
+	viper.AddConfigPath("/usr/share/defaults/etc")
+	viper.AddConfigPath("/etc")
+	viper.AddConfigPath("$HOME/.config")
+
+	viper.SetDefault("socket", "/run/mtmediasrv/sock")
+	viper.SetDefault("webroot", "/var/www/media")
+	viper.SetDefault("mediapath", []string{})
+	viper.SetDefault("mediascan", "true")
+	viper.SetDefault("medialink", "true")
+	viper.SetDefault("mediacopy", "false")
+	viper.SetDefault("extensions", []string{ ".png", ".jpg", ".jpeg", ".ogg", ".x", ".b3d", ".obj"})
+
+	err := viper.ReadInConfig()
+	if err != nil {
+		log.Fatal("Error in confog file: ", err)
+	}
+
+	// step 1, collect media files
+	w := viper.GetString("webroot")
+	ext := viper.GetStringSlice("extensions")
+	extmap := make(map[string]bool)
+	for i := 0; i < len(ext); i++ {
+		extmap[ext[i]] = true
+	}
+	if viper.GetBool("mediascan") {
+		l := viper.GetBool("medialink")
+		c := viper.GetBool("mediacopy")
+		if (!(l || c)) {
+			log.Fatal("mediascan enabled but both medialink and mediacopy are disabled!")
+		}
+		if len(viper.GetStringSlice("mediapath")) == 0 {
+			log.Fatal("empty mediapath list, but mediascan was enabled!")
+		}
+		for _, v := range viper.GetStringSlice("mediapath") {
+			log.Print("Scaning mediapath: ", v)
+			err := filepath.Walk(v, collectMedia(l, c, extmap, w))
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+		log.Print("mediascan linked/copied files: ", newmedia)
+	}
+
+	// step 2, fill our hash table `arr`
+	parseMedia(w)
 	log.Print("mtmediasrv: Number of media files: ", len(arr))
 
-	s := "/run/mtmediasrv/sock"
+	s := viper.GetString("socket")
 	os.Remove(s)
 
 	listener, err := net.Listen("unix", s)

mtmediasrv.yaml

@@ -0,0 +1,31 @@
+#
+# mtmediasrv.yaml - example config file
+#
+
+# Specifies the path to the HTTP content that will be served
+# webroot: /var/www/media
+
+# Socket location/name. Folder must be writable by mtmediasrv
+# socket: /run/mtmediasrv/sock
+
+# Enable media scanning and copying?
+# mediascan: true
+
+# Extensions of files that will be copied/linked if found in the scan:
+# extensions: [ .png, .jpg, .jpeg, .ogg, .x, .b3d, .obj ]
+
+# Enable hard linking of media into webroot
+# medialink: true
+
+# Enable copying of media into webroot (not used if medialink is enabled)
+# mediacopy: false
+
+# List of paths where media should be copied from, recursively
+# mediapath:
+# (default: empty)
+#
+# Example of multiple paths:
+# mediapath:
+# - /home/minetest/git/minetest
+# - /home/minetest/git/minetest_game
+

readme.md

@@ -3,6 +3,13 @@
 
 A Minetest Media server implementation as fcgi server.
 
+This program is useful to distribute minetest media (textures, models,
+sounds) to minetest clients for multiplayer server owners that wish to
+have their media hosted on a `remote media server` URL. Doing this as
+a separate download removes some of the download bandwidth from the
+actual game server and offloads it to a different HTTP server. This
+will work for clients that have cURL support enabled.
+
 
 ### Requirements
 
@@ -10,6 +17,7 @@ A Minetest Media server implementation as fcgi server.
 - webserver must be able to access file sockets in /run/mtmediasrv
 - systemd for controlling the service startup
 - go to build the service
+- mod assets to serve
 
 This program is intended to run as fcgi process and handle POST
 requests for the `/index.mth` URI. It listens on a local unix domain
@@ -17,6 +25,11 @@ socket, and needs to read the media files in the media folder to
 create sha1 hashes. It creates a hash list of files it has available
 and keeps this in memory.
 
+At startup, the program can optionally scan mods and subgames to find
+and copy or hardlink (the default) all the assets into the webroot.
+The hardlink method is better for space, but may fail if the media
+is not on the same filesystem as the webroot.
+
 When a client connects, the client POSTS their list of known sha1
 hashes of files they need.
 
@@ -30,6 +43,8 @@ need to have your web server serve that content as static files.
 
 ### Building
 
+mtmediasrv uses `viper` to read configuration files. You must
+`go get https://github.com/spf13/viper` before building.
 Run `go build` in this folder to create the binary `mtschemsrv`.
 
 Please note the currently hardcoded values in the binary and
@@ -49,6 +64,10 @@ automatically. If the content changes, you need to restart the program.
 You should not have a file called `index.mth` in the media folder,
 although this will not break anything, it will probably be confusing.
 
+Copy and edit the `mtmediasrv.yaml` file and point it at the proper
+webroot, socket path, and mediapath entries. Place it in /etc/ or
+~/.config/.
+
 
 ### logging