Add README.md

README.md

@@ -0,0 +1,29 @@
+# Minetest Mod Store
+
+Written by rubenwardy  
+License: LGPLv2.1+
+
+## Basic concepts and assumptions
+
+* Packages refers to mods, texture packs, and subgames - plus any future additions
+* Packages have a primary key of username/modname
+* Meta data
+	* Packages are either in manual or automatic mode
+	* Automatic packages are updated by workers by checking repositories
+* Download versions
+	* VCS mods: Downloads point to a particular commit, and include a checksum
+	* non-VCS mods: will eventually be copied to a server
+	* Workers are used to poll for updates
+	* Web hooks can be used on GitHub (and all github.com/minetest-mods will use this)
+* RESTful server
+	* Cluster-ised
+* Workers - are API clients to the main server, and are handed tasks to perform.
+            They can be run on separate machines to allow the main server to be
+            as performant as possible
+
+## Why NodeJS?
+
+* Well supported
+* Good with JSON
+* Works nicely with microservices / clusterisation
+* Can reuse code I've already written

pm2.json

@@ -0,0 +1,23 @@
+{
+	"apps" : [
+		{
+			"name": "webapp",
+			"cwd": "./webapp",
+			"script": "./webapp/index.js",
+			"watch": true,
+			"env": {
+				 "PORT": "4100"
+			}
+		},
+
+		{
+			"name": "worker1",
+			"cwd": "./worker",
+			"script": "./worker/index.js",
+			"watch": true,
+			"env": {
+				"FARMER_URL": "http://localhost:4100"
+			}
+		}
+	]
+}

service/farmerapi.js

@@ -1,31 +0,0 @@
-"use strict"
-
-var request = require("request")
-var API_URL = "http://localhost:8080"
-
-class FarmerAPI {
-	getWork() {
-		return new Promise(function(resolve, reject) {
-			request.get({
-				url: API_URL + '/workers/get-work',
-				qs: {
-					token: "foobar"
-				},
-				json: true,
-				// auth: {
-				// 	username: me.username,
-				// 	password: me.password,
-				// 	sendImmediately: true
-				// }
-			}, function (error, response, body) {
-				if (!error && response.statusCode == 200) {
-					resolve(body.work)
-				} else {
-					reject(error)
-				}
-			})
-		})
-	}
-}
-
-module.exports = FarmerAPI

service/fetchjob.js

@@ -1,11 +0,0 @@
-"use strict"
-
-const reposervers = require("./reposervers")
-
-function run(data) {
-	return new Promise(function(resolve, reject) {
-		resolve()
-	})
-}
-
-module.exports = run

webapp/controllers/index.js

@@ -17,6 +17,7 @@ router.get("/", function(req, res) {
 	})
 })
 
+router.use("/static", express.static("static"))
 router.use(require("./mods"))
 router.use("/workers", require("./workerapi"))
 

webapp/controllers/mods.js

@@ -38,12 +38,13 @@ router.get("/mod/:author/:modname", function(req, res) {
 		include: [{
 			model: db.User,
 			where: { username: req.params.author }
-		}]
+		}, db.Work, db.ModDownload]
 	}).then(function(mod) {
 		if (mod) {
 			var mod_c = db.convertRowToMod(mod)
 			switch (req.accepts("html", "json")) {
 				case "html":
+					mod_c.works = mod.works
 					res.render("mod", {
 						title: mod.title,
 						mod: mod_c

webapp/index.js

@@ -1,7 +1,5 @@
 "use strict"
 
-require("process").chdir(__dirname)
-
 var app = require("express")()
 
 // Support JSON and URL encoded bodies
@@ -11,6 +9,7 @@ app.use(bodyParser.urlencoded({	 // to support URL-encoded bodies
 	extended: true
 }))
 
+
 // Caching for JSON APIs
 var apicache = require("apicache").options({ debug: false }).middleware
 
@@ -29,6 +28,7 @@ var options = {
 	// if an error occurred while rendering, show detail or not, default to false
 	traceError: false
 }
+
 app.set("view engine", "liquid")
 app.engine("liquid", expressLiquid(options))
 app.use(expressLiquid.middleware)
@@ -41,6 +41,6 @@ app.set("db", db)
 app.use(require("./controllers"))
 
 // Start server
-app.listen(8080, "127.0.0.1", function () {
+app.listen(require("process").env.PORT, "127.0.0.1", function () {
 	console.log("Minetest Mod Database listening on port 8080!")
 })

webapp/models/database.js

@@ -10,13 +10,14 @@ var sequelize = new Sequelize("database", "", "", {
 	},
 
 	// SQLite only
-	storage: "../db.sqlite"
+	storage: "db.sqlite"
 })
 
 var User = sequelize.define("user", {
 	username: { type: Sequelize.STRING(100), unique: true }
 })
 
+
 var Mod = sequelize.define("mod", {
 	basename: Sequelize.STRING(100),
 	type: Sequelize.ENUM("mod", "game", "texturepack"),
@@ -25,19 +26,25 @@ var Mod = sequelize.define("mod", {
 	description: Sequelize.STRING(900),
 	forum_id: Sequelize.STRING(40),
 
-	download_url: Sequelize.STRING(250),
-	download_hash: Sequelize.STRING(256),
-	download_size: Sequelize.INTEGER,
-
 	repo_url: Sequelize.STRING(250),
 	repo_hash: Sequelize.STRING(250),
 
-	approved: Sequelize.BOOLEAN
+	approved: Sequelize.BOOLEAN,
 })
 
 User.hasMany(Mod)
 Mod.belongsTo(User)
 
+var ModDownload = sequelize.define("moddownload", {
+	url: Sequelize.STRING(250),
+	hash: Sequelize.STRING(256),
+	size: Sequelize.INTEGER,
+
+	approved: Sequelize.BOOLEAN,
+})
+Mod.hasMany(ModDownload)
+ModDownload.belongsTo(Mod)
+
 var Worker = sequelize.define("worker", {
 	token: Sequelize.STRING(128)
 })
@@ -60,11 +67,23 @@ function convertRowToMod(row) {
 	mod.description = row.description
 	mod.forum_id = row.forum_id
 	mod.forum_url = mod.getForumURL()
-	mod.download = {
-		url: row.download_url,
-		hash: row.download_hash || "",
-		size: row.download_size || -1
+	mod.downloads = []
+	mod.download = null
+
+	if (row.moddownloads) {
+		mod.downloads = row.moddownloads.map((download) => {
+			return {
+				url: download.url,
+				hash: download.hash || "",
+				size: download.size || -1
+			}
+		})
+
+		if (mod.downloads.length > 0) {
+			mod.download = mod.downloads[0]
+		}
 	}
+
 	mod.repo = {
 		url: row.repo,
 		hash: row.repo_hash
@@ -77,6 +96,7 @@ function convertRowToMod(row) {
 async.parallel([
 	function(callback) { User.sync().then(callback) },
 	function(callback) { Mod.sync().then(callback) },
+	function(callback) { ModDownload.sync().then(callback) },
 	function(callback) { Worker.sync().then(callback) },
 	function(callback) { Work.sync().then(callback) },
 	function() {
@@ -103,10 +123,6 @@ async.parallel([
 					description: "Adds awards to minetest",
 					forum_id: "4870",
 
-					download_url: "https://github.com/minetest-mods/awards/zipball/master",
-					download_hash: "",
-					download_size: -1,
-
 					repo_url: "https://github.com/minetest-mods/awards/",
 					repo_hash: "c994978683355417783586262914d4be128cbdf0",
 
@@ -115,24 +131,39 @@ async.parallel([
 			}).then(function(mod) {
 				mod = mod[0]
 
-				Worker.findOrCreate({
+				ModDownload.findOrCreate({
 					where: {
-						token: "foobar"
+						modId: mod.id
 					},
-					defaults: {}
-				}).then(function(worker) {
-					worker = worker[0]
+					defaults: {
+						mod: mod,
 
-					Work.findOrCreate({
+						url: "https://github.com/minetest-mods/awards/zipball/master",
+						hash: "",
+						size: -1,
+
+						approved: true
+					}
+				}).then(function(moddownload) {
+					Worker.findOrCreate({
 						where: {
-							modId: mod.id
+							token: "foobar"
 						},
-						defaults: {
-							mod: mod,
-							work_type: "fetch"
-						}
-					}).then(function(mod) {
+						defaults: {}
+					}).then(function(worker) {
+						worker = worker[0]
 
+						Work.findOrCreate({
+							where: {
+								modId: mod.id
+							},
+							defaults: {
+								mod: mod,
+								work_type: "fetch"
+							}
+						}).then(function(mod) {
+
+						})
 					})
 				})
 			})
@@ -142,6 +173,7 @@ async.parallel([
 module.exports = {
 	User: User,
 	Mod: Mod,
+	ModDownload: ModDownload,
 	Work: Work,
 	Worker: Worker,
 	convertRowToMod: convertRowToMod

webapp/package.json

@@ -18,11 +18,11 @@
   "homepage": "https://github.com/rubenwardy/minetest-mod-database#readme",
   "dependencies": {
     "apicache": "^0.7.4",
+    "async": "^2.1.4",
     "body-parser": "^1.15.2",
     "express": "^4.14.0",
     "express-liquid": "^0.2.6",
-    "request": "^2.79.0",
-    "sequelize": "^3.28.0",
+    "sequelize": "^3.30.0",
     "sqlite3": "^3.1.8"
   }
 }

webapp/views/mod.liquid

@@ -14,4 +14,15 @@ by
 <a href="{{ mod.forum_url }}">Forum Topic</a> |
 <a href="/mods/?a={{ mod.author }}">{{ mod.author }}'s mods</a>
 
+<h3>Jobs</h3>
+
+<ul>
+{% for work in mod.works %}
+<li>{{ work.work_type }}
+	[{% if work.workerId %}assigned to worker {{ work.workerId}}{% else %}Queued{% endif %}]</li>
+{% else %}
+<li><i>No jobs in progress</i></li>
+{% endfor %}
+</ul>
+
 {% include "footer" %}

worker/farmerapi.js

@@ -0,0 +1,48 @@
+"use strict"
+
+var request = require("request")
+
+class FarmerAPI {
+	constructor(url) {
+		this.url = url
+	}
+
+	getWork() {
+		return new Promise((resolve, reject) => {
+			request.get({
+				url: this.url + '/workers/job',
+				qs: {
+					token: "foobar"
+				},
+				json: true
+			}, function (error, response, body) {
+				if (!error && response.statusCode == 200) {
+					resolve(body.work)
+				} else {
+					reject(error)
+				}
+			})
+		})
+	}
+
+	sendResults(results) {
+		return new Promise((resolve, reject) => {
+			request.post({
+				url: this.url + '/workers/job',
+				qs: {
+					token: "foobar"
+				},
+				json: true,
+				body: results
+			}, function (error, response, body) {
+				if (!error && response.statusCode == 200) {
+					resolve(body)
+				} else {
+					reject(error)
+				}
+			})
+		})
+	}
+}
+
+module.exports = FarmerAPI

worker/fetchjob.js

@@ -0,0 +1,42 @@
+"use strict"
+
+const reposervers = require("./reposervers")
+
+function run(data, settings) {
+	return new Promise(function(resolve, reject) {
+		var url
+		if (data.repo && data.repo.url) {
+			url = data.repo.url
+		} else if (data.download && data.download.url) {
+			url = data.download.url
+		}
+
+		const github = new reposervers.GithubRepoServer(settings.github_auth, settings.cache_dir)
+		const repo = github.getRepoFromURL(url)
+		if (repo) {
+			Promise.all([
+				github.getDescriptionFromRepo(repo),
+				github.getDependsFromRepo(repo),
+				github.getDownloadAndHash(repo),
+			]).then(function(res) {
+				if (res.length == 3) {
+					resolve({
+						description: res[0],
+						depends: res[1],
+						download: {
+							url: res[2].link,
+							size: -1
+						},
+						commit: res[2].commit
+					})
+				} else {
+					reject()
+				}
+			}).catch(reject)
+		} else {
+			reject("unknown repo server")
+		}
+	})
+}
+
+module.exports = run

worker/index.js

@@ -1,9 +1,10 @@
 "use strict"
 
-require("process").chdir(__dirname)
-
 const FarmerAPI = require("./farmerapi")
-const api = new FarmerAPI()
+
+const api = new FarmerAPI(require("process").env.FARMER_URL)
+const fs = require("fs")
+const settings = JSON.parse(fs.readFileSync("settings.json", 'utf8'))
 
 function startJob(data) {
 	console.log("Starting job of type " + data.type)
@@ -16,10 +17,16 @@ function startJob(data) {
 			return null
 	}
 
-	job().then(function(res) {
+	job(data.mod, settings).then(function(res) {
 		console.log("Job finished!")
 		console.log(res)
-		setTimeout(checkForJobs, 5000)
+		res.type = data.type
+		api.sendResults(res).then(function() {
+			setTimeout(checkForJobs, 5000)
+		}).catch(function() {
+			console.log("Failed to submit results!")
+			console.log(e)
+		})
 	}).catch(function(e) {
 		console.log("Job failed!")
 		console.log(e)
@@ -37,8 +44,9 @@ function checkForJobs(data) {
 		}
 	}).catch(function(e) {
 		console.log("Error whilst checking for jobs")
+		console.log(e)
 		setTimeout(checkForJobs, 5000)
 	})
 }
 
-setTimeout(checkForJobs, 5000)
+setTimeout(checkForJobs, 100)

worker/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "minetest-mod-database-worker",
+  "version": "0.1.0",
+  "description": "Minetest Mod Database Worker",
+  "main": "index.js",
+  "scripts": {},
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/rubenwardy/minetest-mod-database.git"
+  },
+  "author": "rubenwardy",
+  "license": "LGPL-2.1+",
+  "bugs": {
+    "url": "https://github.com/rubenwardy/minetest-mod-database/issues"
+  },
+  "homepage": "https://github.com/rubenwardy/minetest-mod-database#readme",
+  "dependencies": {
+    "bluebird": "^3.4.7",
+    "github": "^8.1.0",
+    "request": "^2.79.0"
+  }
+}

worker/reposervers.js

@@ -115,7 +115,7 @@ class GithubRepoServer extends RepoServer {
 			} else {
 				me.misses++;
 				me.github.repos.getContent({
-					user: repo.user,
+					owner: repo.user,
 					repo: repo.repo,
 					path: "description.txt"
 				}).then(function(data) {
@@ -132,23 +132,25 @@ class GithubRepoServer extends RepoServer {
 						resolve(desc);
 					} else {
 						if (!cached) {
+							cached = {}
 							me.cache[idx] = {};
 						}
 						cached.desc = {
 							text: "",
 							timestamp: new Date().getTime()
 						}
-						reject();
+						resolve(null);
 					}
 				}).catch(function(e) {
 					if (!cached) {
+						cached = {}
 						me.cache[idx] = {};
 					}
 					cached.desc = {
 						text: "",
 						timestamp: new Date().getTime()
 					}
-					reject();
+					reject(e);
 				})
 			}
 		});
@@ -161,12 +163,12 @@ class GithubRepoServer extends RepoServer {
 		var me = this;
 		return new Promise(function(resolve, reject) {
 			me.github.repos.getContent({
-				user: repo.user,
+				owner: repo.user,
 				repo: repo.repo,
 				path: "depends.txt"
 			}).then(function(data) {
 				if (data && data.content) {
-					resolve(new Buffer(data.content, 'base64'));
+					resolve(new Buffer(data.content, 'base64').toString());
 				} else {
 					reject();
 				}
@@ -193,7 +195,7 @@ class GithubRepoServer extends RepoServer {
 			} else {
 				me.misses++;
 				var req = {
-					user: repo.user,
+					owner: repo.user,
 					repo: repo.repo,
 					per_page: 1
 				};
@@ -276,7 +278,7 @@ class GithubRepoServer extends RepoServer {
 
 			cached.hook_registered = true;
 			me.github.repos.createHook({
-				user: repo.user,
+				owner: repo.user,
 				repo: repo.repo,
 				name: "web",
 				config: {

worker/settings.json.example

@@ -0,0 +1,4 @@
+{
+	"github_auth": "GITHUB_TOKEN_HERE",
+	"cache_dir": "/tmp/"
+}