Implement fetching of meta data

package.json

@@ -21,6 +21,7 @@
     "body-parser": "^1.15.2",
     "express": "^4.14.0",
     "express-liquid": "^0.2.6",
+    "github": "^8.1.0",
     "request": "^2.79.0",
     "sequelize": "^3.28.0",
     "sqlite3": "^3.1.8"

service/farmerapi.js

@@ -7,16 +7,11 @@ class FarmerAPI {
 	getWork() {
 		return new Promise(function(resolve, reject) {
 			request.get({
-				url: API_URL + '/workers/get-work',
+				url: API_URL + '/workers/job',
 				qs: {
 					token: "foobar"
 				},
-				json: true,
+				json: true
-				// auth: {
-				// 	username: me.username,
-				// 	password: me.password,
-				// 	sendImmediately: true
-				// }
 			}, function (error, response, body) {
 				if (!error && response.statusCode == 200) {
 					resolve(body.work)
@@ -26,6 +21,25 @@ class FarmerAPI {
 			})
 		})
 	}
+
+	sendResults(results) {
+		return new Promise(function(resolve, reject) {
+			request.post({
+				url: API_URL + '/workers/job',
+				qs: {
+					token: "foobar"
+				},
+				json: true,
+				body: results
+			}, function (error, response, body) {
+				if (!error && response.statusCode == 200) {
+					resolve(body)
+				} else {
+					reject(error)
+				}
+			})
+		})
+	}
 }
 
 module.exports = FarmerAPI

service/fetchjob.js

@@ -2,9 +2,40 @@
 
 const reposervers = require("./reposervers")
 
-function run(data) {
+function run(data, settings) {
 	return new Promise(function(resolve, reject) {
-		resolve()
+		var url
+		if (data.repo && data.repo.url) {
+			url = data.repo.url
+		} else if (data.download && data.download.url) {
+			url = data.download.url
+		}
+
+		const github = new reposervers.GithubRepoServer(settings.github_auth, settings.cache_dir)
+		const repo = github.getRepoFromURL(url)
+		if (repo) {
+			Promise.all([
+				github.getDescriptionFromRepo(repo),
+				github.getDependsFromRepo(repo),
+				github.getDownloadAndHash(repo),
+			]).then(function(res) {
+				if (res.length == 3) {
+					resolve({
+						description: res[0],
+						depends: res[1],
+						download: {
+							url: res[2].link,
+							size: -1
+						},
+						commit: res[2].commit
+					})
+				} else {
+					reject()
+				}
+			}).catch(reject)
+		} else {
+			reject("unknown repo server")
+		}
 	})
 }
 

service/index.js

@@ -4,6 +4,8 @@ require("process").chdir(__dirname)
 
 const FarmerAPI = require("./farmerapi")
 const api = new FarmerAPI()
+const fs = require("fs")
+const settings = JSON.parse(fs.readFileSync("settings.json", 'utf8'))
 
 function startJob(data) {
 	console.log("Starting job of type " + data.type)
@@ -16,10 +18,16 @@ function startJob(data) {
 			return null
 	}
 
-	job().then(function(res) {
+	job(data.mod, settings).then(function(res) {
 		console.log("Job finished!")
 		console.log(res)
-		setTimeout(checkForJobs, 5000)
+		res.type = data.type
+		api.sendResults(res).then(function() {
+			setTimeout(checkForJobs, 5000)
+		}).catch(function() {
+			console.log("Failed to submit results!")
+			console.log(e)
+		})
 	}).catch(function(e) {
 		console.log("Job failed!")
 		console.log(e)
@@ -37,8 +45,9 @@ function checkForJobs(data) {
 		}
 	}).catch(function(e) {
 		console.log("Error whilst checking for jobs")
+		console.log(e)
 		setTimeout(checkForJobs, 5000)
 	})
 }
 
-setTimeout(checkForJobs, 5000)
+setTimeout(checkForJobs, 100)

service/reposervers.js

@@ -115,7 +115,7 @@ class GithubRepoServer extends RepoServer {
 			} else {
 				me.misses++;
 				me.github.repos.getContent({
-					user: repo.user,
+					owner: repo.user,
 					repo: repo.repo,
 					path: "description.txt"
 				}).then(function(data) {
@@ -132,23 +132,25 @@ class GithubRepoServer extends RepoServer {
 						resolve(desc);
 					} else {
 						if (!cached) {
+							cached = {}
 							me.cache[idx] = {};
 						}
 						cached.desc = {
 							text: "",
 							timestamp: new Date().getTime()
 						}
-						reject();
+						resolve(null);
 					}
 				}).catch(function(e) {
 					if (!cached) {
+						cached = {}
 						me.cache[idx] = {};
 					}
 					cached.desc = {
 						text: "",
 						timestamp: new Date().getTime()
 					}
-					reject();
+					reject(e);
 				})
 			}
 		});
@@ -161,12 +163,12 @@ class GithubRepoServer extends RepoServer {
 		var me = this;
 		return new Promise(function(resolve, reject) {
 			me.github.repos.getContent({
-				user: repo.user,
+				owner: repo.user,
 				repo: repo.repo,
 				path: "depends.txt"
 			}).then(function(data) {
 				if (data && data.content) {
-					resolve(new Buffer(data.content, 'base64'));
+					resolve(new Buffer(data.content, 'base64').toString());
 				} else {
 					reject();
 				}
@@ -193,7 +195,7 @@ class GithubRepoServer extends RepoServer {
 			} else {
 				me.misses++;
 				var req = {
-					user: repo.user,
+					owner: repo.user,
 					repo: repo.repo,
 					per_page: 1
 				};
@@ -276,7 +278,7 @@ class GithubRepoServer extends RepoServer {
 
 			cached.hook_registered = true;
 			me.github.repos.createHook({
-				user: repo.user,
+				owner: repo.user,
 				repo: repo.repo,
 				name: "web",
 				config: {

service/settings.json.example

@@ -0,0 +1,4 @@
+{
+	"github_auth": "GITHUB_TOKEN_HERE",
+	"cache_dir": "/tmp/"
+}