Add release contents validation
This commit is contained in:
rubenwardy
parent
bcd003685e
commit
1b1c94ffa0
@ -26,7 +26,8 @@ from app import app
|
||||
from app.models import *
|
||||
from app.tasks import celery, TaskError
|
||||
from app.utils import randomString
|
||||
|
||||
from .minetestcheck import build_tree, MinetestCheckError, ContentType
|
||||
from .minetestcheck.config import parse_conf
|
||||
|
||||
class GithubURLMaker:
|
||||
def __init__(self, url):
|
||||
@ -127,162 +128,6 @@ def findModInfo(author, name, link):
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parseConf(string):
|
||||
retval = {}
|
||||
for line in string.split("\n"):
|
||||
idx = line.find("=")
|
||||
if idx > 0:
|
||||
key = line[:idx].strip()
|
||||
value = line[idx+1:].strip()
|
||||
retval[key] = value
|
||||
|
||||
return retval
|
||||
|
||||
|
||||
class PackageTreeNode:
|
||||
def __init__(self, baseDir, author=None, repo=None, name=None):
|
||||
print("Scanning " + baseDir)
|
||||
self.baseDir = baseDir
|
||||
self.author = author
|
||||
self.name = name
|
||||
self.repo = repo
|
||||
self.meta = None
|
||||
self.children = []
|
||||
|
||||
# Detect type
|
||||
type = None
|
||||
is_modpack = False
|
||||
if os.path.isfile(baseDir + "/game.conf"):
|
||||
type = PackageType.GAME
|
||||
elif os.path.isfile(baseDir + "/init.lua"):
|
||||
type = PackageType.MOD
|
||||
elif os.path.isfile(baseDir + "/modpack.txt") or \
|
||||
os.path.isfile(baseDir + "/modpack.conf"):
|
||||
type = PackageType.MOD
|
||||
is_modpack = True
|
||||
elif os.path.isdir(baseDir + "/mods"):
|
||||
type = PackageType.GAME
|
||||
elif os.listdir(baseDir) == []:
|
||||
# probably a submodule
|
||||
return
|
||||
else:
|
||||
raise TaskError("Unable to detect package type!")
|
||||
|
||||
self.type = type
|
||||
self.readMetaFiles()
|
||||
|
||||
if self.type == PackageType.GAME:
|
||||
self.addChildrenFromModDir(baseDir + "/mods")
|
||||
elif is_modpack:
|
||||
self.addChildrenFromModDir(baseDir)
|
||||
|
||||
|
||||
def readMetaFiles(self):
|
||||
result = {}
|
||||
|
||||
# .conf file
|
||||
try:
|
||||
with open(self.baseDir + "/mod.conf", "r") as myfile:
|
||||
conf = parseConf(myfile.read())
|
||||
for key in ["name", "description", "title", "depends", "optional_depends"]:
|
||||
try:
|
||||
result[key] = conf[key]
|
||||
except KeyError:
|
||||
pass
|
||||
except IOError:
|
||||
print("description.txt does not exist!")
|
||||
|
||||
# description.txt
|
||||
if not "description" in result:
|
||||
try:
|
||||
with open(self.baseDir + "/description.txt", "r") as myfile:
|
||||
result["description"] = myfile.read()
|
||||
except IOError:
|
||||
print("description.txt does not exist!")
|
||||
|
||||
# depends.txt
|
||||
import re
|
||||
pattern = re.compile("^([a-z0-9_]+)\??$")
|
||||
if not "depends" in result and not "optional_depends" in result:
|
||||
try:
|
||||
with open(self.baseDir + "/depends.txt", "r") as myfile:
|
||||
contents = myfile.read()
|
||||
soft = []
|
||||
hard = []
|
||||
for line in contents.split("\n"):
|
||||
line = line.strip()
|
||||
if pattern.match(line):
|
||||
if line[len(line) - 1] == "?":
|
||||
soft.append( line[:-1])
|
||||
else:
|
||||
hard.append(line)
|
||||
|
||||
result["depends"] = hard
|
||||
result["optional_depends"] = soft
|
||||
|
||||
except IOError:
|
||||
print("depends.txt does not exist!")
|
||||
|
||||
else:
|
||||
if "depends" in result:
|
||||
result["depends"] = [x.strip() for x in result["depends"].split(",")]
|
||||
if "optional_depends" in result:
|
||||
result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
|
||||
|
||||
|
||||
# Calculate Title
|
||||
if "name" in result and not "title" in result:
|
||||
result["title"] = result["name"].replace("_", " ").title()
|
||||
|
||||
# Calculate short description
|
||||
if "description" in result:
|
||||
desc = result["description"]
|
||||
idx = desc.find(".") + 1
|
||||
cutIdx = min(len(desc), 200 if idx < 5 else idx)
|
||||
result["short_description"] = desc[:cutIdx]
|
||||
|
||||
# Get forum ID
|
||||
info = findModInfo(self.author, result.get("name"), self.repo)
|
||||
if info is not None:
|
||||
result["forumId"] = info.get("topicId")
|
||||
|
||||
if "name" in result:
|
||||
self.name = result["name"]
|
||||
del result["name"]
|
||||
|
||||
self.meta = result
|
||||
|
||||
def addChildrenFromModDir(self, dir):
|
||||
for entry in next(os.walk(dir))[1]:
|
||||
path = dir + "/" + entry
|
||||
if not entry.startswith('.') and os.path.isdir(path):
|
||||
self.children.append(PackageTreeNode(path, name=entry))
|
||||
|
||||
|
||||
def fold(self, attr, key=None, acc=None):
|
||||
if acc is None:
|
||||
acc = set()
|
||||
|
||||
if self.meta is None:
|
||||
return acc
|
||||
|
||||
at = getattr(self, attr)
|
||||
value = at if key is None else at.get(key)
|
||||
|
||||
if isinstance(value, list):
|
||||
acc |= set(value)
|
||||
elif value is not None:
|
||||
acc.add(value)
|
||||
|
||||
for child in self.children:
|
||||
child.fold(attr, key, acc)
|
||||
|
||||
return acc
|
||||
|
||||
def get(self, key):
|
||||
return self.meta.get(key)
|
||||
|
||||
def generateGitURL(urlstr):
|
||||
scheme, netloc, path, query, frag = urlsplit(urlstr)
|
||||
|
||||
@ -323,7 +168,12 @@ def cloneRepo(urlstr, ref=None, recursive=False):
|
||||
@celery.task()
|
||||
def getMeta(urlstr, author):
|
||||
gitDir, _ = cloneRepo(urlstr, recursive=True)
|
||||
tree = PackageTreeNode(gitDir, author=author, repo=urlstr)
|
||||
|
||||
try:
|
||||
tree = build_tree(gitDir, author=author, repo=urlstr)
|
||||
except MinetestCheckError as err:
|
||||
raise TaskError(str(err))
|
||||
|
||||
shutil.rmtree(gitDir)
|
||||
|
||||
result = {}
|
||||
@ -387,6 +237,12 @@ def makeVCSRelease(id, branch):
|
||||
|
||||
gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
|
||||
|
||||
try:
|
||||
tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \
|
||||
author=release.package.author.username, name=release.package.name)
|
||||
except MinetestCheckError as err:
|
||||
raise TaskError(str(err))
|
||||
|
||||
try:
|
||||
filename = randomString(10) + ".zip"
|
||||
destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
|
||||
@ -464,7 +320,7 @@ def getDepends(package):
|
||||
#
|
||||
try:
|
||||
contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
|
||||
conf = parseConf(contents)
|
||||
conf = parse_conf(contents)
|
||||
for key in ["depends", "optional_depends"]:
|
||||
try:
|
||||
result[key] = conf[key]
|
||||
|
||||
48
app/tasks/minetestcheck/__init__.py
Normal file
48
app/tasks/minetestcheck/__init__.py
Normal file
@ -0,0 +1,48 @@
|
||||
from enum import Enum
|
||||
|
||||
class MinetestCheckError(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
def __str__(self):
|
||||
return repr("Error validating package: " + self.value)
|
||||
|
||||
class ContentType(Enum):
|
||||
UNKNOWN = "unknown"
|
||||
MOD = "mod"
|
||||
MODPACK = "modpack"
|
||||
GAME = "game"
|
||||
TXP = "texture pack"
|
||||
|
||||
def isModLike(self):
|
||||
return self == ContentType.MOD or self == ContentType.MODPACK
|
||||
|
||||
def validate_same(self, other):
|
||||
"""
|
||||
Whether or not `other` is an acceptable type for this
|
||||
"""
|
||||
assert(other)
|
||||
|
||||
if self == ContentType.MOD:
|
||||
if not other.isModLike():
|
||||
raise MinetestCheckError("expected a mod or modpack, found " + other.value)
|
||||
|
||||
elif self == ContentType.TXP:
|
||||
if other != ContentType.UNKNOWN and other != ContentType.TXP:
|
||||
raise MinetestCheckError("expected a " + self.value + ", found a " + other.value)
|
||||
|
||||
elif other != self:
|
||||
raise MinetestCheckError("expected a " + self.value + ", found a " + other.value)
|
||||
|
||||
|
||||
from .tree import PackageTreeNode, get_base_dir
|
||||
|
||||
def build_tree(path, expected_type=None, author=None, repo=None, name=None):
|
||||
path = get_base_dir(path)
|
||||
|
||||
root = PackageTreeNode(path, "/", author=author, repo=repo, name=name)
|
||||
assert(root)
|
||||
|
||||
if expected_type:
|
||||
expected_type.validate_same(root.type)
|
||||
|
||||
return root
|
||||
10
app/tasks/minetestcheck/config.py
Normal file
10
app/tasks/minetestcheck/config.py
Normal file
@ -0,0 +1,10 @@
|
||||
def parse_conf(string):
|
||||
retval = {}
|
||||
for line in string.split("\n"):
|
||||
idx = line.find("=")
|
||||
if idx > 0:
|
||||
key = line[:idx].strip()
|
||||
value = line[idx+1:].strip()
|
||||
retval[key] = value
|
||||
|
||||
return retval
|
||||
162
app/tasks/minetestcheck/tree.py
Normal file
162
app/tasks/minetestcheck/tree.py
Normal file
@ -0,0 +1,162 @@
|
||||
import os
|
||||
from . import MinetestCheckError, ContentType
|
||||
from .config import parse_conf
|
||||
|
||||
def get_base_dir(path):
|
||||
if not os.path.isdir(path):
|
||||
raise IOError("Expected dir")
|
||||
|
||||
root, subdirs, files = next(os.walk(path))
|
||||
if len(subdirs) == 1 and len(files) == 0:
|
||||
return get_base_dir(path + "/" + subdirs[0])
|
||||
else:
|
||||
return path
|
||||
|
||||
|
||||
def detect_type(path):
|
||||
if os.path.isfile(path + "/game.conf"):
|
||||
return ContentType.GAME
|
||||
elif os.path.isfile(path + "/init.lua"):
|
||||
return ContentType.MOD
|
||||
elif os.path.isfile(path + "/modpack.txt") or \
|
||||
os.path.isfile(path + "/modpack.conf"):
|
||||
return ContentType.MODPACK
|
||||
elif os.path.isdir(path + "/mods"):
|
||||
return ContentType.GAME
|
||||
elif os.path.isfile(path + "/texture_pack.conf"):
|
||||
return ContentType.TXP
|
||||
else:
|
||||
return ContentType.UNKNOWN
|
||||
|
||||
|
||||
class PackageTreeNode:
|
||||
def __init__(self, baseDir, relative, author=None, repo=None, name=None):
|
||||
print(baseDir)
|
||||
self.baseDir = baseDir
|
||||
self.relative = relative
|
||||
self.author = author
|
||||
self.name = name
|
||||
self.repo = repo
|
||||
self.meta = None
|
||||
self.children = []
|
||||
|
||||
# Detect type
|
||||
self.type = detect_type(baseDir)
|
||||
self.read_meta()
|
||||
|
||||
if self.type == ContentType.GAME:
|
||||
if not os.path.isdir(baseDir + "/mods"):
|
||||
raise MinetestCheckError(("game at {} does not have a mods/ folder").format(self.relative))
|
||||
self.add_children_from_mod_dir(baseDir + "/mods")
|
||||
elif self.type == ContentType.MODPACK:
|
||||
self.add_children_from_mod_dir(baseDir)
|
||||
|
||||
|
||||
def read_meta(self):
|
||||
result = {}
|
||||
|
||||
# .conf file
|
||||
try:
|
||||
with open(self.baseDir + "/mod.conf", "r") as myfile:
|
||||
conf = parse_conf(myfile.read())
|
||||
for key in ["name", "description", "title", "depends", "optional_depends"]:
|
||||
try:
|
||||
result[key] = conf[key]
|
||||
except KeyError:
|
||||
pass
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
# description.txt
|
||||
if not "description" in result:
|
||||
try:
|
||||
with open(self.baseDir + "/description.txt", "r") as myfile:
|
||||
result["description"] = myfile.read()
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
# depends.txt
|
||||
import re
|
||||
pattern = re.compile("^([a-z0-9_]+)\??$")
|
||||
if not "depends" in result and not "optional_depends" in result:
|
||||
try:
|
||||
with open(self.baseDir + "/depends.txt", "r") as myfile:
|
||||
contents = myfile.read()
|
||||
soft = []
|
||||
hard = []
|
||||
for line in contents.split("\n"):
|
||||
line = line.strip()
|
||||
if pattern.match(line):
|
||||
if line[len(line) - 1] == "?":
|
||||
soft.append( line[:-1])
|
||||
else:
|
||||
hard.append(line)
|
||||
|
||||
result["depends"] = hard
|
||||
result["optional_depends"] = soft
|
||||
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
else:
|
||||
if "depends" in result:
|
||||
result["depends"] = [x.strip() for x in result["depends"].split(",")]
|
||||
if "optional_depends" in result:
|
||||
result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
|
||||
|
||||
|
||||
# Calculate Title
|
||||
if "name" in result and not "title" in result:
|
||||
result["title"] = result["name"].replace("_", " ").title()
|
||||
|
||||
# Calculate short description
|
||||
if "description" in result:
|
||||
desc = result["description"]
|
||||
idx = desc.find(".") + 1
|
||||
cutIdx = min(len(desc), 200 if idx < 5 else idx)
|
||||
result["short_description"] = desc[:cutIdx]
|
||||
|
||||
if "name" in result:
|
||||
self.name = result["name"]
|
||||
del result["name"]
|
||||
|
||||
self.meta = result
|
||||
|
||||
def add_children_from_mod_dir(self, dir):
|
||||
for entry in next(os.walk(dir))[1]:
|
||||
path = os.path.join(dir, entry)
|
||||
if not entry.startswith('.') and os.path.isdir(path):
|
||||
child = PackageTreeNode(path, self.relative + entry + "/", name=entry)
|
||||
if not child.type.isModLike():
|
||||
raise MinetestCheckError(("Expecting mod or modpack, found {} at {} inside {}") \
|
||||
.format(child.type.value, child.relative, self.type.value))
|
||||
|
||||
self.children.append(child)
|
||||
|
||||
|
||||
def fold(self, attr, key=None, acc=None):
|
||||
if acc is None:
|
||||
acc = set()
|
||||
|
||||
if self.meta is None:
|
||||
return acc
|
||||
|
||||
at = getattr(self, attr)
|
||||
value = at if key is None else at.get(key)
|
||||
|
||||
if isinstance(value, list):
|
||||
acc |= set(value)
|
||||
elif value is not None:
|
||||
acc.add(value)
|
||||
|
||||
for child in self.children:
|
||||
child.fold(attr, key, acc)
|
||||
|
||||
return acc
|
||||
|
||||
def get(self, key):
|
||||
return self.meta.get(key)
|
||||
|
||||
def validate(self):
|
||||
for child in self.children:
|
||||
child.validate()
|
||||
Loading…
x
Reference in New Issue
Block a user