diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index 8925f0e..7084909 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -26,7 +26,8 @@ from app import app from app.models import * from app.tasks import celery, TaskError from app.utils import randomString - +from .minetestcheck import build_tree, MinetestCheckError, ContentType +from .minetestcheck.config import parse_conf class GithubURLMaker: def __init__(self, url): @@ -127,162 +128,6 @@ def findModInfo(author, name, link): return None - -def parseConf(string): - retval = {} - for line in string.split("\n"): - idx = line.find("=") - if idx > 0: - key = line[:idx].strip() - value = line[idx+1:].strip() - retval[key] = value - - return retval - - -class PackageTreeNode: - def __init__(self, baseDir, author=None, repo=None, name=None): - print("Scanning " + baseDir) - self.baseDir = baseDir - self.author = author - self.name = name - self.repo = repo - self.meta = None - self.children = [] - - # Detect type - type = None - is_modpack = False - if os.path.isfile(baseDir + "/game.conf"): - type = PackageType.GAME - elif os.path.isfile(baseDir + "/init.lua"): - type = PackageType.MOD - elif os.path.isfile(baseDir + "/modpack.txt") or \ - os.path.isfile(baseDir + "/modpack.conf"): - type = PackageType.MOD - is_modpack = True - elif os.path.isdir(baseDir + "/mods"): - type = PackageType.GAME - elif os.listdir(baseDir) == []: - # probably a submodule - return - else: - raise TaskError("Unable to detect package type!") - - self.type = type - self.readMetaFiles() - - if self.type == PackageType.GAME: - self.addChildrenFromModDir(baseDir + "/mods") - elif is_modpack: - self.addChildrenFromModDir(baseDir) - - - def readMetaFiles(self): - result = {} - - # .conf file - try: - with open(self.baseDir + "/mod.conf", "r") as myfile: - conf = parseConf(myfile.read()) - for key in ["name", "description", "title", "depends", "optional_depends"]: - try: - result[key] = conf[key] - except KeyError: - pass - except IOError: - print("description.txt does not exist!") - - # description.txt - if not "description" in result: - try: - with open(self.baseDir + "/description.txt", "r") as myfile: - result["description"] = myfile.read() - except IOError: - print("description.txt does not exist!") - - # depends.txt - import re - pattern = re.compile("^([a-z0-9_]+)\??$") - if not "depends" in result and not "optional_depends" in result: - try: - with open(self.baseDir + "/depends.txt", "r") as myfile: - contents = myfile.read() - soft = [] - hard = [] - for line in contents.split("\n"): - line = line.strip() - if pattern.match(line): - if line[len(line) - 1] == "?": - soft.append( line[:-1]) - else: - hard.append(line) - - result["depends"] = hard - result["optional_depends"] = soft - - except IOError: - print("depends.txt does not exist!") - - else: - if "depends" in result: - result["depends"] = [x.strip() for x in result["depends"].split(",")] - if "optional_depends" in result: - result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")] - - - # Calculate Title - if "name" in result and not "title" in result: - result["title"] = result["name"].replace("_", " ").title() - - # Calculate short description - if "description" in result: - desc = result["description"] - idx = desc.find(".") + 1 - cutIdx = min(len(desc), 200 if idx < 5 else idx) - result["short_description"] = desc[:cutIdx] - - # Get forum ID - info = findModInfo(self.author, result.get("name"), self.repo) - if info is not None: - result["forumId"] = info.get("topicId") - - if "name" in result: - self.name = result["name"] - del result["name"] - - self.meta = result - - def addChildrenFromModDir(self, dir): - for entry in next(os.walk(dir))[1]: - path = dir + "/" + entry - if not entry.startswith('.') and os.path.isdir(path): - self.children.append(PackageTreeNode(path, name=entry)) - - - def fold(self, attr, key=None, acc=None): - if acc is None: - acc = set() - - if self.meta is None: - return acc - - at = getattr(self, attr) - value = at if key is None else at.get(key) - - if isinstance(value, list): - acc |= set(value) - elif value is not None: - acc.add(value) - - for child in self.children: - child.fold(attr, key, acc) - - return acc - - def get(self, key): - return self.meta.get(key) - def generateGitURL(urlstr): scheme, netloc, path, query, frag = urlsplit(urlstr) @@ -323,7 +168,12 @@ def cloneRepo(urlstr, ref=None, recursive=False): @celery.task() def getMeta(urlstr, author): gitDir, _ = cloneRepo(urlstr, recursive=True) - tree = PackageTreeNode(gitDir, author=author, repo=urlstr) + + try: + tree = build_tree(gitDir, author=author, repo=urlstr) + except MinetestCheckError as err: + raise TaskError(str(err)) + shutil.rmtree(gitDir) result = {} @@ -387,6 +237,12 @@ def makeVCSRelease(id, branch): gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True) + try: + tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \ + author=release.package.author.username, name=release.package.name) + except MinetestCheckError as err: + raise TaskError(str(err)) + try: filename = randomString(10) + ".zip" destPath = os.path.join(app.config["UPLOAD_DIR"], filename) @@ -464,7 +320,7 @@ def getDepends(package): # try: contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") - conf = parseConf(contents) + conf = parse_conf(contents) for key in ["depends", "optional_depends"]: try: result[key] = conf[key] diff --git a/app/tasks/minetestcheck/__init__.py b/app/tasks/minetestcheck/__init__.py new file mode 100644 index 0000000..dbc57ad --- /dev/null +++ b/app/tasks/minetestcheck/__init__.py @@ -0,0 +1,48 @@ +from enum import Enum + +class MinetestCheckError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr("Error validating package: " + self.value) + +class ContentType(Enum): + UNKNOWN = "unknown" + MOD = "mod" + MODPACK = "modpack" + GAME = "game" + TXP = "texture pack" + + def isModLike(self): + return self == ContentType.MOD or self == ContentType.MODPACK + + def validate_same(self, other): + """ + Whether or not `other` is an acceptable type for this + """ + assert(other) + + if self == ContentType.MOD: + if not other.isModLike(): + raise MinetestCheckError("expected a mod or modpack, found " + other.value) + + elif self == ContentType.TXP: + if other != ContentType.UNKNOWN and other != ContentType.TXP: + raise MinetestCheckError("expected a " + self.value + ", found a " + other.value) + + elif other != self: + raise MinetestCheckError("expected a " + self.value + ", found a " + other.value) + + +from .tree import PackageTreeNode, get_base_dir + +def build_tree(path, expected_type=None, author=None, repo=None, name=None): + path = get_base_dir(path) + + root = PackageTreeNode(path, "/", author=author, repo=repo, name=name) + assert(root) + + if expected_type: + expected_type.validate_same(root.type) + + return root diff --git a/app/tasks/minetestcheck/config.py b/app/tasks/minetestcheck/config.py new file mode 100644 index 0000000..a8187d2 --- /dev/null +++ b/app/tasks/minetestcheck/config.py @@ -0,0 +1,10 @@ +def parse_conf(string): + retval = {} + for line in string.split("\n"): + idx = line.find("=") + if idx > 0: + key = line[:idx].strip() + value = line[idx+1:].strip() + retval[key] = value + + return retval diff --git a/app/tasks/minetestcheck/tree.py b/app/tasks/minetestcheck/tree.py new file mode 100644 index 0000000..38c2880 --- /dev/null +++ b/app/tasks/minetestcheck/tree.py @@ -0,0 +1,162 @@ +import os +from . import MinetestCheckError, ContentType +from .config import parse_conf + +def get_base_dir(path): + if not os.path.isdir(path): + raise IOError("Expected dir") + + root, subdirs, files = next(os.walk(path)) + if len(subdirs) == 1 and len(files) == 0: + return get_base_dir(path + "/" + subdirs[0]) + else: + return path + + +def detect_type(path): + if os.path.isfile(path + "/game.conf"): + return ContentType.GAME + elif os.path.isfile(path + "/init.lua"): + return ContentType.MOD + elif os.path.isfile(path + "/modpack.txt") or \ + os.path.isfile(path + "/modpack.conf"): + return ContentType.MODPACK + elif os.path.isdir(path + "/mods"): + return ContentType.GAME + elif os.path.isfile(path + "/texture_pack.conf"): + return ContentType.TXP + else: + return ContentType.UNKNOWN + + +class PackageTreeNode: + def __init__(self, baseDir, relative, author=None, repo=None, name=None): + print(baseDir) + self.baseDir = baseDir + self.relative = relative + self.author = author + self.name = name + self.repo = repo + self.meta = None + self.children = [] + + # Detect type + self.type = detect_type(baseDir) + self.read_meta() + + if self.type == ContentType.GAME: + if not os.path.isdir(baseDir + "/mods"): + raise MinetestCheckError(("game at {} does not have a mods/ folder").format(self.relative)) + self.add_children_from_mod_dir(baseDir + "/mods") + elif self.type == ContentType.MODPACK: + self.add_children_from_mod_dir(baseDir) + + + def read_meta(self): + result = {} + + # .conf file + try: + with open(self.baseDir + "/mod.conf", "r") as myfile: + conf = parse_conf(myfile.read()) + for key in ["name", "description", "title", "depends", "optional_depends"]: + try: + result[key] = conf[key] + except KeyError: + pass + except IOError: + pass + + # description.txt + if not "description" in result: + try: + with open(self.baseDir + "/description.txt", "r") as myfile: + result["description"] = myfile.read() + except IOError: + pass + + # depends.txt + import re + pattern = re.compile("^([a-z0-9_]+)\??$") + if not "depends" in result and not "optional_depends" in result: + try: + with open(self.baseDir + "/depends.txt", "r") as myfile: + contents = myfile.read() + soft = [] + hard = [] + for line in contents.split("\n"): + line = line.strip() + if pattern.match(line): + if line[len(line) - 1] == "?": + soft.append( line[:-1]) + else: + hard.append(line) + + result["depends"] = hard + result["optional_depends"] = soft + + except IOError: + pass + + else: + if "depends" in result: + result["depends"] = [x.strip() for x in result["depends"].split(",")] + if "optional_depends" in result: + result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")] + + + # Calculate Title + if "name" in result and not "title" in result: + result["title"] = result["name"].replace("_", " ").title() + + # Calculate short description + if "description" in result: + desc = result["description"] + idx = desc.find(".") + 1 + cutIdx = min(len(desc), 200 if idx < 5 else idx) + result["short_description"] = desc[:cutIdx] + + if "name" in result: + self.name = result["name"] + del result["name"] + + self.meta = result + + def add_children_from_mod_dir(self, dir): + for entry in next(os.walk(dir))[1]: + path = os.path.join(dir, entry) + if not entry.startswith('.') and os.path.isdir(path): + child = PackageTreeNode(path, self.relative + entry + "/", name=entry) + if not child.type.isModLike(): + raise MinetestCheckError(("Expecting mod or modpack, found {} at {} inside {}") \ + .format(child.type.value, child.relative, self.type.value)) + + self.children.append(child) + + + def fold(self, attr, key=None, acc=None): + if acc is None: + acc = set() + + if self.meta is None: + return acc + + at = getattr(self, attr) + value = at if key is None else at.get(key) + + if isinstance(value, list): + acc |= set(value) + elif value is not None: + acc.add(value) + + for child in self.children: + child.fold(attr, key, acc) + + return acc + + def get(self, key): + return self.meta.get(key) + + def validate(self): + for child in self.children: + child.validate()