From 9a36bb7d727585b023ae288451bb9b93729faaca Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Tue, 5 Jun 2018 00:10:47 +0100 Subject: [PATCH] Add git support for importing meta --- app/public/static/package_create.js | 13 +- app/public/static/polltask.js | 2 +- app/tasks/importtasks.py | 272 ++++++++++++++---------- app/templates/packages/create_edit.html | 4 +- requirements.txt | 1 + 5 files changed, 174 insertions(+), 118 deletions(-) diff --git a/app/public/static/package_create.js b/app/public/static/package_create.js index 771d0fd..8d81206 100644 --- a/app/public/static/package_create.js +++ b/app/public/static/package_create.js @@ -10,11 +10,12 @@ $(function() { } function repoIsSupported(url) { - try { - return URI(url).hostname() == "github.com" - } catch(e) { - return false - } + // try { + // return URI(url).hostname() == "github.com" + // } catch(e) { + // return false + // } + return true } $(".pkg_meta").hide() @@ -36,7 +37,7 @@ $(function() { performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) { $("#name").val(result.name || "") - setSpecial("#provides_str", result.name || "") + setSpecial("#provides_str", result.provides || "") $("#title").val(result.title || "") $("#repo").val(result.repo || repoURL) $("#issueTracker").val(result.issueTracker || "") diff --git a/app/public/static/polltask.js b/app/public/static/polltask.js index f6b528d..57e8e0a 100644 --- a/app/public/static/polltask.js +++ b/app/public/static/polltask.js @@ -22,7 +22,7 @@ function pollTask(poll_url, disableTimeout) { var tries = 0; function retry() { tries++; - if (!disableTimeout && tries > 10) { + if (!disableTimeout && tries > 20) { reject("timeout") } else { const interval = Math.min(tries*100, 1000) diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index cf07a47..c39a008 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -15,7 +15,7 @@ # along with this program. If not, see . -import flask, json, os +import flask, json, os, git, tempfile from flask.ext.sqlalchemy import SQLAlchemy from urllib.error import HTTPError import urllib.request @@ -25,51 +25,6 @@ from app.models import * from app.tasks import celery, TaskError from app.utils import randomString -class GithubURLMaker: - def __init__(self, url): - # Rewrite path - import re - m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path) - if m is None: - return - - user = m.group(1) - repo = m.group(2).replace(".git", "") - self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \ - .format(user, repo) - self.user = user - self.repo = repo - - def isValid(self): - return self.baseUrl is not None - - def getRepoURL(self): - return "https://github.com/{}/{}".format(self.user, self.repo) - - def getIssueTrackerURL(self): - return "https://github.com/{}/{}/issues/".format(self.user, self.repo) - - def getModConfURL(self): - return self.baseUrl + "/mod.conf" - - def getDescURL(self): - return self.baseUrl + "/description.txt" - - def getDependsURL(self): - return self.baseUrl + "/depends.txt" - - def getScreenshotURL(self): - return self.baseUrl + "/screenshot.png" - - def getCommitsURL(self, branch): - return "https://api.github.com/repos/{}/{}/commits?sha={}" \ - .format(self.user, self.repo, urllib.parse.quote_plus(branch)) - - def getCommitDownload(self, commit): - return "https://github.com/{}/{}/archive/{}.zip" \ - .format(self.user, self.repo, commit) - - krock_list_cache = None krock_list_cache_by_name = None def getKrockList(): @@ -97,9 +52,9 @@ def getKrockList(): return { "title": x["title"], "author": x["author"], - "name": x["name"], + "name": x["name"], "topicId": x["topicId"], - "link": x["link"], + "link": x["link"], } krock_list_cache = [g(x) for x in list if h(x)] @@ -143,77 +98,176 @@ def parseConf(string): return retval +class PackageTreeNode: + def __init__(self, baseDir, author=None, repo=None, name=None): + print("Scanning " + baseDir) + self.baseDir = baseDir + self.author = author + self.name = name + self.repo = repo + self.meta = None + self.children = [] + + # Detect type + type = None + is_modpack = False + if os.path.isfile(baseDir + "/game.conf"): + type = PackageType.GAME + elif os.path.isfile(baseDir + "/init.lua"): + type = PackageType.MOD + elif os.path.isfile(baseDir + "/modpack.txt"): + type = PackageType.MOD + is_modpack = True + elif os.path.isdir(baseDir + "/mods"): + type = PackageType.GAME + elif os.listdir(baseDir) == []: + # probably a submodule + return + else: + raise TaskError("Unable to detect package type!") + + self.type = type + self.readMetaFiles() + + if self.type == PackageType.GAME: + self.addChildrenFromModDir(baseDir + "/mods") + elif is_modpack: + self.addChildrenFromModDir(baseDir) + + + def readMetaFiles(self): + result = {} + + # .conf file + try: + with open(self.baseDir + "/mod.conf", "r") as myfile: + conf = parseConf(myfile.read()) + for key in ["name", "description", "title", "depends", "optional_depends"]: + try: + result[key] = conf[key] + except KeyError: + pass + except IOError: + print("description.txt does not exist!") + + # description.txt + if not "description" in result: + try: + with open(self.baseDir + "/description.txt", "r") as myfile: + result["description"] = myfile.read() + except IOError: + print("description.txt does not exist!") + + # depends.txt + import re + pattern = re.compile("^([a-z0-9_]+)\??$") + if not "depends" in result and not "optional_depends" in result: + try: + with open(self.baseDir + "/depends.txt", "r") as myfile: + contents = myfile.read() + soft = [] + hard = [] + for line in contents.split("\n"): + line = line.strip() + if pattern.match(line): + if line[len(line) - 1] == "?": + soft.append( line[:-1]) + else: + hard.append(line) + + result["depends"] = hard + result["optional_depends"] = soft + + except IOError: + print("depends.txt does not exist!") + + else: + if "depends" in result: + result["depends"] = [x.strip() for x in result["depends"].split(",")] + if "optional_depends" in result: + result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")] + + + # Calculate Title + if "name" in result and not "title" in result: + result["title"] = result["name"].replace("_", " ").title() + + # Calculate short description + if "description" in result: + desc = result["description"] + idx = desc.find(".") + 1 + cutIdx = min(len(desc), 200 if idx < 5 else idx) + result["short_description"] = desc[:cutIdx] + + # Get forum ID + info = findModInfo(self.author, result.get("name"), self.repo) + if info is not None: + result["forumId"] = info.get("topicId") + + if "name" in result: + self.name = result["name"] + del result["name"] + + self.meta = result + + def addChildrenFromModDir(self, dir): + for entry in next(os.walk(dir))[1]: + path = dir + "/" + entry + if not entry.startswith('.') and os.path.isdir(path): + self.children.append(PackageTreeNode(path, name=entry)) + + + def fold(self, attr, key=None, acc=None): + if acc is None: + acc = set() + + if self.meta is None: + return acc + + at = getattr(self, attr) + value = at if key is None else at.get(key) + + if isinstance(value, list): + acc |= set(value) + elif value is not None: + acc.add(value) + + for child in self.children: + child.fold(attr, key, acc) + + return acc + + def get(self, key): + return self.meta.get(key) + + @celery.task() def getMeta(urlstr, author): url = urlparse(urlstr) - urlmaker = None - if url.netloc == "github.com": - urlmaker = GithubURLMaker(url) - else: - raise TaskError("Unsupported repo") + gitDir = tempfile.gettempdir() + "/" + randomString(10) + git.Repo.clone_from(urlstr, gitDir, progress=None, env=None, depth=1) - if not urlmaker.isValid(): - raise TaskError("Error! Url maker not valid") + tree = PackageTreeNode(gitDir, author=author, repo=urlstr) result = {} + result["name"] = tree.name + result["provides"] = tree.fold("name") + result["type"] = tree.type.name - result["repo"] = urlmaker.getRepoURL() - result["issueTracker"] = urlmaker.getIssueTrackerURL() + for key in ["depends", "optional_depends"]: + result[key] = tree.fold("meta", key) - try: - contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") - conf = parseConf(contents) - for key in ["name", "description", "title", "depends", "optional_depends"]: - try: - result[key] = conf[key] - except KeyError: - pass - except HTTPError: - print("mod.conf does not exist") + for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]: + result[key] = tree.get(key) - if "name" in result: - result["title"] = result["name"].replace("_", " ").title() + for mod in result["provides"]: + result["depends"].discard(mod) + result["optional_depends"].discard(mod) - if not "description" in result: - try: - contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8") - result["description"] = contents.strip() - except HTTPError: - print("description.txt does not exist!") - - import re - pattern = re.compile("^([a-z0-9_]+)\??$") - if not "depends" in result and not "optional_depends" in result: - try: - contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8") - soft = [] - hard = [] - for line in contents.split("\n"): - line = line.strip() - if pattern.match(line): - if line[len(line) - 1] == "?": - soft.append( line[:-1]) - else: - hard.append(line) - - result["depends"] = ",".join(hard) - result["optional_depends"] = ",".join(soft) - - - except HTTPError: - print("depends.txt does not exist!") - - if "description" in result: - desc = result["description"] - idx = desc.find(".") + 1 - cutIdx = min(len(desc), 200 if idx < 5 else idx) - result["short_description"] = desc[:cutIdx] - - - info = findModInfo(author, result.get("name"), result["repo"]) - if info is not None: - result["forumId"] = info.get("topicId") + for key, value in result.items(): + if isinstance(value, set): + result[key] = list(value) return result @@ -281,7 +335,7 @@ def importRepoScreenshot(id): ss.approved = True ss.package = package ss.title = "screenshot.png" - ss.url = "/uploads/" + filename + ss.url = "/uploads/" + filename db.session.add(ss) db.session.commit() diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index 6cf2324..f443e96 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -49,7 +49,7 @@

Enter the repo URL for the package. - If it's hosted on Github then metadata will automatically be imported.

+ If the repo uses git then the metadata will be automatically imported.

Leave blank if you don't have a repo.

@@ -61,7 +61,7 @@
- Importing... + Importing... (This may take a while)
{{ render_field(form.website, class_="pkg_meta") }} diff --git a/requirements.txt b/requirements.txt index 2caa7fc..4b4f37d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ lxml==4.2.1 Flask-FlatPages==0.6 Flask-Migrate==2.1.1 pillow==5.1.0 +GitPython==2.1.10