diff --git a/app/public/static/package_create.js b/app/public/static/package_create.js
index 771d0fd..8d81206 100644
--- a/app/public/static/package_create.js
+++ b/app/public/static/package_create.js
@@ -10,11 +10,12 @@ $(function() {
}
function repoIsSupported(url) {
- try {
- return URI(url).hostname() == "github.com"
- } catch(e) {
- return false
- }
+ // try {
+ // return URI(url).hostname() == "github.com"
+ // } catch(e) {
+ // return false
+ // }
+ return true
}
$(".pkg_meta").hide()
@@ -36,7 +37,7 @@ $(function() {
performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) {
$("#name").val(result.name || "")
- setSpecial("#provides_str", result.name || "")
+ setSpecial("#provides_str", result.provides || "")
$("#title").val(result.title || "")
$("#repo").val(result.repo || repoURL)
$("#issueTracker").val(result.issueTracker || "")
diff --git a/app/public/static/polltask.js b/app/public/static/polltask.js
index f6b528d..57e8e0a 100644
--- a/app/public/static/polltask.js
+++ b/app/public/static/polltask.js
@@ -22,7 +22,7 @@ function pollTask(poll_url, disableTimeout) {
var tries = 0;
function retry() {
tries++;
- if (!disableTimeout && tries > 10) {
+ if (!disableTimeout && tries > 20) {
reject("timeout")
} else {
const interval = Math.min(tries*100, 1000)
diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py
index cf07a47..c39a008 100644
--- a/app/tasks/importtasks.py
+++ b/app/tasks/importtasks.py
@@ -15,7 +15,7 @@
# along with this program. If not, see .
-import flask, json, os
+import flask, json, os, git, tempfile
from flask.ext.sqlalchemy import SQLAlchemy
from urllib.error import HTTPError
import urllib.request
@@ -25,51 +25,6 @@ from app.models import *
from app.tasks import celery, TaskError
from app.utils import randomString
-class GithubURLMaker:
- def __init__(self, url):
- # Rewrite path
- import re
- m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
- if m is None:
- return
-
- user = m.group(1)
- repo = m.group(2).replace(".git", "")
- self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
- .format(user, repo)
- self.user = user
- self.repo = repo
-
- def isValid(self):
- return self.baseUrl is not None
-
- def getRepoURL(self):
- return "https://github.com/{}/{}".format(self.user, self.repo)
-
- def getIssueTrackerURL(self):
- return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
-
- def getModConfURL(self):
- return self.baseUrl + "/mod.conf"
-
- def getDescURL(self):
- return self.baseUrl + "/description.txt"
-
- def getDependsURL(self):
- return self.baseUrl + "/depends.txt"
-
- def getScreenshotURL(self):
- return self.baseUrl + "/screenshot.png"
-
- def getCommitsURL(self, branch):
- return "https://api.github.com/repos/{}/{}/commits?sha={}" \
- .format(self.user, self.repo, urllib.parse.quote_plus(branch))
-
- def getCommitDownload(self, commit):
- return "https://github.com/{}/{}/archive/{}.zip" \
- .format(self.user, self.repo, commit)
-
-
krock_list_cache = None
krock_list_cache_by_name = None
def getKrockList():
@@ -97,9 +52,9 @@ def getKrockList():
return {
"title": x["title"],
"author": x["author"],
- "name": x["name"],
+ "name": x["name"],
"topicId": x["topicId"],
- "link": x["link"],
+ "link": x["link"],
}
krock_list_cache = [g(x) for x in list if h(x)]
@@ -143,77 +98,176 @@ def parseConf(string):
return retval
+class PackageTreeNode:
+ def __init__(self, baseDir, author=None, repo=None, name=None):
+ print("Scanning " + baseDir)
+ self.baseDir = baseDir
+ self.author = author
+ self.name = name
+ self.repo = repo
+ self.meta = None
+ self.children = []
+
+ # Detect type
+ type = None
+ is_modpack = False
+ if os.path.isfile(baseDir + "/game.conf"):
+ type = PackageType.GAME
+ elif os.path.isfile(baseDir + "/init.lua"):
+ type = PackageType.MOD
+ elif os.path.isfile(baseDir + "/modpack.txt"):
+ type = PackageType.MOD
+ is_modpack = True
+ elif os.path.isdir(baseDir + "/mods"):
+ type = PackageType.GAME
+ elif os.listdir(baseDir) == []:
+ # probably a submodule
+ return
+ else:
+ raise TaskError("Unable to detect package type!")
+
+ self.type = type
+ self.readMetaFiles()
+
+ if self.type == PackageType.GAME:
+ self.addChildrenFromModDir(baseDir + "/mods")
+ elif is_modpack:
+ self.addChildrenFromModDir(baseDir)
+
+
+ def readMetaFiles(self):
+ result = {}
+
+ # .conf file
+ try:
+ with open(self.baseDir + "/mod.conf", "r") as myfile:
+ conf = parseConf(myfile.read())
+ for key in ["name", "description", "title", "depends", "optional_depends"]:
+ try:
+ result[key] = conf[key]
+ except KeyError:
+ pass
+ except IOError:
+ print("description.txt does not exist!")
+
+ # description.txt
+ if not "description" in result:
+ try:
+ with open(self.baseDir + "/description.txt", "r") as myfile:
+ result["description"] = myfile.read()
+ except IOError:
+ print("description.txt does not exist!")
+
+ # depends.txt
+ import re
+ pattern = re.compile("^([a-z0-9_]+)\??$")
+ if not "depends" in result and not "optional_depends" in result:
+ try:
+ with open(self.baseDir + "/depends.txt", "r") as myfile:
+ contents = myfile.read()
+ soft = []
+ hard = []
+ for line in contents.split("\n"):
+ line = line.strip()
+ if pattern.match(line):
+ if line[len(line) - 1] == "?":
+ soft.append( line[:-1])
+ else:
+ hard.append(line)
+
+ result["depends"] = hard
+ result["optional_depends"] = soft
+
+ except IOError:
+ print("depends.txt does not exist!")
+
+ else:
+ if "depends" in result:
+ result["depends"] = [x.strip() for x in result["depends"].split(",")]
+ if "optional_depends" in result:
+ result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
+
+
+ # Calculate Title
+ if "name" in result and not "title" in result:
+ result["title"] = result["name"].replace("_", " ").title()
+
+ # Calculate short description
+ if "description" in result:
+ desc = result["description"]
+ idx = desc.find(".") + 1
+ cutIdx = min(len(desc), 200 if idx < 5 else idx)
+ result["short_description"] = desc[:cutIdx]
+
+ # Get forum ID
+ info = findModInfo(self.author, result.get("name"), self.repo)
+ if info is not None:
+ result["forumId"] = info.get("topicId")
+
+ if "name" in result:
+ self.name = result["name"]
+ del result["name"]
+
+ self.meta = result
+
+ def addChildrenFromModDir(self, dir):
+ for entry in next(os.walk(dir))[1]:
+ path = dir + "/" + entry
+ if not entry.startswith('.') and os.path.isdir(path):
+ self.children.append(PackageTreeNode(path, name=entry))
+
+
+ def fold(self, attr, key=None, acc=None):
+ if acc is None:
+ acc = set()
+
+ if self.meta is None:
+ return acc
+
+ at = getattr(self, attr)
+ value = at if key is None else at.get(key)
+
+ if isinstance(value, list):
+ acc |= set(value)
+ elif value is not None:
+ acc.add(value)
+
+ for child in self.children:
+ child.fold(attr, key, acc)
+
+ return acc
+
+ def get(self, key):
+ return self.meta.get(key)
+
+
@celery.task()
def getMeta(urlstr, author):
url = urlparse(urlstr)
- urlmaker = None
- if url.netloc == "github.com":
- urlmaker = GithubURLMaker(url)
- else:
- raise TaskError("Unsupported repo")
+ gitDir = tempfile.gettempdir() + "/" + randomString(10)
+ git.Repo.clone_from(urlstr, gitDir, progress=None, env=None, depth=1)
- if not urlmaker.isValid():
- raise TaskError("Error! Url maker not valid")
+ tree = PackageTreeNode(gitDir, author=author, repo=urlstr)
result = {}
+ result["name"] = tree.name
+ result["provides"] = tree.fold("name")
+ result["type"] = tree.type.name
- result["repo"] = urlmaker.getRepoURL()
- result["issueTracker"] = urlmaker.getIssueTrackerURL()
+ for key in ["depends", "optional_depends"]:
+ result[key] = tree.fold("meta", key)
- try:
- contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
- conf = parseConf(contents)
- for key in ["name", "description", "title", "depends", "optional_depends"]:
- try:
- result[key] = conf[key]
- except KeyError:
- pass
- except HTTPError:
- print("mod.conf does not exist")
+ for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
+ result[key] = tree.get(key)
- if "name" in result:
- result["title"] = result["name"].replace("_", " ").title()
+ for mod in result["provides"]:
+ result["depends"].discard(mod)
+ result["optional_depends"].discard(mod)
- if not "description" in result:
- try:
- contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
- result["description"] = contents.strip()
- except HTTPError:
- print("description.txt does not exist!")
-
- import re
- pattern = re.compile("^([a-z0-9_]+)\??$")
- if not "depends" in result and not "optional_depends" in result:
- try:
- contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8")
- soft = []
- hard = []
- for line in contents.split("\n"):
- line = line.strip()
- if pattern.match(line):
- if line[len(line) - 1] == "?":
- soft.append( line[:-1])
- else:
- hard.append(line)
-
- result["depends"] = ",".join(hard)
- result["optional_depends"] = ",".join(soft)
-
-
- except HTTPError:
- print("depends.txt does not exist!")
-
- if "description" in result:
- desc = result["description"]
- idx = desc.find(".") + 1
- cutIdx = min(len(desc), 200 if idx < 5 else idx)
- result["short_description"] = desc[:cutIdx]
-
-
- info = findModInfo(author, result.get("name"), result["repo"])
- if info is not None:
- result["forumId"] = info.get("topicId")
+ for key, value in result.items():
+ if isinstance(value, set):
+ result[key] = list(value)
return result
@@ -281,7 +335,7 @@ def importRepoScreenshot(id):
ss.approved = True
ss.package = package
ss.title = "screenshot.png"
- ss.url = "/uploads/" + filename
+ ss.url = "/uploads/" + filename
db.session.add(ss)
db.session.commit()
diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html
index 6cf2324..f443e96 100644
--- a/app/templates/packages/create_edit.html
+++ b/app/templates/packages/create_edit.html
@@ -49,7 +49,7 @@
Enter the repo URL for the package.
- If it's hosted on Github then metadata will automatically be imported.
+ If the repo uses git then the metadata will be automatically imported.
Leave blank if you don't have a repo.
@@ -61,7 +61,7 @@
- Importing...
+ Importing... (This may take a while)
{{ render_field(form.website, class_="pkg_meta") }}
diff --git a/requirements.txt b/requirements.txt
index 2caa7fc..4b4f37d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,3 +13,4 @@ lxml==4.2.1
Flask-FlatPages==0.6
Flask-Migrate==2.1.1
pillow==5.1.0
+GitPython==2.1.10