Add git support for importing meta
This commit is contained in:
rubenwardy
parent
e424dc57e7
commit
9a36bb7d72
|
|
@ -10,11 +10,12 @@ $(function() {
|
|||
}
|
||||
|
||||
function repoIsSupported(url) {
|
||||
try {
|
||||
return URI(url).hostname() == "github.com"
|
||||
} catch(e) {
|
||||
return false
|
||||
}
|
||||
// try {
|
||||
// return URI(url).hostname() == "github.com"
|
||||
// } catch(e) {
|
||||
// return false
|
||||
// }
|
||||
return true
|
||||
}
|
||||
|
||||
$(".pkg_meta").hide()
|
||||
|
|
@ -36,7 +37,7 @@ $(function() {
|
|||
|
||||
performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) {
|
||||
$("#name").val(result.name || "")
|
||||
setSpecial("#provides_str", result.name || "")
|
||||
setSpecial("#provides_str", result.provides || "")
|
||||
$("#title").val(result.title || "")
|
||||
$("#repo").val(result.repo || repoURL)
|
||||
$("#issueTracker").val(result.issueTracker || "")
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ function pollTask(poll_url, disableTimeout) {
|
|||
var tries = 0;
|
||||
function retry() {
|
||||
tries++;
|
||||
if (!disableTimeout && tries > 10) {
|
||||
if (!disableTimeout && tries > 20) {
|
||||
reject("timeout")
|
||||
} else {
|
||||
const interval = Math.min(tries*100, 1000)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import flask, json, os
|
||||
import flask, json, os, git, tempfile
|
||||
from flask.ext.sqlalchemy import SQLAlchemy
|
||||
from urllib.error import HTTPError
|
||||
import urllib.request
|
||||
|
|
@ -25,51 +25,6 @@ from app.models import *
|
|||
from app.tasks import celery, TaskError
|
||||
from app.utils import randomString
|
||||
|
||||
class GithubURLMaker:
|
||||
def __init__(self, url):
|
||||
# Rewrite path
|
||||
import re
|
||||
m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
|
||||
if m is None:
|
||||
return
|
||||
|
||||
user = m.group(1)
|
||||
repo = m.group(2).replace(".git", "")
|
||||
self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
|
||||
.format(user, repo)
|
||||
self.user = user
|
||||
self.repo = repo
|
||||
|
||||
def isValid(self):
|
||||
return self.baseUrl is not None
|
||||
|
||||
def getRepoURL(self):
|
||||
return "https://github.com/{}/{}".format(self.user, self.repo)
|
||||
|
||||
def getIssueTrackerURL(self):
|
||||
return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
|
||||
|
||||
def getModConfURL(self):
|
||||
return self.baseUrl + "/mod.conf"
|
||||
|
||||
def getDescURL(self):
|
||||
return self.baseUrl + "/description.txt"
|
||||
|
||||
def getDependsURL(self):
|
||||
return self.baseUrl + "/depends.txt"
|
||||
|
||||
def getScreenshotURL(self):
|
||||
return self.baseUrl + "/screenshot.png"
|
||||
|
||||
def getCommitsURL(self, branch):
|
||||
return "https://api.github.com/repos/{}/{}/commits?sha={}" \
|
||||
.format(self.user, self.repo, urllib.parse.quote_plus(branch))
|
||||
|
||||
def getCommitDownload(self, commit):
|
||||
return "https://github.com/{}/{}/archive/{}.zip" \
|
||||
.format(self.user, self.repo, commit)
|
||||
|
||||
|
||||
krock_list_cache = None
|
||||
krock_list_cache_by_name = None
|
||||
def getKrockList():
|
||||
|
|
@ -97,9 +52,9 @@ def getKrockList():
|
|||
return {
|
||||
"title": x["title"],
|
||||
"author": x["author"],
|
||||
"name": x["name"],
|
||||
"name": x["name"],
|
||||
"topicId": x["topicId"],
|
||||
"link": x["link"],
|
||||
"link": x["link"],
|
||||
}
|
||||
|
||||
krock_list_cache = [g(x) for x in list if h(x)]
|
||||
|
|
@ -143,77 +98,176 @@ def parseConf(string):
|
|||
return retval
|
||||
|
||||
|
||||
class PackageTreeNode:
|
||||
def __init__(self, baseDir, author=None, repo=None, name=None):
|
||||
print("Scanning " + baseDir)
|
||||
self.baseDir = baseDir
|
||||
self.author = author
|
||||
self.name = name
|
||||
self.repo = repo
|
||||
self.meta = None
|
||||
self.children = []
|
||||
|
||||
# Detect type
|
||||
type = None
|
||||
is_modpack = False
|
||||
if os.path.isfile(baseDir + "/game.conf"):
|
||||
type = PackageType.GAME
|
||||
elif os.path.isfile(baseDir + "/init.lua"):
|
||||
type = PackageType.MOD
|
||||
elif os.path.isfile(baseDir + "/modpack.txt"):
|
||||
type = PackageType.MOD
|
||||
is_modpack = True
|
||||
elif os.path.isdir(baseDir + "/mods"):
|
||||
type = PackageType.GAME
|
||||
elif os.listdir(baseDir) == []:
|
||||
# probably a submodule
|
||||
return
|
||||
else:
|
||||
raise TaskError("Unable to detect package type!")
|
||||
|
||||
self.type = type
|
||||
self.readMetaFiles()
|
||||
|
||||
if self.type == PackageType.GAME:
|
||||
self.addChildrenFromModDir(baseDir + "/mods")
|
||||
elif is_modpack:
|
||||
self.addChildrenFromModDir(baseDir)
|
||||
|
||||
|
||||
def readMetaFiles(self):
|
||||
result = {}
|
||||
|
||||
# .conf file
|
||||
try:
|
||||
with open(self.baseDir + "/mod.conf", "r") as myfile:
|
||||
conf = parseConf(myfile.read())
|
||||
for key in ["name", "description", "title", "depends", "optional_depends"]:
|
||||
try:
|
||||
result[key] = conf[key]
|
||||
except KeyError:
|
||||
pass
|
||||
except IOError:
|
||||
print("description.txt does not exist!")
|
||||
|
||||
# description.txt
|
||||
if not "description" in result:
|
||||
try:
|
||||
with open(self.baseDir + "/description.txt", "r") as myfile:
|
||||
result["description"] = myfile.read()
|
||||
except IOError:
|
||||
print("description.txt does not exist!")
|
||||
|
||||
# depends.txt
|
||||
import re
|
||||
pattern = re.compile("^([a-z0-9_]+)\??$")
|
||||
if not "depends" in result and not "optional_depends" in result:
|
||||
try:
|
||||
with open(self.baseDir + "/depends.txt", "r") as myfile:
|
||||
contents = myfile.read()
|
||||
soft = []
|
||||
hard = []
|
||||
for line in contents.split("\n"):
|
||||
line = line.strip()
|
||||
if pattern.match(line):
|
||||
if line[len(line) - 1] == "?":
|
||||
soft.append( line[:-1])
|
||||
else:
|
||||
hard.append(line)
|
||||
|
||||
result["depends"] = hard
|
||||
result["optional_depends"] = soft
|
||||
|
||||
except IOError:
|
||||
print("depends.txt does not exist!")
|
||||
|
||||
else:
|
||||
if "depends" in result:
|
||||
result["depends"] = [x.strip() for x in result["depends"].split(",")]
|
||||
if "optional_depends" in result:
|
||||
result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
|
||||
|
||||
|
||||
# Calculate Title
|
||||
if "name" in result and not "title" in result:
|
||||
result["title"] = result["name"].replace("_", " ").title()
|
||||
|
||||
# Calculate short description
|
||||
if "description" in result:
|
||||
desc = result["description"]
|
||||
idx = desc.find(".") + 1
|
||||
cutIdx = min(len(desc), 200 if idx < 5 else idx)
|
||||
result["short_description"] = desc[:cutIdx]
|
||||
|
||||
# Get forum ID
|
||||
info = findModInfo(self.author, result.get("name"), self.repo)
|
||||
if info is not None:
|
||||
result["forumId"] = info.get("topicId")
|
||||
|
||||
if "name" in result:
|
||||
self.name = result["name"]
|
||||
del result["name"]
|
||||
|
||||
self.meta = result
|
||||
|
||||
def addChildrenFromModDir(self, dir):
|
||||
for entry in next(os.walk(dir))[1]:
|
||||
path = dir + "/" + entry
|
||||
if not entry.startswith('.') and os.path.isdir(path):
|
||||
self.children.append(PackageTreeNode(path, name=entry))
|
||||
|
||||
|
||||
def fold(self, attr, key=None, acc=None):
|
||||
if acc is None:
|
||||
acc = set()
|
||||
|
||||
if self.meta is None:
|
||||
return acc
|
||||
|
||||
at = getattr(self, attr)
|
||||
value = at if key is None else at.get(key)
|
||||
|
||||
if isinstance(value, list):
|
||||
acc |= set(value)
|
||||
elif value is not None:
|
||||
acc.add(value)
|
||||
|
||||
for child in self.children:
|
||||
child.fold(attr, key, acc)
|
||||
|
||||
return acc
|
||||
|
||||
def get(self, key):
|
||||
return self.meta.get(key)
|
||||
|
||||
|
||||
@celery.task()
|
||||
def getMeta(urlstr, author):
|
||||
url = urlparse(urlstr)
|
||||
|
||||
urlmaker = None
|
||||
if url.netloc == "github.com":
|
||||
urlmaker = GithubURLMaker(url)
|
||||
else:
|
||||
raise TaskError("Unsupported repo")
|
||||
gitDir = tempfile.gettempdir() + "/" + randomString(10)
|
||||
git.Repo.clone_from(urlstr, gitDir, progress=None, env=None, depth=1)
|
||||
|
||||
if not urlmaker.isValid():
|
||||
raise TaskError("Error! Url maker not valid")
|
||||
tree = PackageTreeNode(gitDir, author=author, repo=urlstr)
|
||||
|
||||
result = {}
|
||||
result["name"] = tree.name
|
||||
result["provides"] = tree.fold("name")
|
||||
result["type"] = tree.type.name
|
||||
|
||||
result["repo"] = urlmaker.getRepoURL()
|
||||
result["issueTracker"] = urlmaker.getIssueTrackerURL()
|
||||
for key in ["depends", "optional_depends"]:
|
||||
result[key] = tree.fold("meta", key)
|
||||
|
||||
try:
|
||||
contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
|
||||
conf = parseConf(contents)
|
||||
for key in ["name", "description", "title", "depends", "optional_depends"]:
|
||||
try:
|
||||
result[key] = conf[key]
|
||||
except KeyError:
|
||||
pass
|
||||
except HTTPError:
|
||||
print("mod.conf does not exist")
|
||||
for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
|
||||
result[key] = tree.get(key)
|
||||
|
||||
if "name" in result:
|
||||
result["title"] = result["name"].replace("_", " ").title()
|
||||
for mod in result["provides"]:
|
||||
result["depends"].discard(mod)
|
||||
result["optional_depends"].discard(mod)
|
||||
|
||||
if not "description" in result:
|
||||
try:
|
||||
contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
|
||||
result["description"] = contents.strip()
|
||||
except HTTPError:
|
||||
print("description.txt does not exist!")
|
||||
|
||||
import re
|
||||
pattern = re.compile("^([a-z0-9_]+)\??$")
|
||||
if not "depends" in result and not "optional_depends" in result:
|
||||
try:
|
||||
contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8")
|
||||
soft = []
|
||||
hard = []
|
||||
for line in contents.split("\n"):
|
||||
line = line.strip()
|
||||
if pattern.match(line):
|
||||
if line[len(line) - 1] == "?":
|
||||
soft.append( line[:-1])
|
||||
else:
|
||||
hard.append(line)
|
||||
|
||||
result["depends"] = ",".join(hard)
|
||||
result["optional_depends"] = ",".join(soft)
|
||||
|
||||
|
||||
except HTTPError:
|
||||
print("depends.txt does not exist!")
|
||||
|
||||
if "description" in result:
|
||||
desc = result["description"]
|
||||
idx = desc.find(".") + 1
|
||||
cutIdx = min(len(desc), 200 if idx < 5 else idx)
|
||||
result["short_description"] = desc[:cutIdx]
|
||||
|
||||
|
||||
info = findModInfo(author, result.get("name"), result["repo"])
|
||||
if info is not None:
|
||||
result["forumId"] = info.get("topicId")
|
||||
for key, value in result.items():
|
||||
if isinstance(value, set):
|
||||
result[key] = list(value)
|
||||
|
||||
return result
|
||||
|
||||
|
|
@ -281,7 +335,7 @@ def importRepoScreenshot(id):
|
|||
ss.approved = True
|
||||
ss.package = package
|
||||
ss.title = "screenshot.png"
|
||||
ss.url = "/uploads/" + filename
|
||||
ss.url = "/uploads/" + filename
|
||||
db.session.add(ss)
|
||||
db.session.commit()
|
||||
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@
|
|||
|
||||
<div class="pkg_wiz_1">
|
||||
<p>Enter the repo URL for the package.
|
||||
If it's hosted on Github then metadata will automatically be imported.</p>
|
||||
If the repo uses git then the metadata will be automatically imported.</p>
|
||||
|
||||
<p>Leave blank if you don't have a repo.</p>
|
||||
</div>
|
||||
|
|
@ -61,7 +61,7 @@
|
|||
</div>
|
||||
|
||||
<div class="pkg_wiz_2">
|
||||
Importing...
|
||||
Importing... (This may take a while)
|
||||
</div>
|
||||
|
||||
{{ render_field(form.website, class_="pkg_meta") }}
|
||||
|
|
|
|||
|
|
@ -13,3 +13,4 @@ lxml==4.2.1
|
|||
Flask-FlatPages==0.6
|
||||
Flask-Migrate==2.1.1
|
||||
pillow==5.1.0
|
||||
GitPython==2.1.10
|
||||
|
|
|
|||
Loading…
Reference in New Issue