From 92717f31dd94ad1c763641a397305de9bc143df9 Mon Sep 17 00:00:00 2001
From: rubenwardy <rw@rubenwardy.com>
Date: Fri, 11 May 2018 15:04:17 +0100
Subject: [PATCH] Add VCS import from Github

---
 app/models.py                    |  1 +
 app/tasks/importtasks.py         | 58 +++++++++++++++++++++++++++++---
 app/templates/base.html          |  1 +
 app/templates/packages/view.html | 36 +++++++++++---------
 app/templates/tasks/view.html    | 21 ++++++++++++
 app/views/__init__.py            |  2 +-
 app/views/api.py                 | 35 -------------------
 app/views/packages.py            | 17 ++++++----
 app/views/tasks.py               | 50 +++++++++++++++++++++++++++
 9 files changed, 158 insertions(+), 63 deletions(-)
 create mode 100644 app/templates/tasks/view.html
 delete mode 100644 app/views/api.py
 create mode 100644 app/views/tasks.py

diff --git a/app/models.py b/app/models.py
index 9b64d3b..113a1cb 100644
--- a/app/models.py
+++ b/app/models.py
@@ -322,6 +322,7 @@ class PackageRelease(db.Model):
 	releaseDate  = db.Column(db.DateTime,        nullable=False)
 	url          = db.Column(db.String(100), nullable=False)
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
+	task_id      = db.Column(db.String(32), nullable=True)
 
 
 	def getEditURL(self):
diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py
index 2027f74..1950d5b 100644
--- a/app/tasks/importtasks.py
+++ b/app/tasks/importtasks.py
@@ -1,12 +1,18 @@
-import flask
+import flask, json
 from flask.ext.sqlalchemy import SQLAlchemy
 import urllib.request
-from urllib.parse import urlparse
-
+from urllib.parse import urlparse, quote_plus
 from app import app
 from app.models import *
 from app.tasks import celery
 
+class TaskError(Exception):
+	def __init__(self, value):
+		self.value = value
+	def __str__(self):
+		return repr(self.value)
+
+
 class GithubURLMaker:
 	def __init__(self, url):
 		# Rewrite path
@@ -18,6 +24,8 @@ class GithubURLMaker:
 		user = m.group(1)
 		repo = m.group(2)
 		self.baseUrl = "https://raw.githubusercontent.com/" + user + "/" + repo.replace(".git", "") + "/master"
+		self.user = user
+		self.repo = repo
 
 	def isValid(self):
 		return self.baseUrl is not None
@@ -31,6 +39,13 @@ class GithubURLMaker:
 	def getScreenshotURL(self):
 		return self.baseUrl + "/screenshot.png"
 
+	def getCommitsURL(self, branch):
+		return "https://api.github.com/repos/" + self.user + "/" + self.repo + "/commits?sha" + urllib.parse.quote_plus(branch)
+
+	def getCommitDownload(self, commit):
+		return "https://github.com/" + self.user + "/" + self.repo + "/archive/" + commit + ".zip"
+
+
 def parseConf(string):
 	retval = {}
 	for line in string.split("\n"):
@@ -49,10 +64,11 @@ def getMeta(urlstr):
 	urlmaker = None
 	if url.netloc == "github.com":
 		urlmaker = GithubURLMaker(url)
+	else:
+		raise TaskError("Unsupported repo")
 
 	if not urlmaker.isValid():
-		print("Error! Url maker not valid")
-		return
+		raise TaskError("Error! Url maker not valid")
 
 	print(urlmaker.getModConfURL())
 
@@ -79,3 +95,35 @@ def getMeta(urlstr):
 			print("description.txt does not exist!")
 
 	return result
+
+@celery.task()
+def makeVCSRelease(id, branch):
+	release = PackageRelease.query.get(id)
+	if release is None:
+		raise TaskError("No such release!")
+
+	if release.package is None:
+		raise TaskError("No package attached to release")
+
+	url = urlparse(release.package.repo)
+
+	urlmaker = None
+	if url.netloc == "github.com":
+		urlmaker = GithubURLMaker(url)
+	else:
+		raise TaskError("Unsupported repo")
+
+	if not urlmaker.isValid():
+		raise TaskError("Invalid github repo URL")
+
+	contents = urllib.request.urlopen(urlmaker.getCommitsURL(branch)).read().decode("utf-8")
+	commits = json.loads(contents)
+
+	if len(commits) == 0:
+		raise TaskError("No commits found")
+
+	release.url = urlmaker.getCommitDownload(commits[0]["sha"])
+	release.task_id = None
+	db.session.commit()
+
+	return release.url
diff --git a/app/templates/base.html b/app/templates/base.html
index 07131de..0ab87b4 100644
--- a/app/templates/base.html
+++ b/app/templates/base.html
@@ -7,6 +7,7 @@
 	<meta name="viewport" content="width=device-width, initial-scale=1">
 	<title>{% block title %}title{% endblock %} - {{ config.USER_APP_NAME }}</title>
 	<link rel="stylesheet" type="text/css" href="/static/main.css">
+	{% block headextra %}{% endblock %}
 </head>
 
 <body>
diff --git a/app/templates/packages/view.html b/app/templates/packages/view.html
index 5c68ead..ecd6b35 100644
--- a/app/templates/packages/view.html
+++ b/app/templates/packages/view.html
@@ -78,25 +78,29 @@
 
 	<ul>
 		{% for rel in releases %}
-			<li>
-				{% if not rel.approved %}<i>{% endif %}
+			{% if rel.approved or package.checkPerm(current_user, "MAKE_RELEASE") or package.checkPerm(current_user, "APPROVE_RELEASE")  %}
+				<li>
+					{% if not rel.approved %}<i>{% endif %}
 
-				<a href="{{ rel.url }}">{{ rel.title }}</a>,
-				created {{ rel.releaseDate }}.
-				{% if not rel.approved %}
-					Waiting for approval.
-				{% endif %}
-
-				{% if package.checkPerm(current_user, "MAKE_RELEASE") or package.checkPerm(current_user, "APPROVE_RELEASE")  %}
-					<a href="{{ rel.getEditURL() }}">Edit
-					{% if not rel.approved and package.checkPerm(current_user, "APPROVE_RELEASE") %}
-						/ Approve
+					<a href="{{ rel.url }}">{{ rel.title }}</a>,
+					created {{ rel.releaseDate }}.
+					{% if rel.task_id %}
+						<a href="{{ url_for('check_task', id=rel.task_id, r=package.getDetailsURL()) }}">Importing</a>
+					{% elif not rel.approved %}
+						Waiting for approval.
 					{% endif %}
-					</a>
-				{% endif %}
 
-				{% if not rel.approved %}</i>{% endif %}
-			</li>
+					{% if not rel.task_id and (package.checkPerm(current_user, "MAKE_RELEASE") or package.checkPerm(current_user, "APPROVE_RELEASE"))  %}
+						<a href="{{ rel.getEditURL() }}">Edit
+						{% if not rel.approved and package.checkPerm(current_user, "APPROVE_RELEASE") %}
+							/ Approve
+						{% endif %}
+						</a>
+					{% endif %}
+
+					{% if not rel.approved %}</i>{% endif %}
+				</li>
+			{% endif %}
 		{% else %}
 			<li>No releases available.</li>
 		{% endfor %}
diff --git a/app/templates/tasks/view.html b/app/templates/tasks/view.html
new file mode 100644
index 0000000..a01ca80
--- /dev/null
+++ b/app/templates/tasks/view.html
@@ -0,0 +1,21 @@
+{% extends "base.html" %}
+
+{% block title %}
+Working
+{% endblock %}
+
+{% block headextra %}
+	{% if not "error" in info %}
+		<meta http-equiv="refresh" content="1;URL=">
+	{% endif %}
+{% endblock %}
+
+{% block content %}
+	{% if "error" in info %}
+		<h1>Task Failed</h1>
+
+		<p>{{ info. error }}</p>
+	{% else %}
+		<h1>Working…</h1>
+	{% endif %}
+{% endblock %}
diff --git a/app/views/__init__.py b/app/views/__init__.py
index 51f4908..cd4f0e6 100644
--- a/app/views/__init__.py
+++ b/app/views/__init__.py
@@ -30,4 +30,4 @@ def home_page():
 	packages = Package.query.filter_by(approved=True).all()
 	return render_template("index.html", packages=packages)
 
-from . import users, githublogin, packages, sass, api
+from . import users, githublogin, packages, sass, tasks
diff --git a/app/views/api.py b/app/views/api.py
deleted file mode 100644
index 277b94e..0000000
--- a/app/views/api.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from flask import *
-from flask_user import *
-from flask.ext import menu
-from app import app
-from app.models import *
-from app.tasks import celery
-from app.tasks.importtasks import getMeta
-# from celery.result import AsyncResult
-
-from .utils import *
-
-@app.route("/tasks/getmeta/new/")
-def new_getmeta_page():
-	aresult = getMeta.delay(request.args.get("url"))
-	return jsonify({
-		"poll_url": url_for("check_task", id=aresult.id),
-	})
-
-@app.route("/tasks/<id>/")
-def check_task(id):
-    result = celery.AsyncResult(id)
-    status = result.status
-    traceback = result.traceback
-    result = result.result
-    if isinstance(result, Exception):
-        return jsonify({
-            'status': status,
-            'error': str(result),
-            # 'traceback': traceback,
-        })
-    else:
-        return jsonify({
-            'status': status,
-            'result': result,
-        })
diff --git a/app/views/packages.py b/app/views/packages.py
index 1f9d570..c39cc8e 100644
--- a/app/views/packages.py
+++ b/app/views/packages.py
@@ -3,6 +3,7 @@ from flask_user import *
 from flask.ext import menu
 from app import app
 from app.models import *
+from app.tasks.importtasks import makeVCSRelease
 
 from .utils import *
 
@@ -369,16 +370,17 @@ def create_release_page(type, author, name):
 	# Initial form class from post data and default data
 	form = CreatePackageReleaseForm()
 	if request.method == "POST" and form.validate():
-		for key, value in request.files.items() :
-			print (key, value)
 		if form["uploadOpt"].data == "vcs":
 			rel = PackageRelease()
 			rel.package = package
-			rel.title = form["title"].data
-			rel.url = form["vcsLabel"].data
-			# TODO: get URL to commit from branch name
+			rel.title   = form["title"].data
+			rel.url     = ""
 			db.session.commit()
-			return redirect(package.getDetailsURL())
+
+			rel.task_id = makeVCSRelease.delay(rel.id, form["vcsLabel"].data).id
+			db.session.commit()
+
+			return redirect(url_for("check_task", id=rel.task_id, r=package.getDetailsURL()))
 		else:
 			uploadedPath = doFileUpload(form.fileUpload.data, ["zip"], "a zip file")
 			if uploadedPath is not None:
@@ -412,6 +414,9 @@ def edit_release_page(type, author, name, id):
 	if not (canEdit or canApprove):
 		return redirect(package.getDetailsURL())
 
+	if release.task_id is not None:
+		return redirect(url_for("check_task", id=release.task_id, r=release.getEditURL()))
+
 	# Initial form class from post data and default data
 	form = EditPackageReleaseForm(formdata=request.form, obj=release)
 	if request.method == "POST" and form.validate():
diff --git a/app/views/tasks.py b/app/views/tasks.py
new file mode 100644
index 0000000..b82c326
--- /dev/null
+++ b/app/views/tasks.py
@@ -0,0 +1,50 @@
+from flask import *
+from flask_user import *
+from flask.ext import menu
+from app import app
+from app.models import *
+from app.tasks import celery
+from app.tasks.importtasks import getMeta
+from .utils import shouldReturnJson
+# from celery.result import AsyncResult
+
+from .utils import *
+
+@app.route("/tasks/getmeta/new/")
+def new_getmeta_page():
+	aresult = getMeta.delay(request.args.get("url"))
+	return jsonify({
+		"poll_url": url_for("check_task", id=aresult.id),
+	})
+
+@app.route("/tasks/<id>/")
+def check_task(id):
+	result = celery.AsyncResult(id)
+	status = result.status
+	traceback = result.traceback
+	result = result.result
+
+	info = None
+	if isinstance(result, Exception):
+		info = {
+				'status': status,
+				'error': str(result),
+			}
+	else:
+		info = {
+				'status': status,
+				'result': result,
+			}
+
+	if shouldReturnJson():
+		return jsonify(info)
+	else:
+		r = request.args.get("r")
+		if r is None:
+			abort(422)
+
+		if status == "SUCCESS":
+			flash("Task complete!", "success")
+			return redirect(r)
+		else:
+			return render_template("tasks/view.html", info=info)