From 5e44f3d64c2be1b890e393832efd9fb100adf2c0 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Fri, 11 May 2018 12:57:16 +0100 Subject: [PATCH] Move Github import to backend Fixes #41 --- .gitignore | 2 + app/__init__.py | 2 +- app/static/package_create.js | 128 +++++++++++++++-------------------- app/tasks/__init__.py | 44 ++++++++++++ app/tasks/importtasks.py | 68 +++++++++++++++++++ app/views/__init__.py | 2 +- app/views/api.py | 35 ++++++++++ log.txt | 55 --------------- requirements.txt | 2 + 9 files changed, 209 insertions(+), 129 deletions(-) create mode 100644 app/tasks/__init__.py create mode 100644 app/tasks/importtasks.py create mode 100644 app/views/api.py delete mode 100644 log.txt diff --git a/.gitignore b/.gitignore index b6243fb..011cb94 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ config.prod.cfg *.sqlite main.css tmp +log.txt +*.rdb # Created by https://www.gitignore.io/api/linux,macos,python,windows diff --git a/app/__init__.py b/app/__init__.py index b5aa36c..8e989c4 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -12,5 +12,5 @@ menu.Menu(app=app) markdown.Markdown(app, extensions=["fenced_code"], safe_mode=True, output_format="html5") github = GitHub(app) -from . import models +from . import models, tasks from .views import * diff --git a/app/static/package_create.js b/app/static/package_create.js index 0bea7ca..25cdd3e 100644 --- a/app/static/package_create.js +++ b/app/static/package_create.js @@ -1,20 +1,4 @@ $(function() { - function readConfig(text) { - var retval = {} - - const lines = text.split("\n") - for (var i = 0; i < lines.length; i++) { - const idx = lines[i].indexOf("=") - if (idx > 0) { - const name = lines[i].substring(0, idx - 1).trim() - const value = lines[i].substring(idx + 1).trim() - retval[name] = value - } - } - - return retval - } - function finish() { $(".pkg_wiz_1").hide() $(".pkg_wiz_2").hide() @@ -22,6 +6,49 @@ $(function() { $(".pkg_meta").show() } + function getJSON(url) { + return new Promise(function(resolve, reject) { + fetch(url).then(function(response) { + response.text().then(function(txt) { + resolve(JSON.parse(txt)) + }).catch(reject) + }).catch(reject) + }) + } + + function performTask(url) { + return new Promise(function(resolve, reject) { + getJSON(url).then(function(startResult) { + console.log(startResult) + if (typeof startResult.poll_url == "string") { + var tries = 0; + function retry() { + tries++; + if (tries > 10) { + reject("timeout") + } else { + console.log("Polling task in " + (tries*100) + "ms") + setTimeout(step, tries*100) + } + } + function step() { + getJSON(startResult.poll_url).then(function(res) { + if (res.status == "SUCCESS") { + console.log("Got result") + resolve(res.result) + } else { + retry() + } + }).catch(retry) + } + retry() + } else { + reject("Start task didn't return string!") + } + }).catch(reject) + }) + } + function repoIsSupported(url) { try { return URI(url).hostname() == "github.com" @@ -30,60 +57,6 @@ $(function() { } } - function getFile(url) { - return new Promise(function(resolve, reject) { - fetch(url).then(function(response) { - response.text().then(resolve).catch(reject) - }).catch(reject) - }) - } - - function getInfo(baseUrl) { - return new Promise(function(resolve, reject) { - getFile(baseUrl + "/mod.conf").then(function(text) { - var config = readConfig(text) - - if (config["name"]) { - $("#name").val(config["name"]) - } - - if (config["description"]) { - const desc = config["description"] - const idx = desc.indexOf(".") - $("#shortDesc").val((idx < 5 || idx > 100) ? desc.substring(0, 100) : desc.substring(0, idx)) - $("#desc").val(desc) - } - - resolve() - }).catch(function() { - reject() - }) - }) - } - - function importInfo(urlstr) { - // Convert to HTTPs - try { - var url = URI(urlstr).scheme("https") - .username("") - .password("") - } catch(e) { - return Promise.reject(e) - } - // Change domain - url = url.hostname("raw.githubusercontent.com") - - // Rewrite path - const re = /^\/([^\/]+)\/([^\/]+)\/?$/ - const results = re.exec(url.path()) - if (results == null || results.length != 3) { - return Promise.reject("Unable to parse URL - please provide a direct URL to the repo") - } - url.path("/" + results[1] + "/" + results[2].replace(".git", "") + "/master") - - return getInfo(url.toString()) - } - $(".pkg_meta").hide() $(".pkg_wiz_1").show() $("#pkg_wiz_1_next").click(function() { @@ -93,11 +66,22 @@ $(function() { $(".pkg_wiz_2").show() $(".pkg_repo").hide() - importInfo(repoURL).then(finish).catch(function(x) { - alert(x) + performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) { + console.log(result) + $("#name").val(result.name) + const desc = result.description || "" + if (desc.length > 0) { + const idx = desc.indexOf(".") + $("#shortDesc").val((idx < 5 || idx > 100) ? desc.substring(0, Math.min(desc.length, 100)) : desc.substring(0, idx)) + $("#desc").val(desc) + } + finish() + }).catch(function(e) { + alert(e) $(".pkg_wiz_1").show() $(".pkg_wiz_2").hide() $(".pkg_repo").show() + // finish() }) } else { finish() diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py new file mode 100644 index 0000000..c431fae --- /dev/null +++ b/app/tasks/__init__.py @@ -0,0 +1,44 @@ +import flask +from flask.ext.sqlalchemy import SQLAlchemy +from celery import Celery +from app import app +from app.models import * + +class FlaskCelery(Celery): + def __init__(self, *args, **kwargs): + super(FlaskCelery, self).__init__(*args, **kwargs) + self.patch_task() + + if 'app' in kwargs: + self.init_app(kwargs['app']) + + def patch_task(self): + TaskBase = self.Task + _celery = self + + class ContextTask(TaskBase): + abstract = True + + def __call__(self, *args, **kwargs): + if flask.has_app_context(): + return TaskBase.__call__(self, *args, **kwargs) + else: + with _celery.app.app_context(): + return TaskBase.__call__(self, *args, **kwargs) + + self.Task = ContextTask + + def init_app(self, app): + self.app = app + self.config_from_object(app.config) + +def make_celery(app): + celery = FlaskCelery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'], + broker=app.config['CELERY_BROKER_URL']) + + celery.init_app(app) + return celery + +celery = make_celery(app) + +from . import importtasks diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py new file mode 100644 index 0000000..50b5f5d --- /dev/null +++ b/app/tasks/importtasks.py @@ -0,0 +1,68 @@ +import flask +from flask.ext.sqlalchemy import SQLAlchemy +import urllib.request +from urllib.parse import urlparse + +from app import app +from app.models import * +from app.tasks import celery + +class GithubURLMaker: + def __init__(self, url): + # Rewrite path + import re + m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path) + if m is None: + return + + user = m.group(1) + repo = m.group(2) + self.baseUrl = "https://raw.githubusercontent.com/" + user + "/" + repo.replace(".git", "") + "/master" + + def isValid(self): + return self.baseUrl is not None + + def getModConfURL(self): + return self.baseUrl + "/mod.conf" + +def parseConf(string): + retval = {} + for line in string.split("\n"): + idx = line.find("=") + if idx > 0: + key = line[:idx-1].strip() + value = line[idx+1:].strip() + retval[key] = value + + return retval + +@celery.task() +def getMeta(urlstr): + url = urlparse(urlstr) + + urlmaker = None + if url.netloc == "github.com": + urlmaker = GithubURLMaker(url) + + if not urlmaker.isValid(): + print("Error! Url maker not valid") + return + + print(urlmaker.getModConfURL()) + + result = {} + + try: + contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") + conf = parseConf(contents) + for key in ["name", "description"]: + try: + result[key] = conf[key] + except KeyError: + pass + + print(conf) + except OSError: + print("mod.conf does not exist") + + return result diff --git a/app/views/__init__.py b/app/views/__init__.py index 17b9f00..51f4908 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -30,4 +30,4 @@ def home_page(): packages = Package.query.filter_by(approved=True).all() return render_template("index.html", packages=packages) -from . import users, githublogin, packages, sass +from . import users, githublogin, packages, sass, api diff --git a/app/views/api.py b/app/views/api.py new file mode 100644 index 0000000..277b94e --- /dev/null +++ b/app/views/api.py @@ -0,0 +1,35 @@ +from flask import * +from flask_user import * +from flask.ext import menu +from app import app +from app.models import * +from app.tasks import celery +from app.tasks.importtasks import getMeta +# from celery.result import AsyncResult + +from .utils import * + +@app.route("/tasks/getmeta/new/") +def new_getmeta_page(): + aresult = getMeta.delay(request.args.get("url")) + return jsonify({ + "poll_url": url_for("check_task", id=aresult.id), + }) + +@app.route("/tasks//") +def check_task(id): + result = celery.AsyncResult(id) + status = result.status + traceback = result.traceback + result = result.result + if isinstance(result, Exception): + return jsonify({ + 'status': status, + 'error': str(result), + # 'traceback': traceback, + }) + else: + return jsonify({ + 'status': status, + 'result': result, + }) diff --git a/log.txt b/log.txt deleted file mode 100644 index b10c570..0000000 --- a/log.txt +++ /dev/null @@ -1,55 +0,0 @@ -69efdd7 Add user rank changing -f51224a Add user list -4898b69 Fix script injection using markdown -c9073a8 Update README -287c9e5 Fix suggest changes link -cd77ad6 Use bash script to start server -6be5b3e Add registering using Github -bb9d589 Add EditRequest approval and rejection -a5042a9 Add EditRequest view page -dcfd2b0 Add EditRequest creation -269c8c0 Add packages API -2a836c1 Add dummy Package.getMainScreenshotURL() function -73a79d5 Add file upload for releases -5a9fc51 Add download button and URL -570dd51 Move package templates to subfolder -811e830 added me -8ff5315 Remove access to repos from Github scope -0385590 Use consistent quotes -9ddc29e Fix work queue permissions check -517b8c9 Fix link in README.md -dc31a98 Add list of packages to profile -87d7b14 Add packages page to list all types -4e870bd Add basic search to package list -8a8b0e5 Improve permission checking in work queue -7169170 Improve empty text on work queue -12d58d3 Add more information to approval view -e5de870 Add work queue -aed805d Add new package approval -49a2a91 Add package validation -a8edae1 Add package creation -5cc49f2 Add package release editing and approving -32ac602 Add redirect to full user URL -bbb46ce Fix incomplete datetime being stored in releases, order releases by date -d039474 Add create release page -596f725 Add package releases -623ca3d Simplify PackageType -363f9d8 Add not joined rank -73f24ad Add permission validation to Package.checkPerm() -bd58f9b Clean up permissions code -0fae3a6 Fix bug with PackageType to name form -9fc71a5 User profile: fix typo, add rank -dad980c Add suggest changes button -775850b Implement permissions properly -5a3764f Add edit package -7c628ca Add example info -d3484d9 Add more details to package page -d17535f Fix menu order -07a9b79 Check type and author in package details -bc88027 Add package types -ae60058 Rename mod to package, add README -358fc4e Add package list and package view -84f123a Fix profile page -7d20c49 Add Github login -7f4faf2 Update dependencies -366a230 Initial commit diff --git a/requirements.txt b/requirements.txt index 9bbd7b2..903f984 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,5 @@ Flask-Menu>=0.7.0 Flask-Markdown>=0.3 GitHub-Flask>=3.2.0 pyScss==1.3.4 +celery==4.0.2 +redis==2.10.6