LibreGaming
/
contentdb
14
0
Fork 0
Code Issues 13 Pull Requests Projects Releases Wiki Activity

Add support for zip uploads in the API 

Fixes #261
This commit is contained in:
rubenwardy 2021-02-02 00:07:41 +00:00
parent a78fe8ceb9
commit 033f40c263
13 changed files with 256 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
app/blueprints/api/endpoints.py
View File

@ -19,7 +19,7 @@ from flask import *
from flask_login import current_user, login_required
from . import bp
from .auth import is_api_authd
from .support import error, handleCreateRelease
from .support import error, api_create_vcs_release, api_create_zip_release
from app import csrf
from app.models import *
from app.utils import is_package_page
@ -210,15 +210,23 @@ def create_release(token, package):
if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
error(403, "You do not have the permission to approve releases")
json = request.json
if json is None:
error(400, "JSON post data is required")
data = request.json or request.form
if "title" not in data:
error(400, "Title is required in the POST data")
for option in ["method", "title", "ref"]:
if json.get(option) is None:
error(400, option + " is required in the POST data")
if request.json:
for option in ["method", "ref"]:
if option not in data:
error(400, option + " is required in the POST data")
if json["method"].lower() != "git":
error(400, "Release-creation methods other than git are not supported")
if data["method"].lower() != "git":
error(400, "Release-creation methods other than git are not supported")
return handleCreateRelease(token, package, json["title"], json["ref"])
return api_create_vcs_release(token, package, data["title"], data["ref"])
elif request.files:
file = request.files.get("file")
if file is None:
error(400, "Missing 'file' in multipart body")
return api_create_zip_release(token, package, data["title"], file)

77
app/blueprints/api/support.py
View File

@ -1,44 +1,55 @@
import datetime
# ContentDB
# Copyright (C) 2018-21 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from celery import uuid
from flask import jsonify, abort, make_response, url_for
from app.models import PackageRelease, db, Permission
from app.tasks.importtasks import makeVCSRelease
from app.utils import AuditSeverity, addAuditLog
from app.logic.releases import LogicError, do_create_vcs_release, do_create_zip_release
from app.models import APIToken, Package, MinetestRelease
def error(status, message):
abort(make_response(jsonify({ "success": False, "error": message }), status))
def error(code: int, msg: str):
abort(make_response(jsonify({ "success": False, "error": msg }), code))
# Catches LogicErrors and aborts with JSON error
def run_safe(f, *args, **kwargs):
try:
return f(*args, **kwargs)
except LogicError as e:
error(e.code, e.message)
def handleCreateRelease(token, package, title, ref, reason="API"):
def api_create_vcs_release(token: APIToken, package: Package, title: str, ref: str,
min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API"):
if not token.canOperateOnPackage(package):
return error(403, "API token does not have access to the package")
error(403, "API token does not have access to the package")
if not package.checkPerm(token.owner, Permission.MAKE_RELEASE):
return error(403, "Permission denied. Missing MAKE_RELEASE permission")
five_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=5)
count = package.releases.filter(PackageRelease.releaseDate > five_minutes_ago).count()
if count >= 2:
return error(429, "Too many requests, please wait before trying again")
rel = PackageRelease()
rel.package = package
rel.title = title
rel.url = ""
rel.task_id = uuid()
rel.min_rel = None
rel.max_rel = None
db.session.add(rel)
msg = "Created release {} ({})".format(rel.title, reason)
addAuditLog(AuditSeverity.NORMAL, token.owner, msg, package.getDetailsURL(), package)
db.session.commit()
makeVCSRelease.apply_async((rel.id, ref), task_id=rel.task_id)
rel = run_safe(do_create_vcs_release, token.owner, package, title, ref, None, None, reason)
return jsonify({
"success": True,
"task": url_for("tasks.check", id=rel.task_id),
"release": rel.getAsDictionary()
})
def api_create_zip_release(token: APIToken, package: Package, title: str, file, reason="API"):
if not token.canOperateOnPackage(package):
error(403, "API token does not have access to the package")
rel = run_safe(do_create_zip_release, token.owner, package, title, file, None, None, reason)
return jsonify({
"success": True,

4
app/blueprints/github/__init__.py
View File

@ -24,7 +24,7 @@ from sqlalchemy import func, or_, and_
from app import github, csrf
from app.models import db, User, APIToken, Package, Permission, AuditSeverity
from app.utils import abs_url_for, addAuditLog, login_user_set_active
from app.blueprints.api.support import error, handleCreateRelease
from app.blueprints.api.support import error, api_create_vcs_release
import hmac, requests
@bp.route("/github/start/")
@ -146,4 +146,4 @@ def webhook():
# Perform release
#
return handleCreateRelease(actual_token, package, title, ref, reason="Webhook")
return api_create_vcs_release(actual_token, package, title, ref, reason="Webhook")

4
app/blueprints/gitlab/__init__.py
View File

@ -20,7 +20,7 @@ bp = Blueprint("gitlab", __name__)
from app import csrf
from app.models import Package, APIToken, Permission
from app.blueprints.api.support import error, handleCreateRelease
from app.blueprints.api.support import error, api_create_vcs_release
def webhook_impl():
@ -63,7 +63,7 @@ def webhook_impl():
# Perform release
#
return handleCreateRelease(token, package, title, ref, reason="Webhook")
return api_create_vcs_release(token, package, title, ref, reason="Webhook")
@bp.route("/gitlab/webhook/", methods=["POST"])

54
app/blueprints/packages/releases.py
View File

@ -15,7 +15,6 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from celery import uuid
from flask import *
from flask_login import login_required
from flask_wtf import FlaskForm
@ -23,8 +22,9 @@ from wtforms import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField
from wtforms.validators import *
from app.logic.releases import do_create_vcs_release, LogicError, do_create_zip_release
from app.rediscache import has_key, set_key, make_download_key
from app.tasks.importtasks import makeVCSRelease, checkZipRelease, check_update_config
from app.tasks.importtasks import check_update_config
from app.utils import *
from . import bp
@ -80,48 +80,16 @@ def create_release(package):
form.title.data = request.args.get("title")
if form.validate_on_submit():
if form["uploadOpt"].data == "vcs":
rel = PackageRelease()
rel.package = package
rel.title = form["title"].data
rel.url = ""
rel.task_id = uuid()
rel.min_rel = form["min_rel"].data.getActual()
rel.max_rel = form["max_rel"].data.getActual()
db.session.add(rel)
db.session.commit()
makeVCSRelease.apply_async((rel.id, nonEmptyOrNone(form.vcsLabel.data)), task_id=rel.task_id)
msg = "Created release {}".format(rel.title)
addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT, msg, rel.getEditURL(), package)
addAuditLog(AuditSeverity.NORMAL, current_user, msg, package.getDetailsURL(), package)
db.session.commit()
try:
if form["uploadOpt"].data == "vcs":
rel = do_create_vcs_release(current_user, package, form.title.data,
form.vcsLabel.data, form.min_rel.data.getActual(), form.max_rel.data.getActual())
else:
rel = do_create_zip_release(current_user, package, form.title.data,
form.fileUpload.data, form.min_rel.data.getActual(), form.max_rel.data.getActual())
return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))
else:
uploadedUrl, uploadedPath = doFileUpload(form.fileUpload.data, "zip", "a zip file")
if uploadedUrl is not None:
rel = PackageRelease()
rel.package = package
rel.title = form["title"].data
rel.url = uploadedUrl
rel.task_id = uuid()
rel.min_rel = form["min_rel"].data.getActual()
rel.max_rel = form["max_rel"].data.getActual()
db.session.add(rel)
db.session.commit()
checkZipRelease.apply_async((rel.id, uploadedPath), task_id=rel.task_id)
msg = "Created release {}".format(rel.title)
addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT,
msg, rel.getEditURL(), package)
addAuditLog(AuditSeverity.NORMAL, current_user, msg, package.getDetailsURL(),
package)
db.session.commit()
return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))
except LogicError as e:
flash(e.message, "danger")
return render_template("packages/release_new.html", package=package, form=form)

26
app/blueprints/packages/screenshots.py
View File

@ -24,6 +24,8 @@ from wtforms.validators import *
from app.utils import *
from . import bp
from app.logic.LogicError import LogicError
from app.logic.screenshots import do_create_screenshot
class CreateScreenshotForm(FlaskForm):
@ -88,27 +90,11 @@ def create_screenshot(package):
# Initial form class from post data and default data
form = CreateScreenshotForm()
if form.validate_on_submit():
uploadedUrl, uploadedPath = doFileUpload(form.fileUpload.data, "image",
"a PNG or JPG image file")
if uploadedUrl is not None:
counter = 1
for screenshot in package.screenshots:
screenshot.order = counter
counter += 1
ss = PackageScreenshot()
ss.package = package
ss.title = form["title"].data or "Untitled"
ss.url = uploadedUrl
ss.approved = package.checkPerm(current_user, Permission.APPROVE_SCREENSHOT)
ss.order = counter
db.session.add(ss)
msg = "Screenshot added {}" \
.format(ss.title)
addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT, msg, package.getDetailsURL(), package)
db.session.commit()
try:
do_create_screenshot(current_user, package, form.title.data, form.fileUpload.data)
return redirect(package.getEditScreenshotsURL())
except LogicError as e:
flash(e.message, "danger")
return render_template("packages/screenshot_new.html", package=package, form=form)

9
app/flatpages/help/api.md
View File

@ -46,10 +46,13 @@ Tokens can be attained by visiting [Profile > "API Tokens"](/user/tokens/).
* GET `/api/packages/<username>/<name>/releases/`
* POST `/api/packages/<username>/<name>/releases/new/`
* Requires authentication.
* Body is multipart form if zip upload, JSON otherwise.
* `title`: human-readable name of the release.
* `method`: Release-creation method, only `git` is supported.
* If `git` release-creation method:
* `ref` - git reference, eg: `master`.
* For Git release creation:
* `method`: must be `git`.
* `ref`: (Optional) git reference, eg: `master`.
* For zip upload release creation:
* `file`: multipart file to upload, like `<input type=file>`.
* You can set min and max Minetest Versions [using the content's .conf file](/help/package_config/).

24
app/logic/LogicError.py Normal file
View File

@ -0,0 +1,24 @@
# ContentDB
# Copyright (C) 2021 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
class LogicError(Exception):
def __init__(self, code, message):
self.code = code
self.message = message
def __str__(self):
return repr("LogicError {}: {}".format(self.code, self.message))

0
app/logic/__init__.py Normal file
View File

90
app/logic/releases.py Normal file
View File

@ -0,0 +1,90 @@
# ContentDB
# Copyright (C) 2018-21 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import datetime
from celery import uuid
from app.logic.LogicError import LogicError
from app.logic.uploads import upload_file
from app.models import PackageRelease, db, Permission, User, Package, MinetestRelease
from app.tasks.importtasks import makeVCSRelease, checkZipRelease
from app.utils import AuditSeverity, addAuditLog, nonEmptyOrNone
def check_can_create_release(user: User, package: Package):
if not package.checkPerm(user, Permission.MAKE_RELEASE):
raise LogicError(403, "Permission denied. Missing MAKE_RELEASE permission")
five_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=5)
count = package.releases.filter(PackageRelease.releaseDate > five_minutes_ago).count()
if count >= 2:
raise LogicError(429, "Too many requests, please wait before trying again")
def do_create_vcs_release(user: User, package: Package, title: str, ref: str,
min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason: str = None):
check_can_create_release(user, package)
rel = PackageRelease()
rel.package = package
rel.title = title
rel.url = ""
rel.task_id = uuid()
rel.min_rel = min_v
rel.max_rel = max_v
db.session.add(rel)
if reason is None:
msg = "Created release {}".format(rel.title)
else:
msg = "Created release {} ({})".format(rel.title, reason)
addAuditLog(AuditSeverity.NORMAL, user, msg, package.getDetailsURL(), package)
db.session.commit()
makeVCSRelease.apply_async((rel.id, nonEmptyOrNone(ref)), task_id=rel.task_id)
return rel
def do_create_zip_release(user: User, package: Package, title: str, file,
min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason: str = None):
check_can_create_release(user, package)
uploaded_url, uploaded_path = upload_file(file, "zip", "a zip file")
rel = PackageRelease()
rel.package = package
rel.title = title
rel.url = uploaded_url
rel.task_id = uuid()
rel.min_rel = min_v
rel.max_rel = max_v
db.session.add(rel)
if reason is None:
msg = "Created release {}".format(rel.title)
else:
msg = "Created release {} ({})".format(rel.title, reason)
addAuditLog(AuditSeverity.NORMAL, user, msg, package.getDetailsURL(), package)
db.session.commit()
checkZipRelease.apply_async((rel.id, uploaded_path), task_id=rel.task_id)
return rel

29
app/logic/screenshots.py Normal file
View File

@ -0,0 +1,29 @@
from werkzeug.exceptions import abort
from app.logic.uploads import upload_file
from app.models import User, Package, PackageScreenshot, Permission, NotificationType, db
from app.utils import addNotification
def do_create_screenshot(user: User, package: Package, title: str, file):
uploaded_url, uploaded_path = upload_file(file, "image", "a PNG or JPG image file")
counter = 1
for screenshot in package.screenshots:
screenshot.order = counter
counter += 1
ss = PackageScreenshot()
ss.package = package
ss.title = title or "Untitled"
ss.url = uploaded_url
ss.approved = package.checkPerm(user, Permission.APPROVE_SCREENSHOT)
ss.order = counter
db.session.add(ss)
msg = "Screenshot added {}" \
.format(ss.title)
addNotification(package.maintainers, user, NotificationType.PACKAGE_EDIT, msg, package.getDetailsURL(), package)
db.session.commit()
return ss

31
app/utils/uploads.py → app/logic/uploads.py
View File

@ -17,37 +17,25 @@
import imghdr
import os
import random
import string
from flask import request, flash
from app.logic.LogicError import LogicError
from app.models import *
from app.utils import randomString
def getExtension(filename):
def get_extension(filename):
return filename.rsplit(".", 1)[1].lower() if "." in filename else None
ALLOWED_IMAGES = {"jpeg", "png"}
def isAllowedImage(data):
return imghdr.what(None, data) in ALLOWED_IMAGES
def shouldReturnJson():
return "application/json" in request.accept_mimetypes and \
not "text/html" in request.accept_mimetypes
def randomString(n):
return ''.join(random.choice(string.ascii_lowercase + \
string.ascii_uppercase + string.digits) for _ in range(n))
def doFileUpload(file, fileType, fileTypeDesc):
def upload_file(file, fileType, fileTypeDesc):
if not file or file is None or file.filename == "":
flash("No selected file", "danger")
return None, None
raise LogicError(400, "Expected file")
assert os.path.isdir(app.config["UPLOAD_DIR"]), "UPLOAD_DIR must exist"
allowedExtensions = []
isImage = False
if fileType == "image":
allowedExtensions = ["jpg", "jpeg", "png"]
@ -57,18 +45,17 @@ def doFileUpload(file, fileType, fileTypeDesc):
else:
raise Exception("Invalid fileType")
ext = getExtension(file.filename)
ext = get_extension(file.filename)
if ext is None or not ext in allowedExtensions:
flash("Please upload " + fileTypeDesc, "danger")
return None, None
raise LogicError(400, "Please upload " + fileTypeDesc)
if isImage and not isAllowedImage(file.stream.read()):
flash("Uploaded image isn't actually an image", "danger")
return None, None
raise LogicError(400, "Uploaded image isn't actually an image")
file.stream.seek(0)
filename = randomString(10) + "." + ext
filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
file.save(filepath)
return "/uploads/" + filename, filepath

17
app/utils/__init__.py
View File

@ -13,24 +13,37 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import random
import string
from .flask import *
from .uploads import *
from .models import *
from .user import *
YESES = ["yes", "true", "1", "on"]
def isYes(val):
return val and val.lower() in YESES
def isNo(val):
return val and not isYes(val)
def nonEmptyOrNone(str):
if str is None or str == "":
return None
return str
def shouldReturnJson():
return "application/json" in request.accept_mimetypes and \
not "text/html" in request.accept_mimetypes
def randomString(n):
return ''.join(random.choice(string.ascii_lowercase + \
string.ascii_uppercase + string.digits) for _ in range(n))