LibreGaming
/
contentdb
14
0
Fork 0
Code Issues 13 Pull Requests Projects Releases Wiki Activity

Clean up code

This commit is contained in:
rubenwardy 2020-12-04 02:23:04 +00:00
parent 0c0d3e1715
commit 42f96618e2
62 changed files with 254 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/__init__.py
View File

@ -78,9 +78,9 @@ def send_upload(path):
@menu.register_menu(app, ".help", "Help", order=19, endpoint_arguments_constructor=lambda: { 'path': 'help' }) @menu.register_menu(app, ".help", "Help", order=19, endpoint_arguments_constructor=lambda: { 'path': 'help' })
@app.route('/<path:path>/') @app.route('/<path:path>/')
def flatpage(path): def flatpage(path):
page = pages.get_or_404(path) page = pages.get_or_404(path)
template = page.meta.get('template', 'flatpage.html') template = page.meta.get('template', 'flatpage.html')
return render_template(template, page=page) return render_template(template, page=page)
@app.before_request @app.before_request
def check_for_ban(): def check_for_ban():

18
app/blueprints/admin/admin.py
View File

@ -15,18 +15,20 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
import os
from celery import group
from flask import * from flask import *
from flask_user import * from flask_user import *
import flask_menu as menu
from . import bp
from app.models import *
from celery import uuid, group
from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease, checkZipRelease, updateMetaFromRelease, importForeignDownloads
from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
from app.utils import loginUser, rank_required, addNotification
import datetime, os from app.models import *
from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, updateMetaFromRelease, importForeignDownloads
from app.utils import loginUser, rank_required
from . import bp
@bp.route("/admin/", methods=["GET", "POST"]) @bp.route("/admin/", methods=["GET", "POST"])
@rank_required(UserRank.ADMIN) @rank_required(UserRank.ADMIN)

7
app/blueprints/admin/licenseseditor.py
View File

@ -16,13 +16,14 @@
from flask import * from flask import *
from flask_user import *
from . import bp
from app.models import *
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
from wtforms.validators import * from wtforms.validators import *
from app.models import *
from app.utils import rank_required from app.utils import rank_required
from . import bp
@bp.route("/licenses/") @bp.route("/licenses/")
@rank_required(UserRank.MODERATOR) @rank_required(UserRank.MODERATOR)

7
app/blueprints/admin/tagseditor.py
View File

@ -17,12 +17,13 @@
from flask import * from flask import *
from flask_user import * from flask_user import *
from . import bp
from app.models import *
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
from wtforms.validators import * from wtforms.validators import *
from app.utils import rank_required
from app.models import *
from . import bp
@bp.route("/tags/") @bp.route("/tags/")
@login_required @login_required

7
app/blueprints/admin/versioneditor.py
View File

@ -16,13 +16,14 @@
from flask import * from flask import *
from flask_user import *
from . import bp
from app.models import *
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
from wtforms.validators import * from wtforms.validators import *
from app.models import *
from app.utils import rank_required from app.utils import rank_required
from . import bp
@bp.route("/versions/") @bp.route("/versions/")
@rank_required(UserRank.MODERATOR) @rank_required(UserRank.MODERATOR)

7
app/blueprints/admin/warningseditor.py
View File

@ -16,13 +16,14 @@
from flask import * from flask import *
from flask_user import *
from . import bp
from app.models import *
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
from wtforms.validators import * from wtforms.validators import *
from app.models import *
from app.utils import rank_required from app.utils import rank_required
from . import bp
@bp.route("/admin/warnings/") @bp.route("/admin/warnings/")
@rank_required(UserRank.ADMIN) @rank_required(UserRank.ADMIN)

7
app/blueprints/api/auth.py
View File

@ -14,10 +14,13 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import request, make_response, jsonify, abort from functools import wraps
from flask import request, abort
from app.models import APIToken from app.models import APIToken
from .support import error from .support import error
from functools import wraps
def is_api_authd(f): def is_api_authd(f):
@wraps(f) @wraps(f)

2
app/blueprints/api/endpoints.py
View File

@ -78,7 +78,7 @@ def resolve_package_deps(out, package, only_hard):
# TODO: resolve most likely candidate # TODO: resolve most likely candidate
else: else:
raise "Malformed dependency" raise Exception("Malformed dependency")
ret.append({ ret.append({
"name": name, "name": name,

14
app/blueprints/api/tokens.py
View File

@ -17,19 +17,19 @@
from flask import render_template, redirect, request, session, url_for, abort from flask import render_template, redirect, request, session, url_for, abort
from flask_user import login_required, current_user from flask_user import login_required, current_user
from . import bp
from app.models import db, User, APIToken, Package, Permission
from app.utils import randomString
from app.querybuilder import QueryBuilder
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
from wtforms.validators import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField from wtforms.ext.sqlalchemy.fields import QuerySelectField
from wtforms.validators import *
from app.models import db, User, APIToken, Package, Permission
from app.utils import randomString
from . import bp
class CreateAPIToken(FlaskForm): class CreateAPIToken(FlaskForm):
name = StringField("Name", [InputRequired(), Length(1, 30)]) name = StringField("Name", [InputRequired(), Length(1, 30)])
package = QuerySelectField("Limit to package", allow_blank=True, \ package = QuerySelectField("Limit to package", allow_blank=True,
get_pk=lambda a: a.id, get_label=lambda a: a.title) get_pk=lambda a: a.id, get_label=lambda a: a.title)
submit = SubmitField("Save") submit = SubmitField("Save")

11
app/blueprints/github/__init__.py
View File

@ -21,7 +21,6 @@ bp = Blueprint("github", __name__)
from flask import redirect, url_for, request, flash, abort, render_template, jsonify, current_app from flask import redirect, url_for, request, flash, abort, render_template, jsonify, current_app
from flask_user import current_user, login_required from flask_user import current_user, login_required
from sqlalchemy import func, or_, and_ from sqlalchemy import func, or_, and_
from flask_github import GitHub
from app import github, csrf from app import github, csrf
from app.models import db, User, APIToken, Package, Permission from app.models import db, User, APIToken, Package, Permission
from app.utils import loginUser, randomString, abs_url_for from app.utils import loginUser, randomString, abs_url_for
@ -188,8 +187,8 @@ def setup_webhook():
return redirect(package.getDetailsURL()) return redirect(package.getDetailsURL())
if current_user.github_access_token is None: if current_user.github_access_token is None:
return github.authorize("write:repo_hook", \ return github.authorize("write:repo_hook",
redirect_uri=abs_url_for("github.callback_webhook", pid=pid)) redirect_uri=abs_url_for("github.callback_webhook", pid=pid))
form = SetupWebhookForm(formdata=request.form) form = SetupWebhookForm(formdata=request.form)
if request.method == "POST" and form.validate(): if request.method == "POST" and form.validate():
@ -203,15 +202,15 @@ def setup_webhook():
if event != "push" and event != "create": if event != "push" and event != "create":
abort(500) abort(500)
if handleMakeWebhook(gh_user, gh_repo, package, \ if handleMakeWebhook(gh_user, gh_repo, package,
current_user.github_access_token, event, token): current_user.github_access_token, event, token):
flash("Successfully created webhook", "success") flash("Successfully created webhook", "success")
return redirect(package.getDetailsURL()) return redirect(package.getDetailsURL())
else: else:
return redirect(url_for("github.setup_webhook", pid=package.id)) return redirect(url_for("github.setup_webhook", pid=package.id))
return render_template("github/setup_webhook.html", \ return render_template("github/setup_webhook.html",
form=form, package=package) form=form, package=package)
def handleMakeWebhook(gh_user, gh_repo, package, oauth, event, token): def handleMakeWebhook(gh_user, gh_repo, package, oauth, event, token):

8
app/blueprints/homepage/__init__.py
View File

@ -11,9 +11,9 @@ from sqlalchemy.sql.expression import func
@menu.register_menu(bp, ".", "Home") @menu.register_menu(bp, ".", "Home")
def home(): def home():
def join(query): def join(query):
return query.options( \ return query.options(
joinedload(Package.license), \ joinedload(Package.license),
joinedload(Package.media_license)) joinedload(Package.media_license))
query = Package.query.filter_by(state=PackageState.APPROVED) query = Package.query.filter_by(state=PackageState.APPROVED)
count = query.count() count = query.count()
@ -37,5 +37,5 @@ def home():
tags = db.session.query(func.count(Tags.c.tag_id), Tag) \ tags = db.session.query(func.count(Tags.c.tag_id), Tag) \
.select_from(Tag).outerjoin(Tags).group_by(Tag.id).order_by(db.asc(Tag.title)).all() .select_from(Tag).outerjoin(Tags).group_by(Tag.id).order_by(db.asc(Tag.title)).all()
return render_template("index.html", count=count, downloads=downloads, tags=tags, \ return render_template("index.html", count=count, downloads=downloads, tags=tags,
new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam, reviews=reviews) new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam, reviews=reviews)

7
app/blueprints/metapackages/__init__.py
View File

@ -19,7 +19,6 @@ from flask import *
bp = Blueprint("metapackages", __name__) bp = Blueprint("metapackages", __name__)
from flask_user import *
from app.models import * from app.models import *
@bp.route("/metapackages/") @bp.route("/metapackages/")
@ -59,6 +58,6 @@ def view(name):
.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \ .order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
.all() .all()
return render_template("metapackages/view.html", mpackage=mpackage, \ return render_template("metapackages/view.html", mpackage=mpackage,
dependers=dependers, optional_dependers=optional_dependers, \ dependers=dependers, optional_dependers=optional_dependers,
similar_topics=similar_topics) similar_topics=similar_topics)

23
app/blueprints/metrics/__init__.py
View File

@ -15,9 +15,10 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import Blueprint, make_response from flask import Blueprint, make_response
from app.models import Package, PackageRelease, db, User, UserRank, PackageState
from sqlalchemy.sql.expression import func from sqlalchemy.sql.expression import func
from app.models import Package, db, User, UserRank, PackageState
bp = Blueprint("metrics", __name__) bp = Blueprint("metrics", __name__)
def generate_metrics(full=False): def generate_metrics(full=False):
@ -28,17 +29,17 @@ def generate_metrics(full=False):
def gen_labels(labels): def gen_labels(labels):
pieces = [key + "=" + str(val) for key, val in labels.items()] pieces = [key + "=" + str(val) for key, val in labels.items()]
return (",").join(pieces) return ",".join(pieces)
def write_array_stat(name, help, type, data): def write_array_stat(name, help, type, data):
ret = ("# HELP {name} {help}\n# TYPE {name} {type}\n") \ ret = "# HELP {name} {help}\n# TYPE {name} {type}\n" \
.format(name=name, help=help, type=type) .format(name=name, help=help, type=type)
for entry in data: for entry in data:
assert(len(entry) == 2) assert(len(entry) == 2)
ret += ("{name}{{{labels}}} {value}\n") \ ret += "{name}{{{labels}}} {value}\n" \
.format(name=name, labels=gen_labels(entry[0]), value=entry[1]) .format(name=name, labels=gen_labels(entry[0]), value=entry[1])
return ret + "\n" return ret + "\n"
@ -57,8 +58,8 @@ def generate_metrics(full=False):
scores = Package.query.join(User).with_entities(User.username, Package.name, Package.score) \ scores = Package.query.join(User).with_entities(User.username, Package.name, Package.score) \
.filter(Package.state==PackageState.APPROVED).all() .filter(Package.state==PackageState.APPROVED).all()
ret += write_array_stat("contentdb_package_score", "Package score", "gauge", \ ret += write_array_stat("contentdb_package_score", "Package score", "gauge",
[({ "author": score[0], "name": score[1] }, score[2]) for score in scores]) [({ "author": score[0], "name": score[1] }, score[2]) for score in scores])
else: else:
score_result = db.session.query(func.sum(Package.score)).one_or_none() score_result = db.session.query(func.sum(Package.score)).one_or_none()
score = 0 if not score_result or not score_result[0] else score_result[0] score = 0 if not score_result or not score_result[0] else score_result[0]
@ -68,6 +69,6 @@ def generate_metrics(full=False):
@bp.route("/metrics") @bp.route("/metrics")
def metrics(): def metrics():
response = make_response(generate_metrics(), 200) response = make_response(generate_metrics(), 200)
response.mimetype = "text/plain" response.mimetype = "text/plain"
return response return response

172
app/blueprints/packages/editrequests.py
View File

@ -1,172 +0,0 @@
# ContentDB
# Copyright (C) 2018 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import *
from flask_user import *
from app import app
from app.models import *
from app.utils import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
from . import PackageForm
class EditRequestForm(PackageForm):
edit_title = StringField("Edit Title", [InputRequired(), Length(1, 100)])
edit_desc = TextField("Edit Description", [Optional()])
@app.route("/packages/<author>/<name>/requests/new/", methods=["GET","POST"])
@app.route("/packages/<author>/<name>/requests/<id>/edit/", methods=["GET","POST"])
@login_required
@is_package_page
def create_edit_editrequest_page(package, id=None):
edited_package = package
erequest = None
if id is not None:
erequest = EditRequest.query.get(id)
if erequest.package != package:
abort(404)
if not erequest.checkPerm(current_user, Permission.EDIT_EDITREQUEST):
abort(403)
if erequest.status != 0:
flash("Can't edit EditRequest, it has already been merged or rejected", "danger")
return redirect(erequest.getURL())
edited_package = Package(package)
erequest.applyAll(edited_package)
form = EditRequestForm(request.form, obj=edited_package)
if request.method == "GET":
deps = edited_package.dependencies
form.harddep_str.data = ",".join([str(x) for x in deps if not x.optional])
form.softdep_str.data = ",".join([str(x) for x in deps if x.optional])
form.provides_str.data = MetaPackage.ListToSpec(edited_package.provides)
if request.method == "POST" and form.validate():
if erequest is None:
erequest = EditRequest()
erequest.package = package
erequest.author = current_user
erequest.title = form["edit_title"].data
erequest.desc = form["edit_desc"].data
db.session.add(erequest)
EditRequestChange.query.filter_by(request=erequest).delete()
wasChangeMade = False
for e in PackagePropertyKey:
newValue = form[e.name].data
oldValue = getattr(package, e.name)
newValueComp = newValue
oldValueComp = oldValue
if type(newValue) is str:
newValue = newValue.replace("\r\n", "\n")
newValueComp = newValue.strip()
oldValueComp = "" if oldValue is None else oldValue.strip()
if newValueComp != oldValueComp:
change = EditRequestChange()
change.request = erequest
change.key = e
change.oldValue = e.convert(oldValue)
change.newValue = e.convert(newValue)
db.session.add(change)
wasChangeMade = True
if wasChangeMade:
msg = "Edit request #{} {}" \
.format(erequest.id, "created" if id is None else "edited")
addNotification(package.maintainers, current_user, msg, erequest.getURL(), package)
addNotification(erequest.author, current_user, msg, erequest.getURL(), package)
db.session.commit()
return redirect(erequest.getURL())
else:
flash("No changes detected", "warning")
elif erequest is not None:
form["edit_title"].data = erequest.title
form["edit_desc"].data = erequest.desc
return render_template("packages/editrequest_create_edit.html", package=package, form=form)
@app.route("/packages/<author>/<name>/requests/<id>/")
@is_package_page
def view_editrequest_page(package, id):
erequest = EditRequest.query.get(id)
if erequest is None or erequest.package != package:
abort(404)
return render_template("packages/editrequest_view.html", package=package, request=erequest)
@app.route("/packages/<author>/<name>/requests/<id>/approve/", methods=["POST"])
@is_package_page
def approve_editrequest_page(package, id):
if not package.checkPerm(current_user, Permission.APPROVE_CHANGES):
flash("You don't have permission to do that.", "danger")
return redirect(package.getDetailsURL())
erequest = EditRequest.query.get(id)
if erequest is None or erequest.package != package:
abort(404)
if erequest.status != 0:
flash("Edit request has already been resolved", "danger")
else:
erequest.status = 1
erequest.applyAll(package)
msg = "Edit request #{} merged".format(erequest.id)
addNotification(erequest.author, current_user, msg, erequest.getURL(), package)
addNotification(package.maintainers, current_user, msg, erequest.getURL(), package)
db.session.commit()
return redirect(package.getDetailsURL())
@app.route("/packages/<author>/<name>/requests/<id>/reject/", methods=["POST"])
@is_package_page
def reject_editrequest_page(package, id):
if not package.checkPerm(current_user, Permission.APPROVE_CHANGES):
flash("You don't have permission to do that.", "danger")
return redirect(package.getDetailsURL())
erequest = EditRequest.query.get(id)
if erequest is None or erequest.package != package:
abort(404)
if erequest.status != 0:
flash("Edit request has already been resolved", "danger")
else:
erequest.status = 2
msg = "Edit request #{} rejected".format(erequest.id)
addNotification(erequest.author, current_user, msg, erequest.getURL(), package)
addNotification(package.maintainers, current_user, msg, erequest.getURL(), package)
db.session.commit()
return redirect(package.getDetailsURL())

60
app/blueprints/packages/packages.py
View File

@ -15,27 +15,23 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import render_template, abort, request, redirect, url_for, flash
from flask_user import current_user
import flask_menu as menu
from . import bp
from app.models import *
from app.querybuilder import QueryBuilder
from app.tasks.importtasks import importRepoScreenshot, updateMetaFromRelease
from app.rediscache import has_key, set_key
from app.utils import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
from sqlalchemy import or_, func
from sqlalchemy.orm import joinedload, subqueryload
from urllib.parse import quote as urlescape from urllib.parse import quote as urlescape
import flask_menu as menu
from celery import uuid from celery import uuid
from flask import render_template
from flask_wtf import FlaskForm
from sqlalchemy import or_, func
from sqlalchemy.orm import joinedload, subqueryload
from wtforms import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
from wtforms.validators import *
from app.querybuilder import QueryBuilder
from app.rediscache import has_key, set_key
from app.tasks.importtasks import importRepoScreenshot, updateMetaFromRelease
from app.utils import *
from . import bp
@menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' }) @menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
@ -48,9 +44,9 @@ def list_all():
query = qb.buildPackageQuery() query = qb.buildPackageQuery()
title = qb.title title = qb.title
query = query.options( \ query = query.options(
joinedload(Package.license), \ joinedload(Package.license),
joinedload(Package.media_license), \ joinedload(Package.media_license),
subqueryload(Package.tags)) subqueryload(Package.tags))
ip = request.headers.get("X-Forwarded-For") or request.remote_addr ip = request.headers.get("X-Forwarded-For") or request.remote_addr
@ -103,9 +99,9 @@ def list_all():
selected_tags = set(qb.tags) selected_tags = set(qb.tags)
return render_template("packages/list.html", \ return render_template("packages/list.html",
title=title, packages=query.items, pagination=query, \ title=title, packages=query.items, pagination=query,
query=search, tags=tags, selected_tags=selected_tags, type=type_name, \ query=search, tags=tags, selected_tags=selected_tags, type=type_name,
authors=authors, packages_count=query.total, topics=topics) authors=authors, packages_count=query.total, topics=topics)
@ -176,10 +172,10 @@ def view(package):
has_review = current_user.is_authenticated and PackageReview.query.filter_by(package=package, author=current_user).count() > 0 has_review = current_user.is_authenticated and PackageReview.query.filter_by(package=package, author=current_user).count() > 0
return render_template("packages/view.html", \ return render_template("packages/view.html",
package=package, releases=releases, requests=requests, \ package=package, releases=releases, requests=requests,
alternatives=alternatives, similar_topics=similar_topics, \ alternatives=alternatives, similar_topics=similar_topics,
review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl, \ review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl,
threads=threads.all(), has_review=has_review) threads=threads.all(), has_review=has_review)
@ -366,9 +362,9 @@ def create_edit(author=None, name=None):
package_query = package_query.filter(Package.id != package.id) package_query = package_query.filter(Package.id != package.id)
enableWizard = name is None and request.method != "POST" enableWizard = name is None and request.method != "POST"
return render_template("packages/create_edit.html", package=package, \ return render_template("packages/create_edit.html", package=package,
form=form, author=author, enable_wizard=enableWizard, \ form=form, author=author, enable_wizard=enableWizard,
packages=package_query.all(), \ packages=package_query.all(),
mpackages=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all()) mpackages=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all())
@ -504,7 +500,7 @@ def edit_maintainers(package):
users = User.query.filter(User.rank >= UserRank.NEW_MEMBER).order_by(db.asc(User.username)).all() users = User.query.filter(User.rank >= UserRank.NEW_MEMBER).order_by(db.asc(User.username)).all()
return render_template("packages/edit_maintainers.html", \ return render_template("packages/edit_maintainers.html",
package=package, form=form, users=users) package=package, form=form, users=users)

18
app/blueprints/packages/releases.py
View File

@ -15,21 +15,17 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import *
from flask_user import *
from . import bp
from app.rediscache import has_key, set_key, make_download_key
from app.models import *
from app.tasks.importtasks import makeVCSRelease, checkZipRelease, updateMetaFromRelease
from app.utils import *
from celery import uuid from celery import uuid
from flask import *
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
from wtforms.validators import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField from wtforms.ext.sqlalchemy.fields import QuerySelectField
from wtforms.validators import *
from app.rediscache import has_key, set_key, make_download_key
from app.tasks.importtasks import makeVCSRelease, checkZipRelease, updateMetaFromRelease
from app.utils import *
from . import bp
def get_mt_releases(is_max): def get_mt_releases(is_max):

2
app/blueprints/packages/reviews.py
View File

@ -110,7 +110,7 @@ def review(package):
return redirect(package.getDetailsURL()) return redirect(package.getDetailsURL())
return render_template("packages/review_create_edit.html", \ return render_template("packages/review_create_edit.html",
form=form, package=package, review=review) form=form, package=package, review=review)

12
app/blueprints/packages/screenshots.py
View File

@ -16,17 +16,13 @@
from flask import * from flask import *
from flask_user import *
from . import bp
from app.models import *
from app.utils import *
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
from wtforms.validators import * from wtforms.validators import *
from app.utils import *
from . import bp
class CreateScreenshotForm(FlaskForm): class CreateScreenshotForm(FlaskForm):
title = StringField("Title/Caption", [Optional(), Length(-1, 100)]) title = StringField("Title/Caption", [Optional(), Length(-1, 100)])
@ -43,7 +39,7 @@ class EditScreenshotForm(FlaskForm):
@bp.route("/packages/<author>/<name>/screenshots/new/", methods=["GET", "POST"]) @bp.route("/packages/<author>/<name>/screenshots/new/", methods=["GET", "POST"])
@login_required @login_required
@is_package_page @is_package_page
def create_screenshot(package, id=None): def create_screenshot(package):
if not package.checkPerm(current_user, Permission.ADD_SCREENSHOTS): if not package.checkPerm(current_user, Permission.ADD_SCREENSHOTS):
return redirect(package.getDetailsURL()) return redirect(package.getDetailsURL())

9
app/blueprints/tasks/__init__.py
View File

@ -16,13 +16,10 @@
from flask import * from flask import *
from flask_user import *
import flask_menu as menu
from app import csrf from app import csrf
from app.models import * from app.tasks import celery
from app.tasks import celery, TaskError
from app.tasks.importtasks import getMeta from app.tasks.importtasks import getMeta
from app.utils import shouldReturnJson
from app.utils import * from app.utils import *
bp = Blueprint("tasks", __name__) bp = Blueprint("tasks", __name__)
@ -45,7 +42,7 @@ def check(id):
traceback = result.traceback traceback = result.traceback
result = result.result result = result.result
info = None None
if isinstance(result, Exception): if isinstance(result, Exception):
info = { info = {
'id': id, 'id': id,

6
app/blueprints/threads/__init__.py
View File

@ -21,9 +21,7 @@ bp = Blueprint("threads", __name__)
from flask_user import * from flask_user import *
from app.models import * from app.models import *
from app.utils import addNotification, clearNotifications, isYes, addAuditLog from app.utils import addNotification, isYes, addAuditLog
import datetime
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import * from wtforms import *
@ -199,7 +197,7 @@ def view(id):
flash("Please wait before commenting again", "danger") flash("Please wait before commenting again", "danger")
return redirect(thread.getViewURL()) return redirect(thread.getViewURL())
if len(comment) <= 2000 and len(comment) > 3: if 2000 >= len(comment) > 3:
reply = ThreadReply() reply = ThreadReply()
reply.author = current_user reply.author = current_user
reply.comment = comment reply.comment = comment

24
app/blueprints/todo/__init__.py
View File

@ -16,11 +16,11 @@
from flask import * from flask import *
from flask_user import * from flask_user import *
import flask_menu as menu from sqlalchemy import or_
from app.models import * from app.models import *
from app.querybuilder import QueryBuilder from app.querybuilder import QueryBuilder
from app.utils import get_int_or_abort from app.utils import get_int_or_abort
from sqlalchemy import or_
bp = Blueprint("todo", __name__) bp = Blueprint("todo", __name__)
@ -80,9 +80,9 @@ def view():
return render_template("todo/list.html", title="Reports and Work Queue", return render_template("todo/list.html", title="Reports and Work Queue",
packages=packages, wip_packages=wip_packages, releases=releases, screenshots=screenshots, packages=packages, wip_packages=wip_packages, releases=releases, screenshots=screenshots,
canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn, canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn,
topics_to_add=topics_to_add, total_topics=total_topics, \ topics_to_add=topics_to_add, total_topics=total_topics,
total_packages=total_packages, total_to_tag=total_to_tag, \ total_packages=total_packages, total_to_tag=total_to_tag,
unfulfilled_meta_packages=unfulfilled_meta_packages) unfulfilled_meta_packages=unfulfilled_meta_packages)
@bp.route("/todo/topics/") @bp.route("/todo/topics/")
@ -104,16 +104,16 @@ def topics():
num = 100 num = 100
query = query.paginate(page, num, True) query = query.paginate(page, num, True)
next_url = url_for("todo.topics", page=query.next_num, query=qb.search, \ next_url = url_for("todo.topics", page=query.next_num, query=qb.search,
show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \ show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \
if query.has_next else None if query.has_next else None
prev_url = url_for("todo.topics", page=query.prev_num, query=qb.search, \ prev_url = url_for("todo.topics", page=query.prev_num, query=qb.search,
show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \ show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \
if query.has_prev else None if query.has_prev else None
return render_template("todo/topics.html", topics=query.items, total=total, \ return render_template("todo/topics.html", topics=query.items, total=total,
topic_count=topic_count, query=qb.search, show_discarded=qb.show_discarded, \ topic_count=topic_count, query=qb.search, show_discarded=qb.show_discarded,
next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, \ next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages,
n=num, sort_by=qb.order_by) n=num, sort_by=qb.order_by)

20
app/blueprints/users/profile.py
View File

@ -16,19 +16,19 @@
from flask import * from flask import *
from flask_user import signals, current_user, user_manager from flask_user import signals, current_user, user_manager, login_required
from flask_login import login_user, logout_user
from app.markdown import render_markdown
from . import bp
from app.models import *
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from sqlalchemy import func
from wtforms import * from wtforms import *
from wtforms.validators import * from wtforms.validators import *
from app.utils import randomString, loginUser, rank_required, nonEmptyOrNone, addAuditLog
from app.tasks.forumtasks import checkForumAccount from app.markdown import render_markdown
from app.models import *
from app.tasks.emails import sendVerifyEmail, sendEmailRaw from app.tasks.emails import sendVerifyEmail, sendEmailRaw
from app.tasks.phpbbparser import getProfile from app.tasks.forumtasks import checkForumAccount
from sqlalchemy import func from app.utils import randomString, rank_required, nonEmptyOrNone, addAuditLog
from . import bp
# Define the User profile form # Define the User profile form
class UserProfileForm(FlaskForm): class UserProfileForm(FlaskForm):
@ -198,7 +198,7 @@ def set_password():
return redirect(url_for("user.change_password")) return redirect(url_for("user.change_password"))
form = SetPasswordForm(request.form) form = SetPasswordForm(request.form)
if current_user.email == None: if current_user.email is None:
form.email.validators = [InputRequired(), Email()] form.email.validators = [InputRequired(), Email()]
if request.method == "POST" and form.validate(): if request.method == "POST" and form.validate():

8
app/default_data.py
View File

@ -17,10 +17,10 @@ def populate(session):
session.add(MinetestRelease("5.1", 38)) session.add(MinetestRelease("5.1", 38))
tags = {} tags = {}
for tag in ["Inventory", "Mapgen", "Building", \ for tag in ["Inventory", "Mapgen", "Building",
"Mobs and NPCs", "Tools", "Player effects", \ "Mobs and NPCs", "Tools", "Player effects",
"Environment", "Transport", "Maintenance", "Plants and farming", \ "Environment", "Transport", "Maintenance", "Plants and farming",
"PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer"]: "PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer"]:
row = Tag(tag) row = Tag(tag)
tags[row.name] = row tags[row.name] = row
session.add(row) session.add(row)

3
app/maillogger.py
View File

@ -1,7 +1,8 @@
import logging import logging
from enum import Enum
from app.tasks.emails import sendEmailRaw from app.tasks.emails import sendEmailRaw
def _has_newline(line): def _has_newline(line):
"""Used by has_bad_header to check for \\r or \\n""" """Used by has_bad_header to check for \\r or \\n"""
if line and ("\r" in line or "\n" in line): if line and ("\r" in line or "\n" in line):

6
app/markdown.py
View File

@ -53,9 +53,9 @@ ALLOWED_PROTOCOLS = ['http', 'https', 'mailto']
md = Markdown(extensions=["fenced_code"], output_format="html5") md = Markdown(extensions=["fenced_code"], output_format="html5")
def render_markdown(source): def render_markdown(source):
return bleach.clean(md.convert(source), \ return bleach.clean(md.convert(source),
tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, \ tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES,
styles=[], protocols=ALLOWED_PROTOCOLS) styles=[], protocols=ALLOWED_PROTOCOLS)
def init_app(app): def init_app(app):
@app.template_filter() @app.template_filter()

80
app/models.py
View File

@ -15,20 +15,18 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
import enum, datetime import datetime
import enum
from app import app, gravatar
from urllib.parse import urlparse from urllib.parse import urlparse
from flask import Flask, url_for from flask import url_for
from flask_sqlalchemy import SQLAlchemy, BaseQuery
from flask_migrate import Migrate from flask_migrate import Migrate
from flask_user import login_required, UserManager, UserMixin from flask_sqlalchemy import SQLAlchemy, BaseQuery
from sqlalchemy import func, CheckConstraint from flask_user import UserManager, UserMixin
from sqlalchemy_searchable import SearchQueryMixin from sqlalchemy_searchable import SearchQueryMixin, make_searchable
from sqlalchemy_utils.types import TSVectorType from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
from app import app, gravatar
# Initialise database # Initialise database
db = SQLAlchemy(app) db = SQLAlchemy(app)
@ -130,7 +128,7 @@ class Permission(enum.Enum):
return perm.check(user) return perm.check(user)
def display_name_default(context): def display_name_default(context):
return context.get_current_parameters()["username"] return context.get_current_parameters()["username"]
class User(db.Model, UserMixin): class User(db.Model, UserMixin):
id = db.Column(db.Integer, primary_key=True) id = db.Column(db.Integer, primary_key=True)
@ -163,7 +161,7 @@ class User(db.Model, UserMixin):
donate_url = db.Column(db.String(255), nullable=True, default=None) donate_url = db.Column(db.String(255), nullable=True, default=None)
# Content # Content
notifications = db.relationship("Notification", primaryjoin="User.id==Notification.user_id", \ notifications = db.relationship("Notification", primaryjoin="User.id==Notification.user_id",
order_by="Notification.created_at") order_by="Notification.created_at")
packages = db.relationship("Package", backref=db.backref("author", lazy="joined"), lazy="dynamic") packages = db.relationship("Package", backref=db.backref("author", lazy="joined"), lazy="dynamic")
@ -377,11 +375,11 @@ class PackageState(enum.Enum):
PACKAGE_STATE_FLOW = { PACKAGE_STATE_FLOW = {
PackageState.WIP: set([ PackageState.READY_FOR_REVIEW ]), PackageState.WIP: {PackageState.READY_FOR_REVIEW},
PackageState.CHANGES_NEEDED: set([ PackageState.READY_FOR_REVIEW ]), PackageState.CHANGES_NEEDED: {PackageState.READY_FOR_REVIEW},
PackageState.READY_FOR_REVIEW: set([ PackageState.WIP, PackageState.CHANGES_NEEDED, PackageState.APPROVED ]), PackageState.READY_FOR_REVIEW: {PackageState.WIP, PackageState.CHANGES_NEEDED, PackageState.APPROVED},
PackageState.APPROVED: set([ PackageState.CHANGES_NEEDED ]), PackageState.APPROVED: {PackageState.CHANGES_NEEDED},
PackageState.DELETED: set([ PackageState.READY_FOR_REVIEW ]), PackageState.DELETED: {PackageState.READY_FOR_REVIEW},
} }
@ -410,22 +408,22 @@ class PackagePropertyKey(enum.Enum):
provides = db.Table("provides", provides = db.Table("provides",
db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True), db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True),
db.Column("metapackage_id", db.Integer, db.ForeignKey("meta_package.id"), primary_key=True) db.Column("metapackage_id", db.Integer, db.ForeignKey("meta_package.id"), primary_key=True)
) )
Tags = db.Table("tags", Tags = db.Table("tags",
db.Column("tag_id", db.Integer, db.ForeignKey("tag.id"), primary_key=True), db.Column("tag_id", db.Integer, db.ForeignKey("tag.id"), primary_key=True),
db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True) db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True)
) )
ContentWarnings = db.Table("content_warnings", ContentWarnings = db.Table("content_warnings",
db.Column("content_warning_id", db.Integer, db.ForeignKey("content_warning.id"), primary_key=True), db.Column("content_warning_id", db.Integer, db.ForeignKey("content_warning.id"), primary_key=True),
db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True) db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True)
) )
maintainers = db.Table("maintainers", maintainers = db.Table("maintainers",
db.Column("user_id", db.Integer, db.ForeignKey("user.id"), primary_key=True), db.Column("user_id", db.Integer, db.ForeignKey("user.id"), primary_key=True),
db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True) db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True)
) )
class Dependency(db.Model): class Dependency(db.Model):
@ -471,7 +469,7 @@ class Dependency(db.Model):
raise Exception("Meta and package are both none!") raise Exception("Meta and package are both none!")
@staticmethod @staticmethod
def SpecToList(depender, spec, cache={}): def SpecToList(depender, spec, cache):
retval = [] retval = []
arr = spec.split(",") arr = spec.split(",")
@ -521,7 +519,7 @@ class Package(db.Model):
name_valid = db.CheckConstraint("name ~* '^[a-z0-9_]+$'") name_valid = db.CheckConstraint("name ~* '^[a-z0-9_]+$'")
search_vector = db.Column(TSVectorType("name", "title", "short_desc", "desc", \ search_vector = db.Column(TSVectorType("name", "title", "short_desc", "desc",
weights={ "name": "A", "title": "B", "short_desc": "C", "desc": "D" })) weights={ "name": "A", "title": "B", "short_desc": "C", "desc": "D" }))
license_id = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1) license_id = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1)
@ -548,8 +546,8 @@ class Package(db.Model):
issueTracker = db.Column(db.String(200), nullable=True) issueTracker = db.Column(db.String(200), nullable=True)
forums = db.Column(db.Integer, nullable=True) forums = db.Column(db.Integer, nullable=True)
provides = db.relationship("MetaPackage", \ provides = db.relationship("MetaPackage",
secondary=provides, lazy="select", order_by=db.asc("name"), \ secondary=provides, lazy="select", order_by=db.asc("name"),
backref=db.backref("packages", lazy="dynamic", order_by=db.desc("score"))) backref=db.backref("packages", lazy="dynamic", order_by=db.desc("score")))
dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id]) dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id])
@ -613,7 +611,7 @@ class Package(db.Model):
user = m.group(1) user = m.group(1)
repo = m.group(2).replace(".git", "") repo = m.group(2).replace(".git", "")
return (user,repo) return user, repo
def getSortedDependencies(self, is_hard=None): def getSortedDependencies(self, is_hard=None):
query = self.dependencies query = self.dependencies
@ -770,8 +768,8 @@ class Package(db.Model):
def getDownloadRelease(self, version=None): def getDownloadRelease(self, version=None):
for rel in self.releases: for rel in self.releases:
if rel.approved and (version is None or if rel.approved and (version is None or
((rel.min_rel is None or rel.min_rel_id <= version.id) and \ ((rel.min_rel is None or rel.min_rel_id <= version.id) and
(rel.max_rel is None or rel.max_rel_id >= version.id))): (rel.max_rel is None or rel.max_rel_id >= version.id))):
return rel return rel
return None return None
@ -916,7 +914,7 @@ class MetaPackage(db.Model):
return ",".join([str(x) for x in list]) return ",".join([str(x) for x in list])
@staticmethod @staticmethod
def GetOrCreate(name, cache={}): def GetOrCreate(name, cache):
mp = cache.get(name) mp = cache.get(name)
if mp is None: if mp is None:
mp = MetaPackage.query.filter_by(name=name).first() mp = MetaPackage.query.filter_by(name=name).first()
@ -929,7 +927,7 @@ class MetaPackage(db.Model):
return mp return mp
@staticmethod @staticmethod
def SpecToList(spec, cache={}): def SpecToList(spec, cache):
retval = [] retval = []
arr = spec.split(",") arr = spec.split(",")
@ -1131,7 +1129,7 @@ class PackageScreenshot(db.Model):
id=self.id) id=self.id)
def getThumbnailURL(self, level=2): def getThumbnailURL(self, level=2):
return self.url.replace("/uploads/", ("/thumbnails/{:d}/").format(level)) return self.url.replace("/uploads/", "/thumbnails/{:d}/".format(level))
class APIToken(db.Model): class APIToken(db.Model):
@ -1243,8 +1241,8 @@ class EditRequestChange(db.Model):
watchers = db.Table("watchers", watchers = db.Table("watchers",
db.Column("user_id", db.Integer, db.ForeignKey("user.id"), primary_key=True), db.Column("user_id", db.Integer, db.ForeignKey("user.id"), primary_key=True),
db.Column("thread_id", db.Integer, db.ForeignKey("thread.id"), primary_key=True) db.Column("thread_id", db.Integer, db.ForeignKey("thread.id"), primary_key=True)
) )
class Thread(db.Model): class Thread(db.Model):
@ -1264,10 +1262,10 @@ class Thread(db.Model):
created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
replies = db.relationship("ThreadReply", backref="thread", lazy="dynamic", \ replies = db.relationship("ThreadReply", backref="thread", lazy="dynamic",
order_by=db.asc("thread_reply_id")) order_by=db.asc("thread_reply_id"))
watchers = db.relationship("User", secondary=watchers, lazy="subquery", \ watchers = db.relationship("User", secondary=watchers, lazy="subquery",
backref=db.backref("watching", lazy=True)) backref=db.backref("watching", lazy=True))
def getViewURL(self): def getViewURL(self):
@ -1412,10 +1410,10 @@ class AuditLogEntry(db.Model):
REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com", \ REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com",
"minetest.net", "dropboxusercontent.com", "4shared.com", \ "minetest.net", "dropboxusercontent.com", "4shared.com",
"digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net", \ "digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net",
"imageshack.com", "imgur.com"] "imageshack.com", "imgur.com"]
class ForumTopic(db.Model): class ForumTopic(db.Model):
topic_id = db.Column(db.Integer, primary_key=True, autoincrement=False) topic_id = db.Column(db.Integer, primary_key=True, autoincrement=False)

8
app/querybuilder.py
View File

@ -1,8 +1,10 @@
from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease, User, Tag, Tags, ContentWarning, PackageState
from .utils import isNo, isYes, get_int_or_abort
from sqlalchemy.sql.expression import func
from flask import abort from flask import abort
from sqlalchemy import or_ from sqlalchemy import or_
from sqlalchemy.sql.expression import func
from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease, User, Tag, ContentWarning, PackageState
from .utils import isYes, get_int_or_abort
class QueryBuilder: class QueryBuilder:
title = None title = None

2
app/rediscache.py
View File

@ -4,7 +4,7 @@ from . import r
# and also means that the releases code avoids knowing about `app` # and also means that the releases code avoids knowing about `app`
def make_download_key(ip, package): def make_download_key(ip, package):
return ("{}/{}/{}").format(ip, package.author.username, package.name) return "{}/{}/{}".format(ip, package.author.username, package.name)
def set_key(key, v): def set_key(key, v):
r.set(key, v) r.set(key, v)

6
app/sass.py
View File

@ -53,11 +53,11 @@ def sass(app, inputDir='scss', outputPath='static', force=False, cacheDir="publi
cacheFile = "%s/%s.css" % (cacheDir, filepath) cacheFile = "%s/%s.css" % (cacheDir, filepath)
# Source file exists, and needs regenerating # Source file exists, and needs regenerating
if os.path.isfile(sassfile) and (force or not os.path.isfile(cacheFile) or \ if os.path.isfile(sassfile) and (force or not os.path.isfile(cacheFile) or
os.path.getmtime(sassfile) > os.path.getmtime(cacheFile)): os.path.getmtime(sassfile) > os.path.getmtime(cacheFile)):
_convert(inputDir, sassfile, cacheFile) _convert(inputDir, sassfile, cacheFile)
app.logger.debug('Compiled %s into %s' % (sassfile, cacheFile)) app.logger.debug('Compiled %s into %s' % (sassfile, cacheFile))
return send_from_directory(cacheDir, filepath + ".css") return send_from_directory(cacheDir, filepath + ".css")
app.add_url_rule("/%s/<path:filepath>.css" % (outputPath), 'sass', _sass) app.add_url_rule("/%s/<path:filepath>.css" % outputPath, 'sass', _sass)

2
app/tasks/__init__.py
View File

@ -16,10 +16,8 @@
import flask import flask
from flask_sqlalchemy import SQLAlchemy
from celery import Celery from celery import Celery
from celery.schedules import crontab from celery.schedules import crontab
from app import app
from app.models import * from app.models import *
class TaskError(Exception): class TaskError(Exception):

2
app/tasks/emails.py
View File

@ -15,7 +15,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import render_template, url_for from flask import render_template
from flask_mail import Message from flask_mail import Message
from app import mail from app import mail
from app.tasks import celery from app.tasks import celery

5
app/tasks/forumtasks.py
View File

@ -15,14 +15,11 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
import flask, json, re import json, re
from flask_sqlalchemy import SQLAlchemy
from app import app
from app.models import * from app.models import *
from app.tasks import celery from app.tasks import celery
from .phpbbparser import getProfile, getTopicsFromForum from .phpbbparser import getProfile, getTopicsFromForum
import urllib.request import urllib.request
from urllib.parse import urlparse, quote_plus
@celery.task() @celery.task()
def checkForumAccount(username, forceNoSave=False): def checkForumAccount(username, forceNoSave=False):

20
app/tasks/importtasks.py
View File

@ -15,22 +15,17 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
import flask, json, os, git, tempfile, shutil, gitdb, contextlib import os, git, tempfile, shutil, gitdb, contextlib
from git import GitCommandError from git import GitCommandError
from git_archive_all import GitArchiver from git_archive_all import GitArchiver
from flask_sqlalchemy import SQLAlchemy
from urllib.error import HTTPError from urllib.error import HTTPError
import urllib.request import urllib.request
from urllib.parse import urlparse, quote_plus, urlsplit from urllib.parse import urlsplit
from zipfile import ZipFile from zipfile import ZipFile
from app import app
from app.models import * from app.models import *
from app.tasks import celery, TaskError from app.tasks import celery, TaskError
from app.utils import randomString, getExtension from app.utils import randomString, getExtension
from .minetestcheck import build_tree, MinetestCheckError, ContentType from .minetestcheck import build_tree, MinetestCheckError, ContentType
from .minetestcheck.config import parse_conf
from .krocklist import getKrockList, findModInfo
def generateGitURL(urlstr): def generateGitURL(urlstr):
@ -60,7 +55,7 @@ def cloneRepo(urlstr, ref=None, recursive=False):
print("Cloning from " + gitUrl) print("Cloning from " + gitUrl)
if ref is None: if ref is None:
repo = git.Repo.clone_from(gitUrl, gitDir, \ repo = git.Repo.clone_from(gitUrl, gitDir,
progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15) progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
else: else:
repo = git.Repo.init(gitDir) repo = git.Repo.init(gitDir)
@ -96,10 +91,7 @@ def getMeta(urlstr, author):
except MinetestCheckError as err: except MinetestCheckError as err:
raise TaskError(str(err)) raise TaskError(str(err))
result = {} result = {"name": tree.name, "provides": tree.getModNames(), "type": tree.type.name}
result["name"] = tree.name
result["provides"] = tree.getModNames()
result["type"] = tree.type.name
for key in ["depends", "optional_depends"]: for key in ["depends", "optional_depends"]:
result[key] = tree.fold("meta", key) result[key] = tree.fold("meta", key)
@ -120,8 +112,8 @@ def getMeta(urlstr, author):
def postReleaseCheckUpdate(self, release, path): def postReleaseCheckUpdate(self, release, path):
try: try:
tree = build_tree(path, expected_type=ContentType[release.package.type.name], \ tree = build_tree(path, expected_type=ContentType[release.package.type.name],
author=release.package.author.username, name=release.package.name) author=release.package.author.username, name=release.package.name)
cache = {} cache = {}
def getMetaPackages(names): def getMetaPackages(names):

80
app/tasks/krocklist.py
View File

@ -1,80 +0,0 @@
# ContentDB
# Copyright (C) 2018-20 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import json
import urllib.request
krock_list_cache = None
krock_list_cache_by_name = None
def getKrockList():
global krock_list_cache
global krock_list_cache_by_name
if krock_list_cache is None:
contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
list = json.loads(contents)
def h(x):
if not ("title" in x and "author" in x and \
"topicId" in x and "link" in x and x["link"] != ""):
return False
import re
m = re.search(r"\[([A-Za-z0-9_]+)\]", x["title"])
if m is None:
return False
x["name"] = m.group(1)
return True
def g(x):
return {
"title": x["title"],
"author": x["author"],
"name": x["name"],
"topicId": x["topicId"],
"link": x["link"],
}
krock_list_cache = [g(x) for x in list if h(x)]
krock_list_cache_by_name = {}
for x in krock_list_cache:
if not x["name"] in krock_list_cache_by_name:
krock_list_cache_by_name[x["name"]] = []
krock_list_cache_by_name[x["name"]].append(x)
return krock_list_cache, krock_list_cache_by_name
def findModInfo(author, name, link):
list, lookup = getKrockList()
if name is not None and name in lookup:
if len(lookup[name]) == 1:
return lookup[name][0]
for x in lookup[name]:
if x["author"] == author:
return x
if link is not None and len(link) > 15:
for x in list:
if link in x["link"]:
return x
return None

4
app/tasks/minetestcheck/__init__.py
View File

@ -20,7 +20,7 @@ class ContentType(Enum):
""" """
Whether or not `other` is an acceptable type for this Whether or not `other` is an acceptable type for this
""" """
assert(other) assert other
if self == ContentType.MOD: if self == ContentType.MOD:
if not other.isModLike(): if not other.isModLike():
@ -40,7 +40,7 @@ def build_tree(path, expected_type=None, author=None, repo=None, name=None):
path = get_base_dir(path) path = get_base_dir(path)
root = PackageTreeNode(path, "/", author=author, repo=repo, name=name) root = PackageTreeNode(path, "/", author=author, repo=repo, name=name)
assert(root) assert root
if expected_type: if expected_type:
expected_type.validate_same(root.type) expected_type.validate_same(root.type)

19
app/tasks/minetestcheck/tree.py
View File

@ -54,12 +54,12 @@ class PackageTreeNode:
if self.type == ContentType.GAME: if self.type == ContentType.GAME:
if not os.path.isdir(baseDir + "/mods"): if not os.path.isdir(baseDir + "/mods"):
raise MinetestCheckError(("Game at {} does not have a mods/ folder").format(self.relative)) raise MinetestCheckError("Game at {} does not have a mods/ folder".format(self.relative))
self.add_children_from_mod_dir("mods") self.add_children_from_mod_dir("mods")
elif self.type == ContentType.MOD: elif self.type == ContentType.MOD:
if self.name and not basenamePattern.match(self.name): if self.name and not basenamePattern.match(self.name):
raise MinetestCheckError(("Invalid base name for mod {} at {}, names must only contain a-z0-9_.") \ raise MinetestCheckError("Invalid base name for mod {} at {}, names must only contain a-z0-9_." \
.format(self.name, self.relative)) .format(self.name, self.relative))
elif self.type == ContentType.MODPACK: elif self.type == ContentType.MODPACK:
self.add_children_from_mod_dir(None) self.add_children_from_mod_dir(None)
@ -132,11 +132,12 @@ class PackageTreeNode:
for dep in result["depends"]: for dep in result["depends"]:
if not basenamePattern.match(dep): if not basenamePattern.match(dep):
if " " in dep: if " " in dep:
raise MinetestCheckError(("Invalid dependency name '{}' for mod at {}, did you forget a comma?") \ raise MinetestCheckError("Invalid dependency name '{}' for mod at {}, did you forget a comma?" \
.format(dep, self.relative)) .format(dep, self.relative))
else: else:
raise MinetestCheckError(("Invalid dependency name '{}' for mod at {}, names must only contain a-z0-9_.") \ raise MinetestCheckError(
.format(dep, self.relative)) "Invalid dependency name '{}' for mod at {}, names must only contain a-z0-9_." \
.format(dep, self.relative))
# Check dependencies # Check dependencies
@ -177,11 +178,11 @@ class PackageTreeNode:
if not entry.startswith('.') and os.path.isdir(path): if not entry.startswith('.') and os.path.isdir(path):
child = PackageTreeNode(path, relative + entry + "/", name=entry) child = PackageTreeNode(path, relative + entry + "/", name=entry)
if not child.type.isModLike(): if not child.type.isModLike():
raise MinetestCheckError(("Expecting mod or modpack, found {} at {} inside {}") \ raise MinetestCheckError("Expecting mod or modpack, found {} at {} inside {}" \
.format(child.type.value, child.relative, self.type.value)) .format(child.type.value, child.relative, self.type.value))
if child.name is None: if child.name is None:
raise MinetestCheckError(("Missing base name for mod at {}").format(self.relative)) raise MinetestCheckError("Missing base name for mod at {}".format(self.relative))
self.children.append(child) self.children.append(child)

17
app/tasks/phpbbparser.py
View File

@ -2,15 +2,14 @@
# License: MIT # License: MIT
# Source: https://github.com/rubenwardy/python_phpbb_parser # Source: https://github.com/rubenwardy/python_phpbb_parser
import urllib, socket import re
from bs4 import * import urllib
from urllib.parse import urljoin
from datetime import datetime
import urllib.request
import os.path
import time, re
import urllib.parse as urlparse import urllib.parse as urlparse
import urllib.request
from datetime import datetime
from urllib.parse import urlencode from urllib.parse import urlencode
from bs4 import *
def urlEncodeNonAscii(b): def urlEncodeNonAscii(b):
return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b) return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)
@ -68,7 +67,7 @@ def __extract_properties(profile, soup):
def __extract_signature(soup): def __extract_signature(soup):
res = soup.find_all("div", class_="signature") res = soup.find_all("div", class_="signature")
if (len(res) != 1): if len(res) != 1:
return None return None
else: else:
return res[0] return res[0]
@ -166,7 +165,7 @@ def parseForumListPage(id, page, out, extra=None):
return True return True
def getTopicsFromForum(id, out={}, extra=None): def getTopicsFromForum(id, out, extra=None):
print("Fetching all topics from forum {}".format(id)) print("Fetching all topics from forum {}".format(id))
page = 0 page = 0
while parseForumListPage(id, page, out, extra): while parseForumListPage(id, page, out, extra):

8
app/tests/test_api.py
View File

@ -1,9 +1,7 @@
import pytest
from app import app
from app.default_data import populate_test_data from app.default_data import populate_test_data
from app.models import db, License, Tag, User, UserRank, Package, PackageState from app.models import db, Package, PackageState
from utils import client, recreate_db, parse_json from utils import parse_json, is_str, is_int, is_optional
from utils import is_str, is_int, is_optional
def validate_package_list(packages, strict=False): def validate_package_list(packages, strict=False):
valid_keys = { valid_keys = {

6
app/tests/test_homepage.py
View File

@ -1,8 +1,6 @@
import pytest
from app import app
from app.default_data import populate_test_data from app.default_data import populate_test_data
from app.models import db, License, Tag, User, UserRank from app.models import db
from utils import client, recreate_db
def test_homepage_empty(client): def test_homepage_empty(client):
"""Start with a blank database.""" """Start with a blank database."""

21
app/utils.py
View File

@ -15,15 +15,22 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import request, flash, abort, redirect import imghdr
from flask_user import * import os
from flask_login import login_user, logout_user import random
from .models import * import string
from . import app import user_agents
import random, string, os, imghdr, user_agents
from urllib.parse import urljoin from urllib.parse import urljoin
from flask import request, flash, abort, redirect
from flask_login import login_user
from flask_user import *
from werkzeug.datastructures import MultiDict from werkzeug.datastructures import MultiDict
from . import app
from .models import *
# These are given to Jinja in template_filters.py # These are given to Jinja in template_filters.py
def abs_url_for(path, **kwargs): def abs_url_for(path, **kwargs):
@ -78,7 +85,7 @@ def getExtension(filename):
def isFilenameAllowed(filename, exts): def isFilenameAllowed(filename, exts):
return getExtension(filename) in exts return getExtension(filename) in exts
ALLOWED_IMAGES = set(["jpeg", "png"]) ALLOWED_IMAGES = {"jpeg", "png"}
def isAllowedImage(data): def isAllowedImage(data):
return imghdr.what(None, data) in ALLOWED_IMAGES return imghdr.what(None, data) in ALLOWED_IMAGES

4
migrations/versions/28a427cbd4cf_.py
View File

@ -5,10 +5,6 @@ Revises: e9f534df23a8
Create Date: 2018-06-03 01:47:33.006039 Create Date: 2018-06-03 01:47:33.006039
""" """
from alembic import op
import sqlalchemy as sa
import sqlalchemy.types as ty
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '28a427cbd4cf' revision = '28a427cbd4cf'

5
migrations/versions/2f3c3597c78d_.py
View File

@ -5,11 +5,10 @@ Revises: 9ec17b558413
Create Date: 2019-01-29 02:43:08.865695 Create Date: 2019-01-29 02:43:08.865695
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import sync_trigger from sqlalchemy_searchable import sync_trigger
from sqlalchemy_utils.types import TSVectorType
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '2f3c3597c78d' revision = '2f3c3597c78d'

2
migrations/versions/306ce331a2a7_.py
View File

@ -6,8 +6,6 @@ Create Date: 2020-01-18 23:00:40.487425
""" """
from alembic import op from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '306ce331a2a7' revision = '306ce331a2a7'

2
migrations/versions/3f4d7cd8401f_.py
View File

@ -6,8 +6,6 @@ Create Date: 2018-05-25 17:53:13.215127
""" """
from alembic import op from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '3f4d7cd8401f' revision = '3f4d7cd8401f'

2
migrations/versions/64fee8e5ab34_.py
View File

@ -6,8 +6,6 @@ Create Date: 2020-01-19 02:28:05.432244
""" """
from alembic import op from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '64fee8e5ab34' revision = '64fee8e5ab34'

2
migrations/versions/6dca6eceb04d_.py
View File

@ -6,8 +6,6 @@ Create Date: 2020-01-18 17:32:21.885068
""" """
from alembic import op from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from sqlalchemy_searchable import sync_trigger from sqlalchemy_searchable import sync_trigger

3
migrations/versions/7a48dbd05780_.py
View File

@ -5,9 +5,8 @@ Revises: df66c78e6791
Create Date: 2020-01-24 21:52:49.744404 Create Date: 2020-01-24 21:52:49.744404
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '7a48dbd05780' revision = '7a48dbd05780'

3
migrations/versions/7def3e843d04_.py
View File

@ -5,9 +5,8 @@ Revises: dce69ad1e4eb
Create Date: 2019-01-28 20:27:33.760232 Create Date: 2019-01-28 20:27:33.760232
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '7def3e843d04' revision = '7def3e843d04'

2
migrations/versions/7ff57806ffd5_.py
View File

@ -6,8 +6,6 @@ Create Date: 2019-01-29 02:57:50.279918
""" """
from alembic import op from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '7ff57806ffd5' revision = '7ff57806ffd5'

2
migrations/versions/838081950f27_.py
View File

@ -6,8 +6,6 @@ Create Date: 2020-07-12 01:33:19.499459
""" """
from alembic import op from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '838081950f27' revision = '838081950f27'

3
migrations/versions/97a9c461bc2d_.py
View File

@ -5,9 +5,8 @@ Revises: 7def3e843d04
Create Date: 2019-01-28 20:49:41.831991 Create Date: 2019-01-28 20:49:41.831991
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '97a9c461bc2d' revision = '97a9c461bc2d'

3
migrations/versions/9ec17b558413_.py
View File

@ -5,9 +5,8 @@ Revises: 97a9c461bc2d
Create Date: 2019-01-29 00:37:49.507631 Create Date: 2019-01-29 00:37:49.507631
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '9ec17b558413' revision = '9ec17b558413'

3
migrations/versions/a0f6c8743362_.py
View File

@ -5,9 +5,8 @@ Revises: 64fee8e5ab34
Create Date: 2020-01-19 19:12:39.402679 Create Date: 2020-01-19 19:12:39.402679
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'a0f6c8743362' revision = 'a0f6c8743362'

2
migrations/versions/b254f55eadd2_.py
View File

@ -6,8 +6,6 @@ Create Date: 2018-05-27 23:51:11.008936
""" """
from alembic import op from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'b254f55eadd2' revision = 'b254f55eadd2'

5
migrations/versions/c141a63b2487_.py
View File

@ -5,11 +5,8 @@ Revises: cb6ab141c522
Create Date: 2020-07-09 00:05:39.845465 Create Date: 2020-07-09 00:05:39.845465
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy import orm, func from alembic import op
from app.models import Package, PackageRelease
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'c141a63b2487' revision = 'c141a63b2487'

4
migrations/versions/cb6ab141c522_.py
View File

@ -5,11 +5,9 @@ Revises: 7a48dbd05780
Create Date: 2020-07-08 21:03:51.856561 Create Date: 2020-07-08 21:03:51.856561
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from alembic import op
from sqlalchemy import orm from sqlalchemy import orm
from app.models import Package
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'cb6ab141c522' revision = 'cb6ab141c522'

3
migrations/versions/d6ae9682c45f_.py
View File

@ -5,9 +5,8 @@ Revises: 7ff57806ffd5
Create Date: 2019-07-01 23:27:42.666877 Create Date: 2019-07-01 23:27:42.666877
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'd6ae9682c45f' revision = 'd6ae9682c45f'

3
migrations/versions/df66c78e6791_.py
View File

@ -5,9 +5,8 @@ Revises: a0f6c8743362
Create Date: 2020-01-24 18:39:58.363417 Create Date: 2020-01-24 18:39:58.363417
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'df66c78e6791' revision = 'df66c78e6791'

4
migrations/versions/dff4b87e4a76_.py
View File

@ -5,10 +5,8 @@ Revises: 3a24fc02365e
Create Date: 2020-07-17 23:47:51.096874 Create Date: 2020-07-17 23:47:51.096874
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
import datetime from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'dff4b87e4a76' revision = 'dff4b87e4a76'

2
migrations/versions/ea5a023711e0_.py
View File

@ -6,8 +6,6 @@ Create Date: 2018-05-26 01:55:09.745881
""" """
from alembic import op from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'ea5a023711e0' revision = 'ea5a023711e0'

3
migrations/versions/fd25bf3e57c3_.py
View File

@ -5,9 +5,8 @@ Revises: d6ae9682c45f
Create Date: 2019-11-26 23:43:47.476346 Create Date: 2019-11-26 23:43:47.476346
""" """
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects import postgresql from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'fd25bf3e57c3' revision = 'fd25bf3e57c3'

6
utils/setup.py
View File

@ -15,7 +15,9 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
import os, sys, datetime, inspect import inspect
import os
import sys
if not "FLASK_CONFIG" in os.environ: if not "FLASK_CONFIG" in os.environ:
os.environ["FLASK_CONFIG"] = "../config.cfg" os.environ["FLASK_CONFIG"] = "../config.cfg"
@ -29,7 +31,7 @@ currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentfram
parentdir = os.path.dirname(currentdir) parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir) sys.path.insert(0,parentdir)
from app.models import db, User, UserRank from app.models import db
from app.default_data import populate, populate_test_data from app.default_data import populate, populate_test_data
if delete_db and os.path.isfile("db.sqlite"): if delete_db and os.path.isfile("db.sqlite"):