Import from flatpak action

This commit is contained in:
Armen 2022-02-20 03:06:34 -05:00
parent d4be91caee
commit 52858d4d8e
11 changed files with 318 additions and 46 deletions

View File

@ -1,24 +1,22 @@
FROM python:3.10
RUN groupadd -g 5123 cdb && \
useradd -r -u 5123 -g cdb cdb
WORKDIR /home/cdb
RUN mkdir /var/cdb
RUN chown -R cdb:cdb /var/cdb
COPY requirements.lock.txt requirements.lock.txt
RUN apt update
RUN apt install -y vim
RUN apt install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev gir1.2-gtk-3.0
RUN apt install -y libappstream-glib-dev
RUN pip install -r requirements.lock.txt
RUN pip install gunicorn
RUN pip3 install pycairo PyGObject
COPY utils utils
COPY config.cfg config.cfg
COPY migrations migrations
COPY app app
COPY translations translations
RUN pybabel compile -d translations
RUN chown -R cdb:cdb /home/cdb
USER cdb
USER cdb

View File

@ -29,6 +29,7 @@ from app.models import PackageRelease, db, Package, PackageState, PackageScreens
NotificationType, PackageUpdateConfig, License, UserRank, PackageType, PackageGameSupport
from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, check_for_updates
from app.tasks.appstreamtasks import importFromFlathub
from app.utils import addNotification, get_system_user
from app.utils.image import get_image_size
@ -94,6 +95,10 @@ def import_topic_list():
task = importTopicList.delay()
return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
@action("Import appstream from flathub")
def import_from_flathub():
task = importFromFlathub.delay()
return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
@action("Check all forum accounts")
def check_all_forum_accounts():

View File

@ -228,7 +228,7 @@ def makeLabel(obj):
class PackageForm(FlaskForm):
type = SelectField(lazy_gettext("Type"), [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.GAME)
title = StringField(lazy_gettext("Title (Human-readable)"), [InputRequired(), Length(1, 100)])
name = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_) only"))])
name = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_\-\.]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only"))])
short_desc = StringField(lazy_gettext("Short Description (Plaintext)"), [InputRequired(), Length(1,200)])
dev_state = SelectField(lazy_gettext("Maintenance State"), [InputRequired()], choices=PackageDevState.choices(with_none=True), coerce=PackageDevState.coerce)
@ -546,7 +546,7 @@ def audit(package):
class PackageAliasForm(FlaskForm):
author = StringField(lazy_gettext("Author Name"), [InputRequired(), Length(1, 50)])
name = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100),
Regexp("^[a-z0-9_]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_) only"))])
Regexp("^[a-z0-9_\_\.]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only"))])
submit = SubmitField(lazy_gettext("Save"))

View File

@ -1,33 +1,54 @@
import string
import random
from .models import *
from .utils import make_flask_login_password
def generate_password():
characters = string.ascii_letters + string.digits + string.punctuation
password = ''.join(random.choice(characters) for i in range(16))
return password
def populate(session):
admin_user = User("rubenwardy")
admin_user = User("libregaming")
admin_user.is_active = True
admin_user.password = make_flask_login_password("tuckfrump")
admin_user.github_username = "rubenwardy"
admin_user.forums_username = "rubenwardy"
password = generate_password()
admin_user.password = make_flask_login_password(password)
admin_user.github_username = "libregaming"
admin_user.forums_username = "libregaming"
admin_user.rank = UserRank.ADMIN
session.add(admin_user)
print("#####################################")
print("Admin user : libregaming")
print("Admin password: " + password)
print("#####################################")
system_user = User("ContentDB", active=False)
system_user.email_confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000)
system_user.rank = UserRank.BOT
session.add(system_user)
session.add(MinetestRelease("None", 0))
session.add(MinetestRelease("0.4.16/17", 32))
session.add(MinetestRelease("5.0", 37))
session.add(MinetestRelease("5.1", 38))
session.add(MinetestRelease("5.2", 39))
session.add(MinetestRelease("5.3", 39))
appstream_user = User("AppStreamBot", active=False)
appstream_user.email_confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000)
appstream_user.rank = UserRank.BOT
session.add(appstream_user)
tags = {}
for tag in ["Inventory", "Mapgen", "Building",
"Mobs and NPCs", "Tools", "Player effects",
"Environment", "Transport", "Maintenance", "Plants and farming",
"PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer", "Featured"]:
for tag in [
"Action",
"Adventure",
"Arcade",
"Board",
"Blocks",
"Card",
"Kids",
"Logic",
"RolePlaying",
"Shooter",
"Simulation",
"Sports",
"Strategy"
]:
row = Tag(tag)
tags[row.name] = row
session.add(row)
@ -60,7 +81,7 @@ def populate_test_data(session):
ez.rank = UserRank.EDITOR
session.add(ez)
not1 = Notification(admin_user, ez, NotificationType.PACKAGE_APPROVAL, "Awards approved", "/packages/rubenwardy/awards/")
not1 = Notification(admin_user, ez, NotificationType.PACKAGE_APPROVAL, "Awards approved", "/packages/libregaming/awards/")
session.add(not1)
jeija = User("Jeija")
@ -102,8 +123,8 @@ def populate_test_data(session):
mod1.type = PackageType.TOOL
mod1.author = admin_user
mod1.tags.append(tags["player_effects"])
mod1.repo = "https://github.com/rubenwardy/awards"
mod1.issueTracker = "https://github.com/rubenwardy/awards/issues"
mod1.repo = "https://github.com/libregaming/awards"
mod1.issueTracker = "https://github.com/libregaming/awards/issues"
mod1.forums = 4870
mod1.short_desc = "Adds achievements and an API to register new ones."
mod1.desc = """
@ -126,7 +147,7 @@ awards.register_achievement("award_mesefind",{
rel.package = mod1
rel.min_rel = v51
rel.title = "v1.0.0"
rel.url = "https://github.com/rubenwardy/awards/archive/master.zip"
rel.url = "https://github.com/libregaming/awards/archive/master.zip"
rel.approved = True
session.add(rel)
@ -269,8 +290,8 @@ No warranty is provided, express or implied, for any part of the project.
tool.type = PackageType.TOOL
tool.author = admin_user
tool.tags.append(tags["player_effects"])
tool.repo = "https://github.com/rubenwardy/food/"
tool.issueTracker = "https://github.com/rubenwardy/food/issues/"
tool.repo = "https://github.com/libregaming/food/"
tool.issueTracker = "https://github.com/libregaming/food/issues/"
tool.forums = 2960
tool.short_desc = "Adds lots of food and an API to manage ingredients"
tool.desc = "This is the long desc"
@ -285,8 +306,8 @@ No warranty is provided, express or implied, for any part of the project.
tool.type = PackageType.TOOL
tool.author = admin_user
tool.tags.append(tags["player_effects"])
tool.repo = "https://github.com/rubenwardy/food_sweet/"
tool.issueTracker = "https://github.com/rubenwardy/food_sweet/issues/"
tool.repo = "https://github.com/libregaming/food_sweet/"
tool.issueTracker = "https://github.com/libregaming/food_sweet/issues/"
tool.forums = 9039
tool.short_desc = "Adds sweet food"
tool.desc = "This is the long desc"
@ -304,12 +325,12 @@ No warranty is provided, express or implied, for any part of the project.
game1.tags.append(tags["pvp"])
game1.tags.append(tags["survival"])
game1.tags.append(tags["multiplayer"])
game1.repo = "https://github.com/rubenwardy/capturetheflag"
game1.issueTracker = "https://github.com/rubenwardy/capturetheflag/issues"
game1.repo = "https://github.com/libregaming/capturetheflag"
game1.issueTracker = "https://github.com/libregaming/capturetheflag/issues"
game1.forums = 12835
game1.short_desc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!"
game1.desc = """
As seen on the Capture the Flag server (minetest.rubenwardy.com:30000)
As seen on the Capture the Flag server (minetest.libregaming.com:30000)
` `[`javascript:/*--></title></style></textarea></script></xmp><svg/onload='+/"/+/onmouseover=1/+/`](javascript:/*--%3E%3C/title%3E%3C/style%3E%3C/textarea%3E%3C/script%3E%3C/xmp%3E%3Csvg/onload='+/%22/+/onmouseover=1/+/)`[*/[]/+alert(1)//'>`
@ -351,7 +372,7 @@ Uses the CTF PvP Engine.
rel = PackageRelease()
rel.package = game1
rel.title = "v1.0.0"
rel.url = "https://github.com/rubenwardy/capturetheflag/archive/master.zip"
rel.url = "https://github.com/libregaming/capturetheflag/archive/master.zip"
rel.approved = True
session.add(rel)

View File

@ -41,7 +41,7 @@ def get_license(name):
return license
name_re = re.compile("^[a-z0-9_]+$")
name_re = re.compile("^[a-z0-9_\-\.]+$")
AnyType = "?"
ALLOWED_FIELDS = {

View File

@ -380,7 +380,7 @@ class Package(db.Model):
created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
approved_at = db.Column(db.DateTime, nullable=True, default=None)
name_valid = db.CheckConstraint("name ~* '^[a-z0-9_]+$'")
name_valid = db.CheckConstraint("name ~* '^[a-z0-9_\-\.]+$'")
search_vector = db.Column(TSVectorType("name", "title", "short_desc", "desc",
weights={ "name": "A", "title": "B", "short_desc": "C", "desc": "D" }))
@ -857,7 +857,7 @@ class Tag(db.Model):
self.textColor = textColor
import re
regex = re.compile("[^a-z_]")
regex = re.compile("[^0-9a-z_]")
self.name = regex.sub("", self.title.lower().replace(" ", "_"))
def getAsDictionary(self):

165
app/tasks/appstreamtasks.py Normal file
View File

@ -0,0 +1,165 @@
import json, re, sys
from app.models import *
from app.tasks import celery
from app.utils import is_username_valid
import urllib.request
import gi
import PIL
import requests
import os
import sys
import inspect
import shutil
import urllib.request
from gi.repository import Gio
gi.require_version('AppStreamGlib', '1.0')
from gi.repository import AppStreamGlib
from app.utils.lists import alwaysAccept, alwaysDeny, badLicenses, badCategories, nonFreeAssets, nonFreeNetworkServices
import itertools
from app.utils import make_flask_login_password
from app.utils.image import get_image_size
from app.utils import randomString
#Workaround to get the urls because app.get_urls() doesn't work :|
def get_urls(app):
kinds = [AppStreamGlib.UrlKind(kind) for kind in range(11)]
urls = [(app.get_url_item(kind),kind.value_nick) for kind in kinds]
return list(filter(lambda a: a[0] is not None, urls))
def acceptedGame(app):
#return 'Game' in app.get_categories()
if app.get_id() in alwaysAccept:
return True
if app.get_id() in alwaysDeny:
return False
return app.get_project_license() and \
not [x for x in badLicenses if x in app.get_project_license()] and \
'Game' in app.get_categories() and \
not [x for x in badCategories if x in app.get_categories()]
def getScreenshots(app):
return [images.get_source() for images in app.get_screenshots()]
@celery.task()
def importFromFlathub():
url = "https://flathub.org/repo/appstream/x86_64/appstream.xml.gz"
with urllib.request.urlopen(url) as response, open("/var/cdb/uploads/appstream.xml.gz", 'wb') as out_file:
shutil.copyfileobj(response, out_file)
store = AppStreamGlib.Store()
file = Gio.File.new_for_path("/var/cdb/uploads/appstream.xml.gz")
file.load_contents()
AppStreamGlib.Store.from_file(store, file, ".", None)
apps = list(filter(acceptedGame, store.get_apps()))
session=db.session
licenses = { x.name : x for x in License.query.all() }
tags = { x.name : x for x in Tag.query.all() }
admin_user = User.query.filter_by(username="AppStreamBot").first()
# for ss in PackageScreenshot.query.all():
# if ss.package.cover_image == ss:
# ss.package.cover_image = None
# session.commit()
# session.delete(ss)
# session.commit()
# for game in Package.query.all():
# session.delete(game)
# session.commit()
if not admin_user:
admin_user = User("AppStreamBot")
admin_user.is_active = True
admin_user.password = make_flask_login_password("AppStreamBot")
admin_user.github_username = "AppStreamBot"
admin_user.forums_username = "AppStreamBot"
admin_user.rank = UserRank.ADMIN
session.add(admin_user)
for app in apps:
screenshots = getScreenshots(app)
urls = get_urls(app)
filename = app.get_name().replace(':', '').replace('/','') + ".html"
print("APPLICATION: ",app.get_name())
package_exists = Package.query.filter_by(name=app.get_id()).first()
if package_exists:
print(f"Package {app.get_id()} exists, skipping.")
else:
game1 = Package()
game1.state = PackageState.APPROVED
game1.name = app.get_id()
game1.title = app.get_name()
if "Development" in app.get_categories():
game1.type = PackageType.TOOL
else:
game1.type = PackageType.GAME
license = "Uknown" if app.get_project_license() is None else app.get_project_license().split("AND")[0].split("and")[0]
if license not in licenses:
row = License(license)
licenses[row.name] = row
session.add(row)
session.commit()
for category in app.get_categories():
if category.lower() not in tags:
row = Tag(category.lower())
tags[row.name] = row
print("adding tag: ", row.name)
session.add(row)
game1.tags.append(tags[category.lower()])
game1.license = licenses[license]
game1.media_license = licenses["MIT"]
game1.author = admin_user
for url,t in urls:
if t == "bugtracker":
game1.issueTracker = url
elif t == "homepage":
game1.repo = url
game1.forums = 12835
game1.short_desc = "" or app.get_comment()
game1.desc = app.get_description()
session.add(game1)
for screenshot in screenshots:
counter = 1
url = screenshot.get_url()
try:
r = requests.get(url,timeout=10)
r.raise_for_status()
filename = randomString(10) + "." + "png"
filepath = os.path.join("/var/cdb/uploads", filename)
print("Screenshot url: ", url)
with open(filepath,"wb") as f:
f.write(r.content)
width, height = get_image_size(filepath)
if (width is not None) and (height is not None):
ss = PackageScreenshot()
ss.package = game1
ss.title = "Untitled"
ss.url = "/uploads/" + filename
ss.width = width
ss.height = height
ss.approved = True
ss.order = counter
session.add(ss)
session.commit()
game1.cover_image = ss
session.commit()
counter += 1
except requests.exceptions.HTTPError as err:
print("HTTP error downloading the screenshot ", err)
except requests.exceptions.ConnectionError as err:
print("HTTP error downloading the screenshot ", err)
except requests.exceptions.ReadTimeout as err:
print("Screenshot timeout ", err)
except PIL.UnidentifiedImageError as err:
print("Corrupt image ", err)
session.commit()
def importAppstream():
pass

View File

@ -72,7 +72,7 @@
{{ render_field(form.name, class_="pkg_meta col-sm-4",
readonly=True, hint=_("Please open a thread to request a name change")) }}
{% else %}
{{ render_field(form.name, class_="pkg_meta col-sm-4", pattern="[a-z0-9_]+", title=_("Lower case letters (a-z), digits (0-9), and underscores (_) only")) }}
{{ render_field(form.name, class_="pkg_meta col-sm-4", pattern="[a-z0-9_\-\.]+", title=_("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only")) }}
{% endif %}
</div>
{{ render_field(form.short_desc, class_="pkg_meta") }}

72
app/utils/lists.py Normal file
View File

@ -0,0 +1,72 @@
badLicenses = [
'LicenseRef-proprietary',
'LicenseRef-Proprietary',
'proprietary',
'Proprietary',
'CC-BY-NC-SA-3.0',
'CC-BY-NC-ND-3.0'
]
badCategories = [
'Emulator',
'PackageManager',
'System',
'Utility'
]
nonFreeAssets = [
'jp.yvt.OpenSpades',
'net.openra.OpenRA',
'org.openmw.OpenMW',
'org.zdoom.GZDoom',
'io.github.ezQuake',
'com.etlegacy.ETLegacy',
'com.github.iortcw.iortcw',
'org.yamagi.YamagiQ2',
'org.dhewm3.Dhewm3',
'com.github.bvschaik.julius',
'io.openrct2.OpenRCT2',
'com.github.skullernet.q2pro',
'org.raceintospace.Raceintospace',
'org.srb2.SRB2',
'org.srb2.SRB2Kart',
'io.sourceforge.clonekeenplus',
'io.github.fabiangreffrath.Doom',
'net.dengine.Doomsday',
'com.github.keriew.augustus',
'io.github.yairm210.unciv',
'com.corsixth.corsixth'
]
nonFreeNetworkServices = [
'io.github.yairm210.unciv'
]
alwaysAccept = [
'org.freecol.FreeCol',
'org.freeciv.Freeciv',
'io.github.EndlessSky.endless-sky',
'org.frozen_bubble.frozen-bubble',
'org.kde.ksudoku',
]
alwaysDeny = [
'com.moonlight_stream.Moonlight',
'org.gnome.Games',
'org.ppsspp.PPSSPP',
'org.scummvm.ScummVM',
'org.pegasus_frontend.Pegasus',
'com.gitlab.coringao.cavestory-nx',
'org.sauerbraten.Sauerbraten',
'net.runelite.RuneLite',
'com.zandronum.Zandronum',
'io.mrarm.mcpelauncher',
'org.unitystation.StationHub',
'org.firestormviewer.FirestormViewer',
'com.eduke32.EDuke32',
'io.github.hmlendea.geforcenow-electron',
'io.gdevs.GDLauncher',
'io.github.sharkwouter.Minigalaxy',
'com.katawa_shoujo.KatawaShoujo',
're.chiaki.Chiaki'
]

View File

@ -19,7 +19,7 @@ depends_on = None
def upgrade():
conn = op.get_bind()
sync_trigger(conn, 'package', 'search_vector', ["name", "title", "short_desc", "desc"])
op.create_check_constraint("name_valid", "package", "name ~* '^[a-z0-9_]+$'")
op.create_check_constraint("name_valid", "package", "name ~* '^[a-z0-9_\-\.]+$'")
def downgrade():

View File

@ -18,13 +18,23 @@
import inspect
import os
import sys
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--test-data", "-t", action="store_true", help='Seed test data')
parser.add_argument("--delete", "-d", action="store_true", help='Delete SQLite DB')
parser.add_argument("--create", "-o", action="store_true", help='Create database tables')
parser.add_argument("--seed", "-s", action="store_true", help='Seed minimal data')
args = parser.parse_args()
if not "FLASK_CONFIG" in os.environ:
os.environ["FLASK_CONFIG"] = "../config.cfg"
delete_db = len(sys.argv) >= 2 and sys.argv[1].strip() == "-d"
create_db = not (len(sys.argv) >= 2 and sys.argv[1].strip() == "-o")
test_data = len(sys.argv) >= 2 and sys.argv[1].strip() == "-t" or not create_db
delete_db = args["delete"]
create_db = args["create"]
seed_db = args["seed"]
test_data = args["test-data"]
# Allow finding the `app` module
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
@ -41,9 +51,10 @@ if create_db:
print("Creating database tables...")
db.create_all()
print("Filling database...")
if seed_db:
print("Filling database...")
populate(db.session)
populate(db.session)
if test_data:
populate_test_data(db.session)