Import from flatpak action

Dockerfile

@@ -1,24 +1,22 @@
 FROM python:3.10
-
 RUN groupadd -g 5123 cdb && \
     useradd -r -u 5123 -g cdb cdb
-
 WORKDIR /home/cdb
-
 RUN mkdir /var/cdb
 RUN chown -R cdb:cdb /var/cdb
-
 COPY requirements.lock.txt requirements.lock.txt
+RUN apt update
+RUN apt install -y vim
+RUN apt install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev gir1.2-gtk-3.0 
+RUN apt install -y libappstream-glib-dev
 RUN pip install -r requirements.lock.txt
 RUN pip install gunicorn
-
+RUN pip3 install pycairo PyGObject
 COPY utils utils
 COPY config.cfg config.cfg
 COPY migrations migrations
 COPY app app
 COPY translations translations
-
 RUN pybabel compile -d translations
 RUN chown -R cdb:cdb /home/cdb
-
+USER cdb
-USER cdb

app/blueprints/admin/actions.py

@@ -29,6 +29,7 @@ from app.models import PackageRelease, db, Package, PackageState, PackageScreens
 	NotificationType, PackageUpdateConfig, License, UserRank, PackageType, PackageGameSupport
 from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
 from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, check_for_updates
+from app.tasks.appstreamtasks import importFromFlathub
 from app.utils import addNotification, get_system_user
 from app.utils.image import get_image_size
 
@@ -94,6 +95,10 @@ def import_topic_list():
 	task = importTopicList.delay()
 	return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
 
+@action("Import appstream from flathub")
+def import_from_flathub():
+	task = importFromFlathub.delay()
+	return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
 
 @action("Check all forum accounts")
 def check_all_forum_accounts():

app/blueprints/packages/packages.py

@@ -228,7 +228,7 @@ def makeLabel(obj):
 class PackageForm(FlaskForm):
 	type             = SelectField(lazy_gettext("Type"), [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.GAME)
 	title            = StringField(lazy_gettext("Title (Human-readable)"), [InputRequired(), Length(1, 100)])
-	name             = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_) only"))])
+	name             = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_\-\.]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only"))])
 	short_desc       = StringField(lazy_gettext("Short Description (Plaintext)"), [InputRequired(), Length(1,200)])
 
 	dev_state        = SelectField(lazy_gettext("Maintenance State"), [InputRequired()], choices=PackageDevState.choices(with_none=True), coerce=PackageDevState.coerce)
@@ -546,7 +546,7 @@ def audit(package):
 class PackageAliasForm(FlaskForm):
 	author  = StringField(lazy_gettext("Author Name"), [InputRequired(), Length(1, 50)])
 	name    = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100),
-			Regexp("^[a-z0-9_]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_) only"))])
+			Regexp("^[a-z0-9_\_\.]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only"))])
 	submit  = SubmitField(lazy_gettext("Save"))
 
 

app/default_data.py

@@ -1,33 +1,54 @@
+import string
+import random
 from .models import *
 from .utils import make_flask_login_password
 
+def generate_password():
+	characters = string.ascii_letters + string.digits + string.punctuation
+	password = ''.join(random.choice(characters) for i in range(16))
+	return password
 
 def populate(session):
-	admin_user = User("rubenwardy")
+	admin_user = User("libregaming")
 	admin_user.is_active = True
-	admin_user.password = make_flask_login_password("tuckfrump")
+	password = generate_password()
-	admin_user.github_username = "rubenwardy"
+	admin_user.password = make_flask_login_password(password)
-	admin_user.forums_username = "rubenwardy"
+	admin_user.github_username = "libregaming"
+	admin_user.forums_username = "libregaming"
 	admin_user.rank = UserRank.ADMIN
 	session.add(admin_user)
+	print("#####################################")
+	print("Admin user    : libregaming")
+	print("Admin password: " + password)
+	print("#####################################")
 
 	system_user = User("ContentDB", active=False)
 	system_user.email_confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000)
 	system_user.rank = UserRank.BOT
 	session.add(system_user)
 
-	session.add(MinetestRelease("None", 0))
+	appstream_user = User("AppStreamBot", active=False)
-	session.add(MinetestRelease("0.4.16/17", 32))
+	appstream_user.email_confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000)
-	session.add(MinetestRelease("5.0", 37))
+	appstream_user.rank = UserRank.BOT
-	session.add(MinetestRelease("5.1", 38))
+	session.add(appstream_user)
-	session.add(MinetestRelease("5.2", 39))
+
-	session.add(MinetestRelease("5.3", 39))
 
 	tags = {}
-	for tag in ["Inventory", "Mapgen", "Building",
+	for tag in [
-		"Mobs and NPCs", "Tools", "Player effects",
+		"Action",
-		"Environment", "Transport", "Maintenance", "Plants and farming",
+		"Adventure",
-		"PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer", "Featured"]:
+		"Arcade",
+		"Board",
+		"Blocks",
+		"Card",
+		"Kids",
+		"Logic",
+		"RolePlaying",
+		"Shooter",
+		"Simulation",
+		"Sports",
+		"Strategy"
+		]:
 		row = Tag(tag)
 		tags[row.name] = row
 		session.add(row)
@@ -60,7 +81,7 @@ def populate_test_data(session):
 	ez.rank = UserRank.EDITOR
 	session.add(ez)
 
-	not1 = Notification(admin_user, ez, NotificationType.PACKAGE_APPROVAL, "Awards approved", "/packages/rubenwardy/awards/")
+	not1 = Notification(admin_user, ez, NotificationType.PACKAGE_APPROVAL, "Awards approved", "/packages/libregaming/awards/")
 	session.add(not1)
 
 	jeija = User("Jeija")
@@ -102,8 +123,8 @@ def populate_test_data(session):
 	mod1.type = PackageType.TOOL
 	mod1.author = admin_user
 	mod1.tags.append(tags["player_effects"])
-	mod1.repo = "https://github.com/rubenwardy/awards"
+	mod1.repo = "https://github.com/libregaming/awards"
-	mod1.issueTracker = "https://github.com/rubenwardy/awards/issues"
+	mod1.issueTracker = "https://github.com/libregaming/awards/issues"
 	mod1.forums = 4870
 	mod1.short_desc = "Adds achievements and an API to register new ones."
 	mod1.desc = """
@@ -126,7 +147,7 @@ awards.register_achievement("award_mesefind",{
 	rel.package = mod1
 	rel.min_rel = v51
 	rel.title = "v1.0.0"
-	rel.url = "https://github.com/rubenwardy/awards/archive/master.zip"
+	rel.url = "https://github.com/libregaming/awards/archive/master.zip"
 	rel.approved = True
 	session.add(rel)
 
@@ -269,8 +290,8 @@ No warranty is provided, express or implied, for any part of the project.
 	tool.type = PackageType.TOOL
 	tool.author = admin_user
 	tool.tags.append(tags["player_effects"])
-	tool.repo = "https://github.com/rubenwardy/food/"
+	tool.repo = "https://github.com/libregaming/food/"
-	tool.issueTracker = "https://github.com/rubenwardy/food/issues/"
+	tool.issueTracker = "https://github.com/libregaming/food/issues/"
 	tool.forums = 2960
 	tool.short_desc = "Adds lots of food and an API to manage ingredients"
 	tool.desc = "This is the long desc"
@@ -285,8 +306,8 @@ No warranty is provided, express or implied, for any part of the project.
 	tool.type = PackageType.TOOL
 	tool.author = admin_user
 	tool.tags.append(tags["player_effects"])
-	tool.repo = "https://github.com/rubenwardy/food_sweet/"
+	tool.repo = "https://github.com/libregaming/food_sweet/"
-	tool.issueTracker = "https://github.com/rubenwardy/food_sweet/issues/"
+	tool.issueTracker = "https://github.com/libregaming/food_sweet/issues/"
 	tool.forums = 9039
 	tool.short_desc = "Adds sweet food"
 	tool.desc = "This is the long desc"
@@ -304,12 +325,12 @@ No warranty is provided, express or implied, for any part of the project.
 	game1.tags.append(tags["pvp"])
 	game1.tags.append(tags["survival"])
 	game1.tags.append(tags["multiplayer"])
-	game1.repo = "https://github.com/rubenwardy/capturetheflag"
+	game1.repo = "https://github.com/libregaming/capturetheflag"
-	game1.issueTracker = "https://github.com/rubenwardy/capturetheflag/issues"
+	game1.issueTracker = "https://github.com/libregaming/capturetheflag/issues"
 	game1.forums = 12835
 	game1.short_desc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!"
 	game1.desc = """
-As seen on the Capture the Flag server (minetest.rubenwardy.com:30000)
+As seen on the Capture the Flag server (minetest.libregaming.com:30000)
 
 ` `[`javascript:/*--></title></style></textarea></script></xmp><svg/onload='+/"/+/onmouseover=1/+/`](javascript:/*--%3E%3C/title%3E%3C/style%3E%3C/textarea%3E%3C/script%3E%3C/xmp%3E%3Csvg/onload='+/%22/+/onmouseover=1/+/)`[*/[]/+alert(1)//'>`
 
@@ -351,7 +372,7 @@ Uses the CTF PvP Engine.
 	rel = PackageRelease()
 	rel.package = game1
 	rel.title = "v1.0.0"
-	rel.url = "https://github.com/rubenwardy/capturetheflag/archive/master.zip"
+	rel.url = "https://github.com/libregaming/capturetheflag/archive/master.zip"
 	rel.approved = True
 	session.add(rel)
 

app/logic/packages.py

@@ -41,7 +41,7 @@ def get_license(name):
 	return license
 
 
-name_re = re.compile("^[a-z0-9_]+$")
+name_re = re.compile("^[a-z0-9_\-\.]+$")
 
 AnyType = "?"
 ALLOWED_FIELDS = {

app/models/packages.py

@@ -380,7 +380,7 @@ class Package(db.Model):
 	created_at   = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 	approved_at  = db.Column(db.DateTime, nullable=True, default=None)
 
-	name_valid = db.CheckConstraint("name ~* '^[a-z0-9_]+$'")
+	name_valid = db.CheckConstraint("name ~* '^[a-z0-9_\-\.]+$'")
 
 	search_vector = db.Column(TSVectorType("name", "title", "short_desc", "desc",
 			weights={ "name": "A", "title": "B", "short_desc": "C", "desc": "D" }))
@@ -857,7 +857,7 @@ class Tag(db.Model):
 		self.textColor       = textColor
 
 		import re
-		regex = re.compile("[^a-z_]")
+		regex = re.compile("[^0-9a-z_]")
 		self.name = regex.sub("", self.title.lower().replace(" ", "_"))
 
 	def getAsDictionary(self):

app/tasks/appstreamtasks.py

@@ -0,0 +1,165 @@
+import json, re, sys
+from app.models import *
+from app.tasks import celery
+from app.utils import is_username_valid
+import urllib.request
+import gi
+import PIL
+import requests
+import os
+import sys
+import inspect
+import shutil
+import urllib.request
+from gi.repository import Gio
+gi.require_version('AppStreamGlib', '1.0')
+from gi.repository import AppStreamGlib
+from app.utils.lists import alwaysAccept, alwaysDeny, badLicenses, badCategories,  nonFreeAssets, nonFreeNetworkServices 
+import itertools
+from app.utils import make_flask_login_password
+from app.utils.image import get_image_size
+from app.utils import randomString
+
+#Workaround to get the urls because app.get_urls() doesn't work :|
+def get_urls(app):
+    kinds = [AppStreamGlib.UrlKind(kind) for kind in range(11)]
+    urls = [(app.get_url_item(kind),kind.value_nick) for kind in kinds]
+    return list(filter(lambda a: a[0] is not None, urls))
+
+def acceptedGame(app):
+    #return 'Game' in app.get_categories()
+    if app.get_id() in alwaysAccept:
+        return True
+    if app.get_id() in alwaysDeny:
+        return False
+
+    return app.get_project_license() and \
+        not [x for x in badLicenses if x in app.get_project_license()] and \
+        'Game' in app.get_categories() and \
+        not [x for x in badCategories if x in app.get_categories()]
+
+def getScreenshots(app):
+    return [images.get_source() for images in app.get_screenshots()]
+
+@celery.task()
+def importFromFlathub():
+    url = "https://flathub.org/repo/appstream/x86_64/appstream.xml.gz"
+    with urllib.request.urlopen(url) as response, open("/var/cdb/uploads/appstream.xml.gz", 'wb') as out_file:
+        shutil.copyfileobj(response, out_file)
+    store = AppStreamGlib.Store()
+    file = Gio.File.new_for_path("/var/cdb/uploads/appstream.xml.gz")
+    file.load_contents()
+    AppStreamGlib.Store.from_file(store, file, ".", None)
+    apps = list(filter(acceptedGame, store.get_apps()))
+    session=db.session
+    licenses = { x.name : x for x in License.query.all() }
+    tags = { x.name : x for x in Tag.query.all() }
+    admin_user = User.query.filter_by(username="AppStreamBot").first()
+
+    # for ss in PackageScreenshot.query.all():
+    #     if ss.package.cover_image == ss:
+    #         ss.package.cover_image = None
+    #     session.commit()
+    #     session.delete(ss)
+    #     session.commit()
+
+    # for game in Package.query.all():
+    #     session.delete(game)
+    #     session.commit()
+
+    if not admin_user:
+        admin_user = User("AppStreamBot")
+        admin_user.is_active = True
+        admin_user.password = make_flask_login_password("AppStreamBot")
+        admin_user.github_username = "AppStreamBot"
+        admin_user.forums_username = "AppStreamBot"
+        admin_user.rank = UserRank.ADMIN
+        session.add(admin_user)
+
+
+    for app in apps:
+        screenshots = getScreenshots(app)
+        urls = get_urls(app)
+        filename = app.get_name().replace(':', '').replace('/','') + ".html"
+        print("APPLICATION: ",app.get_name())
+        package_exists = Package.query.filter_by(name=app.get_id()).first()
+        if package_exists:
+            print(f"Package {app.get_id()} exists, skipping.")
+        else:
+            game1 = Package()
+            game1.state = PackageState.APPROVED
+            game1.name = app.get_id()
+            game1.title = app.get_name()
+            if "Development" in app.get_categories():
+                game1.type = PackageType.TOOL
+            else:
+                game1.type = PackageType.GAME
+            license = "Uknown" if app.get_project_license() is None else app.get_project_license().split("AND")[0].split("and")[0]
+            if license not in licenses:
+                row = License(license)
+                licenses[row.name] = row
+                session.add(row)
+                session.commit()
+            for category in app.get_categories():
+                if category.lower() not in tags:
+                    row = Tag(category.lower())
+                    tags[row.name] = row
+                    print("adding tag: ", row.name)
+                    session.add(row)
+                game1.tags.append(tags[category.lower()])
+            game1.license = licenses[license]
+            game1.media_license = licenses["MIT"]
+            game1.author = admin_user
+
+
+            for url,t in urls:
+                if t == "bugtracker":
+                    game1.issueTracker = url
+                elif t == "homepage":
+                    game1.repo = url
+
+            game1.forums = 12835
+            game1.short_desc = "" or app.get_comment()
+            game1.desc = app.get_description()
+            session.add(game1)
+
+            for screenshot in screenshots:
+                counter = 1
+                url = screenshot.get_url()
+                try:
+                    r = requests.get(url,timeout=10)
+                    r.raise_for_status()
+                    filename = randomString(10) + "." + "png"
+                    filepath = os.path.join("/var/cdb/uploads", filename)
+                    print("Screenshot url: ", url)
+                    with open(filepath,"wb") as f:
+                        f.write(r.content)
+
+                    width, height = get_image_size(filepath)
+
+                    if (width is not None) and (height is not None):
+                        ss = PackageScreenshot()
+                        ss.package  = game1
+                        ss.title    =  "Untitled"
+                        ss.url      = "/uploads/" + filename
+                        ss.width = width
+                        ss.height = height
+                        ss.approved = True
+                        ss.order    = counter
+                        session.add(ss)
+                        session.commit()
+                        game1.cover_image = ss
+                        session.commit()
+                        counter += 1
+                except requests.exceptions.HTTPError as err:
+                    print("HTTP error downloading the  screenshot ", err)
+                except requests.exceptions.ConnectionError as err:
+                    print("HTTP error downloading the  screenshot ", err)
+                except requests.exceptions.ReadTimeout as err:
+                    print("Screenshot timeout ", err)
+                except PIL.UnidentifiedImageError as err:
+                    print("Corrupt image ", err)
+
+    session.commit()
+def importAppstream():
+    pass

app/templates/packages/create_edit.html

@@ -72,7 +72,7 @@
 				{{ render_field(form.name, class_="pkg_meta col-sm-4",
 					readonly=True, hint=_("Please open a thread to request a name change")) }}
 			{% else %}
-				{{ render_field(form.name, class_="pkg_meta col-sm-4", pattern="[a-z0-9_]+", title=_("Lower case letters (a-z), digits (0-9), and underscores (_) only")) }}
+				{{ render_field(form.name, class_="pkg_meta col-sm-4", pattern="[a-z0-9_\-\.]+", title=_("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only")) }}
 			{% endif %}
 			</div>
 			{{ render_field(form.short_desc, class_="pkg_meta") }}

app/utils/lists.py

@@ -0,0 +1,72 @@
+badLicenses = [
+    'LicenseRef-proprietary',
+    'LicenseRef-Proprietary',
+    'proprietary',
+    'Proprietary',
+    'CC-BY-NC-SA-3.0',
+    'CC-BY-NC-ND-3.0'
+]
+
+badCategories = [
+    'Emulator',
+    'PackageManager',
+    'System',
+    'Utility'
+]
+
+nonFreeAssets = [
+    'jp.yvt.OpenSpades',
+    'net.openra.OpenRA',
+    'org.openmw.OpenMW',
+    'org.zdoom.GZDoom',
+    'io.github.ezQuake',
+    'com.etlegacy.ETLegacy',
+    'com.github.iortcw.iortcw',
+    'org.yamagi.YamagiQ2',
+    'org.dhewm3.Dhewm3',
+    'com.github.bvschaik.julius',
+    'io.openrct2.OpenRCT2',
+    'com.github.skullernet.q2pro',
+    'org.raceintospace.Raceintospace',
+    'org.srb2.SRB2',
+    'org.srb2.SRB2Kart',
+    'io.sourceforge.clonekeenplus',
+    'io.github.fabiangreffrath.Doom',
+    'net.dengine.Doomsday',
+    'com.github.keriew.augustus',
+    'io.github.yairm210.unciv',
+    'com.corsixth.corsixth'
+]
+
+nonFreeNetworkServices = [
+    'io.github.yairm210.unciv'
+]
+
+alwaysAccept = [
+    'org.freecol.FreeCol',
+    'org.freeciv.Freeciv',
+    'io.github.EndlessSky.endless-sky',
+    'org.frozen_bubble.frozen-bubble',
+    'org.kde.ksudoku',
+]
+
+alwaysDeny = [
+    'com.moonlight_stream.Moonlight',
+    'org.gnome.Games',
+    'org.ppsspp.PPSSPP',
+    'org.scummvm.ScummVM',
+    'org.pegasus_frontend.Pegasus',
+    'com.gitlab.coringao.cavestory-nx',
+    'org.sauerbraten.Sauerbraten',
+    'net.runelite.RuneLite',
+    'com.zandronum.Zandronum',
+    'io.mrarm.mcpelauncher',
+    'org.unitystation.StationHub',
+    'org.firestormviewer.FirestormViewer',
+    'com.eduke32.EDuke32',
+    'io.github.hmlendea.geforcenow-electron',
+    'io.gdevs.GDLauncher',
+    'io.github.sharkwouter.Minigalaxy',
+    'com.katawa_shoujo.KatawaShoujo',
+    're.chiaki.Chiaki'
+]

migrations/versions/6dca6eceb04d_.py

@@ -19,7 +19,7 @@ depends_on = None
 def upgrade():
 	conn = op.get_bind()
 	sync_trigger(conn, 'package', 'search_vector', ["name", "title", "short_desc", "desc"])
-	op.create_check_constraint("name_valid", "package", "name ~* '^[a-z0-9_]+$'")
+	op.create_check_constraint("name_valid", "package", "name ~* '^[a-z0-9_\-\.]+$'")
 
 
 def downgrade():

utils/setup.py

@@ -18,13 +18,23 @@
 import inspect
 import os
 import sys
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--test-data", "-t", action="store_true", help='Seed test data')
+parser.add_argument("--delete", "-d", action="store_true", help='Delete SQLite DB')
+parser.add_argument("--create", "-o", action="store_true", help='Create database tables')
+parser.add_argument("--seed", "-s", action="store_true", help='Seed minimal data')
+
+args = parser.parse_args()
 
 if not "FLASK_CONFIG" in os.environ:
 	os.environ["FLASK_CONFIG"] = "../config.cfg"
 
-delete_db = len(sys.argv) >= 2 and sys.argv[1].strip() == "-d"
+delete_db = args["delete"]
-create_db = not (len(sys.argv) >= 2 and sys.argv[1].strip() == "-o")
+create_db = args["create"]
-test_data = len(sys.argv) >= 2 and sys.argv[1].strip() == "-t" or not create_db
+seed_db = args["seed"]
+test_data = args["test-data"]
 
 # Allow finding the `app` module
 currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
@@ -41,9 +51,10 @@ if create_db:
 	print("Creating database tables...")
 	db.create_all()
 
-print("Filling database...")
+if seed_db:
+	print("Filling database...")
+	populate(db.session)
 
-populate(db.session)
 if test_data:
 	populate_test_data(db.session)