diff --git a/Dockerfile b/Dockerfile index 512fc49..f6eba68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,22 @@ FROM python:3.10 - RUN groupadd -g 5123 cdb && \ useradd -r -u 5123 -g cdb cdb - WORKDIR /home/cdb - RUN mkdir /var/cdb RUN chown -R cdb:cdb /var/cdb - COPY requirements.lock.txt requirements.lock.txt +RUN apt update +RUN apt install -y vim +RUN apt install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev gir1.2-gtk-3.0 +RUN apt install -y libappstream-glib-dev RUN pip install -r requirements.lock.txt RUN pip install gunicorn - +RUN pip3 install pycairo PyGObject COPY utils utils COPY config.cfg config.cfg COPY migrations migrations COPY app app COPY translations translations - RUN pybabel compile -d translations RUN chown -R cdb:cdb /home/cdb - -USER cdb +USER cdb \ No newline at end of file diff --git a/app/blueprints/admin/actions.py b/app/blueprints/admin/actions.py index 8f4e1e7..2a14737 100644 --- a/app/blueprints/admin/actions.py +++ b/app/blueprints/admin/actions.py @@ -29,6 +29,7 @@ from app.models import PackageRelease, db, Package, PackageState, PackageScreens NotificationType, PackageUpdateConfig, License, UserRank, PackageType, PackageGameSupport from app.tasks.forumtasks import importTopicList, checkAllForumAccounts from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, check_for_updates +from app.tasks.appstreamtasks import importFromFlathub from app.utils import addNotification, get_system_user from app.utils.image import get_image_size @@ -94,6 +95,10 @@ def import_topic_list(): task = importTopicList.delay() return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics"))) +@action("Import appstream from flathub") +def import_from_flathub(): + task = importFromFlathub.delay() + return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics"))) @action("Check all forum accounts") def check_all_forum_accounts(): diff --git a/app/blueprints/packages/packages.py b/app/blueprints/packages/packages.py index 620c981..6ecc1cb 100644 --- a/app/blueprints/packages/packages.py +++ b/app/blueprints/packages/packages.py @@ -228,7 +228,7 @@ def makeLabel(obj): class PackageForm(FlaskForm): type = SelectField(lazy_gettext("Type"), [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.GAME) title = StringField(lazy_gettext("Title (Human-readable)"), [InputRequired(), Length(1, 100)]) - name = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_) only"))]) + name = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_\-\.]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only"))]) short_desc = StringField(lazy_gettext("Short Description (Plaintext)"), [InputRequired(), Length(1,200)]) dev_state = SelectField(lazy_gettext("Maintenance State"), [InputRequired()], choices=PackageDevState.choices(with_none=True), coerce=PackageDevState.coerce) @@ -546,7 +546,7 @@ def audit(package): class PackageAliasForm(FlaskForm): author = StringField(lazy_gettext("Author Name"), [InputRequired(), Length(1, 50)]) name = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100), - Regexp("^[a-z0-9_]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_) only"))]) + Regexp("^[a-z0-9_\_\.]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only"))]) submit = SubmitField(lazy_gettext("Save")) diff --git a/app/default_data.py b/app/default_data.py index b14b00f..897e126 100644 --- a/app/default_data.py +++ b/app/default_data.py @@ -1,33 +1,54 @@ +import string +import random from .models import * from .utils import make_flask_login_password +def generate_password(): + characters = string.ascii_letters + string.digits + string.punctuation + password = ''.join(random.choice(characters) for i in range(16)) + return password def populate(session): - admin_user = User("rubenwardy") + admin_user = User("libregaming") admin_user.is_active = True - admin_user.password = make_flask_login_password("tuckfrump") - admin_user.github_username = "rubenwardy" - admin_user.forums_username = "rubenwardy" + password = generate_password() + admin_user.password = make_flask_login_password(password) + admin_user.github_username = "libregaming" + admin_user.forums_username = "libregaming" admin_user.rank = UserRank.ADMIN session.add(admin_user) + print("#####################################") + print("Admin user : libregaming") + print("Admin password: " + password) + print("#####################################") system_user = User("ContentDB", active=False) system_user.email_confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000) system_user.rank = UserRank.BOT session.add(system_user) - session.add(MinetestRelease("None", 0)) - session.add(MinetestRelease("0.4.16/17", 32)) - session.add(MinetestRelease("5.0", 37)) - session.add(MinetestRelease("5.1", 38)) - session.add(MinetestRelease("5.2", 39)) - session.add(MinetestRelease("5.3", 39)) + appstream_user = User("AppStreamBot", active=False) + appstream_user.email_confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000) + appstream_user.rank = UserRank.BOT + session.add(appstream_user) + tags = {} - for tag in ["Inventory", "Mapgen", "Building", - "Mobs and NPCs", "Tools", "Player effects", - "Environment", "Transport", "Maintenance", "Plants and farming", - "PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer", "Featured"]: + for tag in [ + "Action", + "Adventure", + "Arcade", + "Board", + "Blocks", + "Card", + "Kids", + "Logic", + "RolePlaying", + "Shooter", + "Simulation", + "Sports", + "Strategy" + ]: row = Tag(tag) tags[row.name] = row session.add(row) @@ -60,7 +81,7 @@ def populate_test_data(session): ez.rank = UserRank.EDITOR session.add(ez) - not1 = Notification(admin_user, ez, NotificationType.PACKAGE_APPROVAL, "Awards approved", "/packages/rubenwardy/awards/") + not1 = Notification(admin_user, ez, NotificationType.PACKAGE_APPROVAL, "Awards approved", "/packages/libregaming/awards/") session.add(not1) jeija = User("Jeija") @@ -102,8 +123,8 @@ def populate_test_data(session): mod1.type = PackageType.TOOL mod1.author = admin_user mod1.tags.append(tags["player_effects"]) - mod1.repo = "https://github.com/rubenwardy/awards" - mod1.issueTracker = "https://github.com/rubenwardy/awards/issues" + mod1.repo = "https://github.com/libregaming/awards" + mod1.issueTracker = "https://github.com/libregaming/awards/issues" mod1.forums = 4870 mod1.short_desc = "Adds achievements and an API to register new ones." mod1.desc = """ @@ -126,7 +147,7 @@ awards.register_achievement("award_mesefind",{ rel.package = mod1 rel.min_rel = v51 rel.title = "v1.0.0" - rel.url = "https://github.com/rubenwardy/awards/archive/master.zip" + rel.url = "https://github.com/libregaming/awards/archive/master.zip" rel.approved = True session.add(rel) @@ -269,8 +290,8 @@ No warranty is provided, express or implied, for any part of the project. tool.type = PackageType.TOOL tool.author = admin_user tool.tags.append(tags["player_effects"]) - tool.repo = "https://github.com/rubenwardy/food/" - tool.issueTracker = "https://github.com/rubenwardy/food/issues/" + tool.repo = "https://github.com/libregaming/food/" + tool.issueTracker = "https://github.com/libregaming/food/issues/" tool.forums = 2960 tool.short_desc = "Adds lots of food and an API to manage ingredients" tool.desc = "This is the long desc" @@ -285,8 +306,8 @@ No warranty is provided, express or implied, for any part of the project. tool.type = PackageType.TOOL tool.author = admin_user tool.tags.append(tags["player_effects"]) - tool.repo = "https://github.com/rubenwardy/food_sweet/" - tool.issueTracker = "https://github.com/rubenwardy/food_sweet/issues/" + tool.repo = "https://github.com/libregaming/food_sweet/" + tool.issueTracker = "https://github.com/libregaming/food_sweet/issues/" tool.forums = 9039 tool.short_desc = "Adds sweet food" tool.desc = "This is the long desc" @@ -304,12 +325,12 @@ No warranty is provided, express or implied, for any part of the project. game1.tags.append(tags["pvp"]) game1.tags.append(tags["survival"]) game1.tags.append(tags["multiplayer"]) - game1.repo = "https://github.com/rubenwardy/capturetheflag" - game1.issueTracker = "https://github.com/rubenwardy/capturetheflag/issues" + game1.repo = "https://github.com/libregaming/capturetheflag" + game1.issueTracker = "https://github.com/libregaming/capturetheflag/issues" game1.forums = 12835 game1.short_desc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!" game1.desc = """ -As seen on the Capture the Flag server (minetest.rubenwardy.com:30000) +As seen on the Capture the Flag server (minetest.libregaming.com:30000) ` `[`javascript:/*-->` @@ -351,7 +372,7 @@ Uses the CTF PvP Engine. rel = PackageRelease() rel.package = game1 rel.title = "v1.0.0" - rel.url = "https://github.com/rubenwardy/capturetheflag/archive/master.zip" + rel.url = "https://github.com/libregaming/capturetheflag/archive/master.zip" rel.approved = True session.add(rel) diff --git a/app/logic/packages.py b/app/logic/packages.py index baad400..aa5d23e 100644 --- a/app/logic/packages.py +++ b/app/logic/packages.py @@ -41,7 +41,7 @@ def get_license(name): return license -name_re = re.compile("^[a-z0-9_]+$") +name_re = re.compile("^[a-z0-9_\-\.]+$") AnyType = "?" ALLOWED_FIELDS = { diff --git a/app/models/packages.py b/app/models/packages.py index 1d973a9..5158eb0 100644 --- a/app/models/packages.py +++ b/app/models/packages.py @@ -380,7 +380,7 @@ class Package(db.Model): created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) approved_at = db.Column(db.DateTime, nullable=True, default=None) - name_valid = db.CheckConstraint("name ~* '^[a-z0-9_]+$'") + name_valid = db.CheckConstraint("name ~* '^[a-z0-9_\-\.]+$'") search_vector = db.Column(TSVectorType("name", "title", "short_desc", "desc", weights={ "name": "A", "title": "B", "short_desc": "C", "desc": "D" })) @@ -857,7 +857,7 @@ class Tag(db.Model): self.textColor = textColor import re - regex = re.compile("[^a-z_]") + regex = re.compile("[^0-9a-z_]") self.name = regex.sub("", self.title.lower().replace(" ", "_")) def getAsDictionary(self): diff --git a/app/tasks/appstreamtasks.py b/app/tasks/appstreamtasks.py new file mode 100644 index 0000000..0391e90 --- /dev/null +++ b/app/tasks/appstreamtasks.py @@ -0,0 +1,165 @@ +import json, re, sys +from app.models import * +from app.tasks import celery +from app.utils import is_username_valid +import urllib.request +import gi +import PIL +import requests +import os +import sys +import inspect +import shutil +import urllib.request +from gi.repository import Gio +gi.require_version('AppStreamGlib', '1.0') +from gi.repository import AppStreamGlib +from app.utils.lists import alwaysAccept, alwaysDeny, badLicenses, badCategories, nonFreeAssets, nonFreeNetworkServices +import itertools +from app.utils import make_flask_login_password +from app.utils.image import get_image_size +from app.utils import randomString + +#Workaround to get the urls because app.get_urls() doesn't work :| +def get_urls(app): + kinds = [AppStreamGlib.UrlKind(kind) for kind in range(11)] + urls = [(app.get_url_item(kind),kind.value_nick) for kind in kinds] + return list(filter(lambda a: a[0] is not None, urls)) + +def acceptedGame(app): + #return 'Game' in app.get_categories() + if app.get_id() in alwaysAccept: + return True + if app.get_id() in alwaysDeny: + return False + + return app.get_project_license() and \ + not [x for x in badLicenses if x in app.get_project_license()] and \ + 'Game' in app.get_categories() and \ + not [x for x in badCategories if x in app.get_categories()] + +def getScreenshots(app): + return [images.get_source() for images in app.get_screenshots()] + +@celery.task() +def importFromFlathub(): + url = "https://flathub.org/repo/appstream/x86_64/appstream.xml.gz" + with urllib.request.urlopen(url) as response, open("/var/cdb/uploads/appstream.xml.gz", 'wb') as out_file: + shutil.copyfileobj(response, out_file) + store = AppStreamGlib.Store() + file = Gio.File.new_for_path("/var/cdb/uploads/appstream.xml.gz") + file.load_contents() + AppStreamGlib.Store.from_file(store, file, ".", None) + apps = list(filter(acceptedGame, store.get_apps())) + session=db.session + licenses = { x.name : x for x in License.query.all() } + tags = { x.name : x for x in Tag.query.all() } + admin_user = User.query.filter_by(username="AppStreamBot").first() + + # for ss in PackageScreenshot.query.all(): + # if ss.package.cover_image == ss: + # ss.package.cover_image = None + # session.commit() + # session.delete(ss) + # session.commit() + + # for game in Package.query.all(): + # session.delete(game) + # session.commit() + + if not admin_user: + admin_user = User("AppStreamBot") + admin_user.is_active = True + admin_user.password = make_flask_login_password("AppStreamBot") + admin_user.github_username = "AppStreamBot" + admin_user.forums_username = "AppStreamBot" + admin_user.rank = UserRank.ADMIN + session.add(admin_user) + + + for app in apps: + screenshots = getScreenshots(app) + urls = get_urls(app) + filename = app.get_name().replace(':', '').replace('/','') + ".html" + print("APPLICATION: ",app.get_name()) + package_exists = Package.query.filter_by(name=app.get_id()).first() + if package_exists: + print(f"Package {app.get_id()} exists, skipping.") + else: + game1 = Package() + game1.state = PackageState.APPROVED + game1.name = app.get_id() + game1.title = app.get_name() + if "Development" in app.get_categories(): + game1.type = PackageType.TOOL + else: + game1.type = PackageType.GAME + license = "Uknown" if app.get_project_license() is None else app.get_project_license().split("AND")[0].split("and")[0] + if license not in licenses: + row = License(license) + licenses[row.name] = row + session.add(row) + session.commit() + for category in app.get_categories(): + if category.lower() not in tags: + row = Tag(category.lower()) + tags[row.name] = row + print("adding tag: ", row.name) + session.add(row) + game1.tags.append(tags[category.lower()]) + game1.license = licenses[license] + game1.media_license = licenses["MIT"] + game1.author = admin_user + + + for url,t in urls: + if t == "bugtracker": + game1.issueTracker = url + elif t == "homepage": + game1.repo = url + + game1.forums = 12835 + game1.short_desc = "" or app.get_comment() + game1.desc = app.get_description() + session.add(game1) + + for screenshot in screenshots: + counter = 1 + url = screenshot.get_url() + try: + r = requests.get(url,timeout=10) + r.raise_for_status() + filename = randomString(10) + "." + "png" + filepath = os.path.join("/var/cdb/uploads", filename) + print("Screenshot url: ", url) + with open(filepath,"wb") as f: + f.write(r.content) + + width, height = get_image_size(filepath) + + if (width is not None) and (height is not None): + ss = PackageScreenshot() + ss.package = game1 + ss.title = "Untitled" + ss.url = "/uploads/" + filename + ss.width = width + ss.height = height + ss.approved = True + ss.order = counter + session.add(ss) + session.commit() + game1.cover_image = ss + session.commit() + counter += 1 + except requests.exceptions.HTTPError as err: + print("HTTP error downloading the screenshot ", err) + except requests.exceptions.ConnectionError as err: + print("HTTP error downloading the screenshot ", err) + except requests.exceptions.ReadTimeout as err: + print("Screenshot timeout ", err) + except PIL.UnidentifiedImageError as err: + print("Corrupt image ", err) + + session.commit() +def importAppstream(): + pass \ No newline at end of file diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index 3ea2c2b..ae2ff0f 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -72,7 +72,7 @@ {{ render_field(form.name, class_="pkg_meta col-sm-4", readonly=True, hint=_("Please open a thread to request a name change")) }} {% else %} - {{ render_field(form.name, class_="pkg_meta col-sm-4", pattern="[a-z0-9_]+", title=_("Lower case letters (a-z), digits (0-9), and underscores (_) only")) }} + {{ render_field(form.name, class_="pkg_meta col-sm-4", pattern="[a-z0-9_\-\.]+", title=_("Lower case letters (a-z), digits (0-9), and underscores (_), dashes and periods only")) }} {% endif %} {{ render_field(form.short_desc, class_="pkg_meta") }} diff --git a/app/utils/lists.py b/app/utils/lists.py new file mode 100644 index 0000000..d90b090 --- /dev/null +++ b/app/utils/lists.py @@ -0,0 +1,72 @@ +badLicenses = [ + 'LicenseRef-proprietary', + 'LicenseRef-Proprietary', + 'proprietary', + 'Proprietary', + 'CC-BY-NC-SA-3.0', + 'CC-BY-NC-ND-3.0' +] + +badCategories = [ + 'Emulator', + 'PackageManager', + 'System', + 'Utility' +] + +nonFreeAssets = [ + 'jp.yvt.OpenSpades', + 'net.openra.OpenRA', + 'org.openmw.OpenMW', + 'org.zdoom.GZDoom', + 'io.github.ezQuake', + 'com.etlegacy.ETLegacy', + 'com.github.iortcw.iortcw', + 'org.yamagi.YamagiQ2', + 'org.dhewm3.Dhewm3', + 'com.github.bvschaik.julius', + 'io.openrct2.OpenRCT2', + 'com.github.skullernet.q2pro', + 'org.raceintospace.Raceintospace', + 'org.srb2.SRB2', + 'org.srb2.SRB2Kart', + 'io.sourceforge.clonekeenplus', + 'io.github.fabiangreffrath.Doom', + 'net.dengine.Doomsday', + 'com.github.keriew.augustus', + 'io.github.yairm210.unciv', + 'com.corsixth.corsixth' +] + +nonFreeNetworkServices = [ + 'io.github.yairm210.unciv' +] + +alwaysAccept = [ + 'org.freecol.FreeCol', + 'org.freeciv.Freeciv', + 'io.github.EndlessSky.endless-sky', + 'org.frozen_bubble.frozen-bubble', + 'org.kde.ksudoku', +] + +alwaysDeny = [ + 'com.moonlight_stream.Moonlight', + 'org.gnome.Games', + 'org.ppsspp.PPSSPP', + 'org.scummvm.ScummVM', + 'org.pegasus_frontend.Pegasus', + 'com.gitlab.coringao.cavestory-nx', + 'org.sauerbraten.Sauerbraten', + 'net.runelite.RuneLite', + 'com.zandronum.Zandronum', + 'io.mrarm.mcpelauncher', + 'org.unitystation.StationHub', + 'org.firestormviewer.FirestormViewer', + 'com.eduke32.EDuke32', + 'io.github.hmlendea.geforcenow-electron', + 'io.gdevs.GDLauncher', + 'io.github.sharkwouter.Minigalaxy', + 'com.katawa_shoujo.KatawaShoujo', + 're.chiaki.Chiaki' +] diff --git a/migrations/versions/6dca6eceb04d_.py b/migrations/versions/6dca6eceb04d_.py index 7287eeb..f9a002c 100644 --- a/migrations/versions/6dca6eceb04d_.py +++ b/migrations/versions/6dca6eceb04d_.py @@ -19,7 +19,7 @@ depends_on = None def upgrade(): conn = op.get_bind() sync_trigger(conn, 'package', 'search_vector', ["name", "title", "short_desc", "desc"]) - op.create_check_constraint("name_valid", "package", "name ~* '^[a-z0-9_]+$'") + op.create_check_constraint("name_valid", "package", "name ~* '^[a-z0-9_\-\.]+$'") def downgrade(): diff --git a/utils/setup.py b/utils/setup.py index b3c64c5..7b811f2 100644 --- a/utils/setup.py +++ b/utils/setup.py @@ -18,13 +18,23 @@ import inspect import os import sys +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("--test-data", "-t", action="store_true", help='Seed test data') +parser.add_argument("--delete", "-d", action="store_true", help='Delete SQLite DB') +parser.add_argument("--create", "-o", action="store_true", help='Create database tables') +parser.add_argument("--seed", "-s", action="store_true", help='Seed minimal data') + +args = parser.parse_args() if not "FLASK_CONFIG" in os.environ: os.environ["FLASK_CONFIG"] = "../config.cfg" -delete_db = len(sys.argv) >= 2 and sys.argv[1].strip() == "-d" -create_db = not (len(sys.argv) >= 2 and sys.argv[1].strip() == "-o") -test_data = len(sys.argv) >= 2 and sys.argv[1].strip() == "-t" or not create_db +delete_db = args["delete"] +create_db = args["create"] +seed_db = args["seed"] +test_data = args["test-data"] # Allow finding the `app` module currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) @@ -41,9 +51,10 @@ if create_db: print("Creating database tables...") db.create_all() -print("Filling database...") +if seed_db: + print("Filling database...") + populate(db.session) -populate(db.session) if test_data: populate_test_data(db.session)