Import categories from appstream into tags

Dockerfile

@@ -9,8 +9,13 @@ RUN mkdir /var/cdb
 RUN chown -R cdb:cdb /var/cdb
 
 COPY requirements.lock.txt requirements.lock.txt
+RUN apt update
+RUN apt install -y vim
+RUN apt install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev gir1.2-gtk-3.0 
+RUN apt install -y libappstream-glib-dev
 RUN pip install -r requirements.lock.txt
 RUN pip install gunicorn
+RUN pip3 install pycairo PyGObject
 
 COPY utils utils
 COPY config.cfg config.cfg

docker-compose.debug.yml

@@ -0,0 +1,68 @@
+version: '3'
+
+
+services:
+  db:
+    image: "postgres:14.1"
+    volumes:
+      - "./data/db:/var/lib/postgresql/data"
+    environment: &env
+      - POSTGRES_USER=contentdb
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=contentdb
+      - FLASK_DEBUG=1    
+      - FLASK_CONFIG=../config.cfg
+  adminer:
+    image: adminer
+    restart: always
+    environment:
+      - ADMINER_DEFAULT_SERVER=db
+      - ADMINER_DESIGN=pepa-linha
+    ports:
+        - 8081:8080
+
+  redis:
+    image: 'redis:6.2-alpine'
+    command: redis-server
+    volumes:
+      - './data/redis:/data'
+
+  app:
+    build: .
+    container_name: contentdb_app_1
+    command: ./utils/entrypoint.sh
+    environment: *env
+    ports:
+      - 5123:5123
+    volumes:
+      - "./data/uploads:/var/cdb/uploads"
+      - "./app:/source/app"
+      - "./migrations:/source/migrations"
+    depends_on:
+      - db
+      - redis
+
+  worker:
+    build: .
+    command: celery -A app.tasks.celery worker --concurrency 1
+    environment: *env
+    volumes:
+      - "./data/uploads:/var/cdb/uploads"
+      - "./app:/home/cdb/app"
+    depends_on:
+      - redis
+
+  beat:
+    build: .
+    command: celery -A app.tasks.celery beat
+    environment: *env
+    depends_on:
+      - redis
+
+  exporter:
+    image: ovalmoney/celery-exporter
+    environment: *env
+    ports:
+      - 5125:9540
+    depends_on:
+      - redis

docker-compose.yml

@@ -1,11 +1,17 @@
 version: '3'
+
+
 services:
   db:
     image: "postgres:14.1"
     volumes:
       - "./data/db:/var/lib/postgresql/data"
-    env_file:
-      - config.env
+    environment: &env
+      - POSTGRES_USER=contentdb
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=contentdb
+      - FLASK_DEBUG=0
+      - FLASK_CONFIG=../config.cfg
 
   redis:
     image: 'redis:6.2-alpine'
@@ -15,9 +21,9 @@ services:
 
   app:
     build: .
+    container_name: contentdb_app_1
     command: ./utils/entrypoint.sh
-    env_file:
-      - config.env
+    environment: *env
     ports:
       - 5123:5123
     volumes:
@@ -31,10 +37,7 @@ services:
   worker:
     build: .
     command: celery -A app.tasks.celery worker --concurrency 1
-    env_file:
-      - config.env
-    environment:
-      - FLASK_CONFIG=../config.cfg
+    environment: *env
     volumes:
       - "./data/uploads:/var/cdb/uploads"
       - "./app:/home/cdb/app"
@@ -44,17 +47,13 @@ services:
   beat:
     build: .
     command: celery -A app.tasks.celery beat
-    env_file:
-      - config.env
-    environment:
-      - FLASK_CONFIG=../config.cfg
+    environment: *env
     depends_on:
       - redis
 
   exporter:
     image: ovalmoney/celery-exporter
-    env_file:
-      - config.env
+    environment: *env
     ports:
       - 5125:9540
     depends_on:

utils/appstream.py

@@ -0,0 +1,158 @@
+import gi
+import PIL
+import requests
+import os
+import sys
+import inspect
+from gi.repository import Gio
+gi.require_version('AppStreamGlib', '1.0')
+from gi.repository import AppStreamGlib
+from lists import alwaysAccept, alwaysDeny, badLicenses, badCategories,  nonFreeAssets, nonFreeNetworkServices 
+import itertools
+import argparse
+
+#Workaround to get the urls because app.get_urls() doesn't work :|
+def get_urls(app):
+    kinds = [AppStreamGlib.UrlKind(kind) for kind in range(11)]
+    urls = [(app.get_url_item(kind),kind.value_nick) for kind in kinds]
+    return list(filter(lambda a: a[0] is not None, urls))
+
+def acceptedGame(app):
+    #return 'Game' in app.get_categories()
+    if app.get_id() in alwaysAccept:
+        return True
+    if app.get_id() in alwaysDeny:
+        return False
+
+    return app.get_project_license() and \
+        not [x for x in badLicenses if x in app.get_project_license()] and \
+        'Game' in app.get_categories() and \
+        not [x for x in badCategories if x in app.get_categories()]
+
+def getScreenshots(app):
+    return [images.get_source() for images in app.get_screenshots()]
+
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+sys.path.insert(0,parentdir)
+
+from app.models import *
+from app.utils import make_flask_login_password
+from app.utils.image import get_image_size
+from app.utils import randomString
+
+parser = argparse.ArgumentParser(description='Generate HTML files from an Appimage and mako templates.')
+parser.add_argument("-p", '--appstream_path', help='specify the path of the appstream file')
+parser.add_argument("-i", '--icons_path', help='specify the path of the icons')
+parser.add_argument("-m", '--template_path', help='specify the path of the template')
+args = parser.parse_args()
+
+store = AppStreamGlib.Store()
+file = Gio.File.new_for_path(args.appstream_path)
+AppStreamGlib.Store.from_file(store, file, args.icons_path, None)
+apps = list(filter(acceptedGame, store.get_apps()))
+session=db.session
+licenses = { x.name : x for x in License.query.all() }
+tags = { x.name : x for x in Tag.query.all() }
+admin_user = User.query.filter_by(username="appstream").first()
+
+
+for ss in PackageScreenshot.query.all():
+    if ss.package.cover_image == ss:
+        ss.package.cover_image = None
+    session.commit()
+    session.delete(ss)
+    session.commit()
+
+for game in Package.query.all():
+    session.delete(game)
+    session.commit()
+
+if not admin_user:
+    admin_user = User("appstream")
+    admin_user.is_active = True
+    admin_user.password = make_flask_login_password("appstream")
+    admin_user.github_username = "appstream"
+    admin_user.forums_username = "appstream"
+    admin_user.rank = UserRank.ADMIN
+    session.add(admin_user)
+
+
+for app in apps:
+    screenshots = getScreenshots(app)
+    urls = get_urls(app)
+    filename = app.get_name().replace(':', '').replace('/','') + ".html"
+    print("APPLICATION: ",app.get_name())
+    game1 = Package()
+    game1.state = PackageState.APPROVED
+    game1.name = app.get_id().replace('.','_').replace('-','_')
+    game1.title = app.get_name()
+    game1.type = PackageType.GAME
+    license = "Uknown" if app.get_project_license() is None else app.get_project_license().split("AND")[0].split("and")[0]
+    if license not in licenses:
+        row = License(license)
+        licenses[row.name] = row
+        session.add(row)
+        session.commit()
+    for category in app.get_categories():
+        if category.lower() not in tags:
+            row = Tag(category)
+            tags[row.name] = row
+            print("adding tag: ", row.name)
+            session.add(row)
+        game1.tags.append(tags[category.lower()])
+    game1.license = licenses[license]
+    game1.media_license = licenses["MIT"]
+    game1.author = admin_user
+
+
+    for url,t in urls:
+        if t == "bugtracker":
+            game1.issueTracker = url
+        elif t == "homepage":
+            game1.repo = url
+
+    game1.forums = 12835
+    game1.short_desc = "" or app.get_comment()
+    game1.desc = app.get_description()
+    session.add(game1)
+
+    for screenshot in screenshots:
+        counter = 1
+        url = screenshot.get_url()
+        try:
+            r = requests.get(url,timeout=10)
+            r.raise_for_status()
+            filename = randomString(10) + "." + "png"
+            filepath = os.path.join("/var/cdb/uploads", filename)
+            print("Screenshot url: ", url)
+            with open(filepath,"wb") as f:
+                f.write(r.content)
+
+            width, height = get_image_size(filepath)
+
+            if (width is not None) and (height is not None):
+                ss = PackageScreenshot()
+                ss.package  = game1
+                ss.title    =  "Untitled"
+                ss.url      = "/uploads/" + filename
+                ss.width = width
+                ss.height = height
+                ss.approved = True
+                ss.order    = counter
+                session.add(ss)
+                session.commit()
+                game1.cover_image = ss
+                session.commit()
+                counter += 1
+        except requests.exceptions.HTTPError as err:
+            print("HTTP error downloading the  screenshot ", err)
+        except requests.exceptions.ConnectionError as err:
+            print("HTTP error downloading the  screenshot ", err)
+        except requests.exceptions.ReadTimeout as err:
+            print("Screenshot timeout ", err)
+        except PIL.UnidentifiedImageError as err:
+            print("Corrupt image ", err)
+
+session.commit()
+

utils/lists.py

@@ -0,0 +1,72 @@
+badLicenses = [
+    'LicenseRef-proprietary',
+    'LicenseRef-Proprietary',
+    'proprietary',
+    'Proprietary',
+    'CC-BY-NC-SA-3.0',
+    'CC-BY-NC-ND-3.0'
+]
+
+badCategories = [
+    'Emulator',
+    'PackageManager',
+    'System',
+    'Utility'
+]
+
+nonFreeAssets = [
+    'jp.yvt.OpenSpades',
+    'net.openra.OpenRA',
+    'org.openmw.OpenMW',
+    'org.zdoom.GZDoom',
+    'io.github.ezQuake',
+    'com.etlegacy.ETLegacy',
+    'com.github.iortcw.iortcw',
+    'org.yamagi.YamagiQ2',
+    'org.dhewm3.Dhewm3',
+    'com.github.bvschaik.julius',
+    'io.openrct2.OpenRCT2',
+    'com.github.skullernet.q2pro',
+    'org.raceintospace.Raceintospace',
+    'org.srb2.SRB2',
+    'org.srb2.SRB2Kart',
+    'io.sourceforge.clonekeenplus',
+    'io.github.fabiangreffrath.Doom',
+    'net.dengine.Doomsday',
+    'com.github.keriew.augustus',
+    'io.github.yairm210.unciv',
+    'com.corsixth.corsixth'
+]
+
+nonFreeNetworkServices = [
+    'io.github.yairm210.unciv'
+]
+
+alwaysAccept = [
+    'org.freecol.FreeCol',
+    'org.freeciv.Freeciv',
+    'io.github.EndlessSky.endless-sky',
+    'org.frozen_bubble.frozen-bubble',
+    'org.kde.ksudoku',
+]
+
+alwaysDeny = [
+    'com.moonlight_stream.Moonlight',
+    'org.gnome.Games',
+    'org.ppsspp.PPSSPP',
+    'org.scummvm.ScummVM',
+    'org.pegasus_frontend.Pegasus',
+    'com.gitlab.coringao.cavestory-nx',
+    'org.sauerbraten.Sauerbraten',
+    'net.runelite.RuneLite',
+    'com.zandronum.Zandronum',
+    'io.mrarm.mcpelauncher',
+    'org.unitystation.StationHub',
+    'org.firestormviewer.FirestormViewer',
+    'com.eduke32.EDuke32',
+    'io.github.hmlendea.geforcenow-electron',
+    'io.gdevs.GDLauncher',
+    'io.github.sharkwouter.Minigalaxy',
+    'com.katawa_shoujo.KatawaShoujo',
+    're.chiaki.Chiaki'
+]