Compare commits
7 Commits
Author | SHA1 | Date |
---|---|---|
Rampoina | d6f387bcc4 | |
Rampoina | c329a5a428 | |
Rampoina | 6f926a321c | |
Rampoina | 71ca47846d | |
Rampoina | 7180d96567 | |
Rampoina | b3826ad1d7 | |
Armen | e4dcb89d2e |
|
@ -9,8 +9,13 @@ RUN mkdir /var/cdb
|
||||||
RUN chown -R cdb:cdb /var/cdb
|
RUN chown -R cdb:cdb /var/cdb
|
||||||
|
|
||||||
COPY requirements.lock.txt requirements.lock.txt
|
COPY requirements.lock.txt requirements.lock.txt
|
||||||
|
RUN apt update
|
||||||
|
RUN apt install -y vim
|
||||||
|
RUN apt install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev gir1.2-gtk-3.0
|
||||||
|
RUN apt install -y libappstream-glib-dev
|
||||||
RUN pip install -r requirements.lock.txt
|
RUN pip install -r requirements.lock.txt
|
||||||
RUN pip install gunicorn
|
RUN pip install gunicorn
|
||||||
|
RUN pip3 install pycairo PyGObject
|
||||||
|
|
||||||
COPY utils utils
|
COPY utils utils
|
||||||
COPY config.cfg config.cfg
|
COPY config.cfg config.cfg
|
||||||
|
|
|
@ -0,0 +1,68 @@
|
||||||
|
version: '3'
|
||||||
|
|
||||||
|
|
||||||
|
services:
|
||||||
|
db:
|
||||||
|
image: "postgres:14.1"
|
||||||
|
volumes:
|
||||||
|
- "./data/db:/var/lib/postgresql/data"
|
||||||
|
environment: &env
|
||||||
|
- POSTGRES_USER=contentdb
|
||||||
|
- POSTGRES_PASSWORD=password
|
||||||
|
- POSTGRES_DB=contentdb
|
||||||
|
- FLASK_DEBUG=1
|
||||||
|
- FLASK_CONFIG=../config.cfg
|
||||||
|
adminer:
|
||||||
|
image: adminer
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
- ADMINER_DEFAULT_SERVER=db
|
||||||
|
- ADMINER_DESIGN=pepa-linha
|
||||||
|
ports:
|
||||||
|
- 8081:8080
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: 'redis:6.2-alpine'
|
||||||
|
command: redis-server
|
||||||
|
volumes:
|
||||||
|
- './data/redis:/data'
|
||||||
|
|
||||||
|
app:
|
||||||
|
build: .
|
||||||
|
container_name: contentdb_app_1
|
||||||
|
command: ./utils/entrypoint.sh
|
||||||
|
environment: *env
|
||||||
|
ports:
|
||||||
|
- 5123:5123
|
||||||
|
volumes:
|
||||||
|
- "./data/uploads:/var/cdb/uploads"
|
||||||
|
- "./app:/source/app"
|
||||||
|
- "./migrations:/source/migrations"
|
||||||
|
depends_on:
|
||||||
|
- db
|
||||||
|
- redis
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build: .
|
||||||
|
command: celery -A app.tasks.celery worker --concurrency 1
|
||||||
|
environment: *env
|
||||||
|
volumes:
|
||||||
|
- "./data/uploads:/var/cdb/uploads"
|
||||||
|
- "./app:/home/cdb/app"
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
|
||||||
|
beat:
|
||||||
|
build: .
|
||||||
|
command: celery -A app.tasks.celery beat
|
||||||
|
environment: *env
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
|
||||||
|
exporter:
|
||||||
|
image: ovalmoney/celery-exporter
|
||||||
|
environment: *env
|
||||||
|
ports:
|
||||||
|
- 5125:9540
|
||||||
|
depends_on:
|
||||||
|
- redis
|
|
@ -1,11 +1,17 @@
|
||||||
version: '3'
|
version: '3'
|
||||||
|
|
||||||
|
|
||||||
services:
|
services:
|
||||||
db:
|
db:
|
||||||
image: "postgres:14.1"
|
image: "postgres:14.1"
|
||||||
volumes:
|
volumes:
|
||||||
- "./data/db:/var/lib/postgresql/data"
|
- "./data/db:/var/lib/postgresql/data"
|
||||||
env_file:
|
environment: &env
|
||||||
- config.env
|
- POSTGRES_USER=contentdb
|
||||||
|
- POSTGRES_PASSWORD=password
|
||||||
|
- POSTGRES_DB=contentdb
|
||||||
|
- FLASK_DEBUG=0
|
||||||
|
- FLASK_CONFIG=../config.cfg
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: 'redis:6.2-alpine'
|
image: 'redis:6.2-alpine'
|
||||||
|
@ -15,9 +21,9 @@ services:
|
||||||
|
|
||||||
app:
|
app:
|
||||||
build: .
|
build: .
|
||||||
|
container_name: contentdb_app_1
|
||||||
command: ./utils/entrypoint.sh
|
command: ./utils/entrypoint.sh
|
||||||
env_file:
|
environment: *env
|
||||||
- config.env
|
|
||||||
ports:
|
ports:
|
||||||
- 5123:5123
|
- 5123:5123
|
||||||
volumes:
|
volumes:
|
||||||
|
@ -31,10 +37,7 @@ services:
|
||||||
worker:
|
worker:
|
||||||
build: .
|
build: .
|
||||||
command: celery -A app.tasks.celery worker --concurrency 1
|
command: celery -A app.tasks.celery worker --concurrency 1
|
||||||
env_file:
|
environment: *env
|
||||||
- config.env
|
|
||||||
environment:
|
|
||||||
- FLASK_CONFIG=../config.cfg
|
|
||||||
volumes:
|
volumes:
|
||||||
- "./data/uploads:/var/cdb/uploads"
|
- "./data/uploads:/var/cdb/uploads"
|
||||||
- "./app:/home/cdb/app"
|
- "./app:/home/cdb/app"
|
||||||
|
@ -44,17 +47,13 @@ services:
|
||||||
beat:
|
beat:
|
||||||
build: .
|
build: .
|
||||||
command: celery -A app.tasks.celery beat
|
command: celery -A app.tasks.celery beat
|
||||||
env_file:
|
environment: *env
|
||||||
- config.env
|
|
||||||
environment:
|
|
||||||
- FLASK_CONFIG=../config.cfg
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
|
|
||||||
exporter:
|
exporter:
|
||||||
image: ovalmoney/celery-exporter
|
image: ovalmoney/celery-exporter
|
||||||
env_file:
|
environment: *env
|
||||||
- config.env
|
|
||||||
ports:
|
ports:
|
||||||
- 5125:9540
|
- 5125:9540
|
||||||
depends_on:
|
depends_on:
|
||||||
|
|
|
@ -0,0 +1,158 @@
|
||||||
|
import gi
|
||||||
|
import PIL
|
||||||
|
import requests
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import inspect
|
||||||
|
from gi.repository import Gio
|
||||||
|
gi.require_version('AppStreamGlib', '1.0')
|
||||||
|
from gi.repository import AppStreamGlib
|
||||||
|
from lists import alwaysAccept, alwaysDeny, badLicenses, badCategories, nonFreeAssets, nonFreeNetworkServices
|
||||||
|
import itertools
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
#Workaround to get the urls because app.get_urls() doesn't work :|
|
||||||
|
def get_urls(app):
|
||||||
|
kinds = [AppStreamGlib.UrlKind(kind) for kind in range(11)]
|
||||||
|
urls = [(app.get_url_item(kind),kind.value_nick) for kind in kinds]
|
||||||
|
return list(filter(lambda a: a[0] is not None, urls))
|
||||||
|
|
||||||
|
def acceptedGame(app):
|
||||||
|
#return 'Game' in app.get_categories()
|
||||||
|
if app.get_id() in alwaysAccept:
|
||||||
|
return True
|
||||||
|
if app.get_id() in alwaysDeny:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return app.get_project_license() and \
|
||||||
|
not [x for x in badLicenses if x in app.get_project_license()] and \
|
||||||
|
'Game' in app.get_categories() and \
|
||||||
|
not [x for x in badCategories if x in app.get_categories()]
|
||||||
|
|
||||||
|
def getScreenshots(app):
|
||||||
|
return [images.get_source() for images in app.get_screenshots()]
|
||||||
|
|
||||||
|
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
|
||||||
|
parentdir = os.path.dirname(currentdir)
|
||||||
|
sys.path.insert(0,parentdir)
|
||||||
|
|
||||||
|
from app.models import *
|
||||||
|
from app.utils import make_flask_login_password
|
||||||
|
from app.utils.image import get_image_size
|
||||||
|
from app.utils import randomString
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description='Generate HTML files from an Appimage and mako templates.')
|
||||||
|
parser.add_argument("-p", '--appstream_path', help='specify the path of the appstream file')
|
||||||
|
parser.add_argument("-i", '--icons_path', help='specify the path of the icons')
|
||||||
|
parser.add_argument("-m", '--template_path', help='specify the path of the template')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
store = AppStreamGlib.Store()
|
||||||
|
file = Gio.File.new_for_path(args.appstream_path)
|
||||||
|
AppStreamGlib.Store.from_file(store, file, args.icons_path, None)
|
||||||
|
apps = list(filter(acceptedGame, store.get_apps()))
|
||||||
|
session=db.session
|
||||||
|
licenses = { x.name : x for x in License.query.all() }
|
||||||
|
tags = { x.name : x for x in Tag.query.all() }
|
||||||
|
admin_user = User.query.filter_by(username="appstream").first()
|
||||||
|
|
||||||
|
|
||||||
|
for ss in PackageScreenshot.query.all():
|
||||||
|
if ss.package.cover_image == ss:
|
||||||
|
ss.package.cover_image = None
|
||||||
|
session.commit()
|
||||||
|
session.delete(ss)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
for game in Package.query.all():
|
||||||
|
session.delete(game)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
if not admin_user:
|
||||||
|
admin_user = User("appstream")
|
||||||
|
admin_user.is_active = True
|
||||||
|
admin_user.password = make_flask_login_password("appstream")
|
||||||
|
admin_user.github_username = "appstream"
|
||||||
|
admin_user.forums_username = "appstream"
|
||||||
|
admin_user.rank = UserRank.ADMIN
|
||||||
|
session.add(admin_user)
|
||||||
|
|
||||||
|
|
||||||
|
for app in apps:
|
||||||
|
screenshots = getScreenshots(app)
|
||||||
|
urls = get_urls(app)
|
||||||
|
filename = app.get_name().replace(':', '').replace('/','') + ".html"
|
||||||
|
print("APPLICATION: ",app.get_name())
|
||||||
|
game1 = Package()
|
||||||
|
game1.state = PackageState.APPROVED
|
||||||
|
game1.name = app.get_id().replace('.','_').replace('-','_')
|
||||||
|
game1.title = app.get_name()
|
||||||
|
game1.type = PackageType.GAME
|
||||||
|
license = "Uknown" if app.get_project_license() is None else app.get_project_license().split("AND")[0].split("and")[0]
|
||||||
|
if license not in licenses:
|
||||||
|
row = License(license)
|
||||||
|
licenses[row.name] = row
|
||||||
|
session.add(row)
|
||||||
|
session.commit()
|
||||||
|
for category in app.get_categories():
|
||||||
|
if category.lower() not in tags:
|
||||||
|
row = Tag(category)
|
||||||
|
tags[row.name] = row
|
||||||
|
print("adding tag: ", row.name)
|
||||||
|
session.add(row)
|
||||||
|
game1.tags.append(tags[category.lower()])
|
||||||
|
game1.license = licenses[license]
|
||||||
|
game1.media_license = licenses["MIT"]
|
||||||
|
game1.author = admin_user
|
||||||
|
|
||||||
|
|
||||||
|
for url,t in urls:
|
||||||
|
if t == "bugtracker":
|
||||||
|
game1.issueTracker = url
|
||||||
|
elif t == "homepage":
|
||||||
|
game1.repo = url
|
||||||
|
|
||||||
|
game1.forums = 12835
|
||||||
|
game1.short_desc = "" or app.get_comment()
|
||||||
|
game1.desc = app.get_description()
|
||||||
|
session.add(game1)
|
||||||
|
|
||||||
|
for screenshot in screenshots:
|
||||||
|
counter = 1
|
||||||
|
url = screenshot.get_url()
|
||||||
|
try:
|
||||||
|
r = requests.get(url,timeout=10)
|
||||||
|
r.raise_for_status()
|
||||||
|
filename = randomString(10) + "." + "png"
|
||||||
|
filepath = os.path.join("/var/cdb/uploads", filename)
|
||||||
|
print("Screenshot url: ", url)
|
||||||
|
with open(filepath,"wb") as f:
|
||||||
|
f.write(r.content)
|
||||||
|
|
||||||
|
width, height = get_image_size(filepath)
|
||||||
|
|
||||||
|
if (width is not None) and (height is not None):
|
||||||
|
ss = PackageScreenshot()
|
||||||
|
ss.package = game1
|
||||||
|
ss.title = "Untitled"
|
||||||
|
ss.url = "/uploads/" + filename
|
||||||
|
ss.width = width
|
||||||
|
ss.height = height
|
||||||
|
ss.approved = True
|
||||||
|
ss.order = counter
|
||||||
|
session.add(ss)
|
||||||
|
session.commit()
|
||||||
|
game1.cover_image = ss
|
||||||
|
session.commit()
|
||||||
|
counter += 1
|
||||||
|
except requests.exceptions.HTTPError as err:
|
||||||
|
print("HTTP error downloading the screenshot ", err)
|
||||||
|
except requests.exceptions.ConnectionError as err:
|
||||||
|
print("HTTP error downloading the screenshot ", err)
|
||||||
|
except requests.exceptions.ReadTimeout as err:
|
||||||
|
print("Screenshot timeout ", err)
|
||||||
|
except PIL.UnidentifiedImageError as err:
|
||||||
|
print("Corrupt image ", err)
|
||||||
|
|
||||||
|
session.commit()
|
||||||
|
|
|
@ -0,0 +1,72 @@
|
||||||
|
badLicenses = [
|
||||||
|
'LicenseRef-proprietary',
|
||||||
|
'LicenseRef-Proprietary',
|
||||||
|
'proprietary',
|
||||||
|
'Proprietary',
|
||||||
|
'CC-BY-NC-SA-3.0',
|
||||||
|
'CC-BY-NC-ND-3.0'
|
||||||
|
]
|
||||||
|
|
||||||
|
badCategories = [
|
||||||
|
'Emulator',
|
||||||
|
'PackageManager',
|
||||||
|
'System',
|
||||||
|
'Utility'
|
||||||
|
]
|
||||||
|
|
||||||
|
nonFreeAssets = [
|
||||||
|
'jp.yvt.OpenSpades',
|
||||||
|
'net.openra.OpenRA',
|
||||||
|
'org.openmw.OpenMW',
|
||||||
|
'org.zdoom.GZDoom',
|
||||||
|
'io.github.ezQuake',
|
||||||
|
'com.etlegacy.ETLegacy',
|
||||||
|
'com.github.iortcw.iortcw',
|
||||||
|
'org.yamagi.YamagiQ2',
|
||||||
|
'org.dhewm3.Dhewm3',
|
||||||
|
'com.github.bvschaik.julius',
|
||||||
|
'io.openrct2.OpenRCT2',
|
||||||
|
'com.github.skullernet.q2pro',
|
||||||
|
'org.raceintospace.Raceintospace',
|
||||||
|
'org.srb2.SRB2',
|
||||||
|
'org.srb2.SRB2Kart',
|
||||||
|
'io.sourceforge.clonekeenplus',
|
||||||
|
'io.github.fabiangreffrath.Doom',
|
||||||
|
'net.dengine.Doomsday',
|
||||||
|
'com.github.keriew.augustus',
|
||||||
|
'io.github.yairm210.unciv',
|
||||||
|
'com.corsixth.corsixth'
|
||||||
|
]
|
||||||
|
|
||||||
|
nonFreeNetworkServices = [
|
||||||
|
'io.github.yairm210.unciv'
|
||||||
|
]
|
||||||
|
|
||||||
|
alwaysAccept = [
|
||||||
|
'org.freecol.FreeCol',
|
||||||
|
'org.freeciv.Freeciv',
|
||||||
|
'io.github.EndlessSky.endless-sky',
|
||||||
|
'org.frozen_bubble.frozen-bubble',
|
||||||
|
'org.kde.ksudoku',
|
||||||
|
]
|
||||||
|
|
||||||
|
alwaysDeny = [
|
||||||
|
'com.moonlight_stream.Moonlight',
|
||||||
|
'org.gnome.Games',
|
||||||
|
'org.ppsspp.PPSSPP',
|
||||||
|
'org.scummvm.ScummVM',
|
||||||
|
'org.pegasus_frontend.Pegasus',
|
||||||
|
'com.gitlab.coringao.cavestory-nx',
|
||||||
|
'org.sauerbraten.Sauerbraten',
|
||||||
|
'net.runelite.RuneLite',
|
||||||
|
'com.zandronum.Zandronum',
|
||||||
|
'io.mrarm.mcpelauncher',
|
||||||
|
'org.unitystation.StationHub',
|
||||||
|
'org.firestormviewer.FirestormViewer',
|
||||||
|
'com.eduke32.EDuke32',
|
||||||
|
'io.github.hmlendea.geforcenow-electron',
|
||||||
|
'io.gdevs.GDLauncher',
|
||||||
|
'io.github.sharkwouter.Minigalaxy',
|
||||||
|
'com.katawa_shoujo.KatawaShoujo',
|
||||||
|
're.chiaki.Chiaki'
|
||||||
|
]
|
Loading…
Reference in New Issue