import gi import PIL import requests import os import sys import inspect from gi.repository import Gio gi.require_version('AppStreamGlib', '1.0') from gi.repository import AppStreamGlib from lists import alwaysAccept, alwaysDeny, badLicenses, badCategories, nonFreeAssets, nonFreeNetworkServices import itertools import argparse #Workaround to get the urls because app.get_urls() doesn't work :| def get_urls(app): kinds = [AppStreamGlib.UrlKind(kind) for kind in range(11)] urls = [(app.get_url_item(kind),kind.value_nick) for kind in kinds] return list(filter(lambda a: a[0] is not None, urls)) def acceptedGame(app): #return 'Game' in app.get_categories() if app.get_id() in alwaysAccept: return True if app.get_id() in alwaysDeny: return False return app.get_project_license() and \ not [x for x in badLicenses if x in app.get_project_license()] and \ 'Game' in app.get_categories() and \ not [x for x in badCategories if x in app.get_categories()] def getScreenshots(app): return [images.get_source() for images in app.get_screenshots()] currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) parentdir = os.path.dirname(currentdir) sys.path.insert(0,parentdir) from app.models import * from app.utils import make_flask_login_password from app.utils.image import get_image_size from app.utils import randomString parser = argparse.ArgumentParser(description='Generate HTML files from an Appimage and mako templates.') parser.add_argument("-p", '--appstream_path', help='specify the path of the appstream file') parser.add_argument("-i", '--icons_path', help='specify the path of the icons') parser.add_argument("-m", '--template_path', help='specify the path of the template') args = parser.parse_args() store = AppStreamGlib.Store() file = Gio.File.new_for_path(args.appstream_path) AppStreamGlib.Store.from_file(store, file, args.icons_path, None) apps = list(filter(acceptedGame, store.get_apps())) session=db.session licenses = { x.name : x for x in License.query.all() } tags = { x.name : x for x in Tag.query.all() } admin_user = User.query.filter_by(username="appstream").first() for ss in PackageScreenshot.query.all(): if ss.package.cover_image == ss: ss.package.cover_image = None session.commit() session.delete(ss) session.commit() for game in Package.query.all(): session.delete(game) session.commit() if not admin_user: admin_user = User("appstream") admin_user.is_active = True admin_user.password = make_flask_login_password("appstream") admin_user.github_username = "appstream" admin_user.forums_username = "appstream" admin_user.rank = UserRank.ADMIN session.add(admin_user) for app in apps: screenshots = getScreenshots(app) urls = get_urls(app) filename = app.get_name().replace(':', '').replace('/','') + ".html" print("APPLICATION: ",app.get_name()) game1 = Package() game1.state = PackageState.APPROVED game1.name = app.get_id().replace('.','_').replace('-','_') game1.title = app.get_name() game1.type = PackageType.GAME license = "Uknown" if app.get_project_license() is None else app.get_project_license().split("AND")[0].split("and")[0] if license not in licenses: row = License(license) licenses[row.name] = row session.add(row) session.commit() for category in app.get_categories(): if category.lower() not in tags: row = Tag(category) tags[row.name] = row print("adding tag: ", row.name) session.add(row) game1.tags.append(tags[category.lower()]) game1.license = licenses[license] game1.media_license = licenses["MIT"] game1.author = admin_user for url,t in urls: if t == "bugtracker": game1.issueTracker = url elif t == "homepage": game1.repo = url game1.forums = 12835 game1.short_desc = "" or app.get_comment() game1.desc = app.get_description() session.add(game1) for screenshot in screenshots: counter = 1 url = screenshot.get_url() try: r = requests.get(url,timeout=10) r.raise_for_status() filename = randomString(10) + "." + "png" filepath = os.path.join("/var/cdb/uploads", filename) print("Screenshot url: ", url) with open(filepath,"wb") as f: f.write(r.content) width, height = get_image_size(filepath) if (width is not None) and (height is not None): ss = PackageScreenshot() ss.package = game1 ss.title = "Untitled" ss.url = "/uploads/" + filename ss.width = width ss.height = height ss.approved = True ss.order = counter session.add(ss) session.commit() game1.cover_image = ss session.commit() counter += 1 except requests.exceptions.HTTPError as err: print("HTTP error downloading the screenshot ", err) except requests.exceptions.ConnectionError as err: print("HTTP error downloading the screenshot ", err) except requests.exceptions.ReadTimeout as err: print("Screenshot timeout ", err) except PIL.UnidentifiedImageError as err: print("Corrupt image ", err) session.commit()