contentdb/utils/appstream.py

import gi
import PIL
import requests
import os
import sys
import inspect
from gi.repository import Gio
gi.require_version('AppStreamGlib', '1.0')
from gi.repository import AppStreamGlib
from lists import alwaysAccept, alwaysDeny, badLicenses, badCategories,  nonFreeAssets, nonFreeNetworkServices 
import itertools
import argparse

#Workaround to get the urls because app.get_urls() doesn't work :|
def get_urls(app):
    kinds = [AppStreamGlib.UrlKind(kind) for kind in range(11)]
    urls = [(app.get_url_item(kind),kind.value_nick) for kind in kinds]
    return list(filter(lambda a: a[0] is not None, urls))

def acceptedGame(app):
    #return 'Game' in app.get_categories()
    if app.get_id() in alwaysAccept:
        return True
    if app.get_id() in alwaysDeny:
        return False

    return app.get_project_license() and \
        not [x for x in badLicenses if x in app.get_project_license()] and \
        'Game' in app.get_categories() and \
        not [x for x in badCategories if x in app.get_categories()]

def getScreenshots(app):
    return [images.get_source() for images in app.get_screenshots()]

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)

from app.models import *
from app.utils import make_flask_login_password
from app.utils.image import get_image_size
from app.utils import randomString

parser = argparse.ArgumentParser(description='Generate HTML files from an Appimage and mako templates.')
parser.add_argument("-p", '--appstream_path', help='specify the path of the appstream file')
parser.add_argument("-i", '--icons_path', help='specify the path of the icons')
parser.add_argument("-m", '--template_path', help='specify the path of the template')
args = parser.parse_args()

store = AppStreamGlib.Store()
file = Gio.File.new_for_path(args.appstream_path)
AppStreamGlib.Store.from_file(store, file, args.icons_path, None)
apps = list(filter(acceptedGame, store.get_apps()))
session=db.session
licenses = { x.name : x for x in License.query.all() }
tags = { x.name : x for x in Tag.query.all() }
admin_user = User.query.filter_by(username="appstream").first()


for ss in PackageScreenshot.query.all():
    if ss.package.cover_image == ss:
        ss.package.cover_image = None
    session.commit()
    session.delete(ss)
    session.commit()

for game in Package.query.all():
    session.delete(game)
    session.commit()

if not admin_user:
    admin_user = User("appstream")
    admin_user.is_active = True
    admin_user.password = make_flask_login_password("appstream")
    admin_user.github_username = "appstream"
    admin_user.forums_username = "appstream"
    admin_user.rank = UserRank.ADMIN
    session.add(admin_user)


for app in apps:
    screenshots = getScreenshots(app)
    urls = get_urls(app)
    filename = app.get_name().replace(':', '').replace('/','') + ".html"
    print("APPLICATION: ",app.get_name())
    game1 = Package()
    game1.state = PackageState.APPROVED
    game1.name = app.get_id().replace('.','_').replace('-','_')
    game1.title = app.get_name()
    game1.type = PackageType.GAME
    license = "Uknown" if app.get_project_license() is None else app.get_project_license().split("AND")[0].split("and")[0]
    if license not in licenses:
        row = License(license)
        licenses[row.name] = row
        session.add(row)
        session.commit()
    for category in app.get_categories():
        if category.lower() not in tags:
            row = Tag(category)
            tags[row.name] = row
            print("adding tag: ", row.name)
            session.add(row)
        game1.tags.append(tags[category.lower()])
    game1.license = licenses[license]
    game1.media_license = licenses["MIT"]
    game1.author = admin_user


    for url,t in urls:
        if t == "bugtracker":
            game1.issueTracker = url
        elif t == "homepage":
            game1.repo = url

    game1.forums = 12835
    game1.short_desc = "" or app.get_comment()
    game1.desc = app.get_description()
    session.add(game1)

    for screenshot in screenshots:
        counter = 1
        url = screenshot.get_url()
        try:
            r = requests.get(url,timeout=10)
            r.raise_for_status()
            filename = randomString(10) + "." + "png"
            filepath = os.path.join("/var/cdb/uploads", filename)
            print("Screenshot url: ", url)
            with open(filepath,"wb") as f:
                f.write(r.content)

            width, height = get_image_size(filepath)

            if (width is not None) and (height is not None):
                ss = PackageScreenshot()
                ss.package  = game1
                ss.title    =  "Untitled"
                ss.url      = "/uploads/" + filename
                ss.width = width
                ss.height = height
                ss.approved = True
                ss.order    = counter
                session.add(ss)
                session.commit()
                game1.cover_image = ss
                session.commit()
                counter += 1
        except requests.exceptions.HTTPError as err:
            print("HTTP error downloading the  screenshot ", err)
        except requests.exceptions.ConnectionError as err:
            print("HTTP error downloading the  screenshot ", err)
        except requests.exceptions.ReadTimeout as err:
            print("Screenshot timeout ", err)
        except PIL.UnidentifiedImageError as err:
            print("Corrupt image ", err)

session.commit()