From c0112828ebf61e2fbb2588a8515ba0f6ec8b673e Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sat, 30 Jan 2021 19:32:04 +0000 Subject: [PATCH] Optimise package query speed --- app/blueprints/api/endpoints.py | 6 +-- app/models/packages.py | 18 ++++++--- app/querybuilder.py | 66 +++++++++++++++++++++++---------- 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/app/blueprints/api/endpoints.py b/app/blueprints/api/endpoints.py index c0ddfd8..52fd964 100644 --- a/app/blueprints/api/endpoints.py +++ b/app/blueprints/api/endpoints.py @@ -31,15 +31,11 @@ from sqlalchemy.sql.expression import func def packages(): qb = QueryBuilder(request.args) query = qb.buildPackageQuery() - ver = qb.getMinetestVersion() if request.args.get("fmt") == "keys": return jsonify([package.getAsDictionaryKey() for package in query.all()]) - def toJson(package: Package): - return package.getAsDictionaryShort(current_app.config["BASE_URL"], version=ver) - - pkgs = [toJson(package) for package in query.all()] + pkgs = qb.convertToDictionary(query.all()) if "engine_version" in request.args or "protocol_version" in request.args: pkgs = [package for package in pkgs if package.get("release")] return jsonify(pkgs) diff --git a/app/models/packages.py b/app/models/packages.py index 197275a..2fbdd50 100644 --- a/app/models/packages.py +++ b/app/models/packages.py @@ -311,6 +311,10 @@ class Package(db.Model): screenshots = db.relationship("PackageScreenshot", back_populates="package", foreign_keys="PackageScreenshot.package_id", lazy="dynamic", order_by=db.asc("package_screenshot_order"), cascade="all, delete, delete-orphan") + main_screenshot = db.relationship("PackageScreenshot", uselist=False, foreign_keys="PackageScreenshot.package_id", + lazy=True, order_by=db.asc("package_screenshot_order"), + primaryjoin="and_(Package.id==PackageScreenshot.package_id, PackageScreenshot.approved)") + cover_image_id = db.Column(db.Integer, db.ForeignKey("package_screenshot.id"), nullable=True, default=None) cover_image = db.relationship("PackageScreenshot", uselist=False, foreign_keys=[cover_image_id]) @@ -400,16 +404,20 @@ class Package(db.Model): "type": self.type.toName(), } - def getAsDictionaryShort(self, base_url, version=None, release=None): + def getAsDictionaryShort(self, base_url, version=None, release_id=None): tnurl = self.getThumbnailURL(1) - release = release if release else self.getDownloadRelease(version=version) + + if release_id is None: + release = self.getDownloadRelease(version=version) + release_id = release and release.id + return { "name": self.name, "title": self.title, "author": self.author.username, "short_description": self.short_desc, "type": self.type.toName(), - "release": release and release.id, + "release": release_id, "thumbnail": (base_url + tnurl) if tnurl is not None else None } @@ -448,11 +456,11 @@ class Package(db.Model): return self.getThumbnailURL(level) or "/static/placeholder.png" def getThumbnailURL(self, level=2): - screenshot = self.screenshots.filter_by(approved=True).order_by(db.asc(PackageScreenshot.id)).first() + screenshot = self.main_screenshot return screenshot.getThumbnailURL(level) if screenshot is not None else None def getMainScreenshotURL(self, absolute=False): - screenshot = self.screenshots.filter_by(approved=True).order_by(db.asc(PackageScreenshot.id)).first() + screenshot = self.main_screenshot if screenshot is None: return None diff --git a/app/querybuilder.py b/app/querybuilder.py index d29bc41..7c049d5 100644 --- a/app/querybuilder.py +++ b/app/querybuilder.py @@ -1,5 +1,6 @@ -from flask import abort +from flask import abort, current_app from sqlalchemy import or_ +from sqlalchemy.orm import subqueryload from sqlalchemy.sql.expression import func from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease, User, Tag, ContentWarning, PackageState @@ -45,12 +46,16 @@ class QueryBuilder: self.hide_flags.discard("nonfree") # Filters - self.search = args.get("q") - self.minetest_version = args.get("engine_version") - self.protocol_version = get_int_or_abort(args.get("protocol_version")) self.author = args.get("author") + protocol_version = get_int_or_abort(args.get("protocol_version")) + minetest_version = args.get("engine_version") + if protocol_version or minetest_version: + self.version = MinetestRelease.get(minetest_version, protocol_version) + else: + self.version = None + self.show_discarded = isYes(args.get("show_discarded")) self.show_added = args.get("show_added") if self.show_added is not None: @@ -64,11 +69,30 @@ class QueryBuilder: self.order_by = name self.order_dir = dir - def getMinetestVersion(self): - if not self.protocol_version and not self.minetest_version: - return None + def getReleases(self): + releases_query = db.session.query(PackageRelease.package_id, func.max(PackageRelease.id)) \ + .select_from(PackageRelease).filter(PackageRelease.approved) \ + .group_by(PackageRelease.package_id) - return MinetestRelease.get(self.minetest_version, self.protocol_version) + if self.version: + releases_query = releases_query \ + .filter(or_(PackageRelease.min_rel_id == None, + PackageRelease.min_rel_id <= self.version.id)) \ + .filter(or_(PackageRelease.max_rel_id == None, + PackageRelease.max_rel_id >= self.version.id)) + + return releases_query.all() + + def convertToDictionary(self, packages): + releases = {} + for [package_id, release_id] in self.getReleases(): + releases[package_id] = release_id + + def toJson(package: Package): + release_id = releases[package.id] + return package.getAsDictionaryShort(current_app.config["BASE_URL"], release_id=release_id) + + return [toJson(pkg) for pkg in packages] def buildPackageQuery(self): if self.order_by == "last_release": @@ -77,7 +101,14 @@ class QueryBuilder: else: query = Package.query.filter_by(state=PackageState.APPROVED) - return self.filterPackageQuery(self.orderPackageQuery(query)) + query = query.options(subqueryload(Package.main_screenshot)) + + query = self.orderPackageQuery(self.filterPackageQuery(query)) + + if self.limit: + query = query.limit(self.limit) + + return query def filterPackageQuery(self, query): if len(self.types) > 0: @@ -105,16 +136,13 @@ class QueryBuilder: query = query.filter(Package.license.has(License.is_foss == True)) query = query.filter(Package.media_license.has(License.is_foss == True)) - if self.protocol_version or self.minetest_version: - version = self.getMinetestVersion() - if version: - query = query.join(Package.releases) \ - .filter(PackageRelease.approved==True) \ - .filter(or_(PackageRelease.min_rel_id==None, PackageRelease.min_rel_id<=version.id)) \ - .filter(or_(PackageRelease.max_rel_id==None, PackageRelease.max_rel_id>=version.id)) - - if self.limit: - query = query.limit(self.limit) + if self.version: + query = query.join(Package.releases) \ + .filter(PackageRelease.approved == True) \ + .filter(or_(PackageRelease.min_rel_id == None, + PackageRelease.min_rel_id <= self.version.id)) \ + .filter(or_(PackageRelease.max_rel_id == None, + PackageRelease.max_rel_id >= self.version.id)) return query