Optimise package query speed

app/blueprints/api/endpoints.py

@@ -31,15 +31,11 @@ from sqlalchemy.sql.expression import func
 def packages():
 	qb    = QueryBuilder(request.args)
 	query = qb.buildPackageQuery()
-	ver   = qb.getMinetestVersion()
 
 	if request.args.get("fmt") == "keys":
 		return jsonify([package.getAsDictionaryKey() for package in query.all()])
 
-	def toJson(package: Package):
+	pkgs = qb.convertToDictionary(query.all())
-		return package.getAsDictionaryShort(current_app.config["BASE_URL"], version=ver)
-
-	pkgs = [toJson(package) for package in query.all()]
 	if "engine_version" in request.args or "protocol_version" in request.args:
 		pkgs = [package for package in pkgs if package.get("release")]
 	return jsonify(pkgs)

app/models/packages.py

@@ -311,6 +311,10 @@ class Package(db.Model):
 	screenshots = db.relationship("PackageScreenshot", back_populates="package", foreign_keys="PackageScreenshot.package_id",
 			lazy="dynamic", order_by=db.asc("package_screenshot_order"), cascade="all, delete, delete-orphan")
 
+	main_screenshot = db.relationship("PackageScreenshot", uselist=False, foreign_keys="PackageScreenshot.package_id",
+			lazy=True, order_by=db.asc("package_screenshot_order"),
+			primaryjoin="and_(Package.id==PackageScreenshot.package_id, PackageScreenshot.approved)")
+
 	cover_image_id = db.Column(db.Integer, db.ForeignKey("package_screenshot.id"), nullable=True, default=None)
 	cover_image = db.relationship("PackageScreenshot", uselist=False, foreign_keys=[cover_image_id])
 
@@ -400,16 +404,20 @@ class Package(db.Model):
 			"type": self.type.toName(),
 		}
 
-	def getAsDictionaryShort(self, base_url, version=None, release=None):
+	def getAsDictionaryShort(self, base_url, version=None, release_id=None):
 		tnurl = self.getThumbnailURL(1)
-		release = release if release else self.getDownloadRelease(version=version)
+
+		if release_id is None:
+			release = self.getDownloadRelease(version=version)
+			release_id = release and release.id
+
 		return {
 			"name": self.name,
 			"title": self.title,
 			"author": self.author.username,
 			"short_description": self.short_desc,
 			"type": self.type.toName(),
-			"release": release and release.id,
+			"release": release_id,
 			"thumbnail": (base_url + tnurl) if tnurl is not None else None
 		}
 
@@ -448,11 +456,11 @@ class Package(db.Model):
 		return self.getThumbnailURL(level) or "/static/placeholder.png"
 
 	def getThumbnailURL(self, level=2):
-		screenshot = self.screenshots.filter_by(approved=True).order_by(db.asc(PackageScreenshot.id)).first()
+		screenshot = self.main_screenshot
 		return screenshot.getThumbnailURL(level) if screenshot is not None else None
 
 	def getMainScreenshotURL(self, absolute=False):
-		screenshot = self.screenshots.filter_by(approved=True).order_by(db.asc(PackageScreenshot.id)).first()
+		screenshot = self.main_screenshot
 		if screenshot is None:
 			return None
 

app/querybuilder.py

@@ -1,5 +1,6 @@
-from flask import abort
+from flask import abort, current_app
 from sqlalchemy import or_
+from sqlalchemy.orm import subqueryload
 from sqlalchemy.sql.expression import func
 
 from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease, User, Tag, ContentWarning, PackageState
@@ -45,12 +46,16 @@ class QueryBuilder:
 		self.hide_flags.discard("nonfree")
 
 		# Filters
-
 		self.search = args.get("q")
-		self.minetest_version = args.get("engine_version")
-		self.protocol_version = get_int_or_abort(args.get("protocol_version"))
 		self.author = args.get("author")
 
+		protocol_version = get_int_or_abort(args.get("protocol_version"))
+		minetest_version = args.get("engine_version")
+		if protocol_version or minetest_version:
+			self.version = MinetestRelease.get(minetest_version, protocol_version)
+		else:
+			self.version = None
+
 		self.show_discarded = isYes(args.get("show_discarded"))
 		self.show_added = args.get("show_added")
 		if self.show_added is not None:
@@ -64,11 +69,30 @@ class QueryBuilder:
 			self.order_by = name
 			self.order_dir = dir
 
-	def getMinetestVersion(self):
+	def getReleases(self):
-		if not self.protocol_version and not self.minetest_version:
+		releases_query = db.session.query(PackageRelease.package_id, func.max(PackageRelease.id)) \
-			return None
+			.select_from(PackageRelease).filter(PackageRelease.approved) \
+			.group_by(PackageRelease.package_id)
 
-		return MinetestRelease.get(self.minetest_version, self.protocol_version)
+		if self.version:
+			releases_query = releases_query \
+				.filter(or_(PackageRelease.min_rel_id == None,
+					PackageRelease.min_rel_id <= self.version.id)) \
+				.filter(or_(PackageRelease.max_rel_id == None,
+					PackageRelease.max_rel_id >= self.version.id))
+
+		return releases_query.all()
+
+	def convertToDictionary(self, packages):
+		releases = {}
+		for [package_id, release_id] in self.getReleases():
+			releases[package_id] = release_id
+
+		def toJson(package: Package):
+			release_id = releases[package.id]
+			return package.getAsDictionaryShort(current_app.config["BASE_URL"], release_id=release_id)
+
+		return [toJson(pkg) for pkg in packages]
 
 	def buildPackageQuery(self):
 		if self.order_by == "last_release":
@@ -77,7 +101,14 @@ class QueryBuilder:
 		else:
 			query = Package.query.filter_by(state=PackageState.APPROVED)
 
-		return self.filterPackageQuery(self.orderPackageQuery(query))
+		query = query.options(subqueryload(Package.main_screenshot))
+
+		query = self.orderPackageQuery(self.filterPackageQuery(query))
+
+		if self.limit:
+			query = query.limit(self.limit)
+
+		return query
 
 	def filterPackageQuery(self, query):
 		if len(self.types) > 0:
@@ -105,16 +136,13 @@ class QueryBuilder:
 			query = query.filter(Package.license.has(License.is_foss == True))
 			query = query.filter(Package.media_license.has(License.is_foss == True))
 
-		if self.protocol_version or self.minetest_version:
+		if self.version:
-			version = self.getMinetestVersion()
+			query = query.join(Package.releases) \
-			if version:
+				.filter(PackageRelease.approved == True) \
-				query = query.join(Package.releases) \
+				.filter(or_(PackageRelease.min_rel_id == None,
-					.filter(PackageRelease.approved==True) \
+					PackageRelease.min_rel_id <= self.version.id)) \
-					.filter(or_(PackageRelease.min_rel_id==None, PackageRelease.min_rel_id<=version.id)) \
+				.filter(or_(PackageRelease.max_rel_id == None,
-					.filter(or_(PackageRelease.max_rel_id==None, PackageRelease.max_rel_id>=version.id))
+					PackageRelease.max_rel_id >= self.version.id))
-
-		if self.limit:
-			query = query.limit(self.limit)
 
 		return query