Add reimport of package meta from latest release

Fixes #127

app/blueprints/admin/admin.py

@@ -21,7 +21,7 @@ import flask_menu as menu
 from . import bp
 from app.models import *
 from celery import uuid, group
-from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease, checkZipRelease
+from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease, checkZipRelease, updateMetaFromRelease
 from app.tasks.forumtasks  import importTopicList, checkAllForumAccounts
 from flask_wtf import FlaskForm
 from wtforms import *
@@ -47,6 +47,21 @@ def admin_page():
 
 			result = group(tasks).apply_async()
 
+			while not result.ready():
+				import time
+				time.sleep(0.1)
+
+			return redirect(url_for("todo.view"))
+		elif action == "reimportpackages":
+			tasks = []
+			for package in Package.query.filter_by(approved=True, soft_deleted=False).all():
+				release = package.releases.first()
+				if release:
+					zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"])
+					tasks.append(updateMetaFromRelease.s(release.id, zippath))
+
+			result = group(tasks).apply_async()
+
 			while not result.ready():
 				import time
 				time.sleep(0.1)

app/blueprints/packages/packages.py

@@ -23,7 +23,7 @@ from . import bp
 
 from app.models import *
 from app.querybuilder import QueryBuilder
-from app.tasks.importtasks import importRepoScreenshot
+from app.tasks.importtasks import importRepoScreenshot, updateMetaFromRelease
 from app.utils import *
 
 from flask_wtf import FlaskForm
@@ -33,6 +33,8 @@ from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleF
 from sqlalchemy import or_, func
 from sqlalchemy.orm import joinedload, subqueryload
 
+from celery import uuid
+
 
 @menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
 @menu.register_menu(bp, ".games", "Games", order=12, endpoint_arguments_constructor=lambda: { 'type': 'game' })
@@ -466,3 +468,31 @@ def remove_self_maintainers(package):
 		db.session.commit()
 
 	return redirect(package.getDetailsURL())
+
+
+@bp.route("/packages/<author>/<name>/import-meta/", methods=["POST"])
+@login_required
+@is_package_page
+def update_from_release(package):
+	if not package.checkPerm(current_user, Permission.REIMPORT_META):
+		flash("You don't have permission to reimport meta", "danger")
+		return redirect(package.getDetailsURL())
+
+	release = package.releases.first()
+	if not release:
+		flash("Release needed", "danger")
+		return redirect(package.getDetailsURL())
+
+	msg = "Updated meta from latest release"
+	addNotification(package.maintainers, current_user,
+			msg, package.getDetailsURL(), package)
+	severity = AuditSeverity.NORMAL if current_user in package.maintainers else AuditSeverity.EDITOR
+	addAuditLog(severity, current_user, msg, package.getDetailsURL(), package)
+
+	db.session.commit()
+
+	task_id = uuid()
+	zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"])
+	updateMetaFromRelease.apply_async((release.id, zippath), task_id=task_id)
+
+	return redirect(url_for("tasks.check", id=task_id, r=package.getEditURL()))

app/models.py

@@ -80,6 +80,7 @@ class Permission(enum.Enum):
 	MAKE_RELEASE       = "MAKE_RELEASE"
 	DELETE_RELEASE     = "DELETE_RELEASE"
 	ADD_SCREENSHOTS    = "ADD_SCREENSHOTS"
+	REIMPORT_META      = "REIMPORT_META"
 	APPROVE_SCREENSHOT = "APPROVE_SCREENSHOT"
 	APPROVE_RELEASE    = "APPROVE_RELEASE"
 	APPROVE_NEW        = "APPROVE_NEW"
@@ -358,11 +359,12 @@ class Dependency(db.Model):
 	optional        = db.Column(db.Boolean, nullable=False, default=False)
 	__table_args__  = (db.UniqueConstraint("depender_id", "package_id", "meta_package_id", name="_dependency_uc"), )
 
-	def __init__(self, depender=None, package=None, meta=None):
+	def __init__(self, depender=None, package=None, meta=None, optional=False):
 		if depender is None:
 			return
 
 		self.depender = depender
+		self.optional = optional
 
 		packageProvided = package is not None
 		metaProvided = meta is not None
@@ -673,6 +675,10 @@ class Package(db.Model):
 		return url_for("packages.remove_self_maintainers",
 				author=self.author.username, name=self.name)
 
+	def getUpdateFromReleaseURL(self):
+		return url_for("packages.update_from_release",
+				author=self.author.username, name=self.name)
+
 	def getReviewURL(self):
 		return url_for('packages.review',
 				author=self.author.username, name=self.name)
@@ -705,7 +711,8 @@ class Package(db.Model):
 		elif perm == Permission.MAKE_RELEASE or perm == Permission.ADD_SCREENSHOTS:
 			return isMaintainer
 
-		elif perm == Permission.EDIT_PACKAGE or perm == Permission.APPROVE_CHANGES or perm == Permission.APPROVE_RELEASE:
+		elif perm == Permission.EDIT_PACKAGE or perm == Permission.REIMPORT_META or \
+				perm == Permission.APPROVE_CHANGES or perm == Permission.APPROVE_RELEASE:
 			return isMaintainer and user.rank.atLeast(UserRank.MEMBER if self.approved else UserRank.NEW_MEMBER)
 
 		# Anyone can change the package name when not approved, but only editors when approved

app/tasks/importtasks.py

@@ -139,6 +139,60 @@ def cloneRepo(urlstr, ref=None, recursive=False):
 			.replace("Cloning into '" + gitDir + "'...", "") \
 			.strip())
 
+
+@celery.task(bind=True)
+def updateMetaFromRelease(self, id, path):
+	release = PackageRelease.query.get(id)
+	if release is None:
+		raise TaskError("No such release!")
+	elif release.package is None:
+		raise TaskError("No package attached to release")
+
+	temp = getTempDir()
+	try:
+		with ZipFile(path, 'r') as zip_ref:
+			zip_ref.extractall(temp)
+
+		try:
+			tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
+				author=release.package.author.username, name=release.package.name)
+
+			cache = {}
+			def getMetaPackages(names):
+				return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
+
+			provides = getMetaPackages(tree.fold("name"))
+
+			package = release.package
+			package.provides.clear()
+			package.provides.extend(provides)
+
+			for dep in package.dependencies:
+				if dep.meta_package:
+					db.session.delete(dep)
+
+			for meta in getMetaPackages(tree.fold("meta", "depends")):
+				db.session.add(Dependency(package, meta=meta, optional=False))
+
+			for meta in getMetaPackages(tree.fold("meta", "optional_depends")):
+				db.session.add(Dependency(package, meta=meta, optional=True))
+
+			db.session.commit()
+
+		except MinetestCheckError as err:
+			if "Fails validation" not in release.title:
+				release.title += " (Fails validation)"
+
+			release.task_id = self.request.id
+			release.approved = False
+			db.session.commit()
+
+			raise TaskError(str(err))
+
+	finally:
+		shutil.rmtree(temp)
+
+
 @celery.task()
 def getMeta(urlstr, author):
 	gitDir, _ = cloneRepo(urlstr, recursive=True)
@@ -249,6 +303,7 @@ def makeVCSRelease(id, branch):
 	finally:
 		shutil.rmtree(gitDir)
 
+
 @celery.task()
 def importRepoScreenshot(id):
 	package = Package.query.get(id)

app/templates/admin/list.html

@@ -21,6 +21,7 @@
 			<select name="action">
 				<option value="delstuckreleases" selected>Delete stuck releases</option>
 				<option value="checkreleases">Validate all Zip releases</option>
+				<option value="reimportpackages">Reimport meta</option>
 				<option value="importmodlist">Import forum topics</option>
 				<option value="recalcscores">Recalculate package scores</option>
 				<option value="checkusers">Check forum users</option>

app/templates/packages/create_edit.html

@@ -28,6 +28,17 @@
 		{{ _("Have you read the Package Inclusion Policy and Guidance yet?") }}
 	</div>
 
+	{% if package and package.checkPerm(current_user, "REIMPORT_META") and package.releases.first() %}
+		<form class="alert alert-secondary mb-5" method="post" action="{{ package.getUpdateFromReleaseURL() }}">
+			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+			<input class="btn btn-sm btn-warning float-right" type="submit" value="{{ _('Import') }}" />
+
+			<b>{{ _("Reimport meta from latest release.") }}</b>
+			{{ _("This will override 'provides', 'dependencies', and 'optional_dependencies'.") }}
+			<div style="clear:both;"></div>
+		</form>
+	{% endif %}
+
 	<noscript>
 		<div class="alert alert-warning">
 			{{ _("Javascript is needed to improve the user interface, and is needed for features