diff --git a/app/blueprints/packages/releases.py b/app/blueprints/packages/releases.py index 57a2d6a..96db569 100644 --- a/app/blueprints/packages/releases.py +++ b/app/blueprints/packages/releases.py @@ -24,7 +24,7 @@ from wtforms.ext.sqlalchemy.fields import QuerySelectField from wtforms.validators import * from app.rediscache import has_key, set_key, make_download_key -from app.tasks.importtasks import makeVCSRelease, checkZipRelease +from app.tasks.importtasks import makeVCSRelease, checkZipRelease, updateMetaFromRelease, check_for_updates from app.utils import * from . import bp @@ -247,3 +247,34 @@ def delete_release(package, id): db.session.commit() return redirect(package.getDetailsURL()) + + +class PackageUpdateConfigFrom(FlaskForm): + trigger = SelectField("Trigger", [InputRequired()], choices=PackageUpdateTrigger.choices(), coerce=PackageUpdateTrigger.coerce, + default=PackageUpdateTrigger.COMMIT) + make_release = BooleanField("Create Release") + submit = SubmitField("Save") + + +@bp.route("/packages///update-config/", methods=["GET", "POST"]) +@login_required +@is_package_page +def update_config(package): + package.update_config = package.update_config or PackageUpdateConfig() + + if not package.checkPerm(current_user, Permission.MAKE_RELEASE): + return redirect(package.getDetailsURL()) + + form = PackageUpdateConfigFrom(obj=package.update_config) + if form.validate_on_submit(): + flash("Changed update configuration", "success") + form.populate_obj(package.update_config) + db.session.add(package.update_config) + + check_for_updates.delay() + + db.session.commit() + + return redirect(package.getDetailsURL()) + + return render_template("packages/update_config.html", package=package, form=form) diff --git a/app/models/packages.py b/app/models/packages.py index 2048aa7..e42f9ff 100644 --- a/app/models/packages.py +++ b/app/models/packages.py @@ -317,7 +317,7 @@ class Package(db.Model): maintainers = db.relationship("User", secondary=maintainers) threads = db.relationship("Thread", back_populates="package", order_by=db.desc("thread_created_at"), - foreign_keys="Thread.package_id", cascade="all, delete, delete-orphan") + foreign_keys="Thread.package_id", cascade="all, delete, delete-orphan", lazy="dynamic") reviews = db.relationship("PackageReview", back_populates="package", order_by=db.desc("package_review_created_at"), cascade="all, delete, delete-orphan") @@ -331,6 +331,9 @@ class Package(db.Model): tokens = db.relationship("APIToken", foreign_keys="APIToken.package_id", back_populates="package", cascade="all, delete, delete-orphan") + update_config = db.relationship("PackageUpdateConfig", uselist=False, back_populates="package", + cascade="all, delete, delete-orphan") + def __init__(self, package=None): if package is None: return @@ -507,6 +510,10 @@ class Package(db.Model): return url_for("packages.bulk_change_release", author=self.author.username, name=self.name) + def getUpdateConfigURL(self): + return url_for("packages.update_config", + author=self.author.username, name=self.name) + def getDownloadURL(self): return url_for("packages.download", author=self.author.username, name=self.name) @@ -790,7 +797,7 @@ class PackageRelease(db.Model): title = db.Column(db.String(100), nullable=False) releaseDate = db.Column(db.DateTime, nullable=False) - url = db.Column(db.String(200), nullable=False) + url = db.Column(db.String(200), nullable=False, default="") approved = db.Column(db.Boolean, nullable=False, default=False) task_id = db.Column(db.String(37), nullable=True) commit_hash = db.Column(db.String(41), nullable=True, default=None) @@ -903,3 +910,39 @@ class PackageScreenshot(db.Model): def getThumbnailURL(self, level=2): return self.url.replace("/uploads/", "/thumbnails/{:d}/".format(level)) + + +class PackageUpdateTrigger(enum.Enum): + COMMIT = "New Commit" + TAG = "New Tag" + + def toName(self): + return self.name.lower() + + def __str__(self): + return self.name + + @classmethod + def get(cls, name): + try: + return PackageUpdateTrigger[name.upper()] + except KeyError: + return None + + @classmethod + def choices(cls): + return [(choice, choice.value) for choice in cls] + + @classmethod + def coerce(cls, item): + return item if type(item) == PackageUpdateTrigger else PackageUpdateTrigger[item] + + +class PackageUpdateConfig(db.Model): + package_id = db.Column(db.Integer, db.ForeignKey("package.id"), primary_key=True) + package = db.relationship("Package", back_populates="update_config", foreign_keys=[package_id]) + + last_commit = db.Column(db.String(41), nullable=True, default=None) + + trigger = db.Column(db.Enum(PackageUpdateTrigger), nullable=False, default=PackageUpdateTrigger.COMMIT) + make_release = db.Column(db.Boolean, nullable=False, default=False) diff --git a/app/models/threads.py b/app/models/threads.py index 12b3c5d..95e01f7 100644 --- a/app/models/threads.py +++ b/app/models/threads.py @@ -56,7 +56,7 @@ class Thread(db.Model): watchers = db.relationship("User", secondary=watchers, backref="watching") def getViewURL(self): - return url_for("threads.view", id=self.id) + return url_for("threads.view", id=self.id, _external=False) def getSubscribeURL(self): return url_for("threads.subscribe", id=self.id) diff --git a/app/models/users.py b/app/models/users.py index d40d0e9..63f5e52 100644 --- a/app/models/users.py +++ b/app/models/users.py @@ -32,8 +32,9 @@ class UserRank(enum.Enum): MEMBER = 3 TRUSTED_MEMBER = 4 EDITOR = 5 - MODERATOR = 6 - ADMIN = 7 + BOT = 6 + MODERATOR = 7 + ADMIN = 8 def atLeast(self, min): return self.value >= min.value @@ -192,6 +193,8 @@ class User(db.Model, UserMixin): def getProfilePicURL(self): if self.profile_pic: return self.profile_pic + elif self.rank == UserRank.BOT: + return "/static/bot_avatar.png" else: return gravatar(self.email or "") diff --git a/app/public/static/bot_avatar.png b/app/public/static/bot_avatar.png new file mode 100644 index 0000000..96b94d2 Binary files /dev/null and b/app/public/static/bot_avatar.png differ diff --git a/app/scss/comments.scss b/app/scss/comments.scss index 7ba774e..af72bf9 100644 --- a/app/scss/comments.scss +++ b/app/scss/comments.scss @@ -26,4 +26,10 @@ border-width: 14px; } } + + .user-photo { + width: 60px; + height: 60px; + object-fit: cover; + } } diff --git a/app/scss/components.scss b/app/scss/components.scss index 8cd5b1e..8d81eec 100644 --- a/app/scss/components.scss +++ b/app/scss/components.scss @@ -81,6 +81,10 @@ color: #2c2 !important; } +.BOT a, .BOT { + color: #FFDF00 !important; +} + .wiptopic a:not(.btn) { color: #7ac; } diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py index 17409fb..518dae8 100644 --- a/app/tasks/__init__.py +++ b/app/tasks/__init__.py @@ -73,6 +73,11 @@ CELERYBEAT_SCHEDULE = { 'task': 'app.tasks.pkgtasks.updatePackageScores', 'schedule': crontab(minute=10, hour=1), }, + + 'package_score_update': { + 'task': 'app.tasks.importtasks.check_for_updates', + 'schedule': crontab(minute=10, hour=1), + }, 'send_pending_notifications': { 'task': 'app.tasks.emails.send_pending_notifications', 'schedule': crontab(minute='*/5'), diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index e43aa8b..a69ba1c 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -22,9 +22,12 @@ from urllib.error import HTTPError import urllib.request from urllib.parse import urlsplit from zipfile import ZipFile + +from kombu import uuid + from app.models import * from app.tasks import celery, TaskError -from app.utils import randomString, getExtension +from app.utils import randomString, getExtension, post_system_thread from .minetestcheck import build_tree, MinetestCheckError, ContentType @@ -85,6 +88,40 @@ def clone_repo(urlstr, ref=None, recursive=False): .strip()) +def get_commit_hash(urlstr, ref=None): + gitDir = os.path.join(tempfile.gettempdir(), randomString(10)) + + err = None + try: + gitUrl = generateGitURL(urlstr) + print("Cloning from " + gitUrl) + + assert ref != "" + + repo = git.Repo.init(gitDir) + origin: git.Remote = repo.create_remote("origin", url=gitUrl) + assert origin.exists() + origin.fetch() + + if ref: + ref: git.Reference = origin.refs[ref] + else: + ref: git.Reference = origin.refs[0] + + return ref.commit.hexsha + + except GitCommandError as e: + # This is needed to stop the backtrace being weird + err = e.stderr + + except gitdb.exc.BadName as e: + err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr + + raise TaskError(err.replace("stderr: ", "") \ + .replace("Cloning into '" + gitDir + "'...", "") \ + .strip()) + + @celery.task() def getMeta(urlstr, author): with clone_repo(urlstr, recursive=True) as repo: @@ -274,3 +311,50 @@ def importForeignDownloads(self, id): release.task_id = self.request.id release.approved = False db.session.commit() + + +@celery.task +def check_update_config(package_id): + package: Package = Package.query.get(package_id) + if package is None: + raise TaskError("No such package!") + elif package.update_config is None: + raise TaskError("No update config attached to package") + + config = package.update_config + ref = None + hash = get_commit_hash(package.repo, ref) + + if config.last_commit != hash: + if config.make_release: + rel = PackageRelease() + rel.package = package + rel.title = hash[0:5] + rel.url = "" + rel.task_id = uuid() + db.session.add(rel) + db.session.commit() + + makeVCSRelease.apply_async((rel.id, ref), task_id=rel.task_id) + + else: + post_system_thread(package, "New commit detected, package outdated?", + "Commit {} was detected on the Git repository.\n\n[Change update configuration]({})" \ + .format(hash[0:5], package.getUpdateConfigURL())) + + config.last_commit = hash + db.session.commit() + + +@celery.task +def check_for_updates(): + for update_config in PackageUpdateConfig.query.all(): + update_config: PackageUpdateConfig + + if update_config.package.repo is None: + db.session.delete(update_config) + continue + + check_update_config.delay(update_config.package_id) + + db.session.commit() diff --git a/app/templates/macros/threads.html b/app/templates/macros/threads.html index 066c2b1..70d63cf 100644 --- a/app/templates/macros/threads.html +++ b/app/templates/macros/threads.html @@ -11,10 +11,22 @@
- {{ r.author.display_name }} + + {% if r.author in thread.package.maintainers %} + + {{ _("Maintainer") }} + + {% endif %} + {% if r.author.rank == r.author.rank.BOT %} + + {{ r.author.rank.getTitle() }} + + {% endif %} + {{ r.created_at | datetime }} diff --git a/app/templates/packages/update_config.html b/app/templates/packages/update_config.html new file mode 100644 index 0000000..6431efc --- /dev/null +++ b/app/templates/packages/update_config.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} + +{% block title %} + Configure Update Detection | {{ package.title }} +{% endblock %} + +{% block content %} +

{{ _("Configure Update Detection") }}

+ +

+ {{ _("ContentDB will poll your Git repository at 2am UTC every day.") }} + {{ _("You should consider using webhooks or the API for faster rollouts.") }} +

+ + {% from "macros/forms.html" import render_field, render_submit_field, render_checkbox_field %} +
+ {{ form.hidden_tag() }} + +

Triggers

+ {{ render_field(form.trigger) }} + +

Actions

+ {{ render_checkbox_field(form.make_release) }} + +

+ {{ render_submit_field(form.submit) }} +

+
+{% endblock %} diff --git a/app/templates/users/list.html b/app/templates/users/list.html index 90f6227..bfe07fe 100644 --- a/app/templates/users/list.html +++ b/app/templates/users/list.html @@ -38,6 +38,8 @@ {% elif user.rank == user.rank.EDITOR %} + {% elif user.rank == user.rank.BOT %} + {% else %} {% endif %} diff --git a/app/templates/users/profile.html b/app/templates/users/profile.html index 3c13ac4..03a4b96 100644 --- a/app/templates/users/profile.html +++ b/app/templates/users/profile.html @@ -23,7 +23,7 @@ {% endif %} -

+

{{ user.display_name }}

diff --git a/app/utils.py b/app/utils.py index 515f6dc..d8d5874 100644 --- a/app/utils.py +++ b/app/utils.py @@ -253,3 +253,30 @@ def nonEmptyOrNone(str): return None return str + + +def post_system_thread(package: Package, title: str, message: str): + system_user = User.query.filter_by(username="ContentDB").first() + assert system_user + + thread = package.threads.filter_by(author=system_user).first() + if not thread: + thread = Thread() + thread.package = package + thread.title = "System Notifications" + thread.author = system_user + thread.private = True + thread.watchers.append(package.author) + db.session.add(thread) + db.session.flush() + + reply = ThreadReply() + reply.thread = thread + reply.author = system_user + reply.comment = "# {}\n\n{}".format(title, message) + db.session.add(reply) + + addNotification(thread.watchers, system_user, NotificationType.THREAD_REPLY, + title, thread.getViewURL(), thread.package) + + thread.replies.append(reply) diff --git a/migrations/versions/105d4c740ad6_.py b/migrations/versions/105d4c740ad6_.py new file mode 100644 index 0000000..c55c280 --- /dev/null +++ b/migrations/versions/105d4c740ad6_.py @@ -0,0 +1,36 @@ +"""empty message + +Revision ID: 105d4c740ad6 +Revises: 886c92dc6eaa +Create Date: 2020-12-15 17:28:56.559801 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +from sqlalchemy import orm +from app.models import User, UserRank + +revision = '105d4c740ad6' +down_revision = '886c92dc6eaa' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("COMMIT") + op.execute("ALTER TYPE userrank ADD VALUE 'BOT' AFTER 'EDITOR'") + + conn = op.get_bind() + system_user = User("ContentDB", active=False) + system_user.rank = UserRank.BOT + + session = orm.Session(bind=conn) + session.add(system_user) + session.commit() + + +def downgrade(): + pass diff --git a/migrations/versions/886c92dc6eaa_.py b/migrations/versions/886c92dc6eaa_.py new file mode 100644 index 0000000..aaacb46 --- /dev/null +++ b/migrations/versions/886c92dc6eaa_.py @@ -0,0 +1,35 @@ +"""empty message + +Revision ID: 886c92dc6eaa +Revises: 8d22def23c8b +Create Date: 2020-12-15 16:38:54.114559 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '886c92dc6eaa' +down_revision = '8d22def23c8b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('package_update_config', + sa.Column('package_id', sa.Integer(), nullable=False), + sa.Column('last_commit', sa.String(length=41), nullable=True), + sa.Column('trigger', sa.Enum('COMMIT', 'TAG', name='packageupdatetrigger'), nullable=False), + sa.Column('make_release', sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('package_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('package_update_config') + # ### end Alembic commands ### diff --git a/utils/create_migration.sh b/utils/create_migration.sh index 4c73937..59aaea0 100755 --- a/utils/create_migration.sh +++ b/utils/create_migration.sh @@ -2,7 +2,7 @@ # Create a database migration, and copy it back to the host. -docker exec contentdb_app_1 sh -c "FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db migrate" +docker exec contentdb_app_1 sh -c "FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db revision" docker exec -u root contentdb_app_1 sh -c "cp /home/cdb/migrations/versions/* /source/migrations/versions/" USER=$(whoami)