From 7b6ad051c49818b69b32a8a5db834001d5626bfe Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 16:37:54 +0100 Subject: [PATCH 01/14] Remove dependencies, add meta packages --- app/models.py | 85 +++++-------------- app/templates/packages/create_edit.html | 2 - .../packages/editrequest_create_edit.html | 2 - app/templates/packages/view.html | 8 +- app/views/packages/__init__.py | 2 - migrations/versions/900758871713_.py | 57 +++++++++++++ setup.py | 14 ++- 7 files changed, 98 insertions(+), 72 deletions(-) create mode 100644 migrations/versions/900758871713_.py diff --git a/app/models.py b/app/models.py index 142b61c..d715994 100644 --- a/app/models.py +++ b/app/models.py @@ -219,8 +219,7 @@ class PackagePropertyKey(enum.Enum): type = "Type" license = "License" tags = "Tags" - harddeps = "Hard Dependencies" - softdeps = "Soft Dependencies" + provides = "Provides" repo = "Repository" website = "Website" issueTracker = "Issue Tracker" @@ -229,27 +228,22 @@ class PackagePropertyKey(enum.Enum): def convert(self, value): if self == PackagePropertyKey.tags: return ",".join([t.title for t in value]) - elif self == PackagePropertyKey.harddeps or self == PackagePropertyKey.softdeps: - return ",".join([t.author.username + "/" + t.name for t in value]) - + elif self == PackagePropertyKey.provides: + return ",".join([t.name for t in value]) else: return str(value) + +provides = db.Table("provides", + db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True), + db.Column("metapackage_id", db.Integer, db.ForeignKey("meta_package.id"), primary_key=True) +) + tags = db.Table("tags", db.Column("tag_id", db.Integer, db.ForeignKey("tag.id"), primary_key=True), db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True) ) -harddeps = db.Table("harddeps", - db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True), - db.Column("dependency_id", db.Integer, db.ForeignKey("package.id"), primary_key=True) -) - -softdeps = db.Table("softdeps", - db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True), - db.Column("dependency_id", db.Integer, db.ForeignKey("package.id"), primary_key=True) -) - class Package(db.Model): id = db.Column(db.Integer, primary_key=True) @@ -273,20 +267,11 @@ class Package(db.Model): issueTracker = db.Column(db.String(200), nullable=True) forums = db.Column(db.Integer, nullable=True) - tags = db.relationship("Tag", secondary=tags, lazy="subquery", + provides = db.relationship("MetaPackage", secondary=provides, lazy="subquery", backref=db.backref("packages", lazy=True)) - harddeps = db.relationship("Package", - secondary=harddeps, - primaryjoin=id==harddeps.c.package_id, - secondaryjoin=id==harddeps.c.dependency_id, - backref="dependents") - - softdeps = db.relationship("Package", - secondary=softdeps, - primaryjoin=id==softdeps.c.package_id, - secondaryjoin=id==softdeps.c.dependency_id, - backref="softdependents") + tags = db.relationship("Tag", secondary=tags, lazy="subquery", + backref=db.backref("packages", lazy=True)) releases = db.relationship("PackageRelease", backref="package", lazy="dynamic", order_by=db.desc("package_release_releaseDate")) @@ -415,6 +400,16 @@ class Package(db.Model): else: raise Exception("Permission {} is not related to packages".format(perm.name)) +class MetaPackage(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(100), unique=True, nullable=False) + + def __init__(self, name=None): + self.name = name + + def __str__(self): + return self.name + class Tag(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(100), unique=True, nullable=False) @@ -552,42 +547,6 @@ class EditRequestChange(db.Model): tag = Tag.query.filter_by(title=tagTitle.strip()).first() package.tags.append(tag) - elif self.key == PackagePropertyKey.harddeps: - package.harddeps.clear() - for pair in self.newValue.split(","): - key, value = pair.split("/") - if key is None or value is None: - continue - - user = User.query.filter_by(username=key).first() - if user is None: - continue - - dep = Package.query.filter_by(author=user, name=value, soft_deleted=False).first() - if dep is None: - continue - - package.harddeps.append(dep) - - elif self.key == PackagePropertyKey.softdeps: - package.softdeps.clear() - for pair in self.newValue.split(","): - key, value = pair.split("/") - if key is None or value is None: - continue - - user = User.query.filter_by(username=key).first() - if user is None: - raise Exception("No such user!") - continue - - dep = Package.query.filter_by(author=user, name=value).first() - if dep is None: - raise Exception("No such package!") - continue - - package.softdeps.append(dep) - else: setattr(package, self.key.name, self.newValue) diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index fc3715a..4f053dd 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -23,8 +23,6 @@ {{ render_field(form.type, class_="pkg_meta") }} {{ render_field(form.license, class_="pkg_meta") }} {{ render_multiselect_field(form.tags, class_="pkg_meta") }} - {{ render_multiselect_field(form.harddeps, class_="pkg_meta") }} - {{ render_multiselect_field(form.softdeps, class_="pkg_meta") }}

Enter the repo URL for the package. diff --git a/app/templates/packages/editrequest_create_edit.html b/app/templates/packages/editrequest_create_edit.html index 987a292..d245206 100644 --- a/app/templates/packages/editrequest_create_edit.html +++ b/app/templates/packages/editrequest_create_edit.html @@ -18,8 +18,6 @@ {{ render_field(form.type) }} {{ render_field(form.license) }} {{ render_multiselect_field(form.tags) }} - {{ render_multiselect_field(form.harddeps) }} - {{ render_multiselect_field(form.softdeps) }} {{ render_field(form.repo) }} {{ render_field(form.website) }} {{ render_field(form.issueTracker) }} diff --git a/app/templates/packages/view.html b/app/templates/packages/view.html index d6b74a6..1f6ada5 100644 --- a/app/templates/packages/view.html +++ b/app/templates/packages/view.html @@ -67,6 +67,10 @@ Name {{ package.name }} + + Provides + {{ package.provides | join(', ') }} + Author @@ -153,7 +157,7 @@ {% endfor %} - + {% if current_user.is_authenticated or requests %}

Edit Requests

diff --git a/app/views/packages/__init__.py b/app/views/packages/__init__.py index c60d68d..5735d5a 100644 --- a/app/views/packages/__init__.py +++ b/app/views/packages/__init__.py @@ -107,8 +107,6 @@ class PackageForm(FlaskForm): type = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD) license = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query, get_pk=lambda a: a.id, get_label=lambda a: a.name) tags = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title) - harddeps = QuerySelectMultipleField('Dependencies', query_factory=lambda: Package.query.filter_by(soft_deleted=False,approved=True).join(User).order_by(db.asc(Package.title), db.asc(User.display_name)), get_pk=lambda a: a.id, get_label=lambda a: a.title + " by " + a.author.display_name) - softdeps = QuerySelectMultipleField('Soft Dependencies', query_factory=lambda: Package.query.filter_by(soft_deleted=False,approved=True).join(User).order_by(db.asc(Package.title), db.asc(User.display_name)), get_pk=lambda a: a.id, get_label=lambda a: a.title + " by " + a.author.display_name) repo = StringField("Repo URL", [Optional(), URL()]) website = StringField("Website URL", [Optional(), URL()]) issueTracker = StringField("Issue Tracker URL", [Optional(), URL()]) diff --git a/migrations/versions/900758871713_.py b/migrations/versions/900758871713_.py new file mode 100644 index 0000000..ed1ce98 --- /dev/null +++ b/migrations/versions/900758871713_.py @@ -0,0 +1,57 @@ +"""empty message + +Revision ID: 900758871713 +Revises: ea5a023711e0 +Create Date: 2018-05-27 16:36:44.258935 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '900758871713' +down_revision = 'ea5a023711e0' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('meta_package', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=100), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('provides', + sa.Column('package_id', sa.Integer(), nullable=False), + sa.Column('metapackage_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['metapackage_id'], ['meta_package.id'], ), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('package_id', 'metapackage_id') + ) + op.drop_table('harddeps') + op.drop_table('softdeps') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('softdeps', + sa.Column('package_id', sa.INTEGER(), nullable=False), + sa.Column('dependency_id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint(['dependency_id'], ['package.id'], ), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('package_id', 'dependency_id') + ) + op.create_table('harddeps', + sa.Column('package_id', sa.INTEGER(), nullable=False), + sa.Column('dependency_id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint(['dependency_id'], ['package.id'], ), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('package_id', 'dependency_id') + ) + op.drop_table('provides') + op.drop_table('meta_package') + # ### end Alembic commands ### diff --git a/setup.py b/setup.py index d910db7..22b80cf 100644 --- a/setup.py +++ b/setup.py @@ -255,7 +255,6 @@ No warranty is provided, express or implied, for any part of the project. mod.title = "Sweet Foods" mod.license = licenses["CC0"] mod.type = PackageType.MOD - mod.harddeps.append(food) mod.author = ruben mod.tags.append(tags["player_effects"]) mod.repo = "https://github.com/rubenwardy/food_sweet/" @@ -314,6 +313,19 @@ Uses the CTF PvP Engine. rel.approved = True db.session.add(rel) + db.session.commit() + + metas = {} + for package in Package.query.filter_by(type=PackageType.MOD).all(): + meta = None + try: + meta = metas[package.name] + except KeyError: + meta = MetaPackage(package.name) + db.session.add(meta) + metas[package.name] = meta + package.provides.append(meta) + delete_db = len(sys.argv) >= 2 and sys.argv[1].strip() == "-d" if delete_db and os.path.isfile("db.sqlite"): From f4c9348b7f36b31980f7629478fdc8b2877801cc Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 16:51:46 +0100 Subject: [PATCH 02/14] Add metapackages pages --- app/templates/meta/list.html | 15 ++++++++++++++ app/templates/meta/view.html | 12 +++++++++++ app/templates/packages/view.html | 7 ++++++- app/views/__init__.py | 2 +- app/views/meta.py | 34 ++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 app/templates/meta/list.html create mode 100644 app/templates/meta/view.html create mode 100644 app/views/meta.py diff --git a/app/templates/meta/list.html b/app/templates/meta/list.html new file mode 100644 index 0000000..5fec732 --- /dev/null +++ b/app/templates/meta/list.html @@ -0,0 +1,15 @@ +{% extends "base.html" %} + +{% block title %} +Meta Packages +{% endblock %} + +{% block content %} +
    + {% for meta in mpackages %} +
  • {{ meta.name }} ({{ meta.packages | count }} packages)
    • + {% else %} +
  • No meta packages found.
    • + {% endfor %} +
+{% endblock %} diff --git a/app/templates/meta/view.html b/app/templates/meta/view.html new file mode 100644 index 0000000..c5473b9 --- /dev/null +++ b/app/templates/meta/view.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} + +{% block title %} +Packages providing '{{ mpackage.name }}'' +{% endblock %} + +{% block content %} +

Packages providing '{{ mpackage.name }}''

+ + {% from "macros/packagegridtile.html" import render_pkggrid %} + {{ render_pkggrid(mpackage.packages) }} +{% endblock %} diff --git a/app/templates/packages/view.html b/app/templates/packages/view.html index 1f6ada5..1623f1c 100644 --- a/app/templates/packages/view.html +++ b/app/templates/packages/view.html @@ -69,7 +69,12 @@ - + diff --git a/app/views/__init__.py b/app/views/__init__.py index c584bb8..8fff788 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -43,7 +43,7 @@ def home_page(): packages = query.order_by(db.desc(Package.created_at)).limit(15).all() return render_template("index.html", packages=packages, count=count) -from . import users, githublogin, packages, sass, tasks, admin, notifications, tagseditor +from . import users, githublogin, packages, sass, tasks, admin, notifications, tagseditor, meta @menu.register_menu(app, ".help", "Help", order=19, endpoint_arguments_constructor=lambda: { 'path': 'help' }) @app.route('//') diff --git a/app/views/meta.py b/app/views/meta.py new file mode 100644 index 0000000..fe1a05a --- /dev/null +++ b/app/views/meta.py @@ -0,0 +1,34 @@ +# Content DB +# Copyright (C) 2018 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from flask import * +from flask_user import * +from app import app +from app.models import * + +@app.route("/metapackages/") +def meta_package_list_page(): + mpackages = MetaPackage.query.order_by(db.desc(MetaPackage.name)).all() + return render_template("meta/list.html", mpackages=mpackages) + +@app.route("/metapackages//") +def meta_package_page(name): + mpackage = MetaPackage.query.filter_by(name=name).first() + if mpackage is None: + abort(404) + + return render_template("meta/view.html", mpackage=mpackage) From 5e4613a6ef78180f208b0cd58aeffb63f1e19853 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 17:58:09 +0100 Subject: [PATCH 03/14] Add ability to edit provides --- app/models.py | 33 +++++++++++++++++++++++++ app/templates/macros/forms.html | 19 ++++++++++++++ app/templates/packages/create_edit.html | 3 ++- app/views/packages/__init__.py | 10 ++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index d715994..ea8bb18 100644 --- a/app/models.py +++ b/app/models.py @@ -410,6 +410,39 @@ class MetaPackage(db.Model): def __str__(self): return self.name + @staticmethod + def ListToSpec(list): + return ",".join([str(x) for x in list]) + + @staticmethod + def SpecToList(spec, cache={}): + retval = [] + arr = spec.split(",") + + import re + pattern = re.compile("^([a-z0-9_]+)$") + + for x in arr: + x = x.strip() + if x == "": + continue + + if not pattern.match(x): + continue + + mp = cache.get(x) + if mp is None: + mp = MetaPackage.query.filter_by(name=x).first() + + if mp is None: + mp = MetaPackage(x) + db.session.add(mp) + + cache[x] = mp + retval.append(mp) + + return retval + class Tag(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(100), unique=True, nullable=False) diff --git a/app/templates/macros/forms.html b/app/templates/macros/forms.html index b23711a..e0ad2de 100644 --- a/app/templates/macros/forms.html +++ b/app/templates/macros/forms.html @@ -39,6 +39,25 @@ {% endmacro %} +{% macro render_mpackage_field(field, label=None, label_visible=true, right_url=None, right_label=None) -%} +
+ {% if field.type != 'HiddenField' and label_visible %} + {% if not label %}{% set label=field.label.text %}{% endif %} + + {% endif %} +
+ +
+
+ {{ field(class_='form-control', **kwargs) }} + {% if field.errors %} + {% for e in field.errors %} +

{{ e }}

+ {% endfor %} + {% endif %} +
+{% endmacro %} + {% macro render_checkbox_field(field, label=None) -%} {% if not label %}{% set label=field.label.text %}{% endif %}
diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index 4f053dd..8eb7213 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -10,7 +10,7 @@ {% block content %}

Create Package

- {% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field %} + {% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field, render_mpackage_field %} {{ form_includes() }}
@@ -22,6 +22,7 @@ {{ render_field(form.desc, class_="pkg_meta") }} {{ render_field(form.type, class_="pkg_meta") }} {{ render_field(form.license, class_="pkg_meta") }} + {{ render_mpackage_field(form.provides_str, class_="pkg_meta", placeholder="Comma separated list") }} {{ render_multiselect_field(form.tags, class_="pkg_meta") }}
diff --git a/app/views/packages/__init__.py b/app/views/packages/__init__.py index 5735d5a..e5a515d 100644 --- a/app/views/packages/__init__.py +++ b/app/views/packages/__init__.py @@ -106,6 +106,7 @@ class PackageForm(FlaskForm): desc = TextAreaField("Long Description", [Optional(), Length(0,10000)]) type = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD) license = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query, get_pk=lambda a: a.id, get_label=lambda a: a.name) + provides_str = StringField("Provides", [InputRequired(), Length(1,1000)]) tags = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title) repo = StringField("Repo URL", [Optional(), URL()]) website = StringField("Website URL", [Optional(), URL()]) @@ -144,6 +145,9 @@ def create_edit_package_page(author=None, name=None): form = PackageForm(formdata=request.form, obj=package) # Initial form class from post data and default data + if request.method == "GET" and package is not None: + form.provides_str.data = MetaPackage.ListToSpec(package.provides) + if request.method == "POST" and form.validate(): wasNew = False if not package: @@ -164,6 +168,12 @@ def create_edit_package_page(author=None, name=None): form.populate_obj(package) # copy to row + mpackage_cache = {} + package.provides.clear() + mpackages = MetaPackage.SpecToList(form.provides_str.data, mpackage_cache) + for m in mpackages: + package.provides.append(m) + package.tags.clear() for tag in form.tags.raw_data: package.tags.append(Tag.query.get(tag)) From 82159d488d87d204390dc58fdd30ca2167156b79 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 20:15:35 +0100 Subject: [PATCH 04/14] Add meta package selector --- README.md | 8 +- app/models.py | 24 +++-- app/public/static/tagselector.js | 116 +++++++++++++++++++++++- app/scss/components.scss | 16 ++-- app/templates/macros/forms.html | 4 +- app/templates/packages/create_edit.html | 13 +++ app/views/packages/__init__.py | 10 +- 7 files changed, 163 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index c5203e1..8dc588b 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ the current session: If you need to, reset the db like so: - python3 setup.py -d + python3 setup.py -t Then run the server: @@ -43,6 +43,12 @@ Then view in your web browser: http://localhost:5000/ ## How-tos +### Start celery worker + +```sh +FLASK_CONFIG=../config.cfg celery -A app.tasks.celery worker +``` + ### Create migration ```sh diff --git a/app/models.py b/app/models.py index ea8bb18..bfd8a31 100644 --- a/app/models.py +++ b/app/models.py @@ -414,6 +414,19 @@ class MetaPackage(db.Model): def ListToSpec(list): return ",".join([str(x) for x in list]) + @staticmethod + def GetOrCreate(name, cache={}): + mp = cache.get(name) + if mp is None: + mp = MetaPackage.query.filter_by(name=name).first() + + if mp is None: + mp = MetaPackage(name) + db.session.add(mp) + + cache[name] = mp + return mp + @staticmethod def SpecToList(spec, cache={}): retval = [] @@ -430,16 +443,7 @@ class MetaPackage(db.Model): if not pattern.match(x): continue - mp = cache.get(x) - if mp is None: - mp = MetaPackage.query.filter_by(name=x).first() - - if mp is None: - mp = MetaPackage(x) - db.session.add(mp) - - cache[x] = mp - retval.append(mp) + retval.append(MetaPackage.GetOrCreate(x, cache)) return retval diff --git a/app/public/static/tagselector.js b/app/public/static/tagselector.js index d5895bf..2e90657 100644 --- a/app/public/static/tagselector.js +++ b/app/public/static/tagselector.js @@ -5,7 +5,7 @@ * https://petprojects.googlecode.com/svn/trunk/GPL-LICENSE.txt */ (function($) { - $.fn.tagSelector = function(source, name, select) { + $.fn.selectSelector = function(source, name, select) { return this.each(function() { var selector = $(this), input = $('input[type=text]', this); @@ -80,15 +80,115 @@ }); } + $.fn.csvSelector = function(source, name, result, allowSlash) { + return this.each(function() { + var selector = $(this), + input = $('input[type=text]', this); + + var selected = []; + + selector.click(function() { input.focus(); }) + .delegate('.tag a', 'click', function() { + var id = $(this).parent().data("id"); + for (var i = 0; i < selected.length; i++) { + if (selected[i] == id) { + selected.splice(i, 1); + } + } + recreate(); + }); + + + function selectItem(id) { + for (var i = 0; i < selected.length; i++) { + if (selected[i] == id) { + return false; + } + } + selected.push(id); + return true; + } + + function addTag(id, value) { + var tag = $('') + .text(value) + .data("id", id) + .append(' x') + .insertBefore(input); + + input.attr("placeholder", null); + } + + function recreate() { + selector.find("span").remove(); + for (var i = 0; i < selected.length; i++) { + var value = source[selected[i]] || selected[i]; + addTag(selected[i], value); + } + result.val(selected.join(",")) + } + recreate(); + + input.keydown(function(e) { + if (e.keyCode === $.ui.keyCode.TAB && $(this).data('ui-autocomplete').menu.active) + e.preventDefault(); + else if (e.keyCode === $.ui.keyCode.COMMA) { + var item = input.val(); + if (item.match(/^([a-z0-9_]+)$/)) { + selectItem(item); + recreate(); + input.val(""); + } else { + alert("Only lowercase alphanumeric and number names allowed."); + } + e.preventDefault(); + return true; + } else if (e.keyCode === $.ui.keyCode.BACKSPACE) { + if (input.val() == "") { + var item = selected[selected.length - 1]; + selected.splice(selected.length - 1, 1); + recreate(); + input.val(item); + e.preventDefault(); + return true; + } + } + }) + .autocomplete({ + minLength: 0, + source: source, + select: function(event, ui) { + selectItem(ui.item.id); + recreate(); + input.val(""); + return false; + } + }); + + input.data('ui-autocomplete')._renderItem = function(ul, item) { + return $('
  • ') + .data('item.autocomplete', item) + .append($('').text(item.toString())) + .appendTo(ul); + }; + + input.data('ui-autocomplete')._resizeMenu = function(ul, item) { + var ul = this.menu.element; + ul.outerWidth(Math.max( + ul.width('').outerWidth(), + selector.outerWidth() + )); + }; + }); + } + $(function() { $(".multichoice_selector").each(function() { var ele = $(this); var sel = ele.parent().find("select"); - console.log(sel.attr("name")); - sel.css("display", "none"); + sel.hide(); var options = []; - sel.find("option").each(function() { var text = $(this).text(); options.push({ @@ -100,7 +200,13 @@ }); console.log(options); - ele.tagSelector(options, sel.attr("name"), sel); + ele.selectSelector(options, sel.attr("name"), sel); + }); + + $(".metapackage_selector").each(function() { + var input = $(this).parent().children("input[type='text']"); + input.hide(); + $(this).csvSelector(meta_packages, input.attr("name"), input); }) }); })(jQuery); diff --git a/app/scss/components.scss b/app/scss/components.scss index 2cf8af4..a8ec31a 100644 --- a/app/scss/components.scss +++ b/app/scss/components.scss @@ -87,7 +87,7 @@ a:hover { } .button, .buttonset li a, input[type=submit], input[type=text], - input[type=password], textarea, select, .multichoice_selector { + input[type=password], textarea, select, .bulletselector { text-align: center; display: inline-block; padding: 0.4em 1em; @@ -99,7 +99,7 @@ a:hover { font-size: 100%; } -input[type=text], input[type=password], textarea, select, .multichoice_selector { +input[type=text], input[type=password], textarea, select, .bulletselector { text-align: left; } @@ -147,13 +147,13 @@ select:not([multiple]) { padding: 0 8px 8px 0; } -.form-group input, .form-group textarea, .form-group .multichoice_selector { +.form-group input, .form-group textarea, .form-group .bulletselector { display: block; min-width: 100%; max-width: 100%; } -.box .form-group input, .box .form-group textarea, .form-group .multichoice_selector { +.box .form-group input, .box .form-group textarea, .form-group .bulletselector { min-width: 95%; max-width: 95%; } @@ -197,7 +197,7 @@ select:not([multiple]) { } -.multichoice_selector input { +.bulletselector input { border: none; border-radius: 0; -moz-border-radius: 0; @@ -211,7 +211,7 @@ select:not([multiple]) { white-space: nowrap; background: transparent; } -.multichoice_selector .tag { +.bulletselector .tag { background: #375D81; border-radius: 3px; -moz-border-radius: 3px; @@ -223,11 +223,11 @@ select:not([multiple]) { margin-bottom: 0.3em; vertical-align: baseline; } -.multichoice_selector .tag a { +.bulletselector .tag a { color: #FFF; cursor: pointer; } -.multichoice_selector .tag a:hover { +.bulletselector .tag a:hover { color: #0099CC; text-decoration: none; } diff --git a/app/templates/macros/forms.html b/app/templates/macros/forms.html index e0ad2de..430c4e8 100644 --- a/app/templates/macros/forms.html +++ b/app/templates/macros/forms.html @@ -26,7 +26,7 @@ {% if not label %}{% set label=field.label.text %}{% endif %} {% endif %} -
    +
    @@ -45,7 +45,7 @@ {% if not label %}{% set label=field.label.text %}{% endif %} {% endif %} -
    +
    diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index 8eb7213..c7c2dbb 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -10,6 +10,19 @@ {% block content %}

    Create Package

    + + {% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field, render_mpackage_field %} {{ form_includes() }} diff --git a/app/views/packages/__init__.py b/app/views/packages/__init__.py index e5a515d..3c3acf0 100644 --- a/app/views/packages/__init__.py +++ b/app/views/packages/__init__.py @@ -106,7 +106,7 @@ class PackageForm(FlaskForm): desc = TextAreaField("Long Description", [Optional(), Length(0,10000)]) type = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD) license = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query, get_pk=lambda a: a.id, get_label=lambda a: a.name) - provides_str = StringField("Provides", [InputRequired(), Length(1,1000)]) + provides_str = StringField("Provides", [Optional(), Length(0,1000)]) tags = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title) repo = StringField("Repo URL", [Optional(), URL()]) website = StringField("Website URL", [Optional(), URL()]) @@ -174,6 +174,11 @@ def create_edit_package_page(author=None, name=None): for m in mpackages: package.provides.append(m) + if wasNew and package.type == PackageType.MOD and not package.name in mpackage_cache: + m = MetaPackage.GetOrCreate(package.name, mpackage_cache) + package.provides.append(m) + + package.tags.clear() for tag in form.tags.raw_data: package.tags.append(Tag.query.get(tag)) @@ -188,7 +193,8 @@ def create_edit_package_page(author=None, name=None): enableWizard = name is None and request.method != "POST" return render_template("packages/create_edit.html", package=package, \ - form=form, author=author, enable_wizard=enableWizard) + form=form, author=author, enable_wizard=enableWizard, \ + mpackages=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all()) @app.route("/packages///approve/", methods=["POST"]) @login_required From 63af1535b90358f1a26248cae217f4a74fdc1f84 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 21:31:11 +0100 Subject: [PATCH 05/14] Add dependencies --- app/models.py | 72 ++++++++++++++++++++++++- app/public/static/tagselector.js | 31 +++++++++-- app/templates/macros/forms.html | 19 +++++++ app/templates/packages/create_edit.html | 15 +++++- app/templates/packages/view.html | 30 ++++++----- app/views/__init__.py | 4 ++ app/views/packages/__init__.py | 22 +++++++- setup.py | 5 ++ 8 files changed, 178 insertions(+), 20 deletions(-) diff --git a/app/models.py b/app/models.py index bfd8a31..3283595 100644 --- a/app/models.py +++ b/app/models.py @@ -233,7 +233,6 @@ class PackagePropertyKey(enum.Enum): else: return str(value) - provides = db.Table("provides", db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True), db.Column("metapackage_id", db.Integer, db.ForeignKey("meta_package.id"), primary_key=True) @@ -244,6 +243,74 @@ tags = db.Table("tags", db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True) ) +class Dependency(db.Model): + id = db.Column(db.Integer, primary_key=True) + depender_id = db.Column(db.Integer, db.ForeignKey("package.id"), nullable=True) + package_id = db.Column(db.Integer, db.ForeignKey("package.id"), nullable=True) + package = db.relationship("Package", foreign_keys=[package_id]) + meta_package_id = db.Column(db.Integer, db.ForeignKey("meta_package.id"), nullable=True) + optional = db.Column(db.Boolean, nullable=False, default=False) + __table_args__ = (db.UniqueConstraint('depender_id', 'package_id', 'meta_package_id', name='_dependency_uc'), ) + + def __init__(self, depender=None, package=None, meta=None): + if depender is None: + return + + self.depender = depender + + packageProvided = package is not None + metaProvided = meta is not None + + if packageProvided and not metaProvided: + self.package = package + elif metaProvided and not packageProvided: + self.meta_package = meta + else: + raise Exception("Either meta or package must be given, but not both!") + + def __str__(self): + if self.package is not None: + return self.package.author.username + "/" + self.package.name + elif self.meta_package is not None: + return self.meta_package.name + else: + raise Exception("Meta and package are both none!") + + @staticmethod + def SpecToList(depender, spec, cache={}): + retval = [] + arr = spec.split(",") + + import re + pattern1 = re.compile("^([a-z0-9_]+)$") + pattern2 = re.compile("^([A-Za-z0-9_]+)/([a-z0-9_]+)$") + + for x in arr: + x = x.strip() + if x == "": + continue + + if pattern1.match(x): + meta = MetaPackage.GetOrCreate(x, cache) + retval.append(Dependency(depender, meta=meta)) + else: + m = pattern2.match(x) + username = m.group(1) + name = m.group(2) + user = User.query.filter_by(username=username).first() + if user is None: + raise Exception("Unable to find user " + username) + + package = Package.query.filter_by(author=user, name=name).first() + if package is None: + raise Exception("Unable to find package " + name + " by " + username) + + retval.append(Dependency(depender, package=package)) + + return retval + + + class Package(db.Model): id = db.Column(db.Integer, primary_key=True) @@ -270,6 +337,8 @@ class Package(db.Model): provides = db.relationship("MetaPackage", secondary=provides, lazy="subquery", backref=db.backref("packages", lazy=True)) + dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id]) + tags = db.relationship("Tag", secondary=tags, lazy="subquery", backref=db.backref("packages", lazy=True)) @@ -403,6 +472,7 @@ class Package(db.Model): class MetaPackage(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(100), unique=True, nullable=False) + dependencies = db.relationship("Dependency", backref="meta_package", lazy="dynamic") def __init__(self, name=None): self.name = name diff --git a/app/public/static/tagselector.js b/app/public/static/tagselector.js index 2e90657..32e4882 100644 --- a/app/public/static/tagselector.js +++ b/app/public/static/tagselector.js @@ -86,6 +86,18 @@ input = $('input[type=text]', this); var selected = []; + var lookup = {}; + for (var i = 0; i < source.length; i++) { + lookup[source[i].id] = source[i]; + } + + var selected_raw = result.val().split(","); + for (var i = 0; i < selected_raw.length; i++) { + var raw = selected_raw[i].trim(); + if (lookup[raw]) { + selected.push(raw); + } + } selector.click(function() { input.focus(); }) .delegate('.tag a', 'click', function() { @@ -122,8 +134,8 @@ function recreate() { selector.find("span").remove(); for (var i = 0; i < selected.length; i++) { - var value = source[selected[i]] || selected[i]; - addTag(selected[i], value); + var value = lookup[selected[i]] || { value: selected[i] }; + addTag(selected[i], value.value); } result.val(selected.join(",")) } @@ -134,7 +146,9 @@ e.preventDefault(); else if (e.keyCode === $.ui.keyCode.COMMA) { var item = input.val(); - if (item.match(/^([a-z0-9_]+)$/)) { + if (item.length == 0) { + input.data("ui-autocomplete").search(""); + } else if (item.match(/^([a-z0-9_]+)$/)) { selectItem(item); recreate(); input.val(""); @@ -148,7 +162,8 @@ var item = selected[selected.length - 1]; selected.splice(selected.length - 1, 1); recreate(); - input.val(item); + if (!(item.indexOf("/") > 0)) + input.val(item); e.preventDefault(); return true; } @@ -207,6 +222,12 @@ var input = $(this).parent().children("input[type='text']"); input.hide(); $(this).csvSelector(meta_packages, input.attr("name"), input); - }) + }); + + $(".deps_selector").each(function() { + var input = $(this).parent().children("input[type='text']"); + input.hide(); + $(this).csvSelector(all_packages, input.attr("name"), input); + }); }); })(jQuery); diff --git a/app/templates/macros/forms.html b/app/templates/macros/forms.html index 430c4e8..940c4a2 100644 --- a/app/templates/macros/forms.html +++ b/app/templates/macros/forms.html @@ -58,6 +58,25 @@
    {% endmacro %} +{% macro render_deps_field(field, label=None, label_visible=true, right_url=None, right_label=None) -%} +
    + {% if field.type != 'HiddenField' and label_visible %} + {% if not label %}{% set label=field.label.text %}{% endif %} + + {% endif %} +
    + +
    +
    + {{ field(class_='form-control', **kwargs) }} + {% if field.errors %} + {% for e in field.errors %} +

    {{ e }}

    + {% endfor %} + {% endif %} +
    +{% endmacro %} + {% macro render_checkbox_field(field, label=None) -%} {% if not label %}{% set label=field.label.text %}{% endif %}
    diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index c7c2dbb..666d4cd 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -21,9 +21,20 @@ }, {% endfor %} ] + + all_packages = meta_packages.slice(); + + {% for p in packages %} + {# This is safe as name can only contain `[a-z0-9_]` #} + all_packages.push({ + id: "{{ p.author.username }}/{{ p.name }}", + value: {{ p.title | tojson }} + " by " + {{ p.author.display_name | tojson }}, + toString: function() { return {{ p.title | tojson }} + " by " + {{ p.author.display_name | tojson }} + " only"; }, + }); + {% endfor %} - {% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field, render_mpackage_field %} + {% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field, render_mpackage_field, render_deps_field %} {{ form_includes() }} @@ -36,6 +47,8 @@ {{ render_field(form.type, class_="pkg_meta") }} {{ render_field(form.license, class_="pkg_meta") }} {{ render_mpackage_field(form.provides_str, class_="pkg_meta", placeholder="Comma separated list") }} + {{ render_deps_field(form.harddep_str, class_="pkg_meta", placeholder="Comma separated list") }} + {{ render_deps_field(form.softdep_str, class_="pkg_meta", placeholder="Comma separated list") }} {{ render_multiselect_field(form.tags, class_="pkg_meta") }}
    diff --git a/app/templates/packages/view.html b/app/templates/packages/view.html index 1623f1c..834ea63 100644 --- a/app/templates/packages/view.html +++ b/app/templates/packages/view.html @@ -162,27 +162,33 @@ {% endfor %} -
    • -
    Provides{{ package.provides | join(', ') }}{% for meta in package.provides %} + {{ meta.name }} + {%- if not loop.last %} + , + {% endif %} + {% endfor %}
    Author
    --> + {% if current_user.is_authenticated or requests %}

    Edit Requests

    diff --git a/app/views/__init__.py b/app/views/__init__.py index 8fff788..2559969 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -27,6 +27,10 @@ from werkzeug.contrib.cache import SimpleCache from urllib.parse import urlparse cache = SimpleCache() +@app.template_filter() +def throw(err): + raise Exception(err) + @app.template_filter() def domain(url): return urlparse(url).netloc diff --git a/app/views/packages/__init__.py b/app/views/packages/__init__.py index 3c3acf0..07f62a9 100644 --- a/app/views/packages/__init__.py +++ b/app/views/packages/__init__.py @@ -108,6 +108,8 @@ class PackageForm(FlaskForm): license = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query, get_pk=lambda a: a.id, get_label=lambda a: a.name) provides_str = StringField("Provides", [Optional(), Length(0,1000)]) tags = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title) + harddep_str = StringField("Hard Dependencies", [Optional(), Length(0,1000)]) + softdep_str = StringField("Soft Dependencies", [Optional(), Length(0,1000)]) repo = StringField("Repo URL", [Optional(), URL()]) website = StringField("Website URL", [Optional(), URL()]) issueTracker = StringField("Issue Tracker URL", [Optional(), URL()]) @@ -146,6 +148,9 @@ def create_edit_package_page(author=None, name=None): # Initial form class from post data and default data if request.method == "GET" and package is not None: + deps = package.dependencies + form.harddep_str.data = ",".join([str(x) for x in deps if not x.optional]) + form.softdep_str.data = ",".join([str(x) for x in deps if x.optional]) form.provides_str.data = MetaPackage.ListToSpec(package.provides) if request.method == "POST" and form.validate(): @@ -174,11 +179,21 @@ def create_edit_package_page(author=None, name=None): for m in mpackages: package.provides.append(m) + Dependency.query.filter_by(depender=package).delete() + deps = Dependency.SpecToList(package, form.harddep_str.data, mpackage_cache) + for dep in deps: + dep.optional = False + db.session.add(dep) + + deps = Dependency.SpecToList(package, form.softdep_str.data, mpackage_cache) + for dep in deps: + dep.optional = True + db.session.add(dep) + if wasNew and package.type == PackageType.MOD and not package.name in mpackage_cache: m = MetaPackage.GetOrCreate(package.name, mpackage_cache) package.provides.append(m) - package.tags.clear() for tag in form.tags.raw_data: package.tags.append(Tag.query.get(tag)) @@ -191,9 +206,14 @@ def create_edit_package_page(author=None, name=None): return redirect(package.getDetailsURL()) + package_query = Package.query.filter_by(approved=True, soft_deleted=False) + if package is not None: + package_query = package_query.filter(Package.id != package.id) + enableWizard = name is None and request.method != "POST" return render_template("packages/create_edit.html", package=package, \ form=form, author=author, enable_wizard=enableWizard, \ + packages=package_query.all(), \ mpackages=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all()) @app.route("/packages///approve/", methods=["POST"]) diff --git a/setup.py b/setup.py index 22b80cf..5873067 100644 --- a/setup.py +++ b/setup.py @@ -262,6 +262,7 @@ No warranty is provided, express or implied, for any part of the project. mod.forums = 9039 mod.shortDesc = "Adds sweet food" mod.desc = "This is the long desc" + food_sweet = mod db.session.add(mod) game1 = Package() @@ -326,6 +327,10 @@ Uses the CTF PvP Engine. metas[package.name] = meta package.provides.append(meta) + dep = Dependency(food_sweet, meta=metas["food"]) + db.session.add(dep) + + delete_db = len(sys.argv) >= 2 and sys.argv[1].strip() == "-d" if delete_db and os.path.isfile("db.sqlite"): From ca7708437b8cdb5f5719460decc83e22cac5bb68 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 21:33:50 +0100 Subject: [PATCH 06/14] Fix potentiall XSS vulnerability --- app/templates/packages/create_edit.html | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index 666d4cd..8191a17 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -22,14 +22,22 @@ {% endfor %} ] + function escape(unsafe) { + return unsafe + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + } + all_packages = meta_packages.slice(); {% for p in packages %} - {# This is safe as name can only contain `[a-z0-9_]` #} all_packages.push({ id: "{{ p.author.username }}/{{ p.name }}", - value: {{ p.title | tojson }} + " by " + {{ p.author.display_name | tojson }}, - toString: function() { return {{ p.title | tojson }} + " by " + {{ p.author.display_name | tojson }} + " only"; }, + value: escape({{ p.title | tojson }} + " by " + {{ p.author.display_name | tojson }}), + toString: function() { return escape({{ p.title | tojson }} + " by " + {{ p.author.display_name | tojson }} + " only"); }, }); {% endfor %} From 5d944d79d36ca380a67122e9eb306ad223b6ad2a Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 21:36:58 +0100 Subject: [PATCH 07/14] Improve placeholder text --- app/templates/macros/forms.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/templates/macros/forms.html b/app/templates/macros/forms.html index 940c4a2..7700fe2 100644 --- a/app/templates/macros/forms.html +++ b/app/templates/macros/forms.html @@ -46,7 +46,7 @@ {% endif %}
    - +
    {{ field(class_='form-control', **kwargs) }} @@ -65,7 +65,7 @@ {% endif %}
    - +
    {{ field(class_='form-control', **kwargs) }} From fb8aa25b710fe4b6db70a0b54b2d867b737a6cb1 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 21:42:31 +0100 Subject: [PATCH 08/14] Remove required by for now --- app/templates/packages/view.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/templates/packages/view.html b/app/templates/packages/view.html index 834ea63..2844636 100644 --- a/app/templates/packages/view.html +++ b/app/templates/packages/view.html @@ -162,9 +162,9 @@ {% endfor %} - +

    Dependencies

      {% for dep in package.dependencies %} @@ -184,11 +184,11 @@
    • No dependencies
      • {% endfor %}
    - + + {% endfor %} -
    + --> {% if current_user.is_authenticated or requests %}

    Edit Requests

    From fb5cba4cc82efa5cae4e78c5ad75575b615de378 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 22:03:54 +0100 Subject: [PATCH 09/14] Add dependency detection to importer --- app/public/static/package_create.js | 12 ++++++++++++ app/public/static/tagselector.js | 26 ++++++++++++++++---------- app/tasks/importtasks.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/app/public/static/package_create.js b/app/public/static/package_create.js index 2b992f5..771d0fd 100644 --- a/app/public/static/package_create.js +++ b/app/public/static/package_create.js @@ -26,13 +26,25 @@ $(function() { $(".pkg_wiz_2").show() $(".pkg_repo").hide() + function setSpecial(id, value) { + if (value != "") { + var ele = $(id); + ele.val(value); + ele.trigger("change") + } + } + performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) { $("#name").val(result.name || "") + setSpecial("#provides_str", result.name || "") $("#title").val(result.title || "") $("#repo").val(result.repo || repoURL) $("#issueTracker").val(result.issueTracker || "") $("#desc").val(result.description || "") $("#shortDesc").val(result.short_description || "") + setSpecial("#harddep_str", result.depends || "") + setSpecial("#softdep_str", result.optional_depends || "") + $("#shortDesc").val(result.short_description || "") if (result.forumId) { $("#forums").val(result.forumId) } diff --git a/app/public/static/tagselector.js b/app/public/static/tagselector.js index 32e4882..2c69e6d 100644 --- a/app/public/static/tagselector.js +++ b/app/public/static/tagselector.js @@ -91,14 +91,6 @@ lookup[source[i].id] = source[i]; } - var selected_raw = result.val().split(","); - for (var i = 0; i < selected_raw.length; i++) { - var raw = selected_raw[i].trim(); - if (lookup[raw]) { - selected.push(raw); - } - } - selector.click(function() { input.focus(); }) .delegate('.tag a', 'click', function() { var id = $(this).parent().data("id"); @@ -110,7 +102,6 @@ recreate(); }); - function selectItem(id) { for (var i = 0; i < selected.length; i++) { if (selected[i] == id) { @@ -139,7 +130,22 @@ } result.val(selected.join(",")) } - recreate(); + + function readFromResult() { + selected = []; + var selected_raw = result.val().split(","); + for (var i = 0; i < selected_raw.length; i++) { + var raw = selected_raw[i].trim(); + if (lookup[raw] || raw.match(/^([a-z0-9_]+)$/)) { + selected.push(raw); + } + } + + recreate(); + } + readFromResult(); + + result.change(readFromResult); input.keydown(function(e) { if (e.keyCode === $.ui.keyCode.TAB && $(this).data('ui-autocomplete').menu.active) diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index db992b3..022f9b3 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -55,6 +55,9 @@ class GithubURLMaker: def getDescURL(self): return self.baseUrl + "/description.txt" + def getDependsURL(self): + return self.baseUrl + "/depends.txt" + def getScreenshotURL(self): return self.baseUrl + "/screenshot.png" @@ -161,7 +164,7 @@ def getMeta(urlstr, author): try: contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") conf = parseConf(contents) - for key in ["name", "description", "title"]: + for key in ["name", "description", "title", "depends", "optional_depends"]: try: result[key] = conf[key] except KeyError: @@ -179,16 +182,40 @@ def getMeta(urlstr, author): except HTTPError: print("description.txt does not exist!") + import re + pattern = re.compile("^([a-z0-9_]+)\??$") + if not "depends" in result and not "optional_depends" in result: + try: + contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8") + soft = [] + hard = [] + for line in contents.split("\n"): + line = line.strip() + if pattern.match(line): + if line[len(line) - 1] == "?": + soft.append( line[:-1]) + else: + hard.append(line) + + result["depends"] = ",".join(hard) + result["optional_depends"] = ",".join(soft) + + + except HTTPError: + print("depends.txt does not exist!") + if "description" in result: desc = result["description"] idx = desc.find(".") + 1 cutIdx = min(len(desc), 200 if idx < 5 else idx) result["short_description"] = desc[:cutIdx] + info = findModInfo(author, result.get("name"), result["repo"]) if info is not None: result["forumId"] = info.get("topicId") + print(result) return result From 746cf7f4b5bdddd4728bfb7b79abf6edc43d1cc4 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 22:34:24 +0100 Subject: [PATCH 10/14] Add bulk dependency importer from Github --- app/tasks/importtasks.py | 92 ++++++++++++++++++++++++++++++++++- app/templates/admin/list.html | 3 +- app/views/admin.py | 5 +- 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index 022f9b3..7ccd36c 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -215,7 +215,6 @@ def getMeta(urlstr, author): if info is not None: result["forumId"] = info.get("topicId") - print(result) return result @@ -291,3 +290,94 @@ def importRepoScreenshot(id): print("screenshot.png does not exist") return None + + + +def getDepends(package): + url = urlparse(package.repo) + urlmaker = None + if url.netloc == "github.com": + urlmaker = GithubURLMaker(url) + else: + raise TaskError("Unsupported repo") + + result = {} + if urlmaker.isValid(): + # + # Try getting depends on mod.conf + # + try: + contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") + conf = parseConf(contents) + for key in ["depends", "optional_depends"]: + try: + result[key] = conf[key] + except KeyError: + pass + + except HTTPError: + print("mod.conf does not exist") + + if "depends" in result or "optional_depends" in result: + return result + + + # + # Try depends.txt + # + import re + pattern = re.compile("^([a-z0-9_]+)\??$") + try: + contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8") + soft = [] + hard = [] + for line in contents.split("\n"): + line = line.strip() + if pattern.match(line): + if line[len(line) - 1] == "?": + soft.append( line[:-1]) + else: + hard.append(line) + + result["depends"] = ",".join(hard) + result["optional_depends"] = ",".join(soft) + except HTTPError: + print("depends.txt does not exist") + + return result + + else: + print(TaskError("non-github depends detector not implemented yet!")) + return {} + + +def importDependencies(package, mpackage_cache): + if Dependency.query.filter_by(depender=package).count() != 0: + return + + result = getDepends(package) + + if "depends" in result: + deps = Dependency.SpecToList(package, result["depends"], mpackage_cache) + print("{} hard: {}".format(len(deps), result["depends"])) + for dep in deps: + dep.optional = False + db.session.add(dep) + + if "optional_depends" in result: + deps = Dependency.SpecToList(package, result["optional_depends"], mpackage_cache) + print("{} soft: {}".format(len(deps), result["optional_depends"])) + for dep in deps: + dep.optional = True + db.session.add(dep) + +@celery.task() +def importAllDependencies(): + Dependency.query.delete() + mpackage_cache = {} + packages = Package.query.filter_by(type=PackageType.MOD).all() + for i, p in enumerate(packages): + print("============= {} ({}/{}) =============".format(p.name, i, len(packages))) + importDependencies(p, mpackage_cache) + + db.session.commit() diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html index 3c15fa9..284919d 100644 --- a/app/templates/admin/list.html +++ b/app/templates/admin/list.html @@ -17,8 +17,9 @@ diff --git a/app/views/admin.py b/app/views/admin.py index 2320cc6..b1cfed6 100644 --- a/app/views/admin.py +++ b/app/views/admin.py @@ -20,7 +20,7 @@ from flask_user import * from flask.ext import menu from app import app from app.models import * -from app.tasks.importtasks import importRepoScreenshot +from app.tasks.importtasks import importRepoScreenshot, importAllDependencies from app.tasks.forumtasks import importUsersFromModList from flask_wtf import FlaskForm from wtforms import * @@ -52,6 +52,9 @@ def admin_page(): package.soft_deleted = False db.session.commit() return redirect(url_for("admin_page")) + elif action == "importdepends": + task = importAllDependencies.delay() + return redirect(url_for("check_task", id=task.id, r=url_for("admin_page"))) else: flash("Unknown action: " + action, "error") From 92daa87db064c91c05e7c24535a8c09070930831 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 22:39:16 +0100 Subject: [PATCH 11/14] Add migration --- migrations/versions/4e482c47e519_.py | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 migrations/versions/4e482c47e519_.py diff --git a/migrations/versions/4e482c47e519_.py b/migrations/versions/4e482c47e519_.py new file mode 100644 index 0000000..9a23f00 --- /dev/null +++ b/migrations/versions/4e482c47e519_.py @@ -0,0 +1,39 @@ +"""empty message + +Revision ID: 4e482c47e519 +Revises: 900758871713 +Create Date: 2018-05-27 22:38:16.507155 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '4e482c47e519' +down_revision = '900758871713' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('dependency', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('depender_id', sa.Integer(), nullable=True), + sa.Column('package_id', sa.Integer(), nullable=True), + sa.Column('meta_package_id', sa.Integer(), nullable=True), + sa.Column('optional', sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint(['depender_id'], ['package.id'], ), + sa.ForeignKeyConstraint(['meta_package_id'], ['meta_package.id'], ), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('depender_id', 'package_id', 'meta_package_id', name='_dependency_uc') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('dependency') + # ### end Alembic commands ### From 44c9f7e58f912c8b262ac37afcca90379f1cefe9 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 22:48:53 +0100 Subject: [PATCH 12/14] Hide unneeded fields depending on package type --- app/public/static/package_edit.js | 11 +++++++++++ app/templates/packages/create_edit.html | 18 ++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 app/public/static/package_edit.js diff --git a/app/public/static/package_edit.js b/app/public/static/package_edit.js new file mode 100644 index 0000000..40fbe44 --- /dev/null +++ b/app/public/static/package_edit.js @@ -0,0 +1,11 @@ +// @author rubenwardy +// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later + +$(function() { + $("#type").change(function() { + $(".not_mod, .not_game, .not_txp").show() + $(".not_" + this.value.toLowerCase()).hide() + }) + $(".not_mod, .not_game, .not_txp").show() + $(".not_" + $("#type").val().toLowerCase()).hide() +}) diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index 8191a17..ee6e7cf 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -8,7 +8,7 @@ {% endblock %} {% block content %} -

    Create Package

    +

    Create Package

    {% endblock %} From 2d6b55e67bc0f7b7ea621f0f2ab3485585caedb6 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 22:51:50 +0100 Subject: [PATCH 13/14] Reorder package fields --- app/templates/packages/create_edit.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index ee6e7cf..24a23b6 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -50,21 +50,21 @@

    Package

    + {{ render_field(form.type, class_="pkg_meta") }} {{ render_field(form.name, class_="pkg_meta") }} {{ render_field(form.title, class_="pkg_meta") }} {{ render_field(form.shortDesc, class_="pkg_meta") }} {{ render_field(form.desc, class_="pkg_meta") }} - {{ render_field(form.type, class_="pkg_meta") }} - {{ render_field(form.license, class_="pkg_meta") }} {{ render_multiselect_field(form.tags, class_="pkg_meta") }} + {{ render_field(form.license, class_="pkg_meta") }}

    Dependency Info

    -
    - {{ render_mpackage_field(form.provides_str, class_="pkg_meta not_txp", placeholder="Comma separated list") }} - {{ render_deps_field(form.harddep_str, class_="pkg_meta not_txp not_game", placeholder="Comma separated list") }} - {{ render_deps_field(form.softdep_str, class_="pkg_meta not_txp not_game", placeholder="Comma separated list") }} + {{ render_mpackage_field(form.provides_str, class_="not_txp", placeholder="Comma separated list") }} + {{ render_deps_field(form.harddep_str, class_="not_txp not_game", placeholder="Comma separated list") }} + {{ render_deps_field(form.softdep_str, class_="not_txp not_game", placeholder="Comma separated list") }} +

    Repository and Links

    From 05e536b1217633fd612b8ee94360e8afef3c527c Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 22:55:06 +0100 Subject: [PATCH 14/14] Add helpful text to field labels --- app/views/packages/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/packages/__init__.py b/app/views/packages/__init__.py index 07f62a9..bd090c7 100644 --- a/app/views/packages/__init__.py +++ b/app/views/packages/__init__.py @@ -102,11 +102,11 @@ def package_download_page(package): class PackageForm(FlaskForm): name = StringField("Name", [InputRequired(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")]) title = StringField("Title", [InputRequired(), Length(3, 50)]) - shortDesc = StringField("Short Description", [InputRequired(), Length(1,200)]) - desc = TextAreaField("Long Description", [Optional(), Length(0,10000)]) + shortDesc = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)]) + desc = TextAreaField("Long Description (Markdown)", [Optional(), Length(0,10000)]) type = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD) license = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query, get_pk=lambda a: a.id, get_label=lambda a: a.name) - provides_str = StringField("Provides", [Optional(), Length(0,1000)]) + provides_str = StringField("Provides (mods included in package)", [Optional(), Length(0,1000)]) tags = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title) harddep_str = StringField("Hard Dependencies", [Optional(), Length(0,1000)]) softdep_str = StringField("Soft Dependencies", [Optional(), Length(0,1000)])