diff --git a/app/models.py b/app/models.py index e9cd65c..c9b9f5a 100644 --- a/app/models.py +++ b/app/models.py @@ -15,18 +15,29 @@ # along with this program. If not, see . -from flask import Flask, url_for -from flask_sqlalchemy import SQLAlchemy -from flask_migrate import Migrate -from urllib.parse import urlparse -from app import app, gravatar -from sqlalchemy.orm import validates -from flask_user import login_required, UserManager, UserMixin, SQLAlchemyAdapter import enum, datetime +from app import app, gravatar +from urllib.parse import urlparse + +from flask import Flask, url_for +from flask_sqlalchemy import SQLAlchemy, BaseQuery +from flask_migrate import Migrate +from flask_user import login_required, UserManager, UserMixin, SQLAlchemyAdapter +from sqlalchemy.orm import validates +from sqlalchemy_searchable import SearchQueryMixin +from sqlalchemy_utils.types import TSVectorType +from sqlalchemy_searchable import make_searchable + + # Initialise database db = SQLAlchemy(app) migrate = Migrate(app, db) +make_searchable(db.metadata) + + +class ArticleQuery(BaseQuery, SearchQueryMixin): + pass class UserRank(enum.Enum): @@ -246,7 +257,7 @@ class PackageType(enum.Enum): class PackagePropertyKey(enum.Enum): name = "Name" title = "Title" - shortDesc = "Short Description" + short_desc = "Short Description" desc = "Description" type = "Type" license = "License" @@ -343,19 +354,22 @@ class Dependency(db.Model): return retval - class Package(db.Model): + query_class = ArticleQuery + id = db.Column(db.Integer, primary_key=True) # Basic details author_id = db.Column(db.Integer, db.ForeignKey("user.id")) name = db.Column(db.String(100), nullable=False) - title = db.Column(db.String(100), nullable=False) - shortDesc = db.Column(db.String(200), nullable=False) - desc = db.Column(db.Text, nullable=True) + title = db.Column(db.Unicode(100), nullable=False) + short_desc = db.Column(db.Unicode(200), nullable=False) + desc = db.Column(db.UnicodeText, nullable=True) type = db.Column(db.Enum(PackageType)) created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + search_vector = db.Column(TSVectorType("title", "short_desc", "desc")) + license_id = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1) license = db.relationship("License", foreign_keys=[license_id]) media_license_id = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1) @@ -409,7 +423,7 @@ class Package(db.Model): "name": self.name, "title": self.title, "author": self.author.display_name, - "short_description": self.shortDesc, + "short_description": self.short_desc, "type": self.type.toName(), "release": self.getDownloadRelease(protonum).id if self.getDownloadRelease(protonum) is not None else None, "thumbnail": (base_url + tnurl) if tnurl is not None else None, @@ -422,7 +436,7 @@ class Package(db.Model): "author": self.author.display_name, "name": self.name, "title": self.title, - "short_description": self.shortDesc, + "short_description": self.short_desc, "desc": self.desc, "type": self.type.toName(), "created_at": self.created_at, diff --git a/app/public/static/package_create.js b/app/public/static/package_create.js index a115b14..a278953 100644 --- a/app/public/static/package_create.js +++ b/app/public/static/package_create.js @@ -35,10 +35,10 @@ $(function() { setField("#repo", result.repo || repoURL); setField("#issueTracker", result.issueTracker); setField("#desc", result.description); - setField("#shortDesc", result.short_description); + setField("#short_desc", result.short_description); setField("#harddep_str", result.depends); setField("#softdep_str", result.optional_depends); - setField("#shortDesc", result.short_description); + setField("#short_desc", result.short_description); setField("#forums", result.forumId); if (result.type && result.type.length > 2) { $("#type").val(result.type); diff --git a/app/public/static/package_edit.js b/app/public/static/package_edit.js index b997b83..dfd9de0 100644 --- a/app/public/static/package_edit.js +++ b/app/public/static/package_edit.js @@ -41,7 +41,7 @@ $(function() { It's obvious that this adds something to Minetest, there's no need to use phrases such as \"adds X to the game\".` - $("#shortDesc").on("change paste keyup", function() { + $("#short_desc").on("change paste keyup", function() { var val = $(this).val().toLowerCase(); if (val.indexOf("minetest") >= 0 || val.indexOf("mod") >= 0 || val.indexOf("modpack") >= 0 || val.indexOf("mod pack") >= 0) { diff --git a/app/querybuilder.py b/app/querybuilder.py index e524bde..408d95d 100644 --- a/app/querybuilder.py +++ b/app/querybuilder.py @@ -40,7 +40,7 @@ class QueryBuilder: query = query.filter(Package.type.in_(self.types)) if self.search: - query = query.filter(Package.title.ilike('%' + self.search + '%')) + query = query.search(self.search) if self.random: query = query.order_by(func.random()) diff --git a/app/templates/macros/packagegridtile.html b/app/templates/macros/packagegridtile.html index a0f21f4..9f70c1c 100644 --- a/app/templates/macros/packagegridtile.html +++ b/app/templates/macros/packagegridtile.html @@ -12,7 +12,7 @@

- {{ package.shortDesc }} + {{ package.short_desc }}

diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index d0e8174..9a11531 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -49,7 +49,7 @@ {{ render_field(form.title, class_="pkg_meta col-sm-7") }} {{ render_field(form.name, class_="pkg_meta col-sm-3") }} - {{ render_field(form.shortDesc, class_="pkg_meta") }} + {{ render_field(form.short_desc, class_="pkg_meta") }} {{ render_multiselect_field(form.tags, class_="pkg_meta") }}
{{ render_field(form.license, class_="not_txp col-sm-6") }} diff --git a/app/templates/packages/editrequest_create_edit.html b/app/templates/packages/editrequest_create_edit.html index c83bade..7a9052c 100644 --- a/app/templates/packages/editrequest_create_edit.html +++ b/app/templates/packages/editrequest_create_edit.html @@ -17,7 +17,7 @@ {{ render_field(form.type) }} {{ render_field(form.name) }} {{ render_field(form.title) }} - {{ render_field(form.shortDesc) }} + {{ render_field(form.short_desc) }} {{ render_field(form.desc) }} {{ render_multiselect_field(form.tags) }} diff --git a/app/templates/packages/view.html b/app/templates/packages/view.html index f4ef811..b725dbe 100644 --- a/app/templates/packages/view.html +++ b/app/templates/packages/view.html @@ -19,7 +19,7 @@

- {{ package.shortDesc }} + {{ package.short_desc }}

diff --git a/app/views/packages/packages.py b/app/views/packages/packages.py index 0fe4dc1..80c5a97 100644 --- a/app/views/packages/packages.py +++ b/app/views/packages/packages.py @@ -171,7 +171,7 @@ def package_download_page(package): class PackageForm(FlaskForm): name = StringField("Name (Technical)", [InputRequired(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")]) title = StringField("Title (Human-readable)", [InputRequired(), Length(3, 50)]) - shortDesc = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)]) + short_desc = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)]) desc = TextAreaField("Long Description (Markdown)", [Optional(), Length(0,10000)]) type = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD) license = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name) diff --git a/migrations/versions/2f3c3597c78d_.py b/migrations/versions/2f3c3597c78d_.py new file mode 100644 index 0000000..b80945e --- /dev/null +++ b/migrations/versions/2f3c3597c78d_.py @@ -0,0 +1,36 @@ +"""empty message + +Revision ID: 2f3c3597c78d +Revises: 9ec17b558413 +Create Date: 2019-01-29 02:43:08.865695 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from sqlalchemy_utils.types import TSVectorType +from sqlalchemy_searchable import sync_trigger + +# revision identifiers, used by Alembic. +revision = '2f3c3597c78d' +down_revision = '9ec17b558413' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('package', 'short_desc', nullable=False, new_column_name='short_desc') + op.add_column('package', sa.Column('search_vector', TSVectorType("title", "short_desc", "desc"), nullable=True)) + op.create_index('ix_package_search_vector', 'package', ['search_vector'], unique=False, postgresql_using='gin') + + conn = op.get_bind() + sync_trigger(conn, 'package', 'search_vector', ["title", "short_desc", "desc"]) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_package_search_vector', table_name='package') + op.drop_column('package', 'search_vector') + # ### end Alembic commands ### diff --git a/migrations/versions/7ff57806ffd5_.py b/migrations/versions/7ff57806ffd5_.py new file mode 100644 index 0000000..f1e4869 --- /dev/null +++ b/migrations/versions/7ff57806ffd5_.py @@ -0,0 +1,249 @@ +"""empty message + +Revision ID: 7ff57806ffd5 +Revises: 2f3c3597c78d +Create Date: 2019-01-29 02:57:50.279918 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '7ff57806ffd5' +down_revision = '2f3c3597c78d' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.execute(""" + DROP TYPE IF EXISTS tsq_state CASCADE; + +CREATE TYPE tsq_state AS ( + search_query text, + parentheses_stack int, + skip_for int, + current_token text, + current_index int, + current_char text, + previous_char text, + tokens text[] +); + +CREATE OR REPLACE FUNCTION tsq_append_current_token(state tsq_state) +RETURNS tsq_state AS $$ +BEGIN + IF state.current_token != '' THEN + state.tokens := array_append(state.tokens, state.current_token); + state.current_token := ''; + END IF; + RETURN state; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +CREATE OR REPLACE FUNCTION tsq_tokenize_character(state tsq_state) +RETURNS tsq_state AS $$ +BEGIN + IF state.current_char = '(' THEN + state.tokens := array_append(state.tokens, '('); + state.parentheses_stack := state.parentheses_stack + 1; + state := tsq_append_current_token(state); + ELSIF state.current_char = ')' THEN + IF (state.parentheses_stack > 0 AND state.current_token != '') THEN + state := tsq_append_current_token(state); + state.tokens := array_append(state.tokens, ')'); + state.parentheses_stack := state.parentheses_stack - 1; + END IF; + ELSIF state.current_char = '"' THEN + state.skip_for := position('"' IN substring( + state.search_query FROM state.current_index + 1 + )); + + IF state.skip_for > 1 THEN + state.tokens = array_append( + state.tokens, + substring( + state.search_query + FROM state.current_index FOR state.skip_for + 1 + ) + ); + ELSIF state.skip_for = 0 THEN + state.current_token := state.current_token || state.current_char; + END IF; + ELSIF ( + state.current_char = '-' AND + (state.current_index = 1 OR state.previous_char = ' ') + ) THEN + state.tokens := array_append(state.tokens, '-'); + ELSIF state.current_char = ' ' THEN + state := tsq_append_current_token(state); + IF substring( + state.search_query FROM state.current_index FOR 4 + ) = ' or ' THEN + state.skip_for := 2; + + -- remove duplicate OR tokens + IF state.tokens[array_length(state.tokens, 1)] != ' | ' THEN + state.tokens := array_append(state.tokens, ' | '); + END IF; + END IF; + ELSE + state.current_token = state.current_token || state.current_char; + END IF; + RETURN state; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +CREATE OR REPLACE FUNCTION tsq_tokenize(search_query text) RETURNS text[] AS $$ +DECLARE + state tsq_state; +BEGIN + SELECT + search_query::text AS search_query, + 0::int AS parentheses_stack, + 0 AS skip_for, + ''::text AS current_token, + 0 AS current_index, + ''::text AS current_char, + ''::text AS previous_char, + '{}'::text[] AS tokens + INTO state; + + state.search_query := lower(trim( + regexp_replace(search_query, '""+', '""', 'g') + )); + + FOR state.current_index IN ( + SELECT generate_series(1, length(state.search_query)) + ) LOOP + state.current_char := substring( + search_query FROM state.current_index FOR 1 + ); + + IF state.skip_for > 0 THEN + state.skip_for := state.skip_for - 1; + CONTINUE; + END IF; + + state := tsq_tokenize_character(state); + state.previous_char := state.current_char; + END LOOP; + state := tsq_append_current_token(state); + + state.tokens := array_nremove(state.tokens, '(', -state.parentheses_stack); + + RETURN state.tokens; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- Processes an array of text search tokens and returns a tsquery +CREATE OR REPLACE FUNCTION tsq_process_tokens(config regconfig, tokens text[]) +RETURNS tsquery AS $$ +DECLARE + result_query text; + previous_value text; + value text; +BEGIN + result_query := ''; + FOREACH value IN ARRAY tokens LOOP + IF value = '"' THEN + CONTINUE; + END IF; + + IF left(value, 1) = '"' AND right(value, 1) = '"' THEN + value := phraseto_tsquery(config, value); + ELSIF value NOT IN ('(', ' | ', ')', '-') THEN + value := quote_literal(value) || ':*'; + END IF; + + IF previous_value = '-' THEN + IF value = '(' THEN + value := '!' || value; + ELSE + value := '!(' || value || ')'; + END IF; + END IF; + + SELECT + CASE + WHEN result_query = '' THEN value + WHEN ( + previous_value IN ('!(', '(', ' | ') OR + value IN (')', ' | ') + ) THEN result_query || value + ELSE result_query || ' & ' || value + END + INTO result_query; + previous_value := value; + END LOOP; + + RETURN to_tsquery(config, result_query); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +CREATE OR REPLACE FUNCTION tsq_process_tokens(tokens text[]) +RETURNS tsquery AS $$ + SELECT tsq_process_tokens(get_current_ts_config(), tokens); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION tsq_parse(config regconfig, search_query text) +RETURNS tsquery AS $$ + SELECT tsq_process_tokens(config, tsq_tokenize(search_query)); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION tsq_parse(config text, search_query text) +RETURNS tsquery AS $$ + SELECT tsq_parse(config::regconfig, search_query); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION tsq_parse(search_query text) RETURNS tsquery AS $$ + SELECT tsq_parse(get_current_ts_config(), search_query); +$$ LANGUAGE SQL IMMUTABLE; + + +-- remove first N elements equal to the given value from the array (array +-- must be one-dimensional) +-- +-- If negative value is given as the third argument the removal of elements +-- starts from the last array element. +CREATE OR REPLACE FUNCTION array_nremove(anyarray, anyelement, int) +RETURNS ANYARRAY AS $$ + WITH replaced_positions AS ( + SELECT UNNEST( + CASE + WHEN $2 IS NULL THEN + '{}'::int[] + WHEN $3 > 0 THEN + (array_positions($1, $2))[1:$3] + WHEN $3 < 0 THEN + (array_positions($1, $2))[ + (cardinality(array_positions($1, $2)) + $3 + 1): + ] + ELSE + '{}'::int[] + END + ) AS position + ) + SELECT COALESCE(( + SELECT array_agg(value) + FROM unnest($1) WITH ORDINALITY AS t(value, index) + WHERE index NOT IN (SELECT position FROM replaced_positions) + ), $1[1:0]); +$$ LANGUAGE SQL IMMUTABLE; +""") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/migrations/versions/83622276d439_.py b/migrations/versions/83622276d439_.py index 49f5ebb..b275752 100644 --- a/migrations/versions/83622276d439_.py +++ b/migrations/versions/83622276d439_.py @@ -66,7 +66,7 @@ def upgrade(): sa.Column('author_id', sa.Integer(), nullable=True), sa.Column('name', sa.String(length=100), nullable=False), sa.Column('title', sa.String(length=100), nullable=False), - sa.Column('shortDesc', sa.String(length=200), nullable=False), + sa.Column('short_desc', sa.String(length=200), nullable=False), sa.Column('desc', sa.Text(), nullable=True), sa.Column('type', sa.Enum('MOD', 'GAME', 'TXP', name='packagetype'), nullable=True), sa.Column('license_id', sa.Integer(), nullable=True), @@ -141,7 +141,7 @@ def upgrade(): op.create_table('edit_request_change', sa.Column('id', sa.Integer(), nullable=False), sa.Column('request_id', sa.Integer(), nullable=True), - sa.Column('key', sa.Enum('name', 'title', 'shortDesc', 'desc', 'type', 'license', 'tags', 'repo', 'website', 'issueTracker', 'forums', name='packagepropertykey'), nullable=False), + sa.Column('key', sa.Enum('name', 'title', 'short_desc', 'desc', 'type', 'license', 'tags', 'repo', 'website', 'issueTracker', 'forums', name='packagepropertykey'), nullable=False), sa.Column('oldValue', sa.Text(), nullable=True), sa.Column('newValue', sa.Text(), nullable=True), sa.ForeignKeyConstraint(['request_id'], ['edit_request.id'], ), diff --git a/requirements.txt b/requirements.txt index 622232a..03ff5b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ Flask-Migrate~=2.3 Flask-SQLAlchemy~=2.3 Flask-User~=0.6 GitHub-Flask~=3.2 +SQLAlchemy-Searchable==1.0.3 beautifulsoup4~=4.6 celery~=4.2 diff --git a/setup.py b/setup.py index 5d75cc5..4fd5ff1 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def defineDummyData(licenses, tags, ruben): mod.repo = "https://github.com/ezhh/other_worlds" mod.issueTracker = "https://github.com/ezhh/other_worlds/issues" mod.forums = 16015 - mod.shortDesc = "The content library should not be used yet as it is still in alpha" + mod.short_desc = "The content library should not be used yet as it is still in alpha" mod.desc = "This is the long desc" db.session.add(mod) @@ -77,7 +77,7 @@ def defineDummyData(licenses, tags, ruben): mod1.repo = "https://github.com/rubenwardy/awards" mod1.issueTracker = "https://github.com/rubenwardy/awards/issues" mod1.forums = 4870 - mod1.shortDesc = "Adds achievements and an API to register new ones." + mod1.short_desc = "Adds achievements and an API to register new ones." mod1.desc = """ Majority of awards are back ported from Calinou's old fork in Carbone, under same license. @@ -112,7 +112,7 @@ awards.register_achievement("award_mesefind",{ mod2.repo = "https://github.com/minetest-mods/mesecons/" mod2.issueTracker = "https://github.com/minetest-mods/mesecons/issues" mod2.forums = 628 - mod2.shortDesc = "Mesecons adds everything digital, from all kinds of sensors, switches, solar panels, detectors, pistons, lamps, sound blocks to advanced digital circuitry like logic gates and programmable blocks." + mod2.short_desc = "Mesecons adds everything digital, from all kinds of sensors, switches, solar panels, detectors, pistons, lamps, sound blocks to advanced digital circuitry like logic gates and programmable blocks." mod2.desc = """ ######################################################################## ## __ __ _____ _____ _____ _____ _____ _ _ _____ ## @@ -210,7 +210,7 @@ No warranty is provided, express or implied, for any part of the project. mod.repo = "https://github.com/ezhh/handholds" mod.issueTracker = "https://github.com/ezhh/handholds/issues" mod.forums = 17069 - mod.shortDesc = "Adds hand holds and climbing thingies" + mod.short_desc = "Adds hand holds and climbing thingies" mod.desc = "This is the long desc" db.session.add(mod) @@ -233,7 +233,7 @@ No warranty is provided, express or implied, for any part of the project. mod.repo = "https://github.com/ezhh/other_worlds" mod.issueTracker = "https://github.com/ezhh/other_worlds/issues" mod.forums = 16015 - mod.shortDesc = "Adds space with asteroids and comets" + mod.short_desc = "Adds space with asteroids and comets" mod.desc = "This is the long desc" db.session.add(mod) @@ -248,7 +248,7 @@ No warranty is provided, express or implied, for any part of the project. mod.repo = "https://github.com/rubenwardy/food/" mod.issueTracker = "https://github.com/rubenwardy/food/issues/" mod.forums = 2960 - mod.shortDesc = "Adds lots of food and an API to manage ingredients" + mod.short_desc = "Adds lots of food and an API to manage ingredients" mod.desc = "This is the long desc" food = mod db.session.add(mod) @@ -264,7 +264,7 @@ No warranty is provided, express or implied, for any part of the project. mod.repo = "https://github.com/rubenwardy/food_sweet/" mod.issueTracker = "https://github.com/rubenwardy/food_sweet/issues/" mod.forums = 9039 - mod.shortDesc = "Adds sweet food" + mod.short_desc = "Adds sweet food" mod.desc = "This is the long desc" food_sweet = mod db.session.add(mod) @@ -282,7 +282,7 @@ No warranty is provided, express or implied, for any part of the project. game1.repo = "https://github.com/rubenwardy/capturetheflag" game1.issueTracker = "https://github.com/rubenwardy/capturetheflag/issues" game1.forums = 12835 - game1.shortDesc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!" + game1.short_desc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!" game1.desc = """ As seen on the Capture the Flag server (minetest.rubenwardy.com:30000) @@ -307,7 +307,7 @@ Uses the CTF PvP Engine. mod.type = PackageType.TXP mod.author = ruben mod.forums = 14132 - mod.shortDesc = "This is an update of the original PixelBOX texture pack by the brillant artist Gambit" + mod.short_desc = "This is an update of the original PixelBOX texture pack by the brillant artist Gambit" mod.desc = "This is the long desc" db.session.add(mod)