Do some trimming

Author: simmsb

.dockerignore

@@ -1,4 +1,4 @@
-db-data/
+/db-data
 .git
 .gitattributes
 .gitignore

.gitignore

@@ -106,3 +106,5 @@ venv.bak/
 .mypy_cache/
 
 .env
+
+db-data/

alembic/versions/13d2afe94e48_remove_blogs.py

@@ -0,0 +1,49 @@
+"""remove_blogs
+
+Revision ID: 13d2afe94e48
+Revises: 6a56c28c6bf1
+Create Date: 2022-09-06 17:29:55.189057
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy_searchable import drop_trigger
+import sqlalchemy_utils
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '13d2afe94e48'
+down_revision = '6a56c28c6bf1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    drop_trigger(conn, "blogs", "search_vector")
+    op.drop_index('blogs_tags_array_idx', table_name='blogs')
+    op.drop_table('blogs')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('blogs',
+    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('title', sa.TEXT(), autoincrement=False, nullable=False),
+    sa.Column('slug', sa.TEXT(), autoincrement=False, nullable=False),
+    sa.Column('tags', postgresql.ARRAY(sa.TEXT()), autoincrement=False, nullable=False),
+    sa.Column('content', sa.TEXT(), autoincrement=False, nullable=False),
+    sa.Column('creation_date', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False),
+    sa.Column('edit_date', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False),
+    sa.Column('search_vector', postgresql.TSVECTOR(), autoincrement=False, nullable=True),
+    sa.Column('author_id', sa.BIGINT(), autoincrement=False, nullable=True),
+    sa.CheckConstraint("slug <> ''::text", name='blogs_slug_nonempty'),
+    sa.ForeignKeyConstraint(['author_id'], ['users.discord_id'], name='blogs_author_id_fkey', ondelete='SET NULL'),
+    sa.PrimaryKeyConstraint('id', name='blogs_pkey'),
+    sa.UniqueConstraint('slug', name='blogs_slug_key'),
+    sa.UniqueConstraint('title', name='blogs_title_key')
+    )
+    op.create_index('blogs_tags_array_idx', 'blogs', ['tags'], unique=False)
+    # ### end Alembic commands ###

alembic/versions/6a56c28c6bf1_instantiate_search_fns.py

@@ -0,0 +1,29 @@
+"""instantiate_search_fns
+
+Revision ID: 6a56c28c6bf1
+Revises: f78a26022069
+Create Date: 2022-09-06 17:23:48.327005
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+from sqlalchemy_searchable import sql_expressions
+
+
+# revision identifiers, used by Alembic.
+revision = '6a56c28c6bf1'
+down_revision = 'f78a26022069'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    conn = op.get_bind()
+
+    conn.execute(sql_expressions.statement)
+    pass
+
+
+def downgrade():
+    pass

alembic/versions/98b885be8d35_add_writeup_schema.py

@@ -8,7 +8,7 @@
 from alembic import op
 import sqlalchemy as sa
 import sqlalchemy_utils
-from sqlalchemy_searchable import sync_trigger, sql_expressions
+from sqlalchemy_searchable import sync_trigger
 
 # revision identifiers, used by Alembic.
 revision = "98b885be8d35"
@@ -48,8 +48,6 @@ def upgrade():
         postgresql_using="gin",
     )
 
-    conn.execute(sql_expressions.statement)
-
     sync_trigger(conn, "writeups", "search_vector", ["title", "content"])
     # ### end Alembic commands ###
 

alembic/versions/f78a26022069_remove_todos.py

@@ -0,0 +1,59 @@
+"""remove_todos
+
+Revision ID: f78a26022069
+Revises: 276ab5e9da65
+Create Date: 2022-09-06 16:25:19.577355
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+from sqlalchemy_searchable import sync_trigger
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'f78a26022069'
+down_revision = '276ab5e9da65'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('todos')
+
+    conn = op.get_bind()
+
+    sync_trigger(conn,
+                 "challenges", "search_vector", ["title", "content"]
+                 )
+
+    sync_trigger(conn,
+                 "writeups", "search_vector", ["title", "content"]
+                 )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('todos',
+    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('assigned', sa.BIGINT(), autoincrement=False, nullable=True),
+    sa.Column('started', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False),
+    sa.Column('deadline', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('cancelled', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False),
+    sa.Column('completed', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('content', sa.TEXT(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('id', name='todos_pkey')
+    )
+
+    conn = op.get_bind()
+
+    sync_trigger(conn,
+                 "challenges", "search_vector", ["title", "content"]
+                 )
+
+    sync_trigger(conn,
+                 "writeups", "search_vector", ["title", "content"]
+                 )
+    # ### end Alembic commands ###

luhack_bot/__init__.py

@@ -43,32 +43,26 @@ def gen_tokens():
     )
 
 
-def export_writeups_and_blog_posts():
+def export_writeups():
     import asyncio
     import json
 
     from luhack_bot.db.helpers import init_db
-    from luhack_bot.db.models import Writeup, Blog
+    from luhack_bot.db.models import Writeup
 
     def id(x):
         return x
 
     writeup_keys = {"id": id, "author_id": id, "title": id, "slug": id, "tags": id, "content": id, "creation_date": str, "edit_date": str}
-    blog_keys = {"id": id, "title": id, "slug": id, "tags": id, "content": id, "creation_date": str, "edit_date": str}
 
     def t_w(w):
         return {k: f(getattr(w, k)) for k, f in writeup_keys.items()}
 
-    def t_b(b):
-        return {k: f(getattr(b, k)) for k, f in blog_keys.items()}
-
     async def inner():
         await init_db()
 
         writeups = await Writeup.query.gino.all()
-        blogs = await Blog.query.gino.all()
 
         print(json.dumps([t_w(w) for w in writeups]))
-        print(json.dumps([t_b(b) for b in blogs]))
 
     asyncio.run(inner())

luhack_bot/cogs/challenges.py

@@ -7,11 +7,11 @@
 import sqlalchemy as sa
 from discord.ext import commands
 from gino.loader import ColumnLoader
-from sqlalchemy_searchable import search as pg_search
 from discord.ext.alternatives import literal_converter as _
 from tabulate import tabulate
 
 from luhack_bot import constants
+from luhack_bot.db.helpers import text_search
 from luhack_bot.db.models import Challenge, CompletedChallenge, User, db
 from luhack_bot.utils.checks import is_admin, is_authed
 
@@ -74,7 +74,7 @@ async def get_challenge(self, title: str) -> Optional[Challenge]:
     async def search_challenges(self, search: str) -> List[Challenge]:
         """Search for challenges, return top 3 matching."""
         return (
-            await pg_search(
+            await text_search(
                 Challenge.query.where(sa.not_(Challenge.hidden)), search, sort=True
             )
             .limit(3)

luhack_bot/cogs/writeups.py

@@ -7,10 +7,9 @@
 import discord
 import sqlalchemy as sa
 from discord.ext import commands
-from sqlalchemy_searchable import search as pg_search
-from discord.ext.alternatives import literal_converter
 
 from luhack_bot import constants
+from luhack_bot.db.helpers import text_search
 from luhack_bot.db.models import Image, Writeup
 from luhack_bot.utils.checks import is_authed
 
@@ -37,7 +36,7 @@ async def get_writeup(self, title: str) -> Optional[Writeup]:
 
     async def search_writeups(self, search: str) -> List[Writeup]:
         """Search for writeups, return top 3 matching."""
-        return await pg_search(Writeup.query, search, sort=True).limit(3).gino.all()
+        return await text_search(Writeup.query, search, sort=True).limit(3).gino.all()
 
     def can_edit_writeup(self, writeup: Writeup, user_id: int) -> bool:
         return (

luhack_bot/db/helpers.py

@@ -1,6 +1,52 @@
+import sqlalchemy as sa
+from sqlalchemy_searchable import inspect_search_vectors, search_manager
+from sqlalchemy_utils import TSVectorType
+
 from luhack_bot.db.models import db
 from luhack_bot.secrets import db_url
 
 
 async def init_db():
     await db.set_bind(db_url)
+
+def inspect_search_vectors(entity):
+    return [
+        column
+        for column
+        in sa.inspect(entity).columns.values()
+        if isinstance(column.type, TSVectorType)
+    ]
+
+def text_search(query, search_query, vector=None, regconfig=None, sort=False):
+    """
+    Search given query with full text search.
+    :param search_query: the search query
+    :param vector: search vector to use
+    :param regconfig: postgresql regconfig to be used
+    :param sort: order results by relevance (quality of hit)
+    """
+    if not search_query.strip():
+        return query
+
+    if vector is None:
+        entity = query.locate_all_froms()
+        search_vectors = inspect_search_vectors(entity)
+        vector = search_vectors[0]
+
+    if regconfig is None:
+        regconfig = search_manager.options['regconfig']
+
+    query = query.where(
+        vector.op('@@')(sa.func.parse_websearch(regconfig, search_query))
+    )
+    if sort:
+        query = query.order_by(
+            sa.desc(
+                sa.func.ts_rank_cd(
+                    vector,
+                    sa.func.parse_websearch(search_query)
+                )
+            )
+        )
+
+    return query.params(term=search_query)