fix: remove the oai set during the organisation creation

* Removes the automatic oai set creation during the organisation creation.
* Adds the organisation OAI sets during the setup.

Co-Authored-by: Johnny Mariéthoz <[email protected]>

Author: jma

scripts/setup

@@ -96,6 +96,12 @@ invenio flask_wiki index
 section "Create organisations" "info"
 invenio fixtures organisations import "${fixtures_folder}/organisations/data.json"
 
+# Create OAI sets for organisations
+section "Create organisations OAI sets" "info"
+invenio documents oai create-set rero "RERO+" 'organisation.code:rero'
+invenio documents oai create-set fictivededicated "Université des sciences fictives" 'organisation.code:fictivededicated'
+invenio documents oai create-set fictiveshared "École des hautes études partagées" 'organisation.code:fictiveshared'
+
 section "Create users" "info"
 invenio fixtures users import "${fixtures_folder}/users/data.json"
 

sonar/modules/documents/cli/documents.py

@@ -23,11 +23,12 @@
 from invenio_db import db
 from rero_invenio_files.pdf import PDFGenerator
 
-from sonar.modules.documents.cli.rerodoc import rerodoc
-from sonar.modules.documents.cli.urn import urn
 from sonar.modules.documents.serializers.schemas.dc import DublinCoreSchema
 
 from ..api import DocumentIndexer, DocumentRecord
+from .oai import oai
+from .rerodoc import rerodoc
+from .urn import urn
 
 
 @click.group()
@@ -37,6 +38,7 @@ def documents():
 
 documents.add_command(rerodoc)
 documents.add_command(urn)
+documents.add_command(oai)
 
 
 @click.group("documents")

tests/ui/organisations/test_organisations_oai_set.py renamed to sonar/modules/documents/cli/oai.py

@@ -1,5 +1,5 @@
 # Swiss Open Access Repository
-# Copyright (C) 2021 RERO
+# Copyright (C) 2022 RERO
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,25 +13,32 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Test OAI sets corresponding to organisation."""
+"""OAI specific CLI commands."""
 
+import click
+from flask.cli import with_appcontext
+from invenio_db import db
 from invenio_oaiserver.models import OAISet
 
 
-def test_oai_set(organisation, document):
-    """Test OAI set synchronisation with organisation."""
-    # Document has a `_oai` property
-    assert document["_oai"]["id"] == f"oai:sonar.ch:{document['pid']}"
+@click.group()
+def oai():
+    """URN specific commands."""
 
-    # Set for organisation exists
-    sets = OAISet.query.all()
-    assert len(sets) == 1
-    assert sets[0].spec == "org"
-    assert sets[0].name == "org"
 
-    # New name is updated
-    organisation.update({"code": "org", "name": "New name"})
-    sets = OAISet.query.all()
-    assert len(sets) == 1
-    assert sets[0].spec == "org"
-    assert sets[0].name == "New name"
+@oai.command()
+@click.argument("code")
+@click.argument("name")
+@click.argument("pattern")
+@with_appcontext
+def create_set(code, name, pattern):
+    oaiset = OAISet(
+        spec=code,
+        name=name,
+        search_pattern=pattern,
+        system_created=True,
+    )
+    db.session.add(oaiset)
+    db.session.commit()
+    click.secho(f"OAI set '{code}' created.", fg="green")
+    click.secho("Please reindex existing documents if needed.", fg="yellow")

sonar/modules/documents/dojson/rerodoc/overdo.py

@@ -47,6 +47,7 @@ def create_organisation(organisation_key):
                     "isShared": False,
                     "isDedicated": False,
                 },
+                with_bucket=True,
                 dbcommit=True,
             )
             organisation.reindex()

sonar/modules/organisations/api.py

@@ -19,8 +19,6 @@
 
 from flask import has_request_context
 from flask.globals import request_ctx
-from invenio_db import db
-from invenio_oaiserver.models import OAISet
 from werkzeug.local import LocalProxy
 
 from sonar.modules.users.api import current_user_record
@@ -115,21 +113,6 @@ class OrganisationRecord(SonarRecord):
     provider = OrganisationProvider
     schema = "organisations/organisation-v1.0.0.json"
 
-    @classmethod
-    def create(cls, data, id_=None, dbcommit=False, with_bucket=True, **kwargs):
-        """Create organisation record."""
-        # Create OAI set
-        code = data["code"]
-        oaiset = OAISet(
-            spec=code,
-            name=data["name"],
-            search_pattern=f'organisation.code:"{code}"',
-            system_created=True,
-        )
-        db.session.add(oaiset)
-
-        return super().create(data, id_=id_, dbcommit=dbcommit, with_bucket=with_bucket, **kwargs)
-
     @classmethod
     def get_or_create(cls, code, name=None):
         """Get or create an organisation.
@@ -147,18 +130,6 @@ def get_or_create(cls, code, name=None):
         organisation.reindex()
         return organisation
 
-    def update(self, data):
-        """Update data for record."""
-        # Update OAI set name according to organisation's name
-        oaiset = OAISet.query.filter(OAISet.spec == data["code"]).first()
-
-        if oaiset:
-            oaiset.name = data["name"]
-            db.session.commit()
-
-        super().update(data)
-        return self
-
 
 class OrganisationIndexer(SonarIndexer):
     """Indexing documents in Elasticsearch."""

sonar/modules/organisations/cli/organisations.py

@@ -54,7 +54,7 @@ def import_organisations(file):
             files = record.pop("files", [])
 
             # Register record to DB
-            db_record = OrganisationRecord.create(record)
+            db_record = OrganisationRecord.create(record, with_bucket=True)
 
             # Add files
             for file in files:
@@ -68,6 +68,7 @@ def import_organisations(file):
 
             indexer.index(db_record)
         except Exception as error:
+            raise error
             click.secho(
                 f"Organisation {record} could not be imported: {error}",
                 fg="red",

tests/conftest.py

@@ -33,6 +33,7 @@
 from invenio_access.models import ActionUsers, Role
 from invenio_db import db as db_
 from invenio_files_rest.models import Location
+from invenio_oaiserver.models import OAISet
 from invenio_queues.proxies import current_queues
 from invenio_search import current_search
 from sqlalchemy_utils import create_database, database_exists
@@ -87,7 +88,7 @@ def search(appctx):
     should used the function-scoped :py:data:`search_clear` fixture to leave the
     indexes clean for the following tests.
     """
-    from invenio_search import current_search, current_search_client
+    from invenio_search import current_search_client
     from invenio_search.errors import IndexAlreadyExistsError
 
     try:
@@ -236,16 +237,35 @@ def _make_organisation(code, is_shared=True):
         record = OrganisationRecord.get_record_by_pid(code)
 
         if not record:
-            record = OrganisationRecord.create(data, dbcommit=True)
+            record = OrganisationRecord.create(data, dbcommit=True, with_bucket=True)
             db.session.commit()
             record.reindex()
-            current_search.flush_and_refresh(index="documents-document-v1.0.0-percolators")
+            current_search.flush_and_refresh(index="organisations")
 
         return record
 
     return _make_organisation
 
 
+@pytest.fixture()
+def org_oaiset(db, organisation):
+    """Create an OAISet for organisation."""
+    code = organisation["code"]
+    oaiset = OAISet(
+        spec=code,
+        name=organisation["name"],
+        search_pattern=f'organisation.code:"{code}"',
+        system_created=True,
+    )
+    db.session.add(oaiset)
+    db.session.commit()
+    current_search.flush_and_refresh(index="documents-document-v1.0.0-percolators")
+    yield oaiset
+    db.session.delete(oaiset)
+    db.session.commit()
+    current_search.flush_and_refresh(index="documents-document-v1.0.0-percolators")
+
+
 @pytest.fixture()
 def organisation(make_organisation):
     """Create an organisation."""

tests/ui/documents/test_oaipmh.py

@@ -18,7 +18,7 @@
 """Test OAIPMH URLS."""
 
 
-def test_oaipmh_get(client, app, document):
+def test_oaipmh_get(client, org_oaiset, document):
     """Test OAIPMH API."""
     res = client.get("/oai2d?verb=ListSets")
     assert res.status_code == 200