Refactor database and added entity search

.devcontainer/Dockerfile

@@ -80,8 +80,6 @@ RUN apt update && apt install -y \
     default-jre \
     # Pandoc
     pandoc \
-    # Libmagic
-    libmagic1 \
     sqlite3 \
     && apt-get autoremove -y \
     && apt-get clean \
@@ -110,3 +108,4 @@ WORKDIR /workspace
 COPY .devcontainer/entrypoint.sh /home/$DEVCONTAINER_USER/
 
 ENV UV_CACHE_DIR=/resources/cache/uv
+ENV ALAMBIC_CONFIG=/workspace/aymurai/alambic.yml

.devcontainer/devcontainer.json

@@ -11,27 +11,6 @@
 	"workspaceFolder": "/workspace",
 	"customizations": {
 		"vscode": {
-			"settings": {
-				"extensions.verifySignature": false,
-				"files.trimFinalNewlines": true,
-				"files.trimTrailingWhitespace": true,
-				"files.watcherExclude": {
-					"**/.git/objects/**": true,
-					"**/.git/subtree-cache/**": true
-				},
-				"editor.formatOnSave": true,
-				"notebook.formatOnSave.enabled": true,
-				"notebook.output.scrolling": true,
-				"notebook.codeActionsOnSave": {
-					"notebook.source.organizeImports": "explicit"
-				},
-				"[python]": {
-					"editor.defaultFormatter": "charliermarsh.ruff",
-					"editor.codeActionsOnSave": {
-						"source.fixAll": "never"
-					}
-				}
-			},
 			// Add the IDs of extensions you want installed when the container is created.
 			"extensions": [
 				"ms-python.python",
@@ -40,14 +19,13 @@
 				"charliermarsh.ruff",
 				"ms-toolsai.jupyter",
 				"GitHub.vscode-pull-request-github",
-				"njpwerner.autodocstring",
 				"GrapeCity.gc-excelviewer",
 				"tamasfe.even-better-toml",
 				"redhat.vscode-xml",
-				"cweijan.vscode-database-client2",
 				"christian-kohler.path-intellisense",
 				"github.vscode-github-actions",
-				"seatonjiang.gitmoji-vscode"
+				"seatonjiang.gitmoji-vscode",
+				"dbcode.dbcode"
 			]
 		}
 	},

.devcontainer/docker-compose.yml

@@ -10,7 +10,6 @@ x-template: &template
     - /var/run/docker.sock:/var/run/docker.sock
   env_file:
     - ../.env.common
-    - ../.env
   command: /bin/sh -c "while true; do :; done"
 
 services:

.github/workflows/build-docker-image.yml

@@ -15,6 +15,17 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Extract tag name
+        id: envs
+        run: |
+          TAG_NAME="${GITHUB_REF#refs/tags/}"
+          echo "TAG=${TAG_NAME}" >> $GITHUB_OUTPUT
+          if [[ "$TAG_NAME" == *"rc"* ]]; then
+            echo "SUFFIX=-rc" >> $GITHUB_OUTPUT
+          else
+            echo "SUFFIX=" >> $GITHUB_OUTPUT
+          fi
+
       # Step 2: Log in to GitHub Container Registry
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@v2
@@ -23,22 +34,28 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Step 3: Build the Docker image
-      - name: Build Docker Image
-        run: |
-          touch .env
-          docker compose build aymurai-api
-          docker compose build aymurai-api-full
-
-      # Step 4: Push the Docker image to GHCR
-      - name: Push Docker Image
-        run: |
-          docker tag ghcr.io/aymurai/api:latest ghcr.io/aymurai/api:${{ github.ref_name }}
-          docker tag ghcr.io/aymurai/api:full ghcr.io/aymurai/api:full-${{ github.ref_name }}
+      - name: Build and push Docker image (aymurai-api)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/api/Dockerfile
+          push: true
+          target: aymurai-api
+          build-args: |
+            SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.envs.outputs.TAG }}
+          tags: |
+            ghcr.io/aymurai/api:latest${{ steps.envs.outputs.SUFFIX }}
+            ghcr.io/aymurai/api:${{ github.ref_name }}
 
-          docker push ghcr.io/aymurai/api:full-${{ github.ref_name }}
-          docker push ghcr.io/aymurai/api:${{ github.ref_name }}
-          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            docker push ghcr.io/aymurai/api:latest
-            docker push ghcr.io/aymurai/api:full
-          fi
+      - name: Build and push Docker image (aymurai-api-full)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/api/Dockerfile
+          push: true
+          target: aymurai-api-full
+          build-args: |
+            SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.envs.outputs.TAG }}
+          tags: |
+            ghcr.io/aymurai/api:full${{ steps.envs.outputs.SUFFIX }}
+            ghcr.io/aymurai/api:${{ github.ref_name }}-full

.vscode/settings.json

@@ -0,0 +1,45 @@
+{
+    "python.analysis.typeCheckingMode": "off",
+    "extensions.verifySignature": false,
+    "files.trimFinalNewlines": true,
+    "files.trimTrailingWhitespace": true,
+    "files.watcherExclude": {
+        "**/.git/objects/**": true,
+        "**/.git/subtree-cache/**": true
+    },
+    "python.analysis.exclude": [
+        "**/node_modules",
+        "**/__pycache__",
+        "**/build",
+        ".git",
+        ".venv/**"
+    ],
+    "editor.formatOnSave": true,
+    "notebook.formatOnSave.enabled": true,
+    "notebook.output.scrolling": true,
+    "notebook.codeActionsOnSave": {
+        "notebook.source.organizeImports": "explicit"
+    },
+    "[python]": {
+        "editor.defaultFormatter": "charliermarsh.ruff",
+        "editor.codeActionsOnSave": {
+            "source.fixAll": "never",
+            "source.organizeImports.ruff": "always"
+        }
+    },
+    "ruff.format.preview": true,
+    "ruff.organizeImports": true,
+    "dbcode.connections": [
+        {
+            "connectionId": "1oe_iPSN7LlyCERrXeOdw",
+            "name": "aymurai",
+            "driver": "sqlite",
+            "socket": "workspace/resources/cache/sqlite/database.db",
+            "readOnly": false,
+            "driverOptions": {
+                "extensions": []
+            },
+            "connectionType": "socket"
+        }
+    ]
+}

Makefile

@@ -6,6 +6,7 @@ export $(shell sed 's/=.*//' .env.common)
 
 
 api-build:
+	uv build
 	docker compose build aymurai-api-dev
 api-run:
 	docker compose run --service-ports aymurai-api-dev

aymurai/api/core.py

@@ -1,45 +1,28 @@
 from fastapi.routing import APIRouter
 
-from .endpoints.routers.anonymizer import anonymizer
-from .endpoints.routers.anonymizer import database as anonymizer_database
-from .endpoints.routers.datapublic import datapublic
-
-from .endpoints.routers.misc import convert, document_extract
+from .endpoints.routers import anonymizer, database, datapublic, document, paragraph
 from .endpoints.routers.server import stats
 
 router = APIRouter()
 
 
 # Server
-router.include_router(
-    stats.router,
-    prefix="/server/stats",
-    tags=["server"],
-)
+router.include_router(stats.router, prefix="/server/stats", tags=["server"])
 
-# Anonymizer
-router.include_router(
-    anonymizer.router,
-    prefix="/anonymizer",
-    tags=["anonymization/model"],
-)
-# router.include_router(
-#     anonymizer_database.router,
-#     prefix="/anonymizer/database",
-#     tags=["anonymization/database"],
-# )
+# Document
+router.include_router(document.extract.router, prefix="/document", tags=["document"])
+router.include_router(document.convert.router, prefix="/document", tags=["document"])
+router.include_router(document.predict.router, tags=["document"])
 
-# Datapublic
-router.include_router(
-    datapublic.router,
-    prefix="/datapublic",
-    tags=["datapublic/model"],
-)
+# Paragraph
+router.include_router(paragraph.predict.router, tags=["paragraph"])
 
+# Datapublic
+router.include_router(datapublic.validation.router, tags=["datapublic"])
+router.include_router(datapublic.dataset.router, tags=["datapublic"])
 
-# Misc
-router.include_router(document_extract.router, tags=["document"], deprecated=True)
-router.include_router(document_extract.router, prefix="/misc", tags=["document"])
+# Anonymizer
+router.include_router(anonymizer.compilation.router, tags=["anonymizer"])
 
-# Document conversion
-router.include_router(convert.router, tags=["Document conversion"])
+# Database
+router.include_router(database.dump.router, tags=["database"])

aymurai/api/endpoints/routers/anonymizer/__init__.py

@@ -0,0 +1,2 @@
+# ruff: noqa: F401
+from . import compilation