Merge pull request #66 from ral-facilities/DSEGOG-255-realistic-test-data

DSEGOG-255 EPAC Simulated Data

Author: MRichards99

.github/ci_config.yml

@@ -13,6 +13,7 @@ images:
   echo_access_key: access_key
   echo_secret_key: secret_key
   image_bucket_name: test-bucket
+  upload_image_threads: 4
   preferred_colour_map_pref_name: PREFERRED_COLOUR_MAP
 mongodb:
   mongodb_url: mongodb://localhost:27017

.github/ci_ingest_echo_config.yml

@@ -0,0 +1,29 @@
+script_options:
+  wipe_database: false
+  delete_images: false
+  launch_api: true
+  import_users: true
+ssh:
+  enabled: false
+  ssh_connection_url: 127.0.0.1
+database:
+  hostname: localhost
+  port: 27017
+  name: opsgateway
+  remote_experiments_file_path: /tmp/experiments_for_mongoimport.json
+  test_users_file_path: util/users_for_mongoimport.json
+echo:
+  endpoint_url: https://s3.echo.stfc.ac.uk
+  access_key: access_key
+  secret_key: secret_key
+  simulated_data_bucket: og-ci-simulated-data
+  images_bucket: og-bucket
+  page_size: 1
+api:
+  host: 127.0.0.1
+  port: 8000
+  username: backend
+  password: back
+  log_config_path: /home/runner/work/operationsgateway-api/operationsgateway-api/operationsgateway_api/logging.ini
+  gunicorn_num_workers: "1"
+  timeout_seconds: 1000

.github/workflows/ci.yml

@@ -76,7 +76,7 @@ jobs:
           path: ~/.cache/pypoetry/virtualenvs
           key: ${{ runner.os }}-poetry-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
       - name: Install dependencies
-        run: poetry install
+        run: poetry install --without simulated-data
 
       # Configure s4cmd
       - name: Add keys to s4cmd config
@@ -125,16 +125,22 @@ jobs:
         env:
           SSH_KEY_PUBLIC: ${{secrets.SSH_PUBLIC_KEY_FOR_AUTH_OPENSSH}}
 
-      # Clone repo containing test data and use script to ingest the data
-      - name: Checkout OperationsGateway Test Data
-        uses: actions/checkout@v3
-        with:
-          repository: ral-facilities/operationsgateway-test-data
-          path: operationsgateway-test-data
-          ssh-key: ${{ secrets.SSH_PRIV_OG_TEST_DATA_ACTIONS }}
+      # Setup steps for Echo ingestion script
+      - name: Configure echo access key
+        run: yq -i ".echo.access_key = \"$ECHO_S3_ACCESS_KEY\"" .github/ci_ingest_echo_config.yml
+        env:
+          ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
+      - name: Configure echo secret key
+        run: yq -i ".echo.secret_key = \"$ECHO_S3_SECRET_KEY\"" .github/ci_ingest_echo_config.yml
+        env:
+          ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
+      - name: Configure bucket name for current run
+        run: yq -i '.echo.images_bucket = "og-actions-${{ github.sha }}-${{ github.run_id }}-${{ matrix.python-version }}"' .github/ci_ingest_echo_config.yml
+      - name: Copy config for Echo Ingest script to correct place
+        run: cp .github/ci_ingest_echo_config.yml util/realistic_data/config.yml
 
-      - name: Run ingestion script
-        run: poetry run python util/ingest_hdf.py -p operationsgateway-test-data/dev_server -U backend -P back -j util/users_for_mongoimport.json
+      - name: Run Echo Ingest script
+        run: poetry run python util/realistic_data/ingest_echo_data.py
 
       - name: Load Poetry cache for Nox tests session
         uses: actions/cache@v3

.gitignore

@@ -15,3 +15,5 @@ id_rsa
 id_rsa.pub
 logging.ini*
 !logging.ini.example
+util/realistic_data/data
+util/realistic_data/resources

noxfile.py

@@ -6,7 +6,7 @@
 
 # Separating Black away from the rest of the sessions
 nox.options.sessions = "lint", "safety", "tests"
-code_locations = "operationsgateway_api", "test", "noxfile.py"
+code_locations = "operationsgateway_api", "test", "noxfile.py", "util"
 
 
 def install_with_constraints(session, *args, **kwargs):
@@ -110,5 +110,5 @@ def safety(session):
 @nox.session(python=["3.8", "3.9", "3.10", "3.11"], reuse_venv=True)
 def tests(session):
     args = session.posargs
-    session.run("poetry", "install", external=True)
+    session.run("poetry", "install", "--without", "simulated-data", external=True)
     session.run("pytest", *args)

operationsgateway_api/config.yml.example

@@ -17,6 +17,7 @@ images:
   echo_access_key: access_key
   echo_secret_key: secret_key
   image_bucket_name: test-bucket
+  upload_image_threads: 4
   preferred_colour_map_pref_name: PREFERRED_COLOUR_MAP
 mongodb:
   mongodb_url: mongodb://localhost:27017

operationsgateway_api/src/config.py

@@ -37,6 +37,7 @@ class ImagesConfig(BaseModel):
     echo_access_key: StrictStr
     echo_secret_key: StrictStr
     image_bucket_name: StrictStr
+    upload_image_threads: StrictInt
     preferred_colour_map_pref_name: StrictStr
 
 

operationsgateway_api/src/main.py

@@ -93,6 +93,7 @@ def setup_logger():
         "s3transfer",
         "matplotlib.font_manager",
         "zeep",
+        "multipart",
     ]
     for name in libraries_info_logging:
         logging.getLogger(name).setLevel(logging.INFO)

operationsgateway_api/src/models.py

@@ -38,16 +38,17 @@ class ImageModel(BaseModel):
 
 class WaveformModel(BaseModel):
     id_: str = Field(alias="_id")
-    x: str
-    y: str
+    x: List[float]
+    y: List[float]
+
+    class Config:
+        arbitrary_types_allowed = True
 
     @field_validator("x", "y", mode="before")
     def encode_values(cls, value):  # noqa: N805
         if isinstance(value, np.ndarray):
-            return str(list(value))
+            return list(value)
         else:
-            # Typically will be a string when putting waveform data into the model from
-            # results of a MongoDB query
             return value
 
 

operationsgateway_api/src/records/echo_interface.py

@@ -38,7 +38,7 @@ def __init__(self) -> None:
             log.error(
                 "%s: %s. This error happened with bucket '%s'",
                 exc.response["Error"]["Code"],
-                exc.response["Error"]["Message"],
+                exc.response["Error"].get("Message"),
                 Config.config.images.image_bucket_name,
             )
             raise EchoS3Error(
@@ -70,7 +70,7 @@ def download_file_object(self, image_path: str) -> BytesIO:
             log.error(
                 "%s: %s",
                 exc.response["Error"]["Code"],
-                exc.response["Error"]["Message"],
+                exc.response["Error"].get("Message"),
             )
             raise EchoS3Error(
                 f"{exc.response['Error']['Code']} when downloading file at"
@@ -96,7 +96,7 @@ def upload_file_object(self, image_object: BytesIO, image_path: str) -> None:
             log.error(
                 "%s: %s",
                 exc.response["Error"]["Code"],
-                exc.response["Error"]["Message"],
+                exc.response["Error"].get("Message"),
             )
             raise EchoS3Error(
                 f"{exc.response['Error']['Code']} when uploading file at"