Merge pull request #421 from uc-cdis/feat/al2-updates

Feat/al2 updates

Author: piotrsenkow

.secrets.baseline

@@ -136,7 +136,7 @@
         "filename": "bin/settings.py",
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
         "is_verified": false,
-        "line_number": 43
+        "line_number": 51
       }
     ],
     "docs/local_dev_environment.md": [
@@ -354,5 +354,5 @@
       }
     ]
   },
-  "generated_at": "2024-04-22T20:07:28Z"
+  "generated_at": "2025-01-07T20:16:15Z"
 }

Dockerfile

@@ -1,65 +1,56 @@
-# To run:
-# - Create and fill out `creds.json`:
-# {
-#   "fence_host": "",
-#   "fence_username": "",
-#   "fence_password": "",
-#   "fence_database": "",
-#   "db_host": "",
-#   "db_username": "",
-#   "db_password": "",
-#   "db_database": "",
-#   "gdcapi_secret_key": "",
-#   "indexd_password": "",
-#   "hostname": ""
-# }
-# - Build the image: `docker build . -t sheepdog -f Dockerfile`
-# - Run: `docker run -v /full/path/to/creds.json:/var/www/sheepdog/creds.json -p 81:80 sheepdog`
-# To check running container: `docker exec -it sheepdog /bin/bash`
-
-FROM quay.io/cdis/python:python3.9-buster-2.0.0
+ARG AZLINUX_BASE_VERSION=master
+
+# Base stage with python-build-base
+FROM quay.io/cdis/python-nginx-al:${AZLINUX_BASE_VERSION} AS base
 
 ENV appname=sheepdog
 
-RUN pip install --upgrade pip poetry
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential libffi-dev musl-dev gcc libxml2-dev libxslt-dev \
-    curl bash git vim
+WORKDIR /${appname}
 
-RUN mkdir -p /var/www/$appname \
-    && mkdir -p /var/www/.cache/Python-Eggs/ \
-    && mkdir /run/nginx/ \
-    && ln -sf /dev/stdout /var/log/nginx/access.log \
-    && ln -sf /dev/stderr /var/log/nginx/error.log \
-    && chown nginx -R /var/www/.cache/Python-Eggs/ \
-    && chown nginx /var/www/$appname
+RUN chown -R gen3:gen3 /${appname}
 
-EXPOSE 80
+# Builder stage
+FROM base AS builder
 
-WORKDIR /$appname
+RUN yum install -y \
+    gcc \
+    python3-devel \
+    postgresql-devel \
+    libpq-devel && \
+    yum clean all
 
-# copy ONLY poetry artifact, install the dependencies but not indexd
-# this will make sure than the dependencies is cached
-COPY poetry.lock pyproject.toml /$appname/
-RUN poetry config virtualenvs.create false \
-    && poetry install -vv --no-root --without dev --no-interaction \
-    && poetry show -v
+USER gen3
+
+COPY --chown=gen3:gen3 . /${appname}
+
+RUN poetry install -vv --without dev --no-interaction
 
-# copy source code ONLY after installing dependencies
-COPY . /$appname
-COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
-COPY ./bin/settings.py /var/www/$appname/settings.py
-COPY ./bin/confighelper.py /var/www/$appname/confighelper.py
+RUN git config --global --add safe.directory /${appname} && COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" > /${appname}/version_data.py \
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >> /${appname}/version_data.py
+
+# Final stage
+FROM base
+
+# Install runtime dependencies
+RUN yum install -y \
+    gcc \
+    python3-devel \
+    postgresql-devel \
+    libpq-devel && \
+    yum clean all
+
+# Copy poetry artifacts and install the dependencies
+COPY poetry.lock pyproject.toml /$appname/
+RUN poetry config virtualenvs.create false && \
+    poetry install -vv --no-root --without dev --no-interaction && \
+    poetry show -v
 
-# install sheepdog
-RUN poetry config virtualenvs.create false \
-    && poetry install -vv --without dev --no-interaction \
-    && poetry show -v
+# Copy application files from the builder stage
+COPY --from=builder /${appname} /${appname}
 
-RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py
+# Switch to non-root user 'gen3' for the serving process
+USER gen3
 
-WORKDIR /var/www/$appname
+WORKDIR /${appname}
 
-RUN ls
-CMD /dockerrun.sh
+CMD ["/sheepdog/dockerrun.bash"]

README.md

@@ -22,7 +22,7 @@ import sheepdog
 import datamodelutils
 from dictionaryutils import dictionary
 from gdcdictionary import gdcdictionary
-from gdcdatamodel import models, validators
+from gen3datamodel import models, validators
 
 dictionary.init(gdcdictionary)
 datamodelutils.validators.init(validators)

bin/confighelper.py

@@ -49,6 +49,6 @@ def load_json(file_name, app_name, search_folders=None):
     """
     actual_files = find_paths(file_name, app_name, search_folders)
     if not actual_files:
-        return None
+        return {}
     with open(actual_files[0], "r") as reader:
         return json.load(reader)

bin/settings.py

@@ -1,6 +1,7 @@
 from sheepdog.api import app, app_init
 from os import environ
-import confighelper
+import os
+import bin.confighelper as confighelper
 
 APP_NAME = "sheepdog"
 
@@ -12,49 +13,58 @@ def load_json(file_name):
 conf_data = load_json("creds.json")
 config = app.config
 
-config["AUTH"] = "https://auth.service.consul:5000/v3/"
-config["AUTH_ADMIN_CREDS"] = None
-config["INTERNAL_AUTH"] = None
-
 # ARBORIST deprecated, replaced by ARBORIST_URL
 # ARBORIST_URL is initialized in app_init() directly
 config["ARBORIST"] = "http://arborist-service/"
 
-# Signpost: deprecated, replaced by index client.
-config["SIGNPOST"] = {
-    "host": environ.get("SIGNPOST_HOST") or "http://indexd-service",
-    "version": "v0",
-    "auth": ("gdcapi", conf_data.get("indexd_password", "{{indexd_password}}")),
-}
 config["INDEX_CLIENT"] = {
-    "host": environ.get("INDEX_CLIENT_HOST") or "http://indexd-service",
+    "host": os.environ.get("INDEX_CLIENT_HOST") or "http://indexd-service",
     "version": "v0",
-    "auth": ("gdcapi", conf_data.get("indexd_password", "{{indexd_password}}")),
+    # The user should be "sheepdog", but for legacy reasons, we use "gdcapi" instead
+    "auth": (
+        (
+            environ.get("INDEXD_USER", "gdcapi"),
+            environ.get("INDEXD_PASS")
+            or conf_data.get("indexd_password", "{{indexd_password}}"),
+        )
+    ),
 }
-config["FAKE_AUTH"] = False
+
 config["PSQLGRAPH"] = {
-    "host": conf_data["db_host"],
-    "user": conf_data["db_username"],
-    "password": conf_data["db_password"],
-    "database": conf_data["db_database"],
+    "host": conf_data.get("db_host", os.environ.get("PGHOST", "localhost")),
+    "user": conf_data.get("db_username", os.environ.get("PGUSER", "sheepdog")),
+    "password": conf_data.get("db_password", os.environ.get("PGPASSWORD", "sheepdog")),
+    "database": conf_data.get("db_database", os.environ.get("PGDB", "sheepdog")),
 }
 
 config["FLASK_SECRET_KEY"] = conf_data.get("gdcapi_secret_key", "{{gdcapi_secret_key}}")
-config["PSQL_USER_DB_CONNECTION"] = "postgresql://%s:%s@%s:5432/%s" % tuple(
-    [
-        conf_data.get(key, key)
-        for key in ["fence_username", "fence_password", "fence_host", "fence_database"]
-    ]
+fence_username = conf_data.get(
+    "fence_username", os.environ.get("FENCE_DB_USER", "fence")
+)
+fence_password = conf_data.get(
+    "fence_password", os.environ.get("FENCE_DB_PASS", "fence")
+)
+fence_host = conf_data.get("fence_host", os.environ.get("FENCE_DB_HOST", "localhost"))
+fence_database = conf_data.get(
+    "fence_database", os.environ.get("FENCE_DB_DATABASE", "fence")
+)
+config["PSQL_USER_DB_CONNECTION"] = "postgresql://%s:%s@%s:5432/%s" % (
+    fence_username,
+    fence_password,
+    fence_host,
+    fence_database,
 )
 
-config["USER_API"] = "https://%s/user" % conf_data["hostname"]  # for use by authutils
+config["USER_API"] = "https://%s/user" % conf_data.get(
+    "hostname", os.environ.get("CONF_HOSTNAME", "localhost")
+)  # for use by authutils
 # use the USER_API URL instead of the public issuer URL to accquire JWT keys
 config["FORCE_ISSUER"] = True
-config["DICTIONARY_URL"] = environ.get(
+config["DICTIONARY_URL"] = os.environ.get(
     "DICTIONARY_URL",
     "https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json",
 )
 
 app_init(app)
 application = app
-application.debug = environ.get("GEN3_DEBUG") == "True"
+application.debug = os.environ.get("GEN3_DEBUG") == "True"

bin/setup_psqlgraph.py

@@ -3,7 +3,7 @@
 import logging
 from sqlalchemy import create_engine
 
-from gdcdatamodel.models import *
+from gen3datamodel.models import *
 from psqlgraph import create_all, Node, Edge
 
 

bin/setup_transactionlogs.py

@@ -5,7 +5,7 @@
 
 import argparse
 from sqlalchemy import create_engine
-from gdcdatamodel.models.submission import Base
+from gen3datamodel.models.submission import Base
 
 
 def setup(host, port, user, password, database, use_ssl=False):

deployment/uwsgi/uwsgi.ini

@@ -0,0 +1,9 @@
+wsgi_app = "bin.settings:application"
+bind = "0.0.0.0:8000"
+workers = 1
+preload_app = True
+user = "gen3"
+group = "gen3"
+timeout = 300
+keepalive = 2
+keepalive_timeout = 5

deployment/wsgi/gunicorn.conf.py

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+nginx 
+poetry run gunicorn -c "/sheepdog/deployment/wsgi/gunicorn.conf.py"

dockerrun.bash

@@ -195,7 +195,7 @@ For convenience, the minimal usage looks like the following:
 import datamodelutils
 from dictionaryutils import dictionary
 from gdcdictionary import gdcdictionary
-from gdcdatamodel import models, validators
+from gen3datamodel import models, validators
 from flask import Flask
 import sheepdog