Updated readme, python packages transferred to requirements.txt

Author: vladd-bit

Dockerfile_singleuser

@@ -157,43 +157,25 @@ RUN fix-permissions $CONDA_DIR && fix-permissions /home/$NB_USER
 
 # update pip and install basic utilities
 RUN pip3 install --no-cache-dir --upgrade pip
-RUN pip3 install --no-cache-dir setuptools wheel virtualenv
 
-# jupyterhub stuff
-RUN pip3 install --no-cache-dir ipywidgets jupyter jupyterhub jupyterlab jupyterlab-git importlib_metadata
-RUN pip3 install --no-cache-dir jupyterlab_widgets jupyter_contrib_core jupyter_contrib_nbextensions jupyter-server-proxy fastbook
-RUN pip3 install --no-cache-dir docker dockerspawner jupyterhub-firstuseauthenticator jupyterhub-systemdspawner jupyterhub-jwtauthenticator jupyterhub-client jupyterhub-kerberosauthenticator
-RUN pip3 install --no-cache-dir jupyterhub-nanowireauthenticator jupyterhub-ldapauthenticator jupyterhub-kubespawner jupyterhub-nativeauthenticator
+# install the rest of the packages including medcat
+RUN pip3 install --no-cache-dir -r ./requirements.txt
 
-# extra packages for DB connections & utilities (charts, data formats, and other useful tools such as neo4j)
-RUN pip3 install --no-cache-dir pytesseract ipyparallel py7zr cython isort html2text jsoncsv simplejson detect wheel nltk keras bokeh seaborn matplotlib graphviz plotly tqdm
-RUN pip3 install --no-cache-dir pymssql mysql-connector-python cx-Oracle dataclasses numpy matplotlib pandas dill jsonpickle jsonext psycopg2 psycopg2-binary pyodbc openpyxl
-RUN pip3 install --no-cache-dir dvc flask GitPython elasticsearch opensearch-py neo4j eland --ignore-installed PyYAML 
-RUN pip3 install --no-cache-dir opencv-python torchvision eland plotly
-
-# XNAT
-RUN pip3 install --no-cache-dir xnat    
-
-# medcat & models
-RUN pip3 install --no-cache-dir -U spacy click torch thinc
+# install requirements for working with cogstack scripts
+RUN pip3 install --no-cache-dir -r notebooks/demo_working_with_cogstack/requirements.txt
 
 # Get the spacy model
 #ARG SPACY_MODELS="en_core_web_sm en_core_web_md en_core_web_lg"
 #RUN for spacy_model in ${SPACY_MODELS}; do python3 -m spacy download $spacy_model; done
 
-# install requirements for working with cogstack scripts
-# RUN pip3 install --no-cache-dir -r notebooks/demo_working_with_cogstack/requirements.txt
-
-RUN pip3 install --no-cache-dir medcat==1.14.0
-
 # clean up pip
 RUN pip3 cache purge
 
 #######################################################################################################
 
 # install R and other dependencies
-COPY ./scripts/r_kernel_install.sh /etc/jupyterhub/
-# RUN Rscript /etc/jupyterhub/r_kernel_install.sh
+COPY ./scripts/r_kernel_install.sh /srv/jupyterhub/
+RUN Rscript /srv/jupyterhub/r_kernel_install.sh
 
 # create jupyterhub shared folder
 RUN mkdir -p /home/jovyan/scratch

Dockerfile_singleuser_gpu

@@ -187,43 +187,25 @@ RUN fix-permissions $CONDA_DIR && fix-permissions /home/$NB_USER
 
 # update pip and install basic utilities
 RUN pip3 install --no-cache-dir --upgrade pip
-RUN pip3 install --no-cache-dir  setuptools wheel virtualenv
 
-# jupyterhub stuff
-RUN pip3 install --no-cache-dir ipywidgets jupyter jupyterhub jupyterlab jupyterlab-git importlib_metadata
-RUN pip3 install --no-cache-dir jupyterlab_widgets jupyter_contrib_core jupyter_contrib_nbextensions jupyter-server-proxy fastbook
-RUN pip3 install --no-cache-dir docker dockerspawner jupyterhub-firstuseauthenticator jupyterhub-systemdspawner jupyterhub-jwtauthenticator jupyterhub-client jupyterhub-kerberosauthenticator
-RUN pip3 install --no-cache-dir jupyterhub-nanowireauthenticator jupyterhub-ldapauthenticator jupyterhub-kubespawner jupyterhub-nativeauthenticator
+# install the rest of the packages including medcat
+RUN pip3 install --no-cache-dir -r ./requirements.txt
 
-# extra packages for DB connections & utilities (charts, data formats, and other useful tools such as neo4j)
-RUN pip3 install --no-cache-dir pytesseract ipyparallel py7zr cython isort html2text jsoncsv simplejson detect wheel nltk keras bokeh seaborn matplotlib graphviz plotly tqdm
-RUN pip3 install --no-cache-dir pymssql mysql-connector-python cx-Oracle dataclasses numpy matplotlib pandas dill jsonpickle jsonext psycopg2 psycopg2-binary pyodbc openpyxl
-RUN pip3 install --no-cache-dir dvc flask GitPython elasticsearch opensearch-py neo4j eland --ignore-installed PyYAML
-RUN pip3 install --no-cache-dir opencv-python torchvision eland plotly
-
-# XNAT
-RUN pip3 install --no-cache-dir xnat
-
-# medcat & models
-RUN pip3 install --no-cache-dir -U spacy click torch thinc
+# install requirements for working with cogstack scripts
+RUN pip3 install --no-cache-dir -r notebooks/demo_working_with_cogstack/requirements.txt
 
 # Get the spacy model
 #ARG SPACY_MODELS="en_core_web_sm en_core_web_md en_core_web_lg"
 #RUN for spacy_model in ${SPACY_MODELS}; do python3 -m spacy download $spacy_model; done
 
-# install requirements for working with cogstack scripts
-# RUN pip3 install --no-cache-dir -r notebooks/demo_working_with_cogstack/requirements.txt
-
-RUN pip3 install --no-cache-dir medcat==1.14.0
-
 # clean up pip
 RUN pip3 cache purge
 
 #######################################################################################################
 
 # install R and other dependencies
-COPY ./scripts/r_kernel_install.sh /etc/jupyterhub/
-RUN Rscript /etc/jupyterhub/r_kernel_install.sh
+COPY ./scripts/r_kernel_install.sh /srv/jupyterhub/
+RUN Rscript /srv/jupyterhub/r_kernel_install.sh
 
 # create jupyterhub shared folder
 RUN mkdir -p /home/jovyan/scratch

README.md

@@ -25,11 +25,20 @@ Images are available for both x86/ARM architectures (post version 1.2.7):
 
 Full and more in-depth knowledge on the configuration itself is available in the primary repository [official documentation](https://cogstack-nifi.readthedocs.io/en/latest/deploy/services.html#id12).
 
+
 # Usage & configuration
 
 ENV variables are located in: [env/jupyter.env](./env/jupyter.env) and [env/general.env](./env/general.env).\
 Please check the ENV file for additional information, every variable is commented and described.
 
+## Python packages installed
+
+Full list found in [requirements.txt](./requirements.txt).
+
+## Security
+
+Certificates used are located in the `./security/` folder, taken from the [Cogstack-NiFi](https://github.com/CogStack-NiFi) security folder, [root-ca.key](https://raw.githubusercontent.com/CogStack/CogStack-NiFi/refs/heads/master/security/root-ca.key) and [root-ca.pem](https://raw.githubusercontent.com/CogStack/CogStack-NiFi/refs/heads/master/security/root-capem), read the [security section](https://cogstack-nifi.readthedocs.io/en/latest/security.html) for more info on how to generate them from the main NiFi repository.
+
 ## Setting up your own hub
 
 There are two docker compose files:
@@ -50,7 +59,6 @@ Updating certificates and env settings from the main repo:
     - sometimes it is necessary to grab new certificates if the old ones expired (from the main Cogstack-NiFi repo)
     - from the main repo directory, execute `bash scripts/update_env_cert_from_nifi_repo.sh`
 
-
 ## Access and account control
 To access Jupyter Hub on the host machine (e.g.localhost), one can type in the browser `https://localhost:8888`.
 

config/jupyter_notebook_config.py

@@ -1,3 +1,4 @@
+
 # Configuration file for jupyter-notebook.
 
 c = get_config()

config/jupyterhub_config.py

@@ -2,33 +2,26 @@
 # source : https://github.com/jupyterhub/jupyterhub-deploy-docker/blob/main/basic-example/jupyterhub_config.py
 
 import os
-import pwd
-import subprocess
 import sys
 import docker
-import dockerspawner 
+import dockerspawner
+import traceback
+#import traitlets
 from jupyterhub.auth import LocalAuthenticator
 from nativeauthenticator import NativeAuthenticator
+from traitlets.config import Config
 
 
 class LocalNativeAuthenticator(NativeAuthenticator, LocalAuthenticator):
     pass
 
 
-def pre_spawn_hook(spawner):
-    username = str(spawner.user.name).lower()
-    try:
-        pwd.getpwnam(username)
-    except KeyError:
-        subprocess.check_call(["useradd", "-ms", "/bin/bash", username])
-
-
-c = get_config()
+c: Config = get_config()
 
 # Spawn containers from this image
 # Either use the CoGstack one from the repo which is huge and contains all the stuff needed or,
 # use the default official one which is clean.
-c.DockerSpawner.image = os.getenv("DOCKER_NOTEBOOK_IMAGE", "cogstacksystems:jupyterhub/singleuser:latest")
+c.DockerSpawner.image = os.getenv("DOCKER_NOTEBOOK_IMAGE", "cogstacksystems:jupyterhub/singleuser:latest-amd64")
 
 # JupyterHub requires a single-user instance of the Notebook server, so we
 # default to using the `start-singleuser.sh` script included in the
@@ -61,7 +54,7 @@ def pre_spawn_hook(spawner):
 # We follow the same convention.
 notebook_dir = os.environ.get("DOCKER_NOTEBOOK_DIR", "/home/jovyan/work")
 shared_content_dir = os.environ.get("DOCKER_SHARED_DIR", "/home/jovyan/scratch")
-
+work_dir = os.environ.get("JUPYTER_WORK_DIR", "/lab/workspaces/auto-b/tree/" + str(notebook_dir.split("/")[-1]))
 
 #c.DockerSpawner.notebook_dir = notebook_dir
 # Mount the real user"s Docker volume on the host to the notebook user"s
@@ -78,9 +71,9 @@ def pre_spawn_hook(spawner):
 if select_notebook_image_allowed == "true":
     # c.DockerSpawner.image_whitelist has been deprecated for allowed_images
     c.DockerSpawner.allowed_images = {
-        'minimal': 'jupyterhub/singleuser:latest',
-        'cogstack': 'cogstacksystems/jupyter-singleuser:latest',
-        'cogstack-gpu': 'cogstacksystems/jupyter-singleuser-gpu:latest'
+        "minimal": "jupyterhub/singleuser:latest-amd64",
+        "cogstack": "cogstacksystems/jupyter-singleuser:latest-amd64",
+        "cogstack-gpu": "cogstacksystem s/jupyter-singleuser-gpu:latest-amd64"
     }
     # https://github.com/jupyterhub/dockerspawner/issues/423
     c.DockerSpawner.remove = True
@@ -103,27 +96,71 @@ def pre_spawn_hook(spawner):
     "no_proxy": ",".join(list(filter(len, os.environ.get("no_proxy", "").split(",") + [hub_container_ip_or_name]))),
 }
 
-os.environ['NO_PROXY'] = ''
-os.environ['no_proxy'] = ''
-os.environ['HTTP_PROXY'] = ''
-os.environ['HTTPS_PROXY'] = ''
-os.environ['http_proxy'] = ''
-os.environ['https_proxy'] = ''
+os.environ["NO_PROXY"] = ""
+os.environ["no_proxy"] = ""
+os.environ["HTTP_PROXY"] = ""
+os.environ["HTTPS_PROXY"] = ""
+os.environ["http_proxy"] = ""
+os.environ["https_proxy"] = ""
+
+
+"""
+    def pre_spawn_hook(spawner):
+        username = str(spawner.user.name).lower()
+        try:
+            pwd.getpwnam(username)
+        except KeyError:
+            subprocess.check_call(["useradd", "-ms", "/bin/bash", username])
+"""
+
+
+# Spawn single-user servers as Docker containers
+class DockerSpawner(dockerspawner.DockerSpawner):
+    def start(self):
+        # username is self.user.name
+        self.volumes = {"jupyterhub-user-{}".format(self.user.name): notebook_dir}
+
+        if self.user.name not in whitelist:
+            whitelist.add(self.user.name)
+            with open(userlist_path, "a") as f:
+                f.write("\n")
+                f.write(self.user.name)
+
+        if self.user.name in list(team_map.keys()):
+            for team in team_map[self.user.name]:
+                team_dir_path = os.path.join(shared_content_dir, team)
+                self.volumes["jupyterhub-team-{}".format(team)] = {
+                    "bind": team_dir_path,
+                    "mode": "rw",  # or ro for read-only
+                }
+
+        # this is a temporary fix, need to actually check permissions
+        self.mem_limit = resource_allocation_user_ram_limit
+        self.post_start_cmd = "chmod -R 777 " + shared_content_dir
+
+        return super().start()
 
 
-def post_spawn_hook(spawner):
+def pre_spawn_hook(spawner: DockerSpawner):
+    #username = spawner.user.name
+    #spawner.environment["GREETING"] = f"Hello {username}"
     try:
-        with open("/etc/environment", "w+") as f:
+        for key, value in ENV_PROXIES.items():
+            spawner.environment[str(key)] = str(value)
+        with open("/home/jovyan/test.txt", "w+") as f:
             for key, value in ENV_PROXIES.items():
-                f.write(str(key) + "=" + str(value) + "\n")
-    except KeyError:
-        pass
+                f.write("export" + " " + str(key) + "=" + str(value) + "\n")
+    except Exception:
+        traceback.print_exc()
 
 
-c.Spawner.post_spawn_hook = post_spawn_hook
+c.Spawner.default_url = work_dir
+c.Spawner.pre_spawn_hook = pre_spawn_hook
 
-#c.Spawner.pre_spawn_hook = pre_spawn_hook
 #c.Spawner.ip = "127.0.0.1"
+
+# This is buggy, setting the HTTP(s)_PROXY & NO_PROXY variables via pre/post
+#  spawn hook is better
 #c.Spawner.environment = ENV_PROXIES
 
 # AUTHENTICATION
@@ -139,7 +176,10 @@ def post_spawn_hook(spawner):
 c.LocalAuthenticator.create_system_users = True
 c.SystemdSpawner.dynamic_users = True
 c.PAMAuthenticator.admin_groups = {"wheel"}
-c.Authenticator.whitelist = whitelist = set()
+c.Authenticator.allowed_users = whitelist = set()
+
+
+#c.Authenticator.manage_groups = True
 
 #c.Authenticator.allow_all = True
 
@@ -163,32 +203,6 @@ def per_user_limit(role):
     return ram_limits.get(role)
 
 
-# Spawn single-user servers as Docker containers
-class DockerSpawner(dockerspawner.DockerSpawner):
-    def start(self):
-        # username is self.user.name
-        self.volumes = {"jupyterhub-user-{}".format(self.user.name): notebook_dir}
-
-        if self.user.name not in whitelist:
-            whitelist.add(self.user.name)
-            with open(userlist_path , "a") as f:
-                f.write("\n")
-                f.write(self.user.name)
-
-        if self.user.name in list(team_map.keys()):
-            for team in team_map[self.user.name]:
-                team_dir_path = os.path.join(shared_content_dir, team)
-                self.volumes["jupyterhub-team-{}".format(team)] = {
-                    "bind": team_dir_path,
-                    "mode": "rw",  # or ro for read-only
-                }
-
-        # this is a temporary fix, need to actually check permissions
-        self.mem_limit = resource_allocation_user_ram_limit
-        self.post_start_cmd = "chmod -R 777 " + shared_content_dir
-
-        return super().start()
-
 
 # Spawn single-user servers as Docker containers
 c.JupyterHub.spawner_class = DockerSpawner
@@ -252,11 +266,13 @@ def start(self):
 }
 c.DockerSpawner.environment.update(ENV_PROXIES)
 
+# Alternative, use: "nativeauthenticator.NativeAuthenticator"
 #c.JupyterHub.authenticator_class = LocalNativeAuthenticator
 
 c.FirstUseAuthenticator.create_users = True
 c.JupyterHub.authenticator_class = "firstuseauthenticator.FirstUseAuthenticator" 
-# Alternative, use: "nativeauthenticator.NativeAuthenticator"
+
+
 
 # User containers will access hub by container name on the Docker network
 c.JupyterHub.ip = "0.0.0.0"
@@ -327,7 +343,7 @@ def start(self):
 #  Default: False
 # c.Application.show_config_json = False
 
-# Let's start with the least privilege, especially on a single host having limited resources
+# Let"s start with the least privilege, especially on a single host having limited resources
 c.JupyterHub.allow_named_servers = False
 
 # Timeout (in seconds) to wait for spawners to initialize