Add Docker build and static templates (#38)

Integrates the Docker build step into the same repo.

To build the container, run `docker build -f docker/Dockerfile .` from the root project repository.

In addition, this PR adds functionality to enable the Hub menu on newer Dataproc images, even when not proxying through Dataproc Hub.

Author: JerryLeiDing

.github/workflows/linter.yml

@@ -34,7 +34,7 @@ jobs:
           # Only run against changed files
           VALIDATE_ALL_CODEBASE: false
           # Don't lint tests
-          FILTER_REGEX_EXCLUDE: tests/.*
+          FILTER_REGEX_EXCLUDE: (tests/.*)|(docker/*)|(static/*)
           DEFAULT_BRANCH: master
           LINTER_RULES_PATH: .
           VALIDATE_PYTHON_FLAKE8: false

dataprocspawner/spawner.py

@@ -13,36 +13,35 @@
 # limitations under the License.
 """A custom Spawner that creates notebooks backed by Dataproc clusters."""
 
+import asyncio
 import json
-import re
+import math
 import os
 import random
+import re
 import string
-import proto
-import yaml
-import math
-import asyncio
 from datetime import datetime as dt
 from types import SimpleNamespace
 
-from google.protobuf.json_format import MessageToDict
+import proto
+import requests
+import yaml
+from async_generator import aclosing, async_generator, yield_
+from dataprocspawner.customize_cluster import (get_base_cluster_html_form,
+                                               get_custom_cluster_html_form)
+from dataprocspawner.spawnable import DataprocHubServer
 from google.api_core import exceptions
-from google.cloud import storage, logging_v2
-from google.cloud.dataproc_v1beta2 import (
-    ClusterControllerClient, Cluster, ClusterStatus)
+from google.cloud import logging_v2, storage
+from google.cloud.dataproc_v1beta2 import (Cluster, ClusterControllerClient,
+                                           ClusterStatus)
+from google.cloud.dataproc_v1beta2.services.cluster_controller.transports import \
+    ClusterControllerGrpcTransport
 from google.cloud.dataproc_v1beta2.types.shared import Component
-
-from google.cloud.dataproc_v1beta2.services.cluster_controller.transports import ClusterControllerGrpcTransport
-from traitlets import List, Unicode, Dict, Bool
-
-from dataprocspawner.spawnable import DataprocHubServer
-from dataprocspawner.customize_cluster import (
-    get_base_cluster_html_form, get_custom_cluster_html_form)
-
-from async_generator import async_generator, yield_, aclosing
-
+from google.protobuf.json_format import MessageToDict
 from jupyterhub import orm
 from jupyterhub.spawner import Spawner
+from traitlets import Bool, Dict, List, Unicode
+
 
 def url_path_join(*pieces):
   """Join components of url into a relative url.
@@ -329,6 +328,21 @@ def __init__(self, *args, **kwargs):
     self.component_gateway_url = None
     self.progressor = SimpleNamespace(bar=0, logging=set(), start='')
 
+    # Check if we have a notebooks proxy URL
+    self.hub_host = ''
+    if 'hub_host' in kwargs:
+      self.hub_host = kwargs.get('hub_host')
+    else:
+      try:
+        r = requests.get(
+            'http://metadata.google.internal/computeMetadata/v1/instance/attributes/proxy-url',
+            headers={'Metadata-Flavor': 'Google'})
+        r.raise_for_status()
+        self.hub_host = f'https://{r.text}/'
+        self.log.info(f'Got proxy url {r.text} from metadata')
+      except Exception as e:  ## pylint: disable=broad-except
+        self.log.info(f'Failed to get proxy url from metadata: {e}')
+
     if mock:
       # Mock the API
       self.dataproc_client = kwargs.get('dataproc')
@@ -646,6 +660,21 @@ def get_env(self):
     self.log.debug(f'env is {env}')
     return env
 
+  def get_args(self):
+    """Set arguments to pass to Jupyterhub-singleuser on the cluster.
+
+    Note that we don't call the superclass method, as we don't want to
+    set default args like port and ip.
+    """
+    args = []
+
+    if self.debug:
+      args.append('--debug')
+    args.append('--NotebookApp.hub_activity_interval=0')
+    args.append('--NotebookApp.hub_host={}'.format(self.hub_host))
+    args.extend(self.args)
+    return args
+
 ################################################################################
 # Custom Functions
 ################################################################################
@@ -1265,6 +1294,8 @@ def _build_cluster_config(self, cluster_data=None):
                  ['dataproc:jupyter.hub.args']) = self.args_str
     (cluster_data['config']['software_config']['properties']
                  ['dataproc:jupyter.hub.env']) = self.env_str
+    (cluster_data['config']['software_config']['properties']
+                 ['dataproc:jupyter.hub.menu.enabled']) = 'true'
 
     if self.gcs_user_folder:
       (cluster_data['config']['software_config']['properties']

docker/Dockerfile

@@ -0,0 +1,53 @@
+FROM jupyterhub/jupyterhub:latest
+
+# Install gcloud
+RUN apt-get update && apt-get install -y curl apt-transport-https ca-certificates gnupg \
+  && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+  && echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
+  && apt-get update && apt-get install -y google-cloud-sdk
+
+RUN apt-get update \
+  && apt-get remove -y python3-pycurl \
+  && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libcurl4-openssl-dev \
+    python3-wheel \
+    git \
+  && apt-get purge \
+  && apt-get clean -y
+
+RUN pip install --upgrade pip \
+  && pip install --upgrade \
+    psycopg2-binary \
+    google-api-python-client \
+    google-auth-oauthlib \
+    google-api-python-client \
+    oauth2client \
+    google-auth \
+    googleapis-common-protos \
+    google-auth-httplib2
+
+RUN apt-get update && apt-get install -y libpq-dev \
+    && apt-get install -y vim \
+    && apt-get install -y iproute2 \
+    && apt-get autoremove -y \
+    && apt-get clean -y
+
+# Install Dataproc Spawner
+COPY setup.py /tmp/dataprocspawner/setup.py
+COPY dataprocspawner/ /tmp/dataprocspawner/dataprocspawner/
+COPY dataprochub/ /tmp/dataprocspawner/dataprochub/
+RUN pip install git+https://github.com/GoogleCloudPlatform/jupyterhub-gcp-proxies-authenticator
+RUN pip install /tmp/dataprocspawner/
+
+# JupyterHub design
+COPY static/templates/ /etc/jupyterhub/templates/
+
+COPY docker/jupyterhub_config.py .
+
+EXPOSE 8080
+
+LABEL "com.google.environment"="Dataproc Hub"
+
+COPY docker/jupyterhub.sh /usr/local/bin/
+ENTRYPOINT ["bash", "-C", "/usr/local/bin/jupyterhub.sh"]

docker/README.md

@@ -0,0 +1,2 @@
+To build, run `docker build -f docker/Dockerfile .` from the root project
+directory.