Reorganize repo: Move Hue and Superset away from main docker-compose.yml.

Author: johannestang

README.md

@@ -1,4 +1,4 @@
-version: "3"
+version: "3.5"
 
 services:
 
@@ -15,6 +15,8 @@ services:
       - ./config/hadoop-hive.env
     ports:
       - "50070:50070"
+    networks:
+      - base_net
 
   datanode:
     image: johannestang/hadoop-datanode:2.0.0-hadoop2.8.5-java8
@@ -27,6 +29,8 @@ services:
       SERVICE_PRECONDITION: "namenode:50070"
     ports:
       - "50075:50075"
+    networks:
+      - base_net
 
   # Hive
 
@@ -43,6 +47,8 @@ services:
     ports:
       - "10000:10000"
       - "10002:10002"
+    networks:
+      - base_net
 
   hive-metastore:
     image: johannestang/hive:2.3.4-postgresql-metastore-s3
@@ -56,11 +62,15 @@ services:
       HDFS_CONF_fs_s3a_secret_key: ${MINIO_SECRET_KEY}
     ports:
       - "9083:9083"
+    networks:
+      - base_net
 
   hive-metastore-postgresql:
     image: bde2020/hive-metastore-postgresql:2.3.0
     volumes:
       - ${DATADIR}/hive-metastore:/var/lib/postgresql/data
+    networks:
+      - base_net
 
   # Presto
 
@@ -73,6 +83,8 @@ services:
       S3_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       S3_SECRET_KEY: ${MINIO_SECRET_KEY}
       S3_ENDPOINT: "http://minio:9000"
+    networks:
+      - base_net
 
   # Minio
 
@@ -88,57 +100,9 @@ services:
       - ${DATADIR}/minio/data:/data
       - ${DATADIR}/minio/config:/root/.minio
     command: server /data
+    networks:
+      - base_net
 
-  # Superset
-
-  superset:
-    image: amancevice/superset:0.28.1
-    restart: always
-    depends_on:
-      - superset-postgres
-      - superset-redis
-    environment:
-      MAPBOX_API_KEY: ${MAPBOX_API_KEY}
-    ports:
-      - "8088:8088"
-    volumes:
-      - ./config/superset_config.py:/etc/superset/superset_config.py
-
-  superset-redis:
-    image: redis:3.2
-    restart: always
-    volumes:
-      - ${DATADIR}/superset/redis:/data
-
-  superset-postgres:
-    image: postgres:10
-    restart: always
-    environment:
-      POSTGRES_DB: superset
-      POSTGRES_PASSWORD: superset
-      POSTGRES_USER: superset
-    volumes:
-      - ${DATADIR}/superset/postgres:/var/lib/postgresql/data
-
-  # Hue
-
-  hue:
-    image: johannestang/hue:4.3.0-presto0.215
-    restart: always
-    depends_on:
-      - hue-postgres
-    ports:
-      - "8888:8888"
-    volumes:
-      - ./config/pseudo-distributed.ini:/hue/desktop/conf/pseudo-distributed.ini
-
-  hue-postgres:
-    image: postgres:10
-    restart: always
-    environment:
-      POSTGRES_DB: hue
-      POSTGRES_PASSWORD: hue
-      POSTGRES_USER: hue
-    volumes:
-      - ${DATADIR}/hue/postgres:/var/lib/postgresql/data
-
+networks:
+  base_net:
+    name: bigdatanet

docker-compose.yml

@@ -0,0 +1,34 @@
+version: "3.5"
+
+services:
+
+  # Hue
+
+  hue:
+    image: johannestang/hue:4.3.0-presto0.215
+    restart: always
+    depends_on:
+      - hue-postgres
+    ports:
+      - "8888:8888"
+    volumes:
+      - ./config/pseudo-distributed.ini:/hue/desktop/conf/pseudo-distributed.ini
+    networks:
+      - hue_net
+
+  hue-postgres:
+    image: postgres:10
+    restart: always
+    environment:
+      POSTGRES_DB: hue
+      POSTGRES_PASSWORD: hue
+      POSTGRES_USER: hue
+    volumes:
+      - ${DATADIR}/hue/postgres:/var/lib/postgresql/data
+    networks:
+      - hue_net
+
+networks:
+  hue_net:
+    external:
+      name: bigdatanet

hadoop-base/build-local.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+REPO=johannestang
+
+# Hadoop base
+cd hadoop-base
+IMAGE=hadoop-base
+VERSION=2.0.0-hadoop$(grep HADOOP_VERSION Dockerfile | head -n 1 | awk '{print $3}')-java8
+docker build -t ${REPO}/${IMAGE}:${VERSION} .
+cd ..
+
+# Hadoop datanode
+cd hadoop-datanode
+IMAGE=hadoop-datanode
+VERSION=2.0.0-$(grep hadoop-base Dockerfile |  cut -d '-' -f 3)-java8
+docker build -t ${REPO}/${IMAGE}:${VERSION} .
+cd ..
+
+# Hadoop namenode
+cd hadoop-namenode
+IMAGE=hadoop-namenode
+VERSION=2.0.0-$(grep hadoop-base Dockerfile |  cut -d '-' -f 3)-java8
+docker build -t ${REPO}/${IMAGE}:${VERSION} .
+cd ..
+
+# Hive
+cd hadoop-hive
+IMAGE=hive
+VERSION=$(grep HIVE_VERSION Dockerfile | head -n 1 | awk '{print $3}')-postgresql-metastore-s3
+docker build -t ${REPO}/${IMAGE}:${VERSION} .
+cd ..
+
+# Presto
+cd prestodb
+IMAGE=prestodb
+VERSION=$(grep PRESTO_VERSION Dockerfile | head -n 1 | awk '{print $2}' | cut -d '=' -f 2)
+docker build -t ${REPO}/${IMAGE}:${VERSION} .
+cd ..
+
+# Hue
+cd hue
+IMAGE=hue
+HUE_VERSION=$(grep HUE_VERSION Dockerfile | head -n 1 | awk '{print $2}' | cut -d '=' -f 2)
+PRESTO_VERSION=$(grep PRESTO_VERSION Dockerfile | head -n 1 | awk '{print $2}' | cut -d '=' -f 2)
+VERSION=${HUE_VERSION}-presto${PRESTO_VERSION}
+docker build -t ${REPO}/${IMAGE}:${VERSION} .
+cd ..
+

hadoop-datanode/build-local.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 cd "${0%/*}"
-docker run -it --rm --network="bigdata_stack_default" --env-file ../config/hadoop-hive.env -v /:/host johannestang/hadoop-base /bin/bash
+docker run -it --rm --network="bigdatanet" --env-file ../config/hadoop-hive.env -v /:/host johannestang/hadoop-base:2.0.0-hadoop2.8.5-java8 /bin/bash
 

hadoop-hive/build-local.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 cd "${0%/*}"
-docker-compose -f ../docker-compose.yml exec hue ./build/env/bin/hue syncdb --noinput
-docker-compose -f ../docker-compose.yml exec hue ./build/env/bin/hue migrate
+docker-compose -f ../hue/docker-compose.yml exec hue ./build/env/bin/hue syncdb --noinput
+docker-compose -f ../hue/docker-compose.yml exec hue ./build/env/bin/hue migrate

hadoop-namenode/build-local.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 cd "${0%/*}"
-docker-compose -f ../docker-compose.yml exec superset superset-init
-docker cp add-presto.yml $(docker-compose -f ../docker-compose.yml ps -q superset):/tmp
-docker-compose -f ../docker-compose.yml exec superset superset import_datasources -p /tmp/add-presto.yml
+docker-compose -f ../superset/docker-compose.yml exec superset superset-init
+docker cp add-presto.yml $(docker-compose -f ../superset/docker-compose.yml ps -q superset):/tmp
+docker-compose -f ../superset/docker-compose.yml exec superset superset import_datasources -p /tmp/add-presto.yml

hue/build-local.sh

@@ -0,0 +1,45 @@
+version: "3.5"
+
+services:
+
+  # Superset
+
+  superset:
+    image: amancevice/superset:0.28.1
+    restart: always
+    depends_on:
+      - superset-postgres
+      - superset-redis
+    environment:
+      MAPBOX_API_KEY: ${MAPBOX_API_KEY}
+    ports:
+      - "8088:8088"
+    volumes:
+      - ./config/superset_config.py:/etc/superset/superset_config.py
+    networks:
+      - superset_net
+
+  superset-redis:
+    image: redis:3.2
+    restart: always
+    volumes:
+      - ${DATADIR}/superset/redis:/data
+    networks:
+      - superset_net
+
+  superset-postgres:
+    image: postgres:10
+    restart: always
+    environment:
+      POSTGRES_DB: superset
+      POSTGRES_PASSWORD: superset
+      POSTGRES_USER: superset
+    volumes:
+      - ${DATADIR}/superset/postgres:/var/lib/postgresql/data
+    networks:
+      - superset_net
+
+networks:
+  superset_net:
+    external:
+      name: bigdatanet