Add centos7 base image support and refactor the properties file with …

…configurable timeouts for zookeeper
hritvikpatel4 · Jun 18, 2022 · 37633e3 · 37633e3
1 parent 1a811b4
commit 37633e3
Show file tree

Hide file tree

Showing 21 changed files with 260 additions and 260 deletions.
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -14,6 +14,16 @@ on:
 jobs:
   build-and-push-image:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - dockerfile: ./docker/Dockerfile.ubuntu
+            image: ghcr.io/hritvikpatel4/apache-atlas
+            tag: ubuntu-bionic
+          - dockerfile: ./docker/Dockerfile.centos
+            image: ghcr.io/hritvikpatel4/apache-atlas
+            tag: centos7
     permissions:
       contents: read
       packages: write
@@ -23,35 +33,40 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Docker Setup Buildx
-        uses: docker/setup-buildx-action@v1.6.0
+        uses: docker/setup-buildx-action@v2
         with:
           driver-opts: env.BUILDKIT_STEP_LOG_MAX_SIZE=1073741824
 
-      - name: Log in to Docker Hub
-        uses: docker/[email protected]
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+      # - name: Log in to Docker Hub
+      #   if: github.event_name != 'pull_request'
+      #   uses: docker/login-action@v2
+      #   with:
+      #     username: ${{ secrets.DOCKER_USERNAME }}
+      #     password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Log in to the GitHub container registry
-        uses: docker/[email protected]
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
+          username: hritvikpatel4
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker Metadata action
         id: meta
-        uses: docker/metadata-action@v3.6.2
+        uses: docker/metadata-action@v4
         with:
-          images: |
-            ntwine/${{ env.IMAGE_NAME }}
-            ${{ env.REGISTRY }}/${{ github.actor }}/${{ env.IMAGE_NAME }}
+          images: ${{ matrix.image }}
+          tags: type=raw,value=${{ matrix.tag }}
+          # images: |
+          #   ntwine/${{ env.IMAGE_NAME }}
+          #   ${{ env.REGISTRY }}/${{ github.actor }}/${{ env.IMAGE_NAME }}
 
       - name: Build and push Docker images
-        uses: docker/build-push-action@v2.8.0
+        uses: docker/build-push-action@v3
         with:
-          context: .
-          push: true
+          context: ./docker
+          file: ${{ matrix.dockerfile }}
+          push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,4 @@
 ._.DS_Store
 **/.DS_Store
 **/._.DS_Store
+.idea
diff --git a/.dockerignore → docker/.dockerignore b/.dockerignore → docker/.dockerignore
diff --git a/docker/Dockerfile.centos b/docker/Dockerfile.centos
@@ -0,0 +1,99 @@
+# ------------------------------ ATLAS CODE COMPILE STAGE ------------------------------
+FROM maven:3.8.4-openjdk-8 AS atlas_compile
+
+LABEL maintainer="Hritvik Patel <[email protected]>"
+
+ENV ATLAS_VERSION 2.2.0
+
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install apt-utils patch python && \
+    cd /tmp && \
+    wget --retry-connrefused -O apache-atlas-$ATLAS_VERSION-sources.tar.gz https://downloads.apache.org/atlas/$ATLAS_VERSION/apache-atlas-$ATLAS_VERSION-sources.tar.gz && \
+    mkdir -p /tmp/atlas-src && \
+    tar xzf /tmp/apache-atlas-$ATLAS_VERSION-sources.tar.gz -C /tmp/atlas-src --strip 1 && \
+    rm -rf /tmp/apache-atlas-$ATLAS_VERSION-sources.tar.gz
+
+COPY patches/* /tmp/atlas-src
+
+RUN cd /tmp/atlas-src && \
+    patch -u -b pom.xml -i log4j.patch && \
+    patch -u -b webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java -i deprecateNDC.patch && \
+    patch -u -b addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java -i hive_2_X_X_support.patch && \
+    sed -i "s/http:\/\/repo1.maven.org\/maven2/https:\/\/repo1.maven.org\/maven2/g" pom.xml && \
+    export MAVEN_OPTS="-Xms2g -Xmx4g" && \
+    mvn clean -Dhttps.protocols=TLSv1.2 package -Pdist
+
+# ------------------------------ ATLAS IMAGE CREATION STAGE ------------------------------
+FROM centos:7
+
+LABEL maintainer="Hritvik Patel <[email protected]>"
+
+ENV ATLAS_VERSION 2.2.0
+ENV ATLAS_INSTALL_LOCATION /opt/apache-atlas-$ATLAS_VERSION
+
+COPY --from=atlas_compile /tmp/atlas-src/distro/target/apache-atlas-$ATLAS_VERSION-hive-hook.tar.gz /tmp
+COPY --from=atlas_compile /tmp/atlas-src/distro/target/apache-atlas-$ATLAS_VERSION-kafka-hook.tar.gz /tmp
+COPY --from=atlas_compile /tmp/atlas-src/distro/target/apache-atlas-$ATLAS_VERSION-server.tar.gz /tmp
+
+RUN yum clean all && \
+    yum -y update && \
+    yum install -y java-1.8.0-openjdk-devel patch unzip && \
+    yum clean all
+
+RUN tar xzf /tmp/apache-atlas-$ATLAS_VERSION-hive-hook.tar.gz   -C /opt && \
+    tar xzf /tmp/apache-atlas-$ATLAS_VERSION-kafka-hook.tar.gz  -C /opt && \
+    tar xzf /tmp/apache-atlas-$ATLAS_VERSION-server.tar.gz      -C /opt && \
+    cp -R /opt/apache-atlas-hive-hook-$ATLAS_VERSION/*  $ATLAS_INSTALL_LOCATION && \
+    cp -R /opt/apache-atlas-kafka-hook-$ATLAS_VERSION/* $ATLAS_INSTALL_LOCATION && \
+    mkdir -p $ATLAS_INSTALL_LOCATION/conf/patches && \
+    echo $ATLAS_VERSION > $ATLAS_INSTALL_LOCATION/version.txt && \
+    rm -rf /opt/apache-atlas-hive-hook-$ATLAS_VERSION /opt/apache-atlas-kafka-hook-$ATLAS_VERSION && \
+    rm -rf /tmp/*
+
+ENV HBASE_CONF_DIR $ATLAS_INSTALL_LOCATION/conf/hbase
+ENV JAVA_HOME /usr/lib/jvm/java
+
+COPY patches/* $ATLAS_INSTALL_LOCATION/conf/patches
+COPY conf/* $ATLAS_INSTALL_LOCATION/conf
+COPY stop_atlas /
+COPY docker_entrypoint.sh /
+
+RUN chmod +x /docker_entrypoint.sh && \
+    chmod +x /stop_atlas && \
+    mkdir /scripts && \
+    mv stop_atlas /scripts
+
+ENV PATH $PATH:$JAVA_HOME/bin:/scripts
+
+ENV HOSTNAME localhost
+ENV KAFKA_BOOTSTRAP_SERVERS localhost:9092
+ENV ATLAS_ZK_QUORUM localhost:2181
+ENV HBASE_ZK_QUORUM localhost:2181
+ENV KAFKA_ZK_QUORUM localhost:2181
+ENV SOLR_ZK_QUORUM localhost:2181
+ENV SOLR_HOST localhost
+ENV SOLR_PORT 8983
+
+ENV GRAPH_STORAGE_LOCK_WAIT_TIME 10000
+
+ENV CREATE_SOLR_INDICES no
+ENV SOLR_ZOOKEEPER_CONNECT_TIMEOUT 600000
+ENV SOLR_ZOOKEEPER_SESSION_TIMEOUT 600000
+
+ENV KAFKA_ZOOKEEPER_SESSION_TIMEOUT 60000
+ENV KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT 30000
+ENV KAFKA_ZOOKEEPER_SYNC_TIME 20
+ENV KAFKA_AUTO_COMMIT_INTERVAL 1000
+
+ENV AUDIT_ZOOKEEPER_SESSION_TIMEOUT 10000
+
+ENV HA_ZOOKEEPER_RETRY_SLEEPTIME 1000
+ENV HA_ZOOKEEPER_SESSION_TIMEOUT 300000
+ENV CLIENT_HA_SLEEP_INTERVAL 5000
+
+EXPOSE 21000
+
+VOLUME ["$ATLAS_INSTALL_LOCATION/conf", "$ATLAS_INSTALL_LOCATION/data", "$ATLAS_INSTALL_LOCATION/logs"]
+
+ENTRYPOINT ["/docker_entrypoint.sh"]
diff --git a/Dockerfile → docker/Dockerfile.ubuntu b/Dockerfile → docker/Dockerfile.ubuntu
@@ -19,14 +19,10 @@ COPY patches/* /tmp/atlas-src
 RUN cd /tmp/atlas-src && \
     patch -u -b pom.xml -i log4j.patch && \
     patch -u -b webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java -i deprecateNDC.patch && \
-    patch -u -b dashboardv2/public/js/views/graph/LineageLayoutView.js -i 0001_customize_lineage_table.patch && \
-    patch -u -b dashboardv2/public/js/views/graph/LineageLayoutView.js -i 0002_customize_lineage_table.patch && \
     patch -u -b addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java -i hive_2_X_X_support.patch && \
     sed -i "s/http:\/\/repo1.maven.org\/maven2/https:\/\/repo1.maven.org\/maven2/g" pom.xml && \
     export MAVEN_OPTS="-Xms2g -Xmx4g" && \
     mvn clean -Dhttps.protocols=TLSv1.2 package -Pdist
-    # Add option '-DskipTests' to skip unit and integration tests
-    # mvn clean -Dhttps.protocols=TLSv1.2 -DskipTests package -Pdist
 
 # ------------------------------ ATLAS IMAGE CREATION STAGE ------------------------------
 FROM ubuntu:bionic
@@ -35,42 +31,25 @@ LABEL maintainer="Hritvik Patel <[email protected]>"
 
 ENV ATLAS_VERSION 2.2.0
 ENV ATLAS_INSTALL_LOCATION /opt/apache-atlas-$ATLAS_VERSION
-ENV HADOOP_VERSION 2.7.3
-ENV HIVE_VERSION 2.3.6
 
 COPY --from=atlas_compile /tmp/atlas-src/distro/target/apache-atlas-$ATLAS_VERSION-hive-hook.tar.gz /tmp
 COPY --from=atlas_compile /tmp/atlas-src/distro/target/apache-atlas-$ATLAS_VERSION-kafka-hook.tar.gz /tmp
 COPY --from=atlas_compile /tmp/atlas-src/distro/target/apache-atlas-$ATLAS_VERSION-server.tar.gz /tmp
 
 RUN apt-get update && \
     apt-get -y upgrade && \
-    apt-get -y install curl openjdk-8-jdk patch python unzip iputils-ping && \
-    apt-get -y autoclean && \
-    curl -o /tmp/hadoop-$HADOOP_VERSION.tar.gz        https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz && \
-    curl -o /tmp/apache-hive-$HIVE_VERSION-bin.tar.gz https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz && \
-    tar xzf /tmp/hadoop-$HADOOP_VERSION.tar.gz                  -C /opt && \
-    tar xzf /tmp/apache-hive-$HIVE_VERSION-bin.tar.gz           -C /opt && \
-    tar xzf /tmp/apache-atlas-$ATLAS_VERSION-hive-hook.tar.gz   -C /opt && \
+    apt-get -y install curl iputils-ping openjdk-8-jdk patch python unzip vim && \
+    apt-get -y autoclean
+
+RUN tar xzf /tmp/apache-atlas-$ATLAS_VERSION-hive-hook.tar.gz   -C /opt && \
     tar xzf /tmp/apache-atlas-$ATLAS_VERSION-kafka-hook.tar.gz  -C /opt && \
     tar xzf /tmp/apache-atlas-$ATLAS_VERSION-server.tar.gz      -C /opt && \
-    mkdir -p /apache-atlas-$ATLAS_VERSION && \
-    cp /tmp/apache-atlas-$ATLAS_VERSION-hive-hook.tar.gz / && \
-    tar xzf apache-atlas-$ATLAS_VERSION-hive-hook.tar.gz -C /apache-atlas-$ATLAS_VERSION --strip 1 && \
-    tar czf apache-atlas-$ATLAS_VERSION.tar.gz apache-atlas-$ATLAS_VERSION && \
     cp -R /opt/apache-atlas-hive-hook-$ATLAS_VERSION/*  $ATLAS_INSTALL_LOCATION && \
     cp -R /opt/apache-atlas-kafka-hook-$ATLAS_VERSION/* $ATLAS_INSTALL_LOCATION && \
     mkdir -p $ATLAS_INSTALL_LOCATION/conf/patches && \
     echo $ATLAS_VERSION > $ATLAS_INSTALL_LOCATION/version.txt && \
-    rm -rf apache-atlas-$ATLAS_VERSION-hive-hook.tar.gz /apache-atlas-$ATLAS_VERSION && \
     rm -rf /opt/apache-atlas-hive-hook-$ATLAS_VERSION /opt/apache-atlas-kafka-hook-$ATLAS_VERSION && \
     rm -rf /tmp/*
-    # rm -rf /var/lib/apt/lists/*
-
-ENV HADOOP_HOME /opt/hadoop-$HADOOP_VERSION
-ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
-
-ENV HIVE_HOME /opt/apache-hive-$HIVE_VERSION-bin
-ENV HIVE_CONF_DIR $HIVE_HOME/conf
 
 ENV HBASE_CONF_DIR $ATLAS_INSTALL_LOCATION/conf/hbase
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
@@ -80,14 +59,12 @@ COPY conf/* $ATLAS_INSTALL_LOCATION/conf
 COPY stop_atlas /
 COPY docker_entrypoint.sh /
 
-ENV PATH $PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HIVE_HOME/bin
-
 RUN chmod +x /docker_entrypoint.sh && \
     chmod +x /stop_atlas && \
     mkdir /scripts && \
     mv stop_atlas /scripts
 
-ENV PATH $PATH:/scripts
+ENV PATH $PATH:$JAVA_HOME/bin:/scripts
 
 ENV HOSTNAME localhost
 ENV KAFKA_BOOTSTRAP_SERVERS localhost:9092
@@ -98,6 +75,23 @@ ENV SOLR_ZK_QUORUM localhost:2181
 ENV SOLR_HOST localhost
 ENV SOLR_PORT 8983
 
+ENV GRAPH_STORAGE_LOCK_WAIT_TIME 10000
+
+ENV CREATE_SOLR_INDICES no
+ENV SOLR_ZOOKEEPER_CONNECT_TIMEOUT 600000
+ENV SOLR_ZOOKEEPER_SESSION_TIMEOUT 600000
+
+ENV KAFKA_ZOOKEEPER_SESSION_TIMEOUT 60000
+ENV KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT 30000
+ENV KAFKA_ZOOKEEPER_SYNC_TIME 20
+ENV KAFKA_AUTO_COMMIT_INTERVAL 1000
+
+ENV AUDIT_ZOOKEEPER_SESSION_TIMEOUT 10000
+
+ENV HA_ZOOKEEPER_RETRY_SLEEPTIME 1000
+ENV HA_ZOOKEEPER_SESSION_TIMEOUT 300000
+ENV CLIENT_HA_SLEEP_INTERVAL 5000
+
 EXPOSE 21000
 
 VOLUME ["$ATLAS_INSTALL_LOCATION/conf", "$ATLAS_INSTALL_LOCATION/data", "$ATLAS_INSTALL_LOCATION/logs"]

diff --git a/conf/atlas-application.properties → docker/conf/atlas-application.properties b/conf/atlas-application.properties → docker/conf/atlas-application.properties
@@ -42,7 +42,7 @@ atlas.graph.storage.hbase.table=HBASE_TABLE
 #for distributed mode, specify zookeeper quorum here
 atlas.graph.storage.hostname=HBASE_ZK_QUORUM
 atlas.graph.storage.hbase.regions-per-server=1
-atlas.graph.storage.lock.wait-time=10000
+atlas.graph.storage.lock.wait-time=GRAPH_STORAGE_LOCK_WAIT_TIME
 
 #In order to use Cassandra as a backend, comment out the hbase specific properties above, and uncomment the
 #the following properties
@@ -92,9 +92,9 @@ atlas.graph.index.search.solr.mode=cloud
 # ZK quorum setup for solr as comma separated value. Example: 10.1.6.4:2181,10.1.6.5:2181
 atlas.graph.index.search.solr.zookeeper-url=SOLR_ZK_QUORUM
 # SolrCloud Zookeeper Connection Timeout. Default value is 60000 ms
-atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
+atlas.graph.index.search.solr.zookeeper-connect-timeout=SOLR_ZOOKEEPER_CONNECT_TIMEOUT
 # SolrCloud Zookeeper Session Timeout. Default value is 60000 ms
-atlas.graph.index.search.solr.zookeeper-session-timeout=60000
+atlas.graph.index.search.solr.zookeeper-session-timeout=SOLR_ZOOKEEPER_SESSION_TIMEOUT
 atlas.graph.index.search.solr.wait-searcher=true
 
 # Solr-specific configuration property
@@ -126,12 +126,12 @@ atlas.kafka.bootstrap.servers=KAFKA_BOOTSTRAP_SERVERS
 # Zookeeper connect URL for Kafka. Example: localhost:2181
 atlas.kafka.zookeeper.connect=KAFKA_ZK_QUORUM
 # Zookeeper session timeout. Default: 60000
-atlas.kafka.zookeeper.session.timeout.ms=60000
+atlas.kafka.zookeeper.session.timeout.ms=KAFKA_ZOOKEEPER_SESSION_TIMEOUT
 # Zookeeper connection timeout. Default: 30000
-atlas.kafka.zookeeper.connection.timeout.ms=30000
+atlas.kafka.zookeeper.connection.timeout.ms=KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT
 # Zookeeper sync time. Default: 20
-atlas.kafka.zookeeper.sync.time.ms=20
-atlas.kafka.auto.commit.interval.ms=1000
+atlas.kafka.zookeeper.sync.time.ms=KAFKA_ZOOKEEPER_SYNC_TIME
+atlas.kafka.auto.commit.interval.ms=KAFKA_AUTO_COMMIT_INTERVAL
 atlas.kafka.hook.group.id=atlas
 
 atlas.kafka.enable.auto.commit=false
@@ -215,7 +215,7 @@ atlas.server.run.setup.on.start=false
 
 #########  Entity Audit Configs  #########
 atlas.audit.hbase.tablename=apache_atlas_entity_audit
-atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.zookeeper.session.timeout.ms=AUDIT_ZOOKEEPER_SESSION_TIMEOUT
 atlas.audit.hbase.zookeeper.quorum=HBASE_ZK_QUORUM
 
 #########  High Availability Configuration ########

diff --git a/conf/atlas-env.sh → docker/conf/atlas-env.sh b/conf/atlas-env.sh → docker/conf/atlas-env.sh
diff --git a/conf/atlas-log4j.xml → docker/conf/atlas-log4j.xml b/conf/atlas-log4j.xml → docker/conf/atlas-log4j.xml
diff --git a/conf/atlas-simple-authz-policy.json → docker/conf/atlas-simple-authz-policy.json b/conf/atlas-simple-authz-policy.json → docker/conf/atlas-simple-authz-policy.json
diff --git a/conf/hadoop-metrics2.properties → docker/conf/hadoop-metrics2.properties b/conf/hadoop-metrics2.properties → docker/conf/hadoop-metrics2.properties
diff --git a/conf/users-credentials.properties → docker/conf/users-credentials.properties b/conf/users-credentials.properties → docker/conf/users-credentials.properties