Merge pull request skydive-project#2327 from eranra/add_flow_export

add support for flow export
datadope-io · Jan 26, 2021 · 8055a08 · 8055a08
2 parents 8a76bba + e66df73
commit 8055a08
Show file tree

Hide file tree

Showing 4 changed files with 104 additions and 1 deletion.
diff --git a/contrib/openshift/README.md b/contrib/openshift/README.md
@@ -23,14 +23,22 @@ oc adm policy add-cluster-role-to-user cluster-reader -z default
 ```
 
 
-####  Install from OpenShift template
+####  Deploy skydive analyzer and agents from OpenShift template
 
 ```
 # adjust VERSION for the current version - for example: v0.20.1 or master
 VERSION=master
 oc process -f https://raw.githubusercontent.com/skydive-project/skydive/${VERSION}/contrib/openshift/skydive-template.yaml | oc apply -f -
 ```
 
+####  Deploy flow exporter from OpenShift template
+
+```
+# adjust VERSION for the current version - for example: v0.20.1 or master
+VERSION=master
+oc process -f https://raw.githubusercontent.com/skydive-project/skydive/${VERSION}/contrib/openshift/skydive-flow-exporter-template.yaml | oc apply -f -
+```
+
 #### Check that everything is working and created:
 
  - Overall status:  

diff --git a/contrib/openshift/deploy_skydive_ocp.sh b/contrib/openshift/deploy_skydive_ocp.sh
@@ -58,6 +58,18 @@ deploy_skydive() {
   oc process -f $DEPLOY_YAML | oc apply -f -
 }
 
+# deoploy skydive-flow-exporter
+deploy_skydive-flow-exporter() {
+  if [ $USE_LOCAL_YAML = "true" ] ; then
+    DEPLOY_YAML=skydive-flow-exporter-template.yaml
+  else
+    DEPLOY_YAML=https://raw.githubusercontent.com/skydive-project/skydive/${VERSION}/contrib/openshift/skydive-flow-exporter-template.yaml
+  fi
+
+  echo -e "\nDeploying $DEPLOY_YAML\n"
+  oc process -f $DEPLOY_YAML | oc apply -f -
+}
+
 # print pod status
 print_pods_status () {
   echo -e "\n"
@@ -78,6 +90,7 @@ main() {
   create_skydive_project
   set_credentials
   deploy_skydive
+  deploy_skydive-flow-exporter
   print_pods_status
   print_usage_insturctions
 }

diff --git a/contrib/openshift/skydive-flow-exporter-template.yaml b/contrib/openshift/skydive-flow-exporter-template.yaml
@@ -0,0 +1,80 @@
+apiVersion: v1
+kind: Template
+metadata:
+  name: skydive-flow-exporter
+objects:
+# Skydive FlowExporter (Deployment)
+#
+# Containers included in this pod::
+# (1) Skydive Flow Exporter - Flow exporter (connects to Skydive Analyzer)
+# (2) Minio - Object storage destination for flows
+# (3) minio-mc - create bucket in Minio (init only container)
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: skydive-flow-exporter
+  spec:
+    replicas: 1
+    selector:
+      matchLabels:
+        app: skydive-flow-exporter
+        tier: flow-exporter
+    template:
+      metadata:
+        labels:
+          app: skydive-flow-exporter
+          tier: flow-exporter
+      spec:
+        containers:
+        - image: skydive/skydive-flow-exporter
+          imagePullPolicy: Always
+          name: skydive-flow-exporter
+          env:
+          - name: SKYDIVE_LOGGING_LEVEL
+            value: ${SKYDIVE_LOGGING_LEVEL}
+          - name: SKYDIVE_PIPELINE_SUBSCRIBER_URL
+            value: ws://skydive-analyzer:8082/ws/subscriber/flow
+        - image: minio/minio
+          imagePullPolicy: Always
+          name: minio
+          args:
+          - server
+          - /tmp/data
+          env:
+          - name: MINIO_ACCESS_KEY
+            value: user
+          - name: MINIO_SECRET_KEY
+            value: password
+        - image: minio/mc
+          imagePullPolicy: Always
+          name: minio-mc
+          command: ["/bin/sh"]
+          securityContext:
+            privileged: true
+            allowPrivilegeEscalation: true
+          args:
+          - -c
+          - >
+            set -x;
+            while [ $(curl -sw '%{http_code}' "http://localhost:9000" -o /dev/null) -ne 403 ]; do
+              echo "Waiting for minio server to start (autentication failed is expected)";
+              sleep 10;
+            done;
+            echo "Creating bucket";
+            /usr/bin/mc config host add local http://localhost:9000 user password --api S3v4;
+            /usr/bin/mc rm -r --force local/bucket;
+            /usr/bin/mc mb local/bucket;
+            echo "Done.";
+            while true; do
+              echo "====> Listing files in bucket (every 60 seconds)";
+              echo "====> Note: Make sure to working Skydive flow captures";
+              echo "";
+              /usr/bin/mc ls -r local/bucket | tail -n 5;
+              sleep 60;
+            done;
+parameters:
+- description: Loglevel of Skydive-flow-exporter
+  displayName: Loglevel
+  name: SKYDIVE_LOGGING_LEVEL
+  required: true
+  value: INFO
diff --git a/contrib/openshift/skydive-template.yaml b/contrib/openshift/skydive-template.yaml
@@ -45,6 +45,8 @@ objects:
             value: '{"theme":"light","k8s_enabled":"true","topology": {"favorites":{"infrastructure":"G.V().Has(\"Manager\", Without(\"k8s\"))","kubernetes":"G.V().Has(\"Manager\",\"k8s\")"},"default_filter":"infrastructure"}}'
           - name: SKYDIVE_ANALYZER_TOPOLOGY_FABRIC
             value: 'TOR1->*[Type=host]/eth0 TOR1->*[Type=host]/eth1 TOR1->*[Type=host]/ens1 TOR1->*[Type=host]/ens2 TOR1->*[Type=host]/ens3'
+          - name: SKYDIVE_ANALYZER_STARTUP_CAPTURE_GREMLIN
+            value: 'G.V().Has("Type" , "device", "Name", Regex("eth0|ens1|ens2|ens3"))'
           - name: SKYDIVE_ETCD_LISTEN
             value: 0.0.0.0:12379
           - name: SKYDIVE_LOGGING_LEVEL