Merge pull request #1836 from jcantrill/es6_into_master2

Moving feature-es6x branch in master

Author: openshift-merge-robot

Makefile

@@ -1,16 +1,11 @@
+OPERATOR_LOGGING_IMAGE_STREAM?=stable
 # Build the Docker images for Origin Aggregated Logging
 build-images:
 	hack/build-images.sh
 .PHONY: build-images
 
 test:
-	EXCLUDE_SUITE="upgrade|zzz-rsyslog|debug_level_logs|fluentd-forward|remote-syslog|out_rawtcp|zzz-duplicate-entries|read-throttling|viaq-data-model|zzzz-bulk-rejection" hack/testing/entrypoint.sh
+	OPERATOR_LOGGING_IMAGE_STREAM=$(OPERATOR_LOGGING_IMAGE_STREAM) ./hack/test-e2e.sh
 .PHONY: test
 
-test-upgrade:
-	SUITE=test-upgrade hack/testing/entrypoint.sh
-.PHONY: test-upgrade
-
-test-pre-upgrade:
-	SUITE=test-pre-upgrade hack/testing/entrypoint.sh
 .PHONY: test-pre-upgrade

curator/lib/oalconverter/convert.py

@@ -135,7 +135,7 @@ def generate_defaults(self, legacy_config):
         regex = '|'.join(
             map(lambda project: project if legacy_config[project][self.RAW_REGEX] else self.format_regex(project), projects)
             +
-            map(lambda project: self.format_regex(project), ['.searchguard', '.kibana']))
+            map(lambda project: self.format_regex(project), ['.security', '.kibana']))
         return self.generate_action(unit, count, regex, exclude=True)
 
     def validate_internal_config(self, config):
@@ -147,7 +147,7 @@ def validate_internal_config(self, config):
                 except Exception as error:
                     raise ValueError('[{0}] is not a valid regular expression. Message from compiler: {1}'.format(project, error))
             else:
-                if project == '.kibana' or project == '.operations' or project == '.defaults' or project == '.searchguard' or project == '.audit':
+                if project == '.kibana' or project == '.operations' or project == '.defaults' or project == '.security' or project == '.audit':
                     continue
                 # check project's name validity
                 if len(project) > self.projectmaxlen:

docs/com.redhat.viaq-openshift-operations.asciidoc

@@ -207,10 +207,15 @@ will give you the exact document corresponding to the record.
 
 type: keyword
 
-example: project.my-cool-project-in-lab04.748e92c2-70d7-11e9-b387-000d3af2d83b.2019.05.09
+example: container.app-write
 
-Index name in which this message will be stored within the Elasticsearch.
-The value of this field is generated based on the source of the message. 
+For Elasticsearch 6.x and later this is a name of a write index alias. The value depends on a log type
+of this message. Detailed documentation is found at
+https://github.com/openshift/enhancements/blob/master/enhancements/cluster-logging/cluster-logging-es-rollover-data-design.md#data-model
+
+For Elasticsearch 5.x and earlier an index name in which this message will be stored within the Elasticsearch.
+The value of this field is generated based on the source of the message. Example of the value
+is 'project.my-cool-project-in-lab04.748e92c2-70d7-11e9-b387-000d3af2d83b.2019.05.09'.
 
 
 

docs/com.redhat.viaq-openshift-project.asciidoc

@@ -207,10 +207,15 @@ will give you the exact document corresponding to the record.
 
 type: keyword
 
-example: project.my-cool-project-in-lab04.748e92c2-70d7-11e9-b387-000d3af2d83b.2019.05.09
+example: container.app-write
 
-Index name in which this message will be stored within the Elasticsearch.
-The value of this field is generated based on the source of the message. 
+For Elasticsearch 6.x and later this is a name of a write index alias. The value depends on a log type
+of this message. Detailed documentation is found at
+https://github.com/openshift/enhancements/blob/master/enhancements/cluster-logging/cluster-logging-es-rollover-data-design.md#data-model
+
+For Elasticsearch 5.x and earlier an index name in which this message will be stored within the Elasticsearch.
+The value of this field is generated based on the source of the message. Example of the value
+is 'project.my-cool-project-in-lab04.748e92c2-70d7-11e9-b387-000d3af2d83b.2019.05.09'.
 
 
 

docs/org.ovirt.viaq-collectd.asciidoc

@@ -202,10 +202,15 @@ will give you the exact document corresponding to the record.
 
 type: keyword
 
-example: project.my-cool-project-in-lab04.748e92c2-70d7-11e9-b387-000d3af2d83b.2019.05.09
+example: container.app-write
 
-Index name in which this message will be stored within the Elasticsearch.
-The value of this field is generated based on the source of the message. 
+For Elasticsearch 6.x and later this is a name of a write index alias. The value depends on a log type
+of this message. Detailed documentation is found at
+https://github.com/openshift/enhancements/blob/master/enhancements/cluster-logging/cluster-logging-es-rollover-data-design.md#data-model
+
+For Elasticsearch 5.x and earlier an index name in which this message will be stored within the Elasticsearch.
+The value of this field is generated based on the source of the message. Example of the value
+is 'project.my-cool-project-in-lab04.748e92c2-70d7-11e9-b387-000d3af2d83b.2019.05.09'.
 
 
 

elasticsearch/Dockerfile

@@ -1,33 +1,43 @@
-FROM elasticsearch:5.6.16
+FROM registry.svc.ci.openshift.org/openshift/origin-v4.0:base
 
 MAINTAINER OpenShift Development <[email protected]>
 
 EXPOSE 9200
 EXPOSE 9300
 USER 0
 
-ENV ES_CONF=/etc/elasticsearch/ \
+ENV ES_PATH_CONF=/etc/elasticsearch/ \
     ES_HOME=/usr/share/elasticsearch \
-    ES_VER=5.6.16 \
+    ES_VER=6.8.1.redhat-6 \
     HOME=/opt/app-root/src \
     INSTANCE_RAM=512G \
     JAVA_VER=1.8.0 \
+    JAVA_HOME=/usr/lib/jvm/jre \
     NODE_QUORUM=1 \
-    OSE_ES_VER=5.6.16.2-redhat-1 \
-    PROMETHEUS_EXPORTER_VER=5.6.16.0-redhat-1 \
+    PROMETHEUS_EXPORTER_VER=6.8.1.0-redhat-1 \
+    OPENDISTRO_VER=0.10.1.0-redhat-1 \
     PLUGIN_LOGLEVEL=INFO \
     RECOVER_AFTER_NODES=1 \
     RECOVER_EXPECTED_NODES=1 \
     RECOVER_AFTER_TIME=5m \
-    RELEASE_STREAM=prod \
     DHE_TMP_KEY_SIZE=2048 \
-    container=oci
+    RELEASE_STREAM=origin
 
-ARG OSE_ES_VER=5.6.16.2-redhat-1
-ARG OSE_ES_URL
-ARG PROMETHEUS_EXPORTER_VER=5.6.16.0-redhat-1
-ARG PROMETHEUS_EXPORTER_URL
-ARG MAVEN_REPO_URL=http://download-node-02.eng.bos.redhat.com/brewroot/repos/lpc-rhel-7-maven-build/latest/maven/
+ARG MAVEN_REPO_URL=http://download.eng.bos.redhat.com/brewroot/repos/lpc-openjdk-11-rhel-7-maven-build/latest/maven/
+
+RUN packages="java-${JAVA_VER}-openjdk-headless \
+              PyYAML  \
+              hostname \
+              openssl \
+              zip \
+              unzip" && \
+    yum install -y --setopt=tsflags=nodocs ${packages} && \
+    rpm -V ${packages} && \
+    yum clean all
+
+ADD extra-jvm.options install-es.sh /var/tmp
+
+RUN /var/tmp/install-es.sh
 
 ADD sgconfig/ ${HOME}/sgconfig/
 ADD index_templates/ ${ES_HOME}/index_templates/
@@ -37,7 +47,9 @@ ADD kibana_ui_objects/ ${ES_HOME}/kibana_ui_objects/
 ADD probe/ ${ES_HOME}/probe/
 ADD init.sh run.sh prep-install.${RELEASE_STREAM} install.sh ${HOME}/
 COPY utils/** /usr/local/bin/
-RUN ln -s /usr/local/bin/logging ${HOME}/logging
+
+ARG PROMETHEUS_EXPORTER_URL=${MAVEN_REPO_URL}org/elasticsearch/plugin/prometheus/prometheus-exporter/${PROMETHEUS_EXPORTER_VER}/prometheus-exporter-${PROMETHEUS_EXPORTER_VER}.zip
+ARG OPENDISTRO_URL=${MAVEN_REPO_URL}com/amazon/opendistroforelasticsearch/opendistro_security/${OPENDISTRO_VER}/opendistro_security-${OPENDISTRO_VER}.zip
 
 RUN ${HOME}/install.sh
 

elasticsearch/Dockerfile.centos7

@@ -6,43 +6,39 @@ EXPOSE 9200
 EXPOSE 9300
 USER 0
 
-ENV ES_CONF=/etc/elasticsearch/ \
+ENV ES_PATH_CONF=/etc/elasticsearch/ \
     ES_HOME=/usr/share/elasticsearch \
-    ES_VER=5.6.16 \
+    ES_VER=6.8.1 \
     HOME=/opt/app-root/src \
     INSTANCE_RAM=512G \
     JAVA_VER=1.8.0 \
     JAVA_HOME=/usr/lib/jvm/jre \
     NODE_QUORUM=1 \
-    OSE_ES_VER=5.6.16.2 \
-    PROMETHEUS_EXPORTER_VER=5.6.16.0 \
+    PROMETHEUS_EXPORTER_VER=6.8.1.0 \
+    OPENDISTRO_VER=0.10.1.0 \
     PLUGIN_LOGLEVEL=INFO \
     RECOVER_AFTER_NODES=1 \
     RECOVER_EXPECTED_NODES=1 \
     RECOVER_AFTER_TIME=5m \
     DHE_TMP_KEY_SIZE=2048 \
     RELEASE_STREAM=origin
 
-ARG OSE_ES_VER=5.6.16.2
-ARG SG_VER=5.6.16-19.3
+ARG MAVEN_REPO_URL=https://repo1.maven.org/maven2/
 
-LABEL io.k8s.description="Elasticsearch container for EFK aggregated logging storage" \
-      io.k8s.display-name="Elasticsearch ${ES_VER}" \
-      io.openshift.expose-services="9200:https, 9300:https" \
-      io.openshift.tags="logging,elk,elasticsearch" \
-      architecture=x86_64 \
-      name="openshift3/logging-elasticsearch"
-
-ADD elasticsearch.repo /etc/yum.repos.d/elasticsearch.repo
-# install the RPMs in a separate step so it can be cached
-RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
-    yum install -y --setopt=tsflags=nodocs --nogpgcheck \
-                java-${JAVA_VER}-openjdk-headless \
-                elasticsearch-${ES_VER} \
-                openssl \
-                PyYAML && \
+RUN packages="java-${JAVA_VER}-openjdk-headless \
+              PyYAML  \
+              hostname \
+              openssl \
+              zip \
+              unzip" && \
+    yum install -y --setopt=tsflags=nodocs ${packages} && \
+    rpm -V ${packages} && \
     yum clean all
 
+ADD extra-jvm.options install-es.sh /var/tmp/
+
+RUN /var/tmp/install-es.sh
+
 ADD sgconfig/ ${HOME}/sgconfig/
 ADD index_templates/ ${ES_HOME}/index_templates/
 ADD index_patterns/ ${ES_HOME}/index_patterns/
@@ -52,13 +48,12 @@ ADD probe/ ${ES_HOME}/probe/
 ADD init.sh run.sh prep-install.${RELEASE_STREAM} install.sh ${HOME}/
 COPY utils/** /usr/local/bin/
 
-ARG OSE_ES_URL
 ARG PROMETHEUS_EXPORTER_URL=https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/${PROMETHEUS_EXPORTER_VER}/prometheus-exporter-${PROMETHEUS_EXPORTER_VER}.zip
-ARG SG_URL
+#LEAVE FOR FUTURE
+#ARG OPENDISTRO_URL=${MAVEN_REPO_URL}com/amazon/opendistroforelasticsearch/opendistro_security/${OPENDISTRO_VER}/opendistro_security-${OPENDISTRO_VER}.zip
+ARG OPENDISTRO_URL=https://github.com/jcantrill/security/releases/download/v${OPENDISTRO_VER}/opendistro_security-${OPENDISTRO_VER}.zip
 
-RUN ln -s /usr/local/bin/logging ${HOME}/logging && \
-    ${HOME}/install.sh && \
-    rm -rf /tmp/lib
+RUN ${HOME}/install.sh
 
 WORKDIR ${HOME}
 USER 1000

elasticsearch/elasticsearch.repo

@@ -0,0 +1,5 @@
+-XX:+UnlockExperimentalVMOptions
+-XX:+UseCGroupMemoryLimitForHeap
+-XX:MaxRAMFraction=2
+-XX:InitialRAMFraction=2
+-XX:MinRAMFraction=2

elasticsearch/extra-jvm.options

@@ -2,10 +2,14 @@
   "aliases": {
     ".all": {}
   },
+  "index_patterns": [
+    "infra-*",
+    "audit.infra-*"
+  ],
   "mappings": {
-    "_default_": {
+    "_doc": {
       "_meta": {
-        "version": "2019.12.16.0"
+        "version": "2020.01.23.0"
       },
       "date_detection": false,
       "dynamic_templates": [
@@ -970,6 +974,5 @@
   "order": 10,
   "settings": {
     "index.refresh_interval": "5s"
-  },
-  "template": ".operations.*"
+  }
 }