[meta] refactor kitchen tests (elastic#765)

* [meta] refactor kitchen tests

This commit update kitchen tests to match important feature to test:
- default: test a deployment with all default values
- license: test a deployment with a license and default values
- trial: test a deployment with security enabled and trial license
- oss: test a deployment with oss version and default values (6.x only)
- upgrade: test an upgrade from a previous version
- oss-upgrade: test an upgrade with oss version (6.x only)
- oss-to-default-upgrade: test an upgrade from previous oss to latest
default version
- custom_config: test a deployment with custom config files
- security: test a deployment with security enabled

* [meta] fix license test

Author: jmlrt

.kitchen.yml

@@ -149,31 +149,39 @@ platforms:
       use_sudo: false
 
 suites:
+  - name: custom-config
+    provisioner:
+      idempotency_test: true
+      playbook: test/integration/custom-config.yml
+  - name: default
+    provisioner:
+      playbook: test/integration/default.yml
+      idempotency_test: true
+  - name: license
+    provisioner:
+      playbook: test/integration/license.yml
+      idempotency_test: true
   - name: oss
     provisioner:
       idempotency_test: true
       playbook: test/integration/oss.yml
-  - name: oss-upgrade
+  - name: oss-to-default-upgrade
     provisioner:
-      playbook: test/integration/oss-upgrade.yml
+      playbook: test/integration/oss-to-default-upgrade.yml
       idempotency_test: false
-  - name: oss-to-xpack-upgrade
+  - name: oss-upgrade
     provisioner:
-      playbook: test/integration/oss-to-xpack-upgrade.yml
+      playbook: test/integration/oss-upgrade.yml
       idempotency_test: false
-  - name: xpack
+  - name: security
     provisioner:
-      playbook: test/integration/xpack.yml
+      playbook: test/integration/security.yml
       idempotency_test: true
-  - name: xpack-upgrade
+  - name: trial
     provisioner:
-      playbook: test/integration/xpack-upgrade.yml
-      idempotency_test: false
-  - name: issue-test
-    provisioner:
-      playbook: test/integration/issue-test.yml
-      idempotency_test: false
-  - name: xpack-upgrade-trial
+      playbook: test/integration/trial.yml
+      idempotency_test: false # es_xpack_trial is not idempotent currently
+  - name: upgrade
     provisioner:
-      playbook: test/integration/xpack-upgrade-trial.yml
+      playbook: test/integration/upgrade.yml
       idempotency_test: false

Makefile

@@ -2,7 +2,7 @@ default: build
 
 SHELL:=/bin/bash -eux
 export VERSION := 7.x
-PATTERN := xpack-ubuntu-1604
+PATTERN := default-ubuntu-1804
 
 .PHONY: converge verify test login destroy list
 

README.md

@@ -97,7 +97,7 @@ This playbook uses [Kitchen](https://kitchen.ci/) for CI and local testing.
 ### Running the tests
 
 * Ensure you have checked out this repository to `elasticsearch`, not `ansible-elasticsearch`.
-* If you don't have a Gold or Platinum license to test with you can run the trial versions of the `xpack-upgrade` and `issue-test` suites by appending `-trial` to the `PATTERN` variable.
+* If you don't have a Gold or Platinum license to test with you can run the trial versions of the `xpack-upgrade` suites by appending `-trial` to the `PATTERN` variable.
 * You may need to explicitly specify `VERSION=7.x` if some suites are failing.
 
 Install the ruby dependencies with bundler

test/integration/custom-config.yml

@@ -0,0 +1,11 @@
+---
+- name: Setup Elasticsearch with custom config files
+  hosts: localhost
+  post_tasks:
+    - include: elasticsearch/test/integration/debug.yml
+  roles:
+    - elasticsearch
+  vars:
+    es_config_default: "test/integration/files/custom_config/elasticsearch"
+    es_config_log4j2: "test/integration/files/custom_config/log4j2.properties"
+    es_config_jvm: "test/integration/files/custom_config/jvm.options"

test/integration/issue-test/issue-test.yml renamed to test/integration/custom-config/custom_config.yml

@@ -1,9 +1,9 @@
-require 'xpack_upgrade_spec'
+require 'custom_config_spec'
 require 'shared_spec'
 require 'json'
 vars = JSON.parse(File.read('/tmp/vars.json'))
 
-describe 'Xpack upgrade Tests' do
+describe 'Custom Config Tests' do
+  include_examples 'custom_config::init', vars
   include_examples 'shared::init', vars
-  include_examples 'xpack_upgrade::init', vars
 end

test/integration/xpack-upgrade/serverspec/default_spec.rb renamed to test/integration/custom-config/serverspec/default_spec.rb

@@ -0,0 +1,7 @@
+---
+- name: Setup Elasticsearch using default configuration
+  hosts: localhost
+  post_tasks:
+    - include: elasticsearch/test/integration/debug.yml
+  roles:
+    - elasticsearch

test/integration/default.yml

@@ -2,6 +2,6 @@
 require 'json'
 vars = JSON.parse(File.read('/tmp/vars.json'))
 
-describe 'Xpack upgrade Tests' do
+describe 'default tests' do
   include_examples 'shared::init', vars
 end

test/integration/oss-to-xpack-upgrade/oss-to-xpack-upgrade.yml renamed to test/integration/default/default.yml

@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'shared_spec'
 
-shared_examples 'oss::init' do |vars|
+shared_examples 'custom_config::init' do |vars|
   describe file("/etc/elasticsearch/log4j2.properties") do
     it { should be_file }
     it { should be_owned_by 'root' }

test/integration/xpack/serverspec/default_spec.rb renamed to test/integration/default/serverspec/default_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+require 'json'
+require 'pathname'
+vars = JSON.parse(File.read('/tmp/vars.json'))
+
+es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}"
+username = vars['es_api_basic_auth_username']
+password = vars['es_api_basic_auth_password']
+es_keystore = Pathname.new(vars['es_ssl_keystore']).basename.to_s
+es_truststore = Pathname.new(vars['es_ssl_truststore']).basename.to_s
+
+if vars['es_major_version'] == '7.x'
+  es_license_api = "_license"
+else
+  es_license_api = "_xpack/license"
+end
+
+shared_examples 'license::init' do |vars|
+  describe 'License check' do
+    result = curl_json("#{es_api_url}/#{es_license_api}", username=username, password=password)
+    it 'should list the license issued by Elastic' do
+      expect(result['license']['status']).to eq('active')
+      expect(result['license']['type']).to eq('trial')
+      expect(result['license']['issued_to']).to eq('Elastic - INTERNAL (non-production environments)')
+    end
+  end
+end

test/integration/helpers/serverspec/oss_spec.rb renamed to test/integration/helpers/serverspec/custom_config_spec.rb

@@ -15,7 +15,7 @@
   es_security_api = "_xpack/security"
 end
 
-shared_examples 'xpack_upgrade::init' do |vars|
+shared_examples 'security::init' do |vars|
   #Test users file, users_roles and roles.yml
   describe file("/etc/elasticsearch/users_roles") do
     it { should be_owned_by 'root' }
@@ -39,12 +39,9 @@
   describe file("/etc/elasticsearch/elasticsearch.yml") do
     if vars['es_major_version'] == '7.x'
       it { should contain 'security.authc.realms.file.file1.order: 0' }
-      it { should contain 'security.authc.realms.native.native1.order: 1' }
     else
       it { should contain 'security.authc.realms.file1.order: 0' }
       it { should contain 'security.authc.realms.file1.type: file' }
-      it { should contain 'security.authc.realms.native1.order: 1' }
-      it { should contain 'security.authc.realms.native1.type: native' }
     end
     it { should contain 'xpack.security.transport.ssl.enabled: true' }
     it { should contain 'xpack.security.http.ssl.enabled: true' }
@@ -56,18 +53,10 @@
   describe file("/etc/elasticsearch/role_mapping.yml") do
     it { should be_owned_by 'root' }
     it { should contain 'power_user:' }
-    it { should contain '- cn=admins,dc=example,dc=com' }
     it { should contain 'user:' }
-    it { should contain '- cn=admins,dc=example,dc=com' }
   end
 
   #check accounts are correct i.e. we can auth and they have the correct roles
-  describe 'kibana4_server access check' do
-    it 'should be reported as version '+vars['es_version'] do
-      expect(curl_json(es_api_url, username='kibana4_server', password='changeMe')['version']['number']).to eq(vars['es_version'])
-    end
-  end
-
   describe 'security users' do
     result = curl_json("#{es_api_url}/#{es_security_api}/user", username=username, password=password)
     it 'should have the elastic user' do
@@ -80,24 +69,13 @@
       expect(result['kibana']['roles']).to eq(['kibana_system'])
       expect(result['kibana']['enabled']).to eq(true)
     end
-    it 'should have the kibana_server user' do
-      expect(result['kibana4_server']['username']).to eq('kibana4_server')
-      expect(result['kibana4_server']['roles']).to eq(['kibana4_server'])
-      expect(result['kibana4_server']['enabled']).to eq(true)
-    end
     it 'should have the logstash user' do
       expect(result['logstash_system']['username']).to eq('logstash_system')
       expect(result['logstash_system']['roles']).to eq(['logstash_system'])
       expect(result['logstash_system']['enabled']).to eq(true)
     end
   end
 
-  describe 'logstash_system access check' do
-    it 'should be reported as version '+vars['es_version'] do
-      expect(curl_json(es_api_url, username='logstash_system', password='aNewLogstashPassword')['version']['number']).to eq(vars['es_version'])
-    end
-  end
-
   describe 'SSL certificate check' do
     certificates = curl_json("#{es_api_url}/_ssl/certificates", username=username, password=password)
     it 'should list the keystore file' do

test/integration/helpers/serverspec/issue_test_spec.rb

@@ -0,0 +1,57 @@
+---
+- name: Elasticsearch Xpack HTTP different keystore and truststore with password
+  hosts: localhost
+  post_tasks:
+    - include: elasticsearch/test/integration/debug.yml
+  roles:
+    - elasticsearch
+  vars:
+    es_config_6x:
+      xpack.security.authc.realms.file1.order: 0
+      xpack.security.authc.realms.file1.type: file
+    es_config_7x:
+      xpack.security.authc.realms.file.file1.order: 0
+    es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}"
+    es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
+    es_api_basic_auth_username: elastic
+    es_api_basic_auth_password: changeme
+    es_api_sleep: 5
+    es_enable_http_ssl: true
+    es_enable_transport_ssl: true
+    es_ssl_keystore: "test/integration/files/certs/keystore-password.p12"
+    es_ssl_truststore: "test/integration/files/certs/truststore-password.p12"
+    es_ssl_keystore_password: password1
+    es_ssl_truststore_password: password2
+    es_validate_certs: no
+    es_users:
+      file:
+        es_admin:
+          password: changeMe
+          roles:
+            - admin
+        testUser:
+          password: changeMeAlso!
+          roles:
+            - power_user
+            - user
+    es_roles:
+      file:
+        admin:
+          cluster:
+            - all
+          indices:
+            - names: '*'
+              privileges:
+                - all
+        power_user:
+          cluster:
+            - monitor
+          indices:
+            - names: '*'
+              privileges:
+                - all
+        user:
+          indices:
+            - names: '*'
+              privileges:
+                - read

test/integration/helpers/serverspec/license_spec.rb

@@ -0,0 +1,9 @@
+require 'license_spec'
+require 'shared_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
+
+describe 'license tests' do
+  include_examples 'shared::init', vars
+  include_examples 'license::init', vars
+end

test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb

@@ -1,5 +1,5 @@
 ---
-- name: Standard test for single node setup. Tests idempotence.
+- name: Deploy Elasticsearch OSS version
   hosts: localhost
   post_tasks:
     - include: elasticsearch/test/integration/debug.yml
@@ -8,14 +8,12 @@
   vars:
     es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade
     oss_version: true
-    es_heap_size: "1g"
 
-- name: Standard test for single node setup. Tests idempotence.
+- name: Upgrade to Elasticsearch default version
   hosts: localhost
   post_tasks:
     - include: elasticsearch/test/integration/debug.yml
   roles:
     - elasticsearch
   vars:
     oss_version: false
-    es_heap_size: "1g"

test/integration/helpers/serverspec/oss_upgrade_spec.rb

@@ -0,0 +1,7 @@
+require 'shared_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
+
+describe 'oss to default upgrade tests' do
+  include_examples 'shared::init', vars
+end