local /data config generation for docker use (#673)

* local /data generation (wip) * More work with /data generation for docker use
AtlasOfLivingAustralia · Aug 2, 2023 · 8d0815f · 8d0815f
1 parent 6bcc6b0
commit 8d0815f
Show file tree

Hide file tree

Showing 20 changed files with 514 additions and 71 deletions.
diff --git a/.gitignore b/.gitignore
@@ -37,3 +37,4 @@ vagrant/centos/.vagrant/machines/default/virtualbox/id
 *.retry
 
 ansible/roles/alerts/tasks/main.yml
+utils/docopts
diff --git a/ansible/ecodata.yml b/ansible/ecodata.yml
@@ -6,7 +6,7 @@
     es_heap_dump_path: "/data/"
     es_api_host: "{{ elasticsearch_host | default('localhost') }}"
     es_config:
-      network.host: "{{ elasticsearch_host | default('localhost') }}"
+      network.host: "{{ (ecodata_elasticsearch_host | default(elasticsearch_host)) | default('localhost') }}"
       discovery.type: "single-node"
   roles:
     - common
@@ -16,4 +16,4 @@
     - webserver
     - { role: geoserver, log_filename: "geoserver" , when: install_geoserver | bool}
     - { role: ansible-elasticsearch, es_templates: false, es_instance_name: 'ecodata-elasticsearch', es_data_dirs: ['/data/elasticsearch'], tags: ['elasticsearch'], when: install_elasticsearch | bool }
-    - ecodata
+    - ecodata
diff --git a/ansible/events.yml b/ansible/events.yml
@@ -10,17 +10,17 @@
   vars:
     es_version: "7.17.7"
     es_heap_dump_path: "/data/"
-    es_api_host: "{{ elasticsearch_host | default('localhost') }}"
+    es_api_host: "{{ (events_elasticsearch_host | default(elasticsearch_host) | default('localhost') }}"
     es_heap_size: "8g"
     es_config:
       cluster.name: "{{ elasticsearch_cluster_name | default('extended-data') }}"
       node.name: "{{ elasticsearch_host | default('localhost') }}"
-      network.host: "{{ elasticsearch_network_host }}"
-      discovery.seed_hosts: "{{ elasticsearch_seed_hosts }}"
+      network.host: "{{ (events_elasticsearch_network_host | default(elasticsearch_network_host) }}"
+      discovery.seed_hosts: "{{ (events_elasticsearch_seed_host | default(elasticsearch_seed_host) }}"
       bootstrap.memory_lock: false
       cluster.initial_master_nodes: "{{ elasticsearch_cluster_initial_master_nodes }}"
       indices.memory.index_buffer_size: "{{ elasticsearch_index_buffer_size | default('40%') }}"
   roles:
     - common
     - java
-    - { role: ansible-elasticsearch, es_templates: false, es_instance_name: 'ecodata-elasticsearch', es_data_dirs: ['/data/elasticsearch'], tags: ['elasticsearch'], when: install_elasticsearch | bool }
+    - { role: ansible-elasticsearch, es_templates: false, es_instance_name: 'ecodata-elasticsearch', es_data_dirs: ['/data/elasticsearch'], tags: ['elasticsearch'], when: install_elasticsearch | bool }
diff --git a/ansible/roles/apikey/tasks/main.yml b/ansible/roles/apikey/tasks/main.yml
@@ -21,8 +21,8 @@
   file:
     path: "{{item}}"
     state: directory
-    owner: apikey
-    group: apikey
+    owner: "{{ apikey_user | default('apikey') }}"
+    group: "{{ apikey_user | default('apikey') }}"
   with_items:
     - "{{data_dir}}/apikey"
     - "{{data_dir}}/apikey/config"
@@ -35,8 +35,8 @@
 - name: set data ownership
   file:
     path: "{{data_dir}}/apikey"
-    owner: apikey
-    group: apikey
+    owner: "{{ apikey_user | default('apikey') }}"
+    group: "{{ apikey_user | default('apikey') }}"
     recurse: true
   tags:
     - properties
@@ -46,8 +46,8 @@
   template:
     src: apikey-config.yml
     dest: "{{data_dir}}/apikey/config/apikey-config.yml"
-    owner: apikey
-    group: apikey
+    owner: "{{ apikey_user | default('apikey') }}"
+    group: "{{ apikey_user | default('apikey') }}"
   notify:
     - restart apikey
   when: apikey_version is version('1.7.0', '>=')
@@ -59,8 +59,8 @@
   template:
     src: apikey-config-pre-1.7.0.yml
     dest: "{{data_dir}}/apikey/config/apikey-config.yml"
-    owner: apikey
-    group: apikey
+    owner: "{{ apikey_user | default('apikey') }}"
+    group: "{{ apikey_user | default('apikey') }}"
   notify:
     - restart apikey
   when: apikey_version is version('1.7.0', '<')
@@ -72,8 +72,8 @@
   template:
     src: logback.xml
     dest: "{{data_dir}}/apikey/config/logback.xml"
-    owner: apikey
-    group: apikey
+    owner: "{{ apikey_user | default('apikey') }}"
+    group: "{{ apikey_user | default('apikey') }}"
   tags:
     - properties
     - apikey

diff --git a/ansible/roles/cas-management/tasks/main.yml b/ansible/roles/cas-management/tasks/main.yml
@@ -11,8 +11,8 @@
   file:
     path: "{{item}}"
     state: directory
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   with_items:
     - "{{data_dir}}/cas-management"
     - "{{data_dir}}/cas-management/config"
@@ -25,8 +25,8 @@
 - name: set data ownership
   file:
     path: "{{data_dir}}/cas-management"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
     recurse: true
   tags:
     - properties
@@ -37,8 +37,8 @@
 #   template:
 #     src: application.properties
 #     dest: "{{data_dir}}/cas-management/config/application.properties"
-#     owner: cas
-#     group: cas
+#     owner: "{{ cas_user | default('cas') }}"
+#     group: "{{ cas_user | default('cas') }}"
 #   notify:
 #     - restart cas-management
 #   tags:
@@ -49,8 +49,8 @@
   template:
     src: application.yml
     dest: "{{data_dir}}/cas-management/config/application.yml"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   when: cas_management_version is version('6', '>=')
   notify:
     - restart cas-management
@@ -62,8 +62,8 @@
   template:
     src: application.properties
     dest: "{{data_dir}}/cas-management/config/application.properties"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   when: cas_management_version is version('6', '<')
   notify:
     - restart cas-management
@@ -75,8 +75,8 @@
   template:
     src: log4j2-management.xml
     dest: "{{data_dir}}/cas-management/config/log4j2.xml"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   tags:
     - properties
     - cas-management
@@ -90,8 +90,8 @@
       value: /data/cas-management/config/
     log_config_filename: 'log4j2.xml'
     service_name: "cas-management"
-    service_owner: 'cas'
-    service_group: 'cas'
+    service_owner: "{{ cas_user | default('cas') }}"
+    service_group: "{{ cas_user | default('cas') }}"
     use_openjdk11: "{{ cas_management_version is version('6', '>=') }}"
     use_openjdk8: "{{ cas_management_version is version('6', '<') }}"
     java_headless: True

diff --git a/ansible/roles/cas5/tasks/main.yml b/ansible/roles/cas5/tasks/main.yml
@@ -21,8 +21,8 @@
   file:
     path: "{{item}}"
     state: directory
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   with_items:
     - "{{data_dir}}/cas"
     - "{{data_dir}}/cas/config"
@@ -35,8 +35,8 @@
 - name: set data ownership
   file:
     path: "{{data_dir}}/cas"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
     recurse: true
   tags:
     - properties
@@ -46,8 +46,8 @@
   template:
     src: application.yml
     dest: "{{data_dir}}/cas/config/application.yml"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   when: cas_version is version('6', '>=')
   notify:
     - restart cas
@@ -59,8 +59,8 @@
   template:
     src: application-pre-6.yml
     dest: "{{data_dir}}/cas/config/application.yml"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   when: cas_version is version('6', '<')
   notify:
     - restart cas
@@ -72,8 +72,8 @@
   template:
     src: log4j2.xml
     dest: "{{data_dir}}/cas/config/log4j2.xml"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   tags:
     - properties
     - cas
@@ -82,8 +82,8 @@
   template:
     src: pwe.properties
     dest: "{{data_dir}}/cas/config/pwe.properties"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
   notify:
     - restart cas
   tags:
@@ -95,8 +95,8 @@
   copy:
     src:  "{{ inventory_dir }}/{{ cas_jwks_file | default('keystore.jwks') }}"
     dest: "{{data_dir}}/cas/keystore.jwks"
-    owner: cas
-    group: cas
+    owner: "{{ cas_user | default('cas') }}"
+    group: "{{ cas_user | default('cas') }}"
     mode: u=rw,g=,o=
   notify:
     - restart cas
@@ -150,7 +150,7 @@
     - db
 
 - name: ensure target sql directory exist
-  file: path={{item}} state=directory owner=cas group=cas
+  file: path={{item}} state=directory owner="{{ cas_user | default('cas') }}" group="{{ cas_user | default('cas') }}"
   with_items:
     - "{{data_dir}}/cas/setup"
   when: cas_first_admin_email is defined and cas_first_admin_bcrypt_password is defined and cas_first_admin_temp_auth_key is defined

diff --git a/ansible/roles/doi-service/tasks/main.yml b/ansible/roles/doi-service/tasks/main.yml
@@ -28,8 +28,8 @@
   file:
     path: "{{item}}"
     state: directory
-    owner: doi-service
-    group: doi-service
+    owner: "{{ doi_user | default('doi-service') }}"
+    group: "{{ doi_user | default('doi-service') }}"
   with_items:
     - "{{data_dir}}/doi-service"
     - "{{data_dir}}/doi-service/config"
@@ -41,8 +41,8 @@
 - name: set data ownership
   file:
     path: "{{data_dir}}/doi-service"
-    owner: doi-service
-    group: doi-service
+    owner: "{{ doi_user | default('doi-service') }}"
+    group: "{{ doi_user | default('doi-service') }}"
     recurse: true
   tags:
     - properties
@@ -52,8 +52,8 @@
   template:
     src: doi-service-config.yml
     dest: "{{data_dir}}/doi-service/config/doi-service-config.yml"
-    owner: doi-service
-    group: doi-service
+    owner: "{{ doi_user | default('doi-service') }}"
+    group: "{{ doi_user | default('doi-service') }}"
   notify:
     - restart doi-service
   tags:
@@ -64,8 +64,8 @@
   template:
     src: logback.groovy
     dest: "{{data_dir}}/doi-service/config/logback.groovy"
-    owner: doi-service
-    group: doi-service
+    owner: "{{ doi_user | default('doi-service') }}"
+    group: "{{ doi_user | default('doi-service') }}"
   notify:
     - restart doi-service
   tags:

diff --git a/ansible/roles/ecodata/handlers/main.yml b/ansible/roles/ecodata/handlers/main.yml
@@ -1,2 +1,4 @@
 - name: restart ecodata
   service: name=ecodata state=restarted enabled="yes"
+  when:
+  - skip_handlers | default("false") | bool == false
diff --git a/ansible/roles/ecodata/tasks/main.yml b/ansible/roles/ecodata/tasks/main.yml
@@ -20,6 +20,7 @@
   with_items:
     - "{{data_dir}}/mongodb"
   tags:
+    - mongodb-org
     - properties
     - ecodata
 

diff --git a/ansible/roles/events/handlers/main.yml b/ansible/roles/events/handlers/main.yml
@@ -1,2 +1,4 @@
 - name: restart graphql_service
-  service: name={{ env }}-graphql state=restarted enabled="yes"
+  service: name={{ env }}-graphql state=restarted enabled="yes"
+  when:
+  - skip_handlers | default("false") | bool == false
diff --git a/ansible/roles/events/tasks/main.yaml b/ansible/roles/events/tasks/main.yaml
@@ -61,6 +61,7 @@
     - events
     - update-images
     - docker-config
+    - properties
 
 - name: Copy docker YAML files to {{ data_dir }}
   template: src={{ item }} dest={{ data_dir }}/{{ env }}/{{ item }}
@@ -76,6 +77,7 @@
     - events
     - update-images
     - docker-config
+    - properties
 
 - name: Copy service scripts to /usr/bin
   template: src={{ item }} dest=/usr/bin/{{ env }}-{{ item }} mode=777

diff --git a/ansible/roles/geoserver/tasks/ecodata-geoserver.yml b/ansible/roles/geoserver/tasks/ecodata-geoserver.yml
@@ -80,6 +80,13 @@
   tags:
     - geoserver
 
+# Used for /data local generation
+- name: set ownership of geoserver data
+  shell: "chown -R {{ geoserver_user }}:{{ geoserver_user }} {{data_dir}}/geoserver*"
+  when: geoserver_user is defined
+  tags:
+    - geoserver
+
 #
 # Override the default memory settings for Tomcat to increase heap space and change the garbage collector
 #

diff --git a/ansible/roles/geoserver/tasks/geoserver.yml b/ansible/roles/geoserver/tasks/geoserver.yml
@@ -254,6 +254,14 @@
     - geoserver
     - properties
 
+# Used for /data local generation
+- name: set ownership of geoserver data
+  shell: "chown -R {{ geoserver_user }}:{{ geoserver_user }} {{data_dir}}/geoserver*"
+  when: geoserver_user is defined
+  tags:
+    - geoserver
+    - properties
+
 - name: Running geoserver.sh
   command: "{{data_dir}}/geoserver.sh"
   args:
Original file line number	Diff line number	Diff line change
Expand Up		@@ -37,3 +37,4 @@ vagrant/centos/.vagrant/machines/default/virtualbox/id
		*.retry

		ansible/roles/alerts/tasks/main.yml
		utils/docopts