From 8d0815f6a7c6d771951350e38eebd144d1216145 Mon Sep 17 00:00:00 2001 From: vjrj Date: Wed, 2 Aug 2023 17:34:54 +0200 Subject: [PATCH] local /data config generation for docker use (#673) * local /data generation (wip) * More work with /data generation for docker use --- .gitignore | 1 + ansible/ecodata.yml | 4 +- ansible/events.yml | 8 +- ansible/roles/apikey/tasks/main.yml | 20 +- ansible/roles/cas-management/tasks/main.yml | 28 +-- ansible/roles/cas5/tasks/main.yml | 30 +-- ansible/roles/doi-service/tasks/main.yml | 16 +- ansible/roles/ecodata/handlers/main.yml | 2 + ansible/roles/ecodata/tasks/main.yml | 1 + ansible/roles/events/handlers/main.yml | 4 +- ansible/roles/events/tasks/main.yaml | 2 + .../geoserver/tasks/ecodata-geoserver.yml | 7 + ansible/roles/geoserver/tasks/geoserver.yml | 8 + ansible/roles/image-service/tasks/main.yml | 12 +- .../templates/config/image-service-config.yml | 2 +- ansible/roles/pipelines/tasks/main.yml | 7 + ansible/roles/userdetails/tasks/main.yml | 20 +- utils/ala-data-generator | 94 +++++++++ utils/la-data-generator | 187 ++++++++++++++++++ utils/local.ini | 132 +++++++++++++ 20 files changed, 514 insertions(+), 71 deletions(-) create mode 100755 utils/ala-data-generator create mode 100755 utils/la-data-generator create mode 100644 utils/local.ini diff --git a/.gitignore b/.gitignore index 8d6e1dd52..b84481997 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ vagrant/centos/.vagrant/machines/default/virtualbox/id *.retry ansible/roles/alerts/tasks/main.yml +utils/docopts diff --git a/ansible/ecodata.yml b/ansible/ecodata.yml index 6ab9e8a12..3312c66ef 100644 --- a/ansible/ecodata.yml +++ b/ansible/ecodata.yml @@ -6,7 +6,7 @@ es_heap_dump_path: "/data/" es_api_host: "{{ elasticsearch_host | default('localhost') }}" es_config: - network.host: "{{ elasticsearch_host | default('localhost') }}" + network.host: "{{ (ecodata_elasticsearch_host | default(elasticsearch_host)) | default('localhost') }}" discovery.type: "single-node" roles: - common @@ -16,4 +16,4 @@ - webserver - { role: geoserver, log_filename: "geoserver" , when: install_geoserver | bool} - { role: ansible-elasticsearch, es_templates: false, es_instance_name: 'ecodata-elasticsearch', es_data_dirs: ['/data/elasticsearch'], tags: ['elasticsearch'], when: install_elasticsearch | bool } - - ecodata \ No newline at end of file + - ecodata diff --git a/ansible/events.yml b/ansible/events.yml index ed3c5270c..807e5ab62 100644 --- a/ansible/events.yml +++ b/ansible/events.yml @@ -10,17 +10,17 @@ vars: es_version: "7.17.7" es_heap_dump_path: "/data/" - es_api_host: "{{ elasticsearch_host | default('localhost') }}" + es_api_host: "{{ (events_elasticsearch_host | default(elasticsearch_host) | default('localhost') }}" es_heap_size: "8g" es_config: cluster.name: "{{ elasticsearch_cluster_name | default('extended-data') }}" node.name: "{{ elasticsearch_host | default('localhost') }}" - network.host: "{{ elasticsearch_network_host }}" - discovery.seed_hosts: "{{ elasticsearch_seed_hosts }}" + network.host: "{{ (events_elasticsearch_network_host | default(elasticsearch_network_host) }}" + discovery.seed_hosts: "{{ (events_elasticsearch_seed_host | default(elasticsearch_seed_host) }}" bootstrap.memory_lock: false cluster.initial_master_nodes: "{{ elasticsearch_cluster_initial_master_nodes }}" indices.memory.index_buffer_size: "{{ elasticsearch_index_buffer_size | default('40%') }}" roles: - common - java - - { role: ansible-elasticsearch, es_templates: false, es_instance_name: 'ecodata-elasticsearch', es_data_dirs: ['/data/elasticsearch'], tags: ['elasticsearch'], when: install_elasticsearch | bool } \ No newline at end of file + - { role: ansible-elasticsearch, es_templates: false, es_instance_name: 'ecodata-elasticsearch', es_data_dirs: ['/data/elasticsearch'], tags: ['elasticsearch'], when: install_elasticsearch | bool } diff --git a/ansible/roles/apikey/tasks/main.yml b/ansible/roles/apikey/tasks/main.yml index fafb37cff..62a11b7d3 100644 --- a/ansible/roles/apikey/tasks/main.yml +++ b/ansible/roles/apikey/tasks/main.yml @@ -21,8 +21,8 @@ file: path: "{{item}}" state: directory - owner: apikey - group: apikey + owner: "{{ apikey_user | default('apikey') }}" + group: "{{ apikey_user | default('apikey') }}" with_items: - "{{data_dir}}/apikey" - "{{data_dir}}/apikey/config" @@ -35,8 +35,8 @@ - name: set data ownership file: path: "{{data_dir}}/apikey" - owner: apikey - group: apikey + owner: "{{ apikey_user | default('apikey') }}" + group: "{{ apikey_user | default('apikey') }}" recurse: true tags: - properties @@ -46,8 +46,8 @@ template: src: apikey-config.yml dest: "{{data_dir}}/apikey/config/apikey-config.yml" - owner: apikey - group: apikey + owner: "{{ apikey_user | default('apikey') }}" + group: "{{ apikey_user | default('apikey') }}" notify: - restart apikey when: apikey_version is version('1.7.0', '>=') @@ -59,8 +59,8 @@ template: src: apikey-config-pre-1.7.0.yml dest: "{{data_dir}}/apikey/config/apikey-config.yml" - owner: apikey - group: apikey + owner: "{{ apikey_user | default('apikey') }}" + group: "{{ apikey_user | default('apikey') }}" notify: - restart apikey when: apikey_version is version('1.7.0', '<') @@ -72,8 +72,8 @@ template: src: logback.xml dest: "{{data_dir}}/apikey/config/logback.xml" - owner: apikey - group: apikey + owner: "{{ apikey_user | default('apikey') }}" + group: "{{ apikey_user | default('apikey') }}" tags: - properties - apikey diff --git a/ansible/roles/cas-management/tasks/main.yml b/ansible/roles/cas-management/tasks/main.yml index 7022c1a95..677b8da72 100644 --- a/ansible/roles/cas-management/tasks/main.yml +++ b/ansible/roles/cas-management/tasks/main.yml @@ -11,8 +11,8 @@ file: path: "{{item}}" state: directory - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" with_items: - "{{data_dir}}/cas-management" - "{{data_dir}}/cas-management/config" @@ -25,8 +25,8 @@ - name: set data ownership file: path: "{{data_dir}}/cas-management" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" recurse: true tags: - properties @@ -37,8 +37,8 @@ # template: # src: application.properties # dest: "{{data_dir}}/cas-management/config/application.properties" -# owner: cas -# group: cas +# owner: "{{ cas_user | default('cas') }}" +# group: "{{ cas_user | default('cas') }}" # notify: # - restart cas-management # tags: @@ -49,8 +49,8 @@ template: src: application.yml dest: "{{data_dir}}/cas-management/config/application.yml" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" when: cas_management_version is version('6', '>=') notify: - restart cas-management @@ -62,8 +62,8 @@ template: src: application.properties dest: "{{data_dir}}/cas-management/config/application.properties" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" when: cas_management_version is version('6', '<') notify: - restart cas-management @@ -75,8 +75,8 @@ template: src: log4j2-management.xml dest: "{{data_dir}}/cas-management/config/log4j2.xml" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" tags: - properties - cas-management @@ -90,8 +90,8 @@ value: /data/cas-management/config/ log_config_filename: 'log4j2.xml' service_name: "cas-management" - service_owner: 'cas' - service_group: 'cas' + service_owner: "{{ cas_user | default('cas') }}" + service_group: "{{ cas_user | default('cas') }}" use_openjdk11: "{{ cas_management_version is version('6', '>=') }}" use_openjdk8: "{{ cas_management_version is version('6', '<') }}" java_headless: True diff --git a/ansible/roles/cas5/tasks/main.yml b/ansible/roles/cas5/tasks/main.yml index 496bba44d..15b55bbf1 100644 --- a/ansible/roles/cas5/tasks/main.yml +++ b/ansible/roles/cas5/tasks/main.yml @@ -21,8 +21,8 @@ file: path: "{{item}}" state: directory - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" with_items: - "{{data_dir}}/cas" - "{{data_dir}}/cas/config" @@ -35,8 +35,8 @@ - name: set data ownership file: path: "{{data_dir}}/cas" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" recurse: true tags: - properties @@ -46,8 +46,8 @@ template: src: application.yml dest: "{{data_dir}}/cas/config/application.yml" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" when: cas_version is version('6', '>=') notify: - restart cas @@ -59,8 +59,8 @@ template: src: application-pre-6.yml dest: "{{data_dir}}/cas/config/application.yml" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" when: cas_version is version('6', '<') notify: - restart cas @@ -72,8 +72,8 @@ template: src: log4j2.xml dest: "{{data_dir}}/cas/config/log4j2.xml" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" tags: - properties - cas @@ -82,8 +82,8 @@ template: src: pwe.properties dest: "{{data_dir}}/cas/config/pwe.properties" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" notify: - restart cas tags: @@ -95,8 +95,8 @@ copy: src: "{{ inventory_dir }}/{{ cas_jwks_file | default('keystore.jwks') }}" dest: "{{data_dir}}/cas/keystore.jwks" - owner: cas - group: cas + owner: "{{ cas_user | default('cas') }}" + group: "{{ cas_user | default('cas') }}" mode: u=rw,g=,o= notify: - restart cas @@ -150,7 +150,7 @@ - db - name: ensure target sql directory exist - file: path={{item}} state=directory owner=cas group=cas + file: path={{item}} state=directory owner="{{ cas_user | default('cas') }}" group="{{ cas_user | default('cas') }}" with_items: - "{{data_dir}}/cas/setup" when: cas_first_admin_email is defined and cas_first_admin_bcrypt_password is defined and cas_first_admin_temp_auth_key is defined diff --git a/ansible/roles/doi-service/tasks/main.yml b/ansible/roles/doi-service/tasks/main.yml index 99d9fe111..c47bf2da1 100644 --- a/ansible/roles/doi-service/tasks/main.yml +++ b/ansible/roles/doi-service/tasks/main.yml @@ -28,8 +28,8 @@ file: path: "{{item}}" state: directory - owner: doi-service - group: doi-service + owner: "{{ doi_user | default('doi-service') }}" + group: "{{ doi_user | default('doi-service') }}" with_items: - "{{data_dir}}/doi-service" - "{{data_dir}}/doi-service/config" @@ -41,8 +41,8 @@ - name: set data ownership file: path: "{{data_dir}}/doi-service" - owner: doi-service - group: doi-service + owner: "{{ doi_user | default('doi-service') }}" + group: "{{ doi_user | default('doi-service') }}" recurse: true tags: - properties @@ -52,8 +52,8 @@ template: src: doi-service-config.yml dest: "{{data_dir}}/doi-service/config/doi-service-config.yml" - owner: doi-service - group: doi-service + owner: "{{ doi_user | default('doi-service') }}" + group: "{{ doi_user | default('doi-service') }}" notify: - restart doi-service tags: @@ -64,8 +64,8 @@ template: src: logback.groovy dest: "{{data_dir}}/doi-service/config/logback.groovy" - owner: doi-service - group: doi-service + owner: "{{ doi_user | default('doi-service') }}" + group: "{{ doi_user | default('doi-service') }}" notify: - restart doi-service tags: diff --git a/ansible/roles/ecodata/handlers/main.yml b/ansible/roles/ecodata/handlers/main.yml index 467542d79..dae48007e 100644 --- a/ansible/roles/ecodata/handlers/main.yml +++ b/ansible/roles/ecodata/handlers/main.yml @@ -1,2 +1,4 @@ - name: restart ecodata service: name=ecodata state=restarted enabled="yes" + when: + - skip_handlers | default("false") | bool == false diff --git a/ansible/roles/ecodata/tasks/main.yml b/ansible/roles/ecodata/tasks/main.yml index a0ab065fc..2255d1491 100644 --- a/ansible/roles/ecodata/tasks/main.yml +++ b/ansible/roles/ecodata/tasks/main.yml @@ -20,6 +20,7 @@ with_items: - "{{data_dir}}/mongodb" tags: + - mongodb-org - properties - ecodata diff --git a/ansible/roles/events/handlers/main.yml b/ansible/roles/events/handlers/main.yml index 1f799657f..1f922817f 100644 --- a/ansible/roles/events/handlers/main.yml +++ b/ansible/roles/events/handlers/main.yml @@ -1,2 +1,4 @@ - name: restart graphql_service - service: name={{ env }}-graphql state=restarted enabled="yes" \ No newline at end of file + service: name={{ env }}-graphql state=restarted enabled="yes" + when: + - skip_handlers | default("false") | bool == false diff --git a/ansible/roles/events/tasks/main.yaml b/ansible/roles/events/tasks/main.yaml index d65b0917f..1ead21b57 100644 --- a/ansible/roles/events/tasks/main.yaml +++ b/ansible/roles/events/tasks/main.yaml @@ -61,6 +61,7 @@ - events - update-images - docker-config + - properties - name: Copy docker YAML files to {{ data_dir }} template: src={{ item }} dest={{ data_dir }}/{{ env }}/{{ item }} @@ -76,6 +77,7 @@ - events - update-images - docker-config + - properties - name: Copy service scripts to /usr/bin template: src={{ item }} dest=/usr/bin/{{ env }}-{{ item }} mode=777 diff --git a/ansible/roles/geoserver/tasks/ecodata-geoserver.yml b/ansible/roles/geoserver/tasks/ecodata-geoserver.yml index c668c2230..12ce1e813 100644 --- a/ansible/roles/geoserver/tasks/ecodata-geoserver.yml +++ b/ansible/roles/geoserver/tasks/ecodata-geoserver.yml @@ -80,6 +80,13 @@ tags: - geoserver +# Used for /data local generation +- name: set ownership of geoserver data + shell: "chown -R {{ geoserver_user }}:{{ geoserver_user }} {{data_dir}}/geoserver*" + when: geoserver_user is defined + tags: + - geoserver + # # Override the default memory settings for Tomcat to increase heap space and change the garbage collector # diff --git a/ansible/roles/geoserver/tasks/geoserver.yml b/ansible/roles/geoserver/tasks/geoserver.yml index 32501575a..63a0ff2f6 100644 --- a/ansible/roles/geoserver/tasks/geoserver.yml +++ b/ansible/roles/geoserver/tasks/geoserver.yml @@ -254,6 +254,14 @@ - geoserver - properties +# Used for /data local generation +- name: set ownership of geoserver data + shell: "chown -R {{ geoserver_user }}:{{ geoserver_user }} {{data_dir}}/geoserver*" + when: geoserver_user is defined + tags: + - geoserver + - properties + - name: Running geoserver.sh command: "{{data_dir}}/geoserver.sh" args: diff --git a/ansible/roles/image-service/tasks/main.yml b/ansible/roles/image-service/tasks/main.yml index 455d556ec..fa0856449 100644 --- a/ansible/roles/image-service/tasks/main.yml +++ b/ansible/roles/image-service/tasks/main.yml @@ -48,7 +48,7 @@ - image-service - name: ensure target directories exist [data subdirectories etc.] - file: path={{item}} state=directory owner=image-service group=image-service + file: path={{item}} state=directory owner="{{ image_user | default('image-service') }}" group="{{ image_user | default('image-service') }}" with_items: - "{{data_dir}}/image-service/setup" - "{{data_dir}}/image-service/config" @@ -90,7 +90,7 @@ - image-service - name: ensure target directories exist [data subdirectories etc.] - file: path={{item}} state=directory owner=image-service group=image-service + file: path={{item}} state=directory owner="{{ image_user | default('image-service') }}" group="{{ image_user | default('image-service') }}" with_items: - "{{data_dir}}/image-service/setup" - "{{data_dir}}/image-service/config" @@ -106,7 +106,7 @@ - properties - name: set data ownership for {{data_dir}}/image-service - file: path={{data_dir}}/image-service owner="image-service" group="image-service" + file: path={{data_dir}}/image-service owner="{{ image_user | default('image-service') }}" group="{{ image_user | default('image-service') }}" notify: - restart image-service tags: @@ -115,7 +115,7 @@ - image-service - name: set data ownership postgres for {{data_dir}}/image-service/exports - file: path={{data_dir}}/image-service/exports owner="image-service" group="image-service" recurse=true + file: path={{data_dir}}/image-service/exports owner="{{ image_user | default('image-service') }}" group="{{ image_user | default('image-service') }}" recurse=true notify: - restart image-service tags: @@ -124,7 +124,7 @@ - image-service - name: set data ownership for /opt/atlas/image-service - file: path=/opt/atlas/image-service owner="image-service" group="image-service" recurse=true + file: path=/opt/atlas/image-service owner="{{ image_user | default('image-service') }}" group="{{ image_user | default('image-service') }}" recurse=true notify: - restart image-service tags: @@ -133,7 +133,7 @@ - image-service - name: set data ownership for /var/log/atlas/image-service - file: path=/var/log/atlas/image-service owner="image-service" group="image-service" recurse=true + file: path=/var/log/atlas/image-service owner="{{ image_user | default('image-service') }}" group="{{ image_user | default('image-service') }}" recurse=true notify: - restart image-service tags: diff --git a/ansible/roles/image-service/templates/config/image-service-config.yml b/ansible/roles/image-service/templates/config/image-service-config.yml index 4cca08bd5..a97c57701 100644 --- a/ansible/roles/image-service/templates/config/image-service-config.yml +++ b/ansible/roles/image-service/templates/config/image-service-config.yml @@ -43,7 +43,7 @@ flyway: baselineOnMigrate: {{ image_service_baseline_on_migrate | default('true') }} dataSource: - url: "jdbc:postgresql://localhost/{{ image_service_db_name }}?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8" + url: "jdbc:postgresql://{{ image_service_db_host | default('localhost') }}/{{ image_service_db_name }}?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8" driverClassName: org.postgresql.Driver username: {{ image_service_db_username }} password: {{ image_service_db_password }} diff --git a/ansible/roles/pipelines/tasks/main.yml b/ansible/roles/pipelines/tasks/main.yml index 04568d6a0..67cb77d97 100644 --- a/ansible/roles/pipelines/tasks/main.yml +++ b/ansible/roles/pipelines/tasks/main.yml @@ -11,12 +11,14 @@ apt: name=aptitude state=latest update_cache=yes force_apt_get=yes tags: - pipelines + - apt - name: Install required system packages apt: name={{ item }} state=latest update_cache=yes loop: [ 'apt-transport-https', 'ca-certificates', 'curl', 'software-properties-common', 'python3-pip', 'virtualenv', 'python3-setuptools', 'unzip'] tags: - pipelines + - apt - name: Add Docker GPG apt Key apt_key: @@ -92,6 +94,7 @@ - la-pipelines - pipelines - yq + - apt - name: Add YQ Repository apt_repository: @@ -101,6 +104,7 @@ - la-pipelines - pipelines - yq + - apt - name: Add an apt key by id from a keyserver apt_key: @@ -111,6 +115,7 @@ - la-pipelines - pipelines - yq + - apt - name: Update apt and install pipelines apt: update_cache=yes name=yq={{ yq_version }} @@ -119,6 +124,7 @@ - pipelines - la-pipelines - yq + - apt # docopts is needed by the la-pipelines script @@ -137,6 +143,7 @@ - la-pipelines - pipelines - docopts + - apt - name: Make docopts executable shell: chmod +x /usr/local/bin/docopts diff --git a/ansible/roles/userdetails/tasks/main.yml b/ansible/roles/userdetails/tasks/main.yml index 847519585..6ff0c7edc 100644 --- a/ansible/roles/userdetails/tasks/main.yml +++ b/ansible/roles/userdetails/tasks/main.yml @@ -21,8 +21,8 @@ file: path: "{{item}}" state: directory - owner: userdetails - group: userdetails + owner: "{{ userdetails_user | default('userdetails') }}" + group: "{{ userdetails_user | default('userdetails') }}" with_items: - "{{data_dir}}/userdetails" - "{{data_dir}}/userdetails/config" @@ -35,8 +35,8 @@ - name: set data ownership file: path: "{{data_dir}}/userdetails" - owner: userdetails - group: userdetails + owner: "{{ userdetails_user | default('userdetails') }}" + group: "{{ userdetails_user | default('userdetails') }}" recurse: true tags: - properties @@ -46,8 +46,8 @@ template: src: userdetails-config.yml dest: "{{data_dir}}/userdetails/config/userdetails-config.yml" - owner: userdetails - group: userdetails + owner: "{{ userdetails_user | default('userdetails') }}" + group: "{{ userdetails_user | default('userdetails') }}" notify: - restart userdetails when: user_details_version is version('3', '>=') @@ -59,8 +59,8 @@ template: src: userdetails-config-pre-3.yml dest: "{{data_dir}}/userdetails/config/userdetails-config.yml" - owner: userdetails - group: userdetails + owner: "{{ userdetails_user | default('userdetails') }}" + group: "{{ userdetails_user | default('userdetails') }}" notify: - restart userdetails when: user_details_version is version('3', '<') @@ -72,8 +72,8 @@ template: src: logback.xml dest: "{{data_dir}}/userdetails/config/logback.xml" - owner: userdetails - group: userdetails + owner: "{{ userdetails_user | default('userdetails') }}" + group: "{{ userdetails_user | default('userdetails') }}" tags: - properties - userdetails diff --git a/utils/ala-data-generator b/utils/ala-data-generator new file mode 100755 index 000000000..0d3b5b19c --- /dev/null +++ b/utils/ala-data-generator @@ -0,0 +1,94 @@ +#!/bin/bash + +# Run with: +# ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate doi alerts +# or for all services: +# ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate + +# It will fail if some playbook fails +set -e + +CMD=$(basename "$0") +GEN_LOC="$(dirname "$(realpath "$0")")" + +if [[ ! -e $GEN_LOC/docopts ]] ; then + curl -s -o "$GEN_LOC/docopts" -LJO https://github.com/docopt/docopts/releases/download/v0.6.3-rc2/docopts_linux_amd64 + chmod +x "$GEN_LOC/docopts" +fi + +eval "$("$GEN_LOC/docopts" -V - -h - : "$@" < --inv= generate [...] + $CMD -h | --help + $CMD -v | --version + +Options: + --verbose Verbose output. + -h --help Show this help. + -v --version Show version. +---- +$CMD $VER +License Apache-2.0 +EOF +)" + +list=( + "./alerts/alerts-prod" + "./auth/aws-auth-prod.yml" + "./bie/bie-hub-prod-2022" + "./bie/bie-ws-solr-prod-2022" + "./biocache/biocache-hub-2021" + "./biocache/biocache-service-2021" + "./biocache/cassandra-cluster-2021" + "./biocache/solrcloud-2021-1" + "./biocollect/biocollect-prod" + "./calendars/calendars-prod" + "./collections/collections-prod" + "./dashboard/dashboard-prod" + "./data_quality_filter_service/data_quality_filter_service_prod" + "./doi/doi-prod" + "./events/events-prod-2023" + "./fieldcapture/fieldcapture-prod" + "./image-service/image-service-prod" + "./logger/logger-prod" + "./pdf-service/pdf-service-prod" + "./profiles/profiles-prod" + "./regions/regions-prod" + "./sampling/sampling-prod" + "./sandbox/sandbox-prod" + "./spatial/spatial-prod" + "./specieslists/specieslists-prod" + "./ecodata/ecodata-prod" + "./fieldguide/fieldguide-prod" + # This needs extra role work: + "./pipelines/databox-pipelines.yml" + "./namematching/namematching-prod" + "./sensitive-data-service/sensitive-data-service-prod-2022" +) + #"./pipelines/aws-spark-quoll-pipelines.yml" + #"./pipelines/nci3-spark-pipelines.yml" + +if [[ -n $service ]] ; then + services=("${service[@]}") +fi + +for el in "${list[@]}" +do + match=false + for arg in "${services[@]}" + do + if [[ $el == *"$arg"* ]]; then + match=true + break + fi + done + + if [[ -z "${services[*]}" ]] || $match; then + # echo -n "$el " + "$GEN_LOC/la-data-generator" --inv=$inv --data=$data generate "$el" + fi +done + diff --git a/utils/la-data-generator b/utils/la-data-generator new file mode 100755 index 000000000..023c5a51d --- /dev/null +++ b/utils/la-data-generator @@ -0,0 +1,187 @@ +#!/bin/bash + +# It will fail if some playbook fails +set -e + +CMD=$(basename "$0") +GEN_LOC="$(dirname "$(realpath "$0")")" +ALA_INSTALL="${GEN_LOC%/*}" +A_ALA_INSTALL="${GEN_LOC%/*}/ansible" +#echo $GEN_LOC +#echo $A_ALA_INSTALL + + +if [[ ! -e $GEN_LOC/docopts ]] ; then + curl -s -o "$GEN_LOC/docopts" -LJO https://github.com/docopt/docopts/releases/download/v0.6.3-rc2/docopts_linux_amd64 + chmod +x "$GEN_LOC/docopts" +fi + +eval "$("$GEN_LOC/docopts" -V - -h - : "$@" < --inv= generate [...] + $CMD -h | --help + $CMD -v | --version + +Options: + --verbose Verbose output. + -d --dry-run Print the commands without actually running them. + -h --help Show this help. + -v --version Show version. +---- +$CMD $VER +License Apache-2.0 +EOF +)" + +# shellcheck disable=SC2154 +if ($dry_run); then _D="echo"; else _D=; fi +if ($dry_run); then echo "Only printing the commands:"; fi + +# shellcheck disable=SC2154 +if $verbose; then + # shellcheck disable=SC2154 + echo build: "$build" + # shellcheck disable=SC2154 + echo run: "$run" + # shellcheck disable=SC2154 + echo data: "$data" + # shellcheck disable=SC2154 + echo inv: "$inv" +fi + +use_an_inventory=false +user=$USER + +# shellcheck disable=SC2154 +if [[ -d $inv && -f $inv/ansiblew ]]; then + # use_ansiblew=true + if $verbose; then echo "It seems a standard generated inventory"; fi +elif [[ -f $inv/$service ]]; then + use_an_inventory=true +elif [[ ! -d $inv || ! -f $inv/ansiblew ]]; then + >&2 echo "ERR: It seems that '$inv' is not a generated inventory as we expect or an ALA inventory directory + service inventory provided" + exit 1; +fi + +TAGS=common,properties,mongodb-org,namematching-service,pipelines,pipelines-layers +SKIP_TAGS=restart,image-stored-procedures,db,mongodb-org-restart,mongodb-users,sslcerts,pipelines-services,hadoop_dir,hadoop_vocab,apt_update,apt,mongodb-org,docopts,yq + EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo cas_audit_host=auth-mongo cas_spring_session_host=auth-mongo \ + user_store_db_hostname=auth-mysql apikey_db_hostname=auth-mysql cas_db_hostname=auth-mysql cas_tickets_host=auth-mongo \ + cas_tickets_host=auth-mongo cas_services_host=auth-mongo cas_services_host=auth-mongo collectory_db_host_address=collectory-mysql \ + dq_db_url=jdbc:postgresql://dq-postgresql/data-quality specieslist_db_hostname=lists-mysql logger_db_hostname=logger-mysql \ + alerts_db_hostname=alerts-mysql phylolink_db_hostname=phylolink-mysql geonetwork_database=spatial-postgresql layers_db_host=spatial-postgresql \ + doi_db_hostname=doi-postgresql image_service_db_host=images-postgresql \ + mail_smtp_host=biocache-smtp mail_host=biocollect-smtp \ + ecodata_elasticsearch_host=ecodata-es events_elasticsearch_host=events-es events_elasticsearch_network_host=events-es-network \ + events_elasticsearch_seed_host=events-es-seed solr_url=FIXME_solr_url biocache_db_host=FIXME_cassandra_host \ + data_dir=/data/docker skip_handlers=true tomcat_apr=false use_docker_with_pipelines=false install_elasticsearch=false ecodata_url=FIXME_ECODATA_URL \ + tomcat=$user tomcat_user=$user biocollect_user=$user ecodata_user=$user merit_user=$user fieldguide_app=fieldguide spark_user=$user \ + profile_service_user=$user profile_hub_user=$user doi_user=$user cas_user=$user userdetails_user=$user apikey_user=$user image_user=$user \ + geoserver_user=$user" + +function gen() { + local what + what="$1" + echo "Generating config for '$what'" + if $verbose; then V="-vvvv" ; else V=""; fi + cp "$GEN_LOC/local.ini" "$inv" + bash -c "cd $inv; ./ansiblew --alainstall=$ALA_INSTALL -i local.ini $what --tags=$TAGS --skip=$SKIP_TAGS -e '$EXTRAS' --nodryrun --limit localhost $V" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + >&2 echo "The generation failed, are you inventories and/or your ala-install repo up-to-date?" + fi + rm "$inv/local.ini" +} + +function genCustom() { + # local dinv="$(dirname "$1")" + local cinv + local play + local dinv + cinv="$(basename "$1")" + play="$2" + dinv="$(dirname "$1")" + echo "Generating config for $dinv/$cinv and '$play' in '$inv'" + + if $verbose; then V="-vvvv" ; else V=""; fi + + cp "$GEN_LOC/local.ini" "$inv/$dinv" + bash -c "cd $inv/$dinv; ansible-playbook -u ubuntu --become -i $cinv -i local.ini $A_ALA_INSTALL/$play --tags $TAGS --skip-tags $SKIP_TAGS --extra-vars '$EXTRAS' --limit localhost $V" + rm "$inv/$dinv/local.ini" +exit + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + >&2 echo "The generation failed, are you inventories and/or your ala-install repo up-to-date?" + fi +} + +function run_checks() { + if [[ ! -d $data ]]; then + >&2 echo "Directory '$data' does not exists" + exit 1 + fi + + if [[ ! $data =~ ^/.* ]]; then + >&2 echo "Use an /absolute path for directory '$data' " + exit 1 + fi + + if [[ ! $inv =~ ^/.* ]]; then + >&2 echo "Use an /absolute path for directory '$inv' " + exit 1 + fi + + if [ -z "$(find "$inv" -mindepth 1 -print -quit)" ]; then + >&2 echo "WARN: It seems that '$inv' is empty" + exit 1 + fi + + # shellcheck disable=SC2154 + if [[ -d $ala_install && ! -d $ala_install/ansible ]]; then + >&2 echo "It seems that '$ala_install' is not the ala-install repository as we expect" + exit 1 + fi +} + +run_checks + +if [[ -n $service ]] ; then + services=("${service[@]}") +fi + +if $use_an_inventory ; then + echo "Processing $inv$service" + set +e + + output=$(bash -c "grep 'ansible-playbook -i' $inv$service 2>&1") + + set -e + + playbook_pattern="ala-install/ansible" + + while IFS= read -r line; do + if [[ $line == *"ansible-playbook -i"* ]]; then + playbook=$(awk -F "$playbook_pattern" '{print $2}' <<< "$line" | awk '{gsub(/--.*$/, ""); print}') + playbook=${playbook// /} + fi + done <<< "$output" + + if [[ -z "$playbook" ]]; then + >&2 echo "Playbook not detected in inventory comments" + exit 1 + fi + + genCustom "$service" "$playbook" + +else + if [[ -n $service ]] ; then + for s in "${services[@]}"; do + gen "$s" + done + else + gen all + fi +fi diff --git a/utils/local.ini b/utils/local.ini new file mode 100644 index 000000000..9c8098c27 --- /dev/null +++ b/utils/local.ini @@ -0,0 +1,132 @@ +# +# This extra inventory is needed for /data/ local generation, +# in order to don't modificate the ansible/playbooks groups. +# +# Also this local inventory should we placed in the same directory of your real +# inventories, and if not, file accessing to files references in inventories, +# like: +# blacklist_file=blacklist-prod.json +# fails. Because of this, we copy this file there and remove it on each run. +# +# Warn: be sure that your playbook group is included here to generate its +# config in local +# +[collectory] +127.0.0.1 ansible_connection=local + +[biocache-hub] +127.0.0.1 ansible_connection=local + +[biocache-service-clusterdb] +127.0.0.1 ansible_connection=local + +[bie-hub] +127.0.0.1 ansible_connection=local + +[bie-index] +127.0.0.1 ansible_connection=local + +[image-service] +127.0.0.1 ansible_connection=local + +[species-list] +127.0.0.1 ansible_connection=local + +[regions] +127.0.0.1 ansible_connection=local + +[logger-service] +127.0.0.1 ansible_connection=local + +[solr7-server] +127.0.0.1 ansible_connection=local + +[solrcloud] +127.0.0.1 ansible_connection=local + +[zookeeper] +127.0.0.1 ansible_connection=local + +[cas-servers] +127.0.0.1 ansible_connection=local + +[biocache-db] +127.0.0.1 ansible_connection=local + +[biocache-cli] +127.0.0.1 ansible_connection=local + +[spatial] +127.0.0.1 ansible_connection=local + +[webapi_standalone] +127.0.0.1 ansible_connection=local + +[dashboard] +127.0.0.1 ansible_connection=local + +[alerts-service] +127.0.0.1 ansible_connection=local + +[doi-service] +127.0.0.1 ansible_connection=local + +[nameindexer] +127.0.0.1 ansible_connection=local + +[sds] +127.0.0.1 ansible_connection=local + +[pipelines] +127.0.0.1 ansible_connection=local + +[spark] +127.0.0.1 ansible_connection=local + +[hadoop] +127.0.0.1 ansible_connection=local + +[jenkins] +127.0.0.1 ansible_connection=local + +[pipelines_jenkins] +127.0.0.1 ansible_connection=local + +[data_quality_filter_service] +127.0.0.1 ansible_connection=local + +[namematching-service] +127.0.0.1 ansible_connection=local + +[sensitive-data-service] +127.0.0.1 ansible_connection=local + +[branding] +127.0.0.1 ansible_connection=local + +[biocollect] +127.0.0.1 ansible_connection=local + +[pdfgen] +127.0.0.1 ansible_connection=local + +[ecodata] +127.0.0.1 ansible_connection=local + +[ecodata-reporting] +127.0.0.1 ansible_connection=local + +[primary] +127.0.0.1 ansible_connection=local + +[secondary] +127.0.0.1 ansible_connection=local + +[events] +127.0.0.1 ansible_connection=local + +[events_elasticsearch] +127.0.0.1 ansible_connection=local + +[cassandra3] +127.0.0.1 ansible_connection=local