From 4e08f0cb9ada46f758506eea9c693c9ede9fbcb2 Mon Sep 17 00:00:00 2001 From: vjrj Date: Wed, 29 Nov 2023 12:00:36 +0100 Subject: [PATCH] More work to generate /data/ configs (#732) * More work to generate /data/ configs * Additional option for fast generation * Use of new cognito inventories --- .../templates/config/image-service-config.yml | 2 +- ansible/roles/pipelines/tasks/main.yml | 15 ++++++++++++++- ansible/roles/profile-hub/handlers/main.yml | 2 ++ utils/ala-data-generator | 16 +++++++--------- utils/la-data-generator | 15 ++++++++++----- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/ansible/roles/image-service/templates/config/image-service-config.yml b/ansible/roles/image-service/templates/config/image-service-config.yml index 79db90b6e..fc4b3f2e3 100644 --- a/ansible/roles/image-service/templates/config/image-service-config.yml +++ b/ansible/roles/image-service/templates/config/image-service-config.yml @@ -32,7 +32,7 @@ security: discoveryUri: {{ discoveryUri | default('') }} scope: {{ oidc_scopes | default('openid profile email ala roles') }} alaUseridClaim: {{ alaUseridClaim | security_jwt_userIdClaim | default('userid') }} -{% if logout_action == 'cognito' %} +{% if logout_action is defined and logout_action == 'cognito' %} logoutUrl: {{ cognito_logout_url | default('') }} logoutAction: cognito {% endif %} diff --git a/ansible/roles/pipelines/tasks/main.yml b/ansible/roles/pipelines/tasks/main.yml index ba17099d6..9c3cffe44 100644 --- a/ansible/roles/pipelines/tasks/main.yml +++ b/ansible/roles/pipelines/tasks/main.yml @@ -53,6 +53,7 @@ with_items: - "{{ data_dir }}/la-pipelines/config" - "{{ data_dir }}/spark-tmp" + - "{{ data_dir }}/spark/conf" - "{{ data_dir }}/pipelines-shp" - "{{ data_dir }}/dwca-tmp" - "{{ data_dir }}/dwca-export" @@ -405,11 +406,23 @@ with_items: - "{{ data_dir }}/pipelines-shp/pipelines-shapefiles.zip" - "{{ data_dir }}/pipelines-shp/sds-layers.tgz" + tags: + - pipelines + - pipelines-layers + +- name: Unpack SHP extra file + unarchive: + src: "{{ item }}" + dest: "{{ data_dir }}/pipelines-shp" + remote_src: yes + owner: "{{ spark_user }}" + group: "{{ spark_user }}" + with_items: - "{{ data_dir }}/pipelines-shp/extra-layers.tgz" tags: - pipelines - pipelines-layers - when: item is defined + when: pipelines_shapefiles_extra_url is defined - name: Copy additional logging configs copy: src={{ item.src }} dest={{ item.dest }} owner={{ item.owner }} group={{ item.group }} diff --git a/ansible/roles/profile-hub/handlers/main.yml b/ansible/roles/profile-hub/handlers/main.yml index 54797a5b8..7203e6012 100644 --- a/ansible/roles/profile-hub/handlers/main.yml +++ b/ansible/roles/profile-hub/handlers/main.yml @@ -1,2 +1,4 @@ - name: restart profile-hub service: name=profile-hub state=restarted enabled="yes" + when: + - skip_handlers | default("false") | bool == false diff --git a/utils/ala-data-generator b/utils/ala-data-generator index a7340a223..194d1ed75 100755 --- a/utils/ala-data-generator +++ b/utils/ala-data-generator @@ -45,15 +45,15 @@ EOF list=( "./alerts/alerts-prod" "./auth/aws-auth-prod.yml" - "./bie/bie-hub-prod-2022" - "./bie/bie-ws-solr-prod-2022" - "./biocache/biocache-hub-2021" - "./biocache/biocache-service-2021" + "./bie/bie-hub-prod-2022-cognito" + "./bie/bie-ws-solr-prod-2022-cognito" + "./biocache/biocache-hub-2021-cognito" + "./biocache/biocache-service-2021-cognito" "./biocache/cassandra-cluster-2021" "./biocache/solrcloud-2021-1" "./biocollect/biocollect-prod" "./calendars/calendars-prod" - "./collections/collections-prod" + "./collections/collections-prod-cognito" "./dashboard/dashboard-prod" "./data_quality_filter_service/data_quality_filter_service_prod" "./doi/doi-prod" @@ -65,18 +65,16 @@ list=( "./profiles/profiles-prod" "./regions/regions-prod" "./sampling/sampling-prod" - "./sandbox/sandbox-prod" "./spatial/spatial-prod" "./specieslists/specieslists-prod" "./ecodata/ecodata-prod" "./fieldguide/fieldguide-prod" - # This needs extra role work: "./pipelines/databox-pipelines.yml" "./namematching/namematching-prod" "./sensitive-data-service/sensitive-data-service-prod-2022" + # This needs extra role work: + "./sandbox/sandbox-prod" ) - #"./pipelines/aws-spark-quoll-pipelines.yml" - #"./pipelines/nci3-spark-pipelines.yml" # shellcheck disable=SC2154 if [[ -n $extra_args ]] ; then diff --git a/utils/la-data-generator b/utils/la-data-generator index 06482259b..8bebfaa6a 100755 --- a/utils/la-data-generator +++ b/utils/la-data-generator @@ -21,13 +21,14 @@ eval "$("$GEN_LOC/docopts" -V - -h - : "$@" < --inv= generate [...] + $CMD [options] [--fast] --data= --inv= generate [...] $CMD -h | --help $CMD -v | --version Options: --extra-args= Additional "arg1=values arg2=value" to get preference over defaul extra variables values --verbose Verbose output. + --fast Only exec the properties-file tag -d --dry-run Print the commands without actually running them. -h --help Show this help. -v --version Show version. @@ -69,7 +70,11 @@ elif [[ ! -d $inv || ! -f $inv/ansiblew ]]; then fi -TAGS=common,properties,mongodb-org,namematching-service,pipelines,pipelines-layers +if [[ -n $fast ]] ; then + TAGS=properties-file +else + TAGS=common,properties,mongodb-org,namematching-service,pipelines,pipelines-layers +fi SKIP_TAGS=restart,image-stored-procedures,db,mongodb-org-restart,mongodb-users,sslcerts,pipelines-services,hadoop_dir,hadoop_vocab,apt_update,apt,mongodb-org,docopts,yq # shellcheck disable=SC2154 EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo cas_audit_host=auth-mongo cas_spring_session_host=auth-mongo \ @@ -80,11 +85,11 @@ EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo doi_db_hostname=doi-postgresql image_service_db_host=images-postgresql \ mail_smtp_host=biocache-smtp mail_host=biocollect-smtp \ ecodata_elasticsearch_host=ecodata-es events_elasticsearch_host=events-es events_elasticsearch_network_host=events-es-network \ - events_elasticsearch_seed_host=events-es-seed solr_url=solr_host:2181 biocache_db_host=cassandra_host \ - data_dir=/data/docker skip_handlers=true tomcat_apr=false use_docker_with_pipelines=false install_elasticsearch=false ecodata_url=FIXME_ECODATA_URL \ + events_elasticsearch_seed_host=events-es-seed solr_user=$user solr_url=solr_host:2181 biocache_db_host=cassandra_host \ + data_dir=$data skip_handlers=true tomcat_apr=false use_docker_with_pipelines=false install_elasticsearch=false ecodata_url=FIXME_ECODATA_URL \ tomcat=$user tomcat_user=$user biocollect_user=$user ecodata_user=$user merit_user=$user fieldguide_app=fieldguide spark_user=$user \ profile_service_user=$user profile_hub_user=$user doi_user=$user cas_user=$user userdetails_user=$user apikey_user=$user image_user=$user \ - geoserver_user=$user \ + geoserver_user=$user namematching_user=$user sensitive_data_user=$user\ $extra_args" function gen() {