More work to generate /data/ configs (#732)

* More work to generate /data/ configs

* Additional option for fast generation

* Use of new cognito inventories

ansible/roles/image-service/templates/config/image-service-config.yml

@@ -32,7 +32,7 @@ security:
     discoveryUri: {{ discoveryUri | default('') }}
     scope: {{ oidc_scopes | default('openid profile email ala roles') }}
     alaUseridClaim: {{ alaUseridClaim | security_jwt_userIdClaim | default('userid') }}
-{% if logout_action == 'cognito' %}
+{% if logout_action is defined and logout_action == 'cognito' %}
     logoutUrl: {{ cognito_logout_url | default('') }}
     logoutAction: cognito
 {% endif %}

ansible/roles/pipelines/tasks/main.yml

@@ -53,6 +53,7 @@
   with_items:
     - "{{ data_dir }}/la-pipelines/config"
     - "{{ data_dir }}/spark-tmp"
+    - "{{ data_dir }}/spark/conf"
     - "{{ data_dir }}/pipelines-shp"
     - "{{ data_dir }}/dwca-tmp"
     - "{{ data_dir }}/dwca-export"
@@ -405,11 +406,23 @@
   with_items:
     - "{{ data_dir }}/pipelines-shp/pipelines-shapefiles.zip"
     - "{{ data_dir }}/pipelines-shp/sds-layers.tgz"
+  tags:
+    - pipelines
+    - pipelines-layers
+
+- name: Unpack SHP extra file
+  unarchive:
+    src: "{{ item }}"
+    dest: "{{ data_dir }}/pipelines-shp"
+    remote_src: yes
+    owner: "{{ spark_user }}"
+    group: "{{ spark_user }}"
+  with_items:
     - "{{ data_dir }}/pipelines-shp/extra-layers.tgz"
   tags:
     - pipelines
     - pipelines-layers
-  when: item is defined
+  when: pipelines_shapefiles_extra_url is defined
 
 - name: Copy additional logging configs
   copy: src={{ item.src }} dest={{ item.dest }} owner={{ item.owner }} group={{ item.group }}

ansible/roles/profile-hub/handlers/main.yml

@@ -1,2 +1,4 @@
 - name: restart profile-hub
   service: name=profile-hub state=restarted enabled="yes"
+  when:
+  - skip_handlers | default("false") | bool == false

utils/ala-data-generator

@@ -45,15 +45,15 @@ EOF
 list=(
   "./alerts/alerts-prod"
   "./auth/aws-auth-prod.yml"
-  "./bie/bie-hub-prod-2022"
-  "./bie/bie-ws-solr-prod-2022"
-  "./biocache/biocache-hub-2021"
-  "./biocache/biocache-service-2021"
+  "./bie/bie-hub-prod-2022-cognito"
+  "./bie/bie-ws-solr-prod-2022-cognito"
+  "./biocache/biocache-hub-2021-cognito"
+  "./biocache/biocache-service-2021-cognito"
   "./biocache/cassandra-cluster-2021"
   "./biocache/solrcloud-2021-1"
   "./biocollect/biocollect-prod"
   "./calendars/calendars-prod"
-  "./collections/collections-prod"
+  "./collections/collections-prod-cognito"
   "./dashboard/dashboard-prod"
   "./data_quality_filter_service/data_quality_filter_service_prod"
   "./doi/doi-prod"
@@ -65,18 +65,16 @@ list=(
   "./profiles/profiles-prod"
   "./regions/regions-prod"
   "./sampling/sampling-prod"
-  "./sandbox/sandbox-prod"
   "./spatial/spatial-prod"
   "./specieslists/specieslists-prod"
   "./ecodata/ecodata-prod"
   "./fieldguide/fieldguide-prod"
-  # This needs extra role work:
   "./pipelines/databox-pipelines.yml"
   "./namematching/namematching-prod"
   "./sensitive-data-service/sensitive-data-service-prod-2022"
+  # This needs extra role work:
+  "./sandbox/sandbox-prod"
 )
-  #"./pipelines/aws-spark-quoll-pipelines.yml"
-  #"./pipelines/nci3-spark-pipelines.yml"
 
 # shellcheck disable=SC2154
 if [[ -n $extra_args ]] ; then

utils/la-data-generator

@@ -21,13 +21,14 @@ eval "$("$GEN_LOC/docopts" -V - -h - : "$@" <<EOF
 $CMD: LA data generator
 
 Usage:
-  $CMD [options] --data=<dir> --inv=<dir> generate [<service>...]
+  $CMD [options] [--fast] --data=<dir> --inv=<dir> generate [<service>...]
   $CMD -h | --help
   $CMD -v | --version
 
 Options:
   --extra-args=<args>  Additional "arg1=values arg2=value" to get preference over defaul extra variables values
   --verbose            Verbose output.
+  --fast               Only exec the properties-file tag
   -d --dry-run         Print the commands without actually running them.
   -h --help            Show this help.
   -v --version         Show version.
@@ -69,7 +70,11 @@ elif [[ ! -d $inv || ! -f $inv/ansiblew ]]; then
 fi
 
 
-TAGS=common,properties,mongodb-org,namematching-service,pipelines,pipelines-layers
+if [[ -n $fast ]] ; then
+  TAGS=properties-file
+else
+  TAGS=common,properties,mongodb-org,namematching-service,pipelines,pipelines-layers
+fi
 SKIP_TAGS=restart,image-stored-procedures,db,mongodb-org-restart,mongodb-users,sslcerts,pipelines-services,hadoop_dir,hadoop_vocab,apt_update,apt,mongodb-org,docopts,yq
 # shellcheck disable=SC2154
 EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo cas_audit_host=auth-mongo cas_spring_session_host=auth-mongo \
@@ -80,11 +85,11 @@ EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo
     doi_db_hostname=doi-postgresql image_service_db_host=images-postgresql \
     mail_smtp_host=biocache-smtp mail_host=biocollect-smtp \
     ecodata_elasticsearch_host=ecodata-es events_elasticsearch_host=events-es events_elasticsearch_network_host=events-es-network \
-    events_elasticsearch_seed_host=events-es-seed solr_url=solr_host:2181 biocache_db_host=cassandra_host \
-    data_dir=/data/docker skip_handlers=true tomcat_apr=false use_docker_with_pipelines=false install_elasticsearch=false ecodata_url=FIXME_ECODATA_URL \
+    events_elasticsearch_seed_host=events-es-seed solr_user=$user solr_url=solr_host:2181 biocache_db_host=cassandra_host \
+    data_dir=$data skip_handlers=true tomcat_apr=false use_docker_with_pipelines=false install_elasticsearch=false ecodata_url=FIXME_ECODATA_URL \
     tomcat=$user tomcat_user=$user biocollect_user=$user ecodata_user=$user merit_user=$user fieldguide_app=fieldguide spark_user=$user \
     profile_service_user=$user profile_hub_user=$user doi_user=$user cas_user=$user userdetails_user=$user apikey_user=$user image_user=$user \
-    geoserver_user=$user \
+    geoserver_user=$user namematching_user=$user sensitive_data_user=$user\
     $extra_args"
 
 function gen() {