From ca2b6d8bf1e6bfa484b365511e08372ccb29b634 Mon Sep 17 00:00:00 2001 From: vjrj Date: Thu, 3 Aug 2023 11:07:08 +0200 Subject: [PATCH] More local /data generation options (#674) * More /data generation options * Added groups for local generation * Added extra-args for solr_url and similar portal specific variables * Remove FIXME from now configurable variables --- utils/ala-data-generator | 64 ++++++++++++++++++++++++++-------------- utils/la-data-generator | 11 +++++-- utils/local.ini | 9 ++++++ 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/utils/ala-data-generator b/utils/ala-data-generator index 0d3b5b19c..a7340a223 100755 --- a/utils/ala-data-generator +++ b/utils/ala-data-generator @@ -1,11 +1,6 @@ #!/bin/bash -# Run with: -# ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate doi alerts -# or for all services: -# ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate - -# It will fail if some playbook fails +# Let fail if some service generation fails set -e CMD=$(basename "$0") @@ -22,13 +17,25 @@ $CMD: ALA data generator Usage: $CMD [options] --data= --inv= generate [...] + $CMD [options] --data= $CMD -h | --help $CMD -v | --version Options: + --extra-args= Additional "arg1=values arg2=value" to get preference over defaul extra variables values --verbose Verbose output. -h --help Show this help. -v --version Show version. + +For instance, run for production inventories with: + ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate doi alerts +or for all services: + ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate +or for some specific service: + ./ala-data-generator --data=/data/docker ./collections-test +or in other directory and using some extra arg: + /home/youruser/ala-install/utils/ala-data-generator --extra-args="solr_url=some_solr_host:2181" --data=/data/docker ./biocache-service-test-databox + ---- $CMD $VER License Apache-2.0 @@ -71,24 +78,37 @@ list=( #"./pipelines/aws-spark-quoll-pipelines.yml" #"./pipelines/nci3-spark-pipelines.yml" -if [[ -n $service ]] ; then - services=("${service[@]}") +# shellcheck disable=SC2154 +if [[ -n $extra_args ]] ; then + E="--extra-args $extra_args" +else + E="" fi -for el in "${list[@]}" -do - match=false - for arg in "${services[@]}" +# shellcheck disable=SC2154 +if [[ -n $inventory ]]; then + # shellcheck disable=SC2086 + "$GEN_LOC/la-data-generator" --inv="$PWD/" --data="$data" $E generate "$inventory" +else + if [[ -n $service ]] ; then + services=("${service[@]}") + fi + + for el in "${list[@]}" do - if [[ $el == *"$arg"* ]]; then - match=true - break + match=false + for arg in "${services[@]}" + do + if [[ $el == *"$arg"* ]]; then + match=true + break + fi + done + + if [[ -z "${services[*]}" ]] || $match; then + # echo -n "$el " + # shellcheck disable=SC2086 + "$GEN_LOC/la-data-generator" --inv="$inv" --data="$data" $E generate "$el" fi done - - if [[ -z "${services[*]}" ]] || $match; then - # echo -n "$el " - "$GEN_LOC/la-data-generator" --inv=$inv --data=$data generate "$el" - fi -done - +fi diff --git a/utils/la-data-generator b/utils/la-data-generator index 023c5a51d..06482259b 100755 --- a/utils/la-data-generator +++ b/utils/la-data-generator @@ -26,10 +26,12 @@ Usage: $CMD -v | --version Options: + --extra-args= Additional "arg1=values arg2=value" to get preference over defaul extra variables values --verbose Verbose output. -d --dry-run Print the commands without actually running them. -h --help Show this help. -v --version Show version. + ---- $CMD $VER License Apache-2.0 @@ -66,9 +68,11 @@ elif [[ ! -d $inv || ! -f $inv/ansiblew ]]; then exit 1; fi + TAGS=common,properties,mongodb-org,namematching-service,pipelines,pipelines-layers SKIP_TAGS=restart,image-stored-procedures,db,mongodb-org-restart,mongodb-users,sslcerts,pipelines-services,hadoop_dir,hadoop_vocab,apt_update,apt,mongodb-org,docopts,yq - EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo cas_audit_host=auth-mongo cas_spring_session_host=auth-mongo \ +# shellcheck disable=SC2154 +EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo cas_audit_host=auth-mongo cas_spring_session_host=auth-mongo \ user_store_db_hostname=auth-mysql apikey_db_hostname=auth-mysql cas_db_hostname=auth-mysql cas_tickets_host=auth-mongo \ cas_tickets_host=auth-mongo cas_services_host=auth-mongo cas_services_host=auth-mongo collectory_db_host_address=collectory-mysql \ dq_db_url=jdbc:postgresql://dq-postgresql/data-quality specieslist_db_hostname=lists-mysql logger_db_hostname=logger-mysql \ @@ -76,11 +80,12 @@ SKIP_TAGS=restart,image-stored-procedures,db,mongodb-org-restart,mongodb-users,s doi_db_hostname=doi-postgresql image_service_db_host=images-postgresql \ mail_smtp_host=biocache-smtp mail_host=biocollect-smtp \ ecodata_elasticsearch_host=ecodata-es events_elasticsearch_host=events-es events_elasticsearch_network_host=events-es-network \ - events_elasticsearch_seed_host=events-es-seed solr_url=FIXME_solr_url biocache_db_host=FIXME_cassandra_host \ + events_elasticsearch_seed_host=events-es-seed solr_url=solr_host:2181 biocache_db_host=cassandra_host \ data_dir=/data/docker skip_handlers=true tomcat_apr=false use_docker_with_pipelines=false install_elasticsearch=false ecodata_url=FIXME_ECODATA_URL \ tomcat=$user tomcat_user=$user biocollect_user=$user ecodata_user=$user merit_user=$user fieldguide_app=fieldguide spark_user=$user \ profile_service_user=$user profile_hub_user=$user doi_user=$user cas_user=$user userdetails_user=$user apikey_user=$user image_user=$user \ - geoserver_user=$user" + geoserver_user=$user \ + $extra_args" function gen() { local what diff --git a/utils/local.ini b/utils/local.ini index 9c8098c27..1601f53f2 100644 --- a/utils/local.ini +++ b/utils/local.ini @@ -130,3 +130,12 @@ [cassandra3] 127.0.0.1 ansible_connection=local + +[solrcloud_config] +127.0.0.1 ansible_connection=local + +[sandbox] +127.0.0.1 ansible_connection=local + +[profiles] +127.0.0.1 ansible_connection=local