More local /data generation options (#674)

* More /data generation options

* Added groups for local generation

* Added extra-args for solr_url and similar portal specific variables

* Remove FIXME from now configurable variables

utils/ala-data-generator

@@ -1,11 +1,6 @@
 #!/bin/bash
 
-# Run with:
-#   ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate doi alerts
-# or for all services:
-#   ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate
-
-# It will fail if some playbook fails 
+# Let fail if some service generation fails
 set -e
 
 CMD=$(basename "$0")
@@ -22,13 +17,25 @@ $CMD: ALA data generator
 
 Usage:
   $CMD [options] --data=<dir> --inv=<dir> generate [<service>...]
+  $CMD [options] --data=<dir> <inventory>
   $CMD -h | --help
   $CMD -v | --version
 
 Options:
+  --extra-args=<args>  Additional "arg1=values arg2=value" to get preference over defaul extra variables values
   --verbose            Verbose output.
   -h --help            Show this help.
   -v --version         Show version.
+
+For instance, run for production inventories with:
+  ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate doi alerts
+or for all services:
+  ./ala-data-generator --inv=/home/youruser/ala/ansible-inventories/ --data=/data/docker generate
+or for some specific service:
+  ./ala-data-generator --data=/data/docker ./collections-test
+or in other directory and using some extra arg:
+  /home/youruser/ala-install/utils/ala-data-generator --extra-args="solr_url=some_solr_host:2181" --data=/data/docker ./biocache-service-test-databox
+
 ----
 $CMD $VER
 License Apache-2.0
@@ -71,24 +78,37 @@ list=(
   #"./pipelines/aws-spark-quoll-pipelines.yml"
   #"./pipelines/nci3-spark-pipelines.yml"
 
-if [[ -n $service ]] ; then
-    services=("${service[@]}")
+# shellcheck disable=SC2154
+if [[ -n $extra_args ]] ; then
+  E="--extra-args $extra_args"
+else
+  E=""
 fi
 
-for el in "${list[@]}"
-do
-  match=false
-  for arg in "${services[@]}"
+# shellcheck disable=SC2154
+if [[ -n $inventory ]]; then
+   # shellcheck disable=SC2086
+   "$GEN_LOC/la-data-generator" --inv="$PWD/" --data="$data" $E generate "$inventory"
+else
+  if [[ -n $service ]] ; then
+    services=("${service[@]}")
+  fi
+
+  for el in "${list[@]}"
   do
-    if [[ $el == *"$arg"* ]]; then
-      match=true
-      break
+    match=false
+    for arg in "${services[@]}"
+    do
+      if [[ $el == *"$arg"* ]]; then
+        match=true
+        break
+      fi
+    done
+
+    if [[ -z "${services[*]}" ]] || $match; then
+      # echo -n "$el "
+      # shellcheck disable=SC2086
+      "$GEN_LOC/la-data-generator" --inv="$inv" --data="$data" $E generate "$el"
     fi
   done
-
-  if [[ -z "${services[*]}" ]] || $match; then
-    # echo -n "$el "
-    "$GEN_LOC/la-data-generator" --inv=$inv --data=$data generate "$el"
-  fi
-done
-
+fi

utils/la-data-generator

@@ -26,10 +26,12 @@ Usage:
   $CMD -v | --version
 
 Options:
+  --extra-args=<args>  Additional "arg1=values arg2=value" to get preference over defaul extra variables values
   --verbose            Verbose output.
   -d --dry-run         Print the commands without actually running them.
   -h --help            Show this help.
   -v --version         Show version.
+
 ----
 $CMD $VER
 License Apache-2.0
@@ -66,21 +68,24 @@ elif [[ ! -d $inv || ! -f $inv/ansiblew ]]; then
       exit 1;
 fi
 
+
 TAGS=common,properties,mongodb-org,namematching-service,pipelines,pipelines-layers
 SKIP_TAGS=restart,image-stored-procedures,db,mongodb-org-restart,mongodb-users,sslcerts,pipelines-services,hadoop_dir,hadoop_vocab,apt_update,apt,mongodb-org,docopts,yq
- EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo cas_audit_host=auth-mongo cas_spring_session_host=auth-mongo \
+# shellcheck disable=SC2154
+EXTRAS="user_store_db_hostname=auth-mysql ticket_registry_db_hostname=auth-mongo cas_audit_host=auth-mongo cas_spring_session_host=auth-mongo \
     user_store_db_hostname=auth-mysql apikey_db_hostname=auth-mysql cas_db_hostname=auth-mysql cas_tickets_host=auth-mongo \
     cas_tickets_host=auth-mongo cas_services_host=auth-mongo cas_services_host=auth-mongo collectory_db_host_address=collectory-mysql \
     dq_db_url=jdbc:postgresql://dq-postgresql/data-quality specieslist_db_hostname=lists-mysql logger_db_hostname=logger-mysql \
     alerts_db_hostname=alerts-mysql phylolink_db_hostname=phylolink-mysql geonetwork_database=spatial-postgresql layers_db_host=spatial-postgresql \
     doi_db_hostname=doi-postgresql image_service_db_host=images-postgresql \
     mail_smtp_host=biocache-smtp mail_host=biocollect-smtp \
     ecodata_elasticsearch_host=ecodata-es events_elasticsearch_host=events-es events_elasticsearch_network_host=events-es-network \
-    events_elasticsearch_seed_host=events-es-seed solr_url=FIXME_solr_url biocache_db_host=FIXME_cassandra_host \
+    events_elasticsearch_seed_host=events-es-seed solr_url=solr_host:2181 biocache_db_host=cassandra_host \
     data_dir=/data/docker skip_handlers=true tomcat_apr=false use_docker_with_pipelines=false install_elasticsearch=false ecodata_url=FIXME_ECODATA_URL \
     tomcat=$user tomcat_user=$user biocollect_user=$user ecodata_user=$user merit_user=$user fieldguide_app=fieldguide spark_user=$user \
     profile_service_user=$user profile_hub_user=$user doi_user=$user cas_user=$user userdetails_user=$user apikey_user=$user image_user=$user \
-    geoserver_user=$user"
+    geoserver_user=$user \
+    $extra_args"
 
 function gen() {
     local what

utils/local.ini

@@ -130,3 +130,12 @@
 
 [cassandra3]
 127.0.0.1 ansible_connection=local
+
+[solrcloud_config]
+127.0.0.1 ansible_connection=local
+
+[sandbox]
+127.0.0.1 ansible_connection=local
+
+[profiles]
+127.0.0.1 ansible_connection=local