ci: refactor bootstrapping to avoid needing to invoke apt get (Azure#2336)

* refactor: Stop invoking jq in infra scripts

* refactor: Remove some commented out code from infra

* refactor: Don't install jq

    Ubuntu runners come with it pre-installed:
        https://github.com/actions/runner-images/blob/4fe7f6bc86111e8ae1721eeecaa8df3e9a3c2dbd/images/linux/Ubuntu2204-Readme.md

* refactor: Replace `az command | jq '.QUERY'` with `az command --query QUERY`

* refactor: Collapse `jq | jq | jq` into a single jq invocation

* refactor: Do not install xmlstarlet

    Seem to be entirely unused

* refactor: Don't install uuid-runtime and remove install_packages function

Author: kdestin

cli/batch-score-rest.sh

@@ -1,9 +1,9 @@
 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
 
-LOCATION=$(az ml workspace show| jq -r '.location')
+LOCATION=$(az ml workspace show --query location -o tsv)
 
 API_VERSION="2022-05-01"
 
@@ -149,7 +149,7 @@ response=$(curl --location --request PUT "https://management.azure.com/subscript
 }")
 #</create_endpoint>
 
-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id $TOKEN
 
 # <create_deployment>
@@ -201,7 +201,7 @@ response=$(curl --location --request PUT "https://management.azure.com/subscript
     \"location\": \"$LOCATION\"
 }")
 
-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id $TOKEN
 #</set_endpoint_defaults>
 

cli/deploy-rest.sh

@@ -1,10 +1,10 @@
 set -x
 
 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-LOCATION=$(az ml workspace show --query location | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+LOCATION=$(az ml workspace show --query location -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
 
 #</create_variables>
 
@@ -124,7 +124,7 @@ response=$(curl --location --request PUT "https://management.azure.com/subscript
 #</create_endpoint>
 
 echo "Endpoint response: $response"
-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id
 
 # <create_deployment>
@@ -153,15 +153,15 @@ response=$(curl --location --request PUT "https://management.azure.com/subscript
 #</create_deployment>
 
 echo "Endpoint response: $response"
-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id
 
 # <get_endpoint>
 response=$(curl --location --request GET "https://management.azure.com/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.MachineLearningServices/workspaces/$WORKSPACE/onlineEndpoints/$ENDPOINT_NAME?api-version=$API_VERSION" \
 --header "Content-Type: application/json" \
 --header "Authorization: Bearer $TOKEN")
 
-scoringUri=$(echo $response | jq -r '.properties' | jq -r '.scoringUri')
+scoringUri=$(echo $response | jq -r '.properties.scoringUri')
 # </get_endpoint>
 
 # <get_access_token>

cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh

@@ -50,7 +50,7 @@ az ml batch-deployment show --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NA
 
 echo "Invoking batch endpoint"
 # <start_batch_scoring_job>
-JOB_NAME = $(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/heart-disease-uci/data | jq -r '.name')
+JOB_NAME = $(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/heart-disease-uci/data --query name -o tsv)
 # </start_batch_scoring_job>
 
 echo "Showing job detail"

cli/endpoints/batch/deploy-models/heart-classifier-mlflow/deploy-and-run.sh

@@ -54,7 +54,7 @@ az ml batch-deployment show --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NA
 
 echo "Invoking batch endpoint"
 # <start_batch_scoring_job>
-JOB_NAME = $(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input azureml:heart-dataset-unlabeled@latest | jq -r '.name')
+JOB_NAME = $(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input azureml:heart-dataset-unlabeled@latest --query name -o tsv)
 # </start_batch_scoring_job>
 
 echo "Showing job detail"

cli/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/deploy-and-run.sh

@@ -53,15 +53,15 @@ az ml batch-deployment create --endpoint $ENDPOINT_NAME -f deployment.yml --set-
 #</create_deployment>
 
 #<invoke_deployment_file>
-JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --f inputs.yml | jq -r ".name")
+JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --f inputs.yml --query name -o tsv)
 #</invoke_deployment_file>
 
 #<stream_job_logs>
 az ml job stream -n $JOB_NAME
 #</stream_job_logs>
 
 #<get_child_jobs>
-az ml job list --parent-job-name $JOB_NAME | jq -r ".[].name"
+az ml job list --parent-job-name $JOB_NAME --query "[].name" -o tsv
 #</get_child_jobs>
 
 #<download_outputs>

cli/endpoints/batch/deploy-pipelines/hello-batch/deploy-and-run.sh

@@ -33,15 +33,15 @@ az ml batch-deployment create --endpoint $ENDPOINT_NAME -f deployment.yml --set-
 #</create_deployment>
 
 #<invoke_deployment_inline>
-JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME | jq -r ".name")
+JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --query name -o tsv)
 #</invoke_deployment_inline>
 
 #<stream_job_logs>
 az ml job stream -n $JOB_NAME
 #</stream_job_logs>
 
 #<run_pipeline_job_deployment>
-JOB_NAME=$(az ml job create -f pipeline-job.yml | jq -r ".name")
+JOB_NAME=$(az ml job create -f pipeline-job.yml --query name -o tsv)
 #</run_pipeline_job_deployment>
 
 #<create_deployment_from_job>

cli/endpoints/batch/deploy-pipelines/training-with-components/deploy-and-run.sh

@@ -41,15 +41,15 @@ az ml batch-deployment create --endpoint $ENDPOINT_NAME -f deployment-ordinal/de
 #</create_deployment>
 
 #<invoke_deployment_file>
-JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --f inputs.yml | jq -r ".name")
+JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --f inputs.yml --query name -o tsv)
 #</invoke_deployment_file>
 
 #<stream_job_logs>
 az ml job stream -n $JOB_NAME
 #</stream_job_logs>
 
 #<get_child_jobs>
-az ml job list --parent-job-name $JOB_NAME | jq -r ".[].name"
+az ml job list --parent-job-name $JOB_NAME --query "[].name" -o tsv
 #</get_child_jobs>
 
 #<download_outputs>
@@ -64,7 +64,7 @@ az ml batch-deployment create --endpoint $ENDPOINT_NAME -f deployment-onehot/dep
 
 #<invoke_nondefault_deployment_file>
 DEPLOYMENT_NAME="uci-classifier-train-onehot"
-JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME -d $DEPLOYMENT_NAME --f inputs.yml | jq -r ".name")
+JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME -d $DEPLOYMENT_NAME --f inputs.yml --query name -o tsv)
 #</invoke_nondefault_deployment_file>
 
 #<stream_nondefault_job_logs>

cli/train-rest.sh

@@ -1,11 +1,11 @@
 exit 0 # TODO - update script to new API
 
 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-LOCATION=$(az group show --query location | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+LOCATION=$(az group show --query location -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
 
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
 API_VERSION="2022-02-01-preview"
 COMPUTE_NAME="cpu-cluster"
 TOKEN=$(az account get-access-token --query accessToken -o tsv)
@@ -24,7 +24,7 @@ wait_for_completion () {
         job=$(curl --location --request GET "https://management.azure.com/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.MachineLearningServices/workspaces/$WORKSPACE/jobs/$1?api-version=$API_VERSION" \
             --header "Authorization: Bearer $TOKEN")
         # TODO error handling here
-        job_status=$(echo $job | jq -r '.properties' | jq -r '.status')
+        job_status=$(echo $job | jq -r '.properties.status')
         echo "Current job status: $job_status"
         sleep 5
     done

deploy-arm-templates-az-cli.sh

@@ -5,10 +5,10 @@ TOKEN=$(az account get-access-token --query accessToken -o tsv)
 #</get_access_token>
 
 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-LOCATION=$(az ml workspace show --query location | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+LOCATION=$(az ml workspace show --query location -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
 #</create_variables>
 
 # <set_endpoint_name>
@@ -115,7 +115,7 @@ response=$(curl --location --request GET "https://management.azure.com/subscript
 --header "Content-Type: application/json" \
 --header "Authorization: Bearer $TOKEN")
 
-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id
 # </get_endpoint>
 
@@ -142,10 +142,10 @@ response=$(curl --location --request GET "https://management.azure.com/subscript
 --header "Content-Type: application/json" \
 --header "Authorization: Bearer $TOKEN")
 
-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id
 
-scoringUri=$(echo $response | jq -r '.properties' | jq -r '.scoringUri')
+scoringUri=$(echo $response | jq -r '.properties.scoringUri')
 # </get_endpoint>
 
 # <get_endpoint_access_token>

deploy-arm-templates-rest.sh

@@ -1,10 +1,10 @@
 set -x
 
 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-LOCATION=$(az ml workspace show --query location | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+LOCATION=$(az ml workspace show --query location -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
 schema='$schema'
 #</create_variables>
 
@@ -35,7 +35,7 @@ wait_for_completion () {
     do
         sleep 5
         response=$($1)
-        operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+        operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
     done
 
   while [[ $status != "Succeeded" && $status != "Failed" ]]

infra/bootstrapping/bootstrap.sh

@@ -50,9 +50,6 @@ else
   echo "---------------------------------------------------------"
 fi
 
-echo_title "Ensuring dependent packages"
-"$SCRIPT_DIR"/sdk_helpers.sh install_packages
-
 echo_title "Installing tools"
 "$SCRIPT_DIR"/sdk_helpers.sh install_tools
 
@@ -61,7 +58,6 @@ echo_title "Installing tools"
 ###################
 
 "$SCRIPT_DIR"/sdk_helpers.sh validate_tool az || exit 1
-"$SCRIPT_DIR"/sdk_helpers.sh validate_tool jq || exit 1
 "$SCRIPT_DIR"/sdk_helpers.sh validate_tool sed || exit 1
 
 #login to azure using your credentials

infra/bootstrapping/sdk_helpers.sh

@@ -260,43 +260,6 @@ function IsInstalled {
     sudo dpkg -S "$1" &> /dev/null
 }
 
-function install_packages() {
-    echo_info "------------------------------------------------"
-    echo_info ">>> Updating packages index"
-    echo_info "------------------------------------------------"
-
-    echo_info ">>> Executing: apt-get update"
-    # https://github.com/orgs/community/discussions/47863
-    sudo apt-mark hold grub-efi-amd64-signed
-    sudo apt-get update --fix-missing
-    echo_info ">>> Executing: apt-get upgrade"
-    sudo apt-get upgrade -y
-    echo_info ">>> Executing: apt-get dist-upgrade"
-    sudo apt-get dist-upgrade -y
-
-    echo_info ">>> Installing packages"
-
-    # jq                - Required for running filters on a stream of JSON data from az
-    # uuid-runtime      - Required for containers
-    # uuid-runtime      - Required for aks/arc
-    packages_to_install=(
-      jq
-      uuid-runtime
-      xmlstarlet
-    )
-    for package in "${packages_to_install[@]}"; do
-      echo_info "Installing '$package'"
-      if ! IsInstalled "$package"; then
-          sudo apt-get install -y --no-install-recommends "${package}" > /dev/null 2>&1
-      else
-          echo_info "$package is already installed"
-      fi
-    done
-    echo_info ">>> Clean local cache for packages"
-
-    sudo apt-get autoclean && sudo apt-get autoremove > /dev/null 2>&1
-}
-
 function add_extension() {
     echo_info "az extension add -n $1 "
     az extension add -n "$1" -y
@@ -489,7 +452,7 @@ connect_arc(){
     get_kubeconfig "$AKS_CLUSTER_NAME"
 
     if
-        [[ $(az connectedk8s show --resource-group "${RESOURCE_GROUP_NAME}" --name "${ARC_CLUSTER_NAME}" | jq -r .name) == ${ARC_CLUSTER_NAME} ]]
+        [[ $(az connectedk8s show --resource-group "${RESOURCE_GROUP_NAME}" --name "${ARC_CLUSTER_NAME}" --query name --output tsv) == ${ARC_CLUSTER_NAME} ]]
     then
         echo_info "Cluster: ${ARC_CLUSTER_NAME} is already connected..."
         clusterState=$(az connectedk8s show --resource-group "${RESOURCE_GROUP_NAME}" --name "${ARC_CLUSTER_NAME}" --query connectivityStatus -o json)
@@ -527,7 +490,7 @@ function setup_compute() {
     fi
 
     if
-        [[ $(az ml compute show --resource-group "${RESOURCE_GROUP_NAME}" --name "${COMPUTE_NAME}" | jq -r .provisioning_state) == "Succeeded" ]]
+        [[ $(az ml compute show --resource-group "${RESOURCE_GROUP_NAME}" --name "${COMPUTE_NAME}" --query provisioning_state --output tsv) == "Succeeded" ]]
     then
         echo_info "Cluster is already attached to workspace for the cluster: ${CLUSTER_NAME} as ${COMPUTE_NAME} in workspace:${WORKSPACE_NAME} under namespace: ${COMPUTE_NS}..."
     else
@@ -633,7 +596,7 @@ install_k8s_extension(){
     fi
 
     if
-        [[ $(az k8s-extension show --cluster-type "${CLUSTER_TYPE}" -c "${ARC_CLUSTER_NAME}" -g "${RESOURCE_GROUP_NAME}" --name "${EXTENSION_NAME}" | jq -r .provisioningState) == "Succeeded" ]]
+        [[ $(az k8s-extension show --cluster-type "${CLUSTER_TYPE}" -c "${ARC_CLUSTER_NAME}" -g "${RESOURCE_GROUP_NAME}" --name "${EXTENSION_NAME}" --output tsv --query provisioningState) == "Succeeded" ]]
     then
         echo "Extension:${EXTENSION_NAME} already installed on cluster: ${ARC_CLUSTER_NAME}"
     else
@@ -745,18 +708,16 @@ function vmss_upgrade_policy_automatic() {
     local LOCAL_RESOURCE_GROUP_NAME=${1:-testrg}
     printf "Update VMSS upgrade policy in resource group %s\n" ${LOCAL_RESOURCE_GROUP_NAME}
     # get list of all scale sets
-    # VM_SCALE_SETS_JSON=$(az vmss list --resource-group ${LOCAL_RESOURCE_GROUP_NAME} -o json)
-    # VM_SCALE_SETS_LIST=$(echo $VM_SCALE_SETS_JSON | jq -r '.[] | .name')
-    VM_SCALE_SETS=$(az vmss list --subscription "${SUBSCRIPTION_ID}" --resource-group ${LOCAL_RESOURCE_GROUP_NAME} | jq -r '.[].name')
+    VM_SCALE_SETS=$(az vmss list --subscription "${SUBSCRIPTION_ID}" --resource-group ${LOCAL_RESOURCE_GROUP_NAME} --query '[].name' --output tsv)
 
     printf "Checking scalesets %s in resource-group %s\n" "${VM_SCALE_SETS}" "${LOCAL_RESOURCE_GROUP_NAME}"
     # temporarily disable the flag
     set +e
     for VMSS in ${VM_SCALE_SETS}; do
-        VMSS_PROPERTIES=$(az vmss show --subscription "${SUBSCRIPTION_ID}" --resource-group ${LOCAL_RESOURCE_GROUP_NAME} --name $VMSS)
-        # echo SKU_TEMP $VMSS_PROPERTIES
+        VMSS_UPGRADE_POLICY_MODE=$(az vmss show --subscription "${SUBSCRIPTION_ID}" --resource-group ${LOCAL_RESOURCE_GROUP_NAME} --name $VMSS --query upgradePolicy.mode --output tsv)
+
         # az vmss show -g "${LOCAL_RESOURCE_GROUP_NAME}" -n "${VMSS}" -o json
-        if [[ $(echo $VMSS_PROPERTIES | jq -r '.upgradePolicy.mode') == "Automatic" ]]; then
+        if [[ "$VMSS_UPGRADE_POLICY_MODE" == "Automatic" ]]; then
             echo_info "Skipping to update upgradePolicy for VMSS $VMSS in resource-group ${LOCAL_RESOURCE_GROUP_NAME}..."
             continue
         else
@@ -778,7 +739,7 @@ function vmss_upgrade_policy_all_rg() {
     # checking Resource group name to ensure we're in a managed cluster RG
     echo "Number of Resource groups starting with ${RG_PREFIX}:" $(az group list --subscription "${SUBSCRIPTION_ID}" --query "[? starts_with(@.name, '${RG_PREFIX}')] | length(@)")
     # az group list --query "[? starts_with(@.name, '${RG_PREFIX}')].name" -o tsv | xargs -i "$SCRIPT_DIR"/sdk_helpers.sh check_vmss "{}"
-    for LOCAL_RESOURCE_GROUP_NAME in $(az group list --subscription "${SUBSCRIPTION_ID}" --query "[? starts_with(@.name, '${RG_PREFIX}')].name" --output json | jq .[] -r); do
+    for LOCAL_RESOURCE_GROUP_NAME in $(az group list --subscription "${SUBSCRIPTION_ID}" --query "[? starts_with(@.name, '${RG_PREFIX}')].name" --output tsv); do
         # resource_id=$(az resource list --resource-group "${LOCAL_RESOURCE_GROUP_NAME}" --query [].id --output tsv)
         RESOURCE_GROUP_ID=$(az group show --subscription "${SUBSCRIPTION_ID}" --name "${LOCAL_RESOURCE_GROUP_NAME}" --query id -o tsv | tail -n1 | tr -d "[:cntrl:]")
         echo "Current tags for resource-group ${LOCAL_RESOURCE_GROUP_NAME}"