feat: Allow specifying Helm values for operator installations (#425)

* allow specifying a values file that will be used for operator installations

* minor refactor

* rename FileTransfer error variant

* Remove common key in yaml and deep merge logic; recommend yaml anchors

* Improve helm value file error output

* Fix values file example

* explain why serialization statement won't fail

* fix pre-commit

Author: labrenbe

docs/modules/stackablectl/partials/commands/cache.adoc

@@ -61,6 +61,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -90,6 +115,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]

docs/modules/stackablectl/partials/commands/completions.adoc

@@ -64,6 +64,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -93,6 +118,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]

docs/modules/stackablectl/partials/commands/demo.adoc

@@ -65,6 +65,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -94,6 +119,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]

docs/modules/stackablectl/partials/commands/experimental-debug.adoc

@@ -78,6 +78,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -107,6 +132,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]

docs/modules/stackablectl/partials/commands/index.adoc

@@ -68,6 +68,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -97,6 +122,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]

docs/modules/stackablectl/partials/commands/operator.adoc

@@ -64,6 +64,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -93,6 +118,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]

docs/modules/stackablectl/partials/commands/release.adoc

@@ -64,6 +64,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -93,6 +118,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]

docs/modules/stackablectl/partials/commands/stack.adoc

@@ -65,6 +65,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -94,6 +119,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]

docs/modules/stackablectl/partials/commands/stacklet.adoc

@@ -66,6 +66,31 @@ File options:
           Use "stackablectl [OPTIONS] <COMMAND> -r path/to/releases1.yaml -r path/to/releases2.yaml"
           to provide multiple additional release files.
 
+  -f, --operator-values <VALUES_FILE>
+          Path to a Helm values file that will be used for the installation of operators
+
+          The file is a YAML file containing Helm values used to deploy operators.
+          Operator-specific keys (e.g. 'airflow-operator', 'zookeeper-operator') map
+          to the Helm values for that operator. Use YAML anchors and aliases to share
+          values across operators.
+
+          Example values file:
+
+            airflow-operator:
+              tolerations: &default-tolerations
+                - key: "example"
+                  operator: "Exists"
+                  effect: "NoSchedule"
+              podAnnotations:
+                example.com/team: "data-engineering"
+            zookeeper-operator:
+              tolerations: *default-tolerations
+              podAnnotations:
+                example.com/team: "platform"
+
+          Use "stackablectl [OPTIONS] <COMMAND> -f path/to/values.yaml" to provide a
+          values file.
+
 Helm repository options:
       --helm-repo-stable <URL>
           Provide a custom Helm stable repository URL
@@ -95,6 +120,8 @@ Operator specific configurations:
       --listener-class-preset <LISTENER_CLASS_PRESET>
           Choose the ListenerClass preset (`none`, `ephemeral-nodes` or `stable-nodes`).
 
+          This takes precedence over values provided via the '-f/--operator-values' flag.
+
           This maps to the listener-operator Helm Chart preset value, see [the listener-operator documentation](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass/#presets) for details.
 
           [possible values: none, stable-nodes, ephemeral-nodes]