Updated kit pipelines for kubeconfig workspaces #385
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: PipelineRun | ||
| metadata: | ||
| name: kit-cl2-load-example-run | ||
| namespace: tekton-pipelines | ||
| spec: | ||
| timeout: "12h" | ||
| workspaces: | ||
| - name: source | ||
| emptyDir: {} | ||
| - name: results | ||
| emptyDir: {} | ||
| - name: config | ||
| volumeClaimTemplate: | ||
| spec: | ||
| accessModes: | ||
| - ReadWriteOnce | ||
| storageClassName: kit-gp2 | ||
| resources: | ||
| requests: | ||
| storage: 1Gi | ||
| params: | ||
| - name: name | ||
| value: "my-cluster-name" | ||
| - name: "results-bucket" | ||
| value: "my-bucket" | ||
| - name: "node_count" | ||
| value: "100" | ||
| - name: "amp-workspace-id" | ||
| value: "my-AMP-workspace" | ||
| podTemplate: | ||
| nodeSelector: | ||
| kubernetes.io/arch: amd64 | ||
| # TODO: The mounted EBS PV is only writable to root, ideally we should avoid to run containers as root. | ||
| securityContext: | ||
| runAsNonRoot: false | ||
| runAsUser: 0 | ||
| serviceAccountName: tekton-pipelines-executor | ||
| pipelineRef: | ||
| name: kit-cl2-load | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,201 @@ | ||
| --- | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Pipeline | ||
| metadata: | ||
| name: kit-cl2-load | ||
| namespace: tekton-pipelines | ||
| spec: | ||
| workspaces: | ||
| - name: source | ||
| - name: results | ||
| - name: config | ||
| params: | ||
| - name: name | ||
| description: The name of the test cluster. | ||
| default: "guest" | ||
| - name: amp-workspace-id | ||
| description: The AMP workspace ID where remote write needs to happen. | ||
| - name: node_count | ||
| description: desired node count for Dataplane, min is 1000 to create DP nodes. | ||
| - name: kubernetes-version | ||
| default: "1.24" | ||
| description: Kubernetes version for the guest cluster. | ||
| - name: apiserver-replicas | ||
| description: "Desired replicas of the APIServer" | ||
| default: "2" | ||
| - name: apiserver-image | ||
| default: "" | ||
| description: Image of apiserver | ||
| - name: apiserver-parameters | ||
| default: "" | ||
| description: | | ||
| Parameters of the apiserver. e.g. '["--goaway-chance=0.01"]' | ||
| - name: apiserver-instance-type | ||
| default: "m5.2xlarge" | ||
| description: Instance type for the apiserver | ||
| - name: etcd-replicas | ||
| default: "3" | ||
| description: Number of ETCD replicas | ||
| - name: etcd-image | ||
| default: "" | ||
| description: Image of ETCD | ||
| - name: etcd-parameters | ||
| default: "" | ||
| description: Parameters of the ETCD container | ||
| - name: etcd-instance-type | ||
| default: "m5.xlarge" | ||
| description: Instance type for the ETCD | ||
| - name: aws-vpc-cni-version | ||
| default: release-1.10 | ||
| description: The release version for aws vpc cni. | ||
| - name: aws-ebs-csi-driver-version | ||
| default: release-1.13 | ||
| description: The release version for aws ebs csi driver. | ||
| - name: monitoring-node-instance-types | ||
| default: | | ||
| ["m5.4xlarge"] | ||
| description: The desired instance type of the monitoring node. | ||
| - name: monitoring-node-selectors | ||
| default: 'node.kubernetes.io/instance-type=m5.4xlarge' | ||
| description: The selectors to select the monitoring node for tainting. | ||
| - name: giturl | ||
| description: "git url to clone the package" | ||
| default: https://github.com/kubernetes/perf-tests.git | ||
| - name: cl2-branch | ||
| description: "The branch of clusterloader2 you want to use" | ||
| default: "master" | ||
| - name: pods-per-node | ||
| description: "pod density" | ||
| default: "10" | ||
| - name: nodes-per-namespace | ||
| description: "nodes per namespace to get created for load test " | ||
| default: "100" | ||
| - name: cl2-load-test-throughput | ||
| description: "throughput used for mutate operations" | ||
| default: "15" | ||
| - name: results-bucket | ||
| description: "Results bucket with path of s3 to upload results" | ||
| tasks: | ||
| - name: setup-control-plane | ||
| taskRef: | ||
| kind: Task | ||
| name: control-plane-setup | ||
| params: | ||
| - name: cluster-name | ||
| value: '$(params.name)' | ||
| - name: kubernetes-version | ||
| value: '$(params.kubernetes-version)' | ||
| - name: apiserver-replicas | ||
| value: '$(params.apiserver-replicas)' | ||
| - name: apiserver-image | ||
| value: '$(params.apiserver-image)' | ||
| - name: apiserver-parameters | ||
| value: '$(params.apiserver-parameters)' | ||
| - name: apiserver-instance-type | ||
| value: '$(params.apiserver-instance-type)' | ||
| - name: etcd-replicas | ||
| value: '$(params.etcd-replicas)' | ||
| - name: etcd-image | ||
| value: '$(params.etcd-image)' | ||
| - name: etcd-parameters | ||
| value: '$(params.etcd-parameters)' | ||
| - name: etcd-instance-type | ||
| value: '$(params.etcd-instance-type)' | ||
|
|
||
| - name: setup-monitoring-node | ||
| runAfter: [setup-control-plane] | ||
| taskRef: | ||
| kind: Task | ||
| name: data-plane-setup | ||
| params: | ||
| - name: cluster-name | ||
| value: '$(params.name)' | ||
| - name: dataplane-name | ||
| value: 'monitoring-$(params.name)' | ||
| - name: desired-nodes | ||
| value: '1' | ||
| - name: instance-types | ||
| value: '$(params.monitoring-node-instance-types)' | ||
|
|
||
| - name: label-and-taint-minitoring-node | ||
| runAfter: [setup-monitoring-node] | ||
| taskRef: | ||
| kind: Task | ||
| name: label-and-taint-node | ||
| params: | ||
| - name: cluster-name | ||
| value: '$(params.name)' | ||
| - name: selectors | ||
| value: '$(params.monitoring-node-selectors)' | ||
| - name: taint | ||
| value: 'monitoring=true:NoSchedule' | ||
| # We add another label to ensure Prometheus stack only run on the dedicated node in | ||
| # https://github.com/awslabs/kubernetes-iteration-toolkit/blob/e95b99e09e89f23a4aa8162632ba78b6c351dec5/tests/tasks/generators/clusterloader/load.yaml#L84 | ||
| - name: labels | ||
| value: 'eks.amazonaws.com/nodegroup=monitoring-$(params.name)-nodes-1' | ||
| workspaces: | ||
| - name: config | ||
| workspace: config | ||
|
|
||
| - name: setup-data-plane | ||
| runAfter: [label-and-taint-minitoring-node] | ||
| taskRef: | ||
| kind: Task | ||
| name: data-plane-setup | ||
| params: | ||
| - name: cluster-name | ||
| value: '$(params.name)' | ||
| - name: desired-nodes | ||
| value: '$(params.node_count)' | ||
|
|
||
| - name: validate-data-plane | ||
| runAfter: [setup-data-plane] | ||
| taskRef: | ||
| kind: Task | ||
| name: validate-data-plane | ||
| params: | ||
| - name: desired-nodes | ||
| value: '$(($(params.node_count)+1))' | ||
| workspaces: | ||
| - name: config | ||
| workspace: config | ||
|
|
||
| - name: load | ||
| runAfter: [validate-data-plane] | ||
| taskRef: | ||
| kind: Task | ||
| name: load | ||
| params: | ||
| - name: cluster-name | ||
| value: '$(params.name)' | ||
| - name: cl2-branch | ||
| value: '$(params.cl2-branch)' | ||
| - name: giturl | ||
| value: '$(params.giturl)' | ||
| - name: pods-per-node | ||
| value: '$(params.pods-per-node)' | ||
| - name: nodes-per-namespace | ||
| value: '$(params.nodes-per-namespace)' | ||
| - name: cl2-load-test-throughput | ||
| value: '$(params.cl2-load-test-throughput)' | ||
| - name: results-bucket | ||
| value: '$(params.results-bucket)' | ||
| - name: nodes | ||
| value: '$(params.node_count)' | ||
| - name: amp-workspace-id | ||
| value: '$(params.amp-workspace-id)' | ||
| workspaces: | ||
| - name: source | ||
| workspace: source | ||
| - name: results | ||
| workspace: results | ||
| - name: config | ||
| workspace: config | ||
|
|
||
| # finally: | ||
|
There was a problem hiding this comment. why is this commented out ? |
||
| # - name: teardown | ||
| # taskRef: | ||
| # name: teardown | ||
| # params: | ||
| # - name: name | ||
| # value: '$(params.name)' | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| --- | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Task | ||
| metadata: | ||
| name: control-plane-setup | ||
| namespace: tekton-pipelines | ||
| annotations: | ||
| tekton.dev/pipelines.minVersion: "0.17.0" | ||
| tekton.dev/categories: Kubernetes | ||
| tekton.dev/tags: CLI, kubectl | ||
| tekton.dev/displayName: "kubernetes actions" | ||
| tekton.dev/platforms: "linux/amd64" | ||
| spec: | ||
| description: | | ||
| Setup a kubernetes control plane in the guest cluster. | ||
| params: | ||
| - name: cluster-name | ||
| default: "guest" | ||
| description: Name of the guest cluster | ||
| - name: kubernetes-version | ||
| default: "1.24" | ||
| description: Kubernetes version for the guest cluster | ||
| - name: apiserver-replicas | ||
| default: "1" | ||
| description: Number of APIserver replicas | ||
| - name: apiserver-image | ||
| default: "" | ||
| description: Image of apiserver | ||
| - name: apiserver-parameters | ||
| default: "" | ||
| description: | | ||
| Parameters of the apiserver. e.g. '["--goaway-chance=0.01"]' | ||
| - name: apiserver-instance-type | ||
| default: "m5.2xlarge" | ||
| description: Instance type for the apiserver | ||
| - name: etcd-replicas | ||
| default: "3" | ||
| description: Number of ETCD replicas | ||
| - name: etcd-image | ||
| default: "" | ||
| description: Image of ETCD | ||
| - name: etcd-parameters | ||
| default: "" | ||
| description: Parameters of the ETCD container | ||
| - name: etcd-instance-type | ||
| default: "m5.xlarge" | ||
| description: Instance type for the ETCD | ||
| - name: cni-version | ||
| default: "v1.9" | ||
| description: Version of the CNI to be installed | ||
| - name: aws-ebs-csi-driver-version | ||
| default: release-1.13 | ||
| description: The release version for aws ebs csi driver. | ||
| workspaces: | ||
| - name: config | ||
| mountPath: /config/ | ||
| steps: | ||
| - name: setup-control-plane | ||
| image: bitnami/kubectl:1.24.5 # curl was removed in more recent versions | ||
|
There was a problem hiding this comment. can't we use Alpine regular image that we use in other tasks ? |
||
| script: | | ||
| #!/bin/bash | ||
| echo "Approving CSRs" | ||
| kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}') 2>/dev/null || true | ||
| namespace=$(kubectl get ns $(params.cluster-name) -o yaml 2>/dev/null | grep phase | awk '{print $2}') | ||
| if [[ $namespace != "Active" ]] | ||
| then | ||
| echo "Create namespace" | ||
| kubectl create namespace $(params.cluster-name) | ||
| fi | ||
| echo "Setting up control plane" | ||
| cat <<EOF > /tmp/controlplane.yaml | ||
|
There was a problem hiding this comment. let's take these files as params which will reduce lot's of params in these tasks and boils down to file locations, it's much cleaner that way and also it's much more dynamic than embedding it here. Example - like how we take it for service role tasks etc. |
||
| apiVersion: kit.k8s.sh/v1alpha1 | ||
| kind: ControlPlane | ||
| metadata: | ||
| name: $(params.cluster-name) # Desired Cluster name | ||
| namespace: $(params.cluster-name) | ||
| spec: | ||
| etcd: | ||
| replicas: $(params.etcd-replicas) | ||
| spec: | ||
| nodeSelector: | ||
| node.kubernetes.io/instance-type: $(params.etcd-instance-type) | ||
| containers: | ||
| - name: etcd | ||
| EOF | ||
| if [ -n "$(params.etcd-image)" ]; then | ||
| cat <<EOF >> /tmp/controlplane.yaml | ||
| image: $(params.etcd-image) | ||
| EOF | ||
| fi | ||
| if [ -n "$(params.etcd-parameters)" ]; then | ||
| cat <<EOF >> /tmp/controlplane.yaml | ||
| args: $(params.etcd-parameters) | ||
| EOF | ||
| fi | ||
| cat <<EOF >> /tmp/controlplane.yaml | ||
| master: | ||
| apiServer: | ||
| replicas: $(params.apiserver-replicas) | ||
| spec: | ||
| nodeSelector: | ||
| node.kubernetes.io/instance-type: $(params.apiserver-instance-type) | ||
| containers: | ||
| - name: apiserver | ||
| EOF | ||
| if [ -n "$(params.apiserver-image)" ]; then | ||
| cat <<EOF >> /tmp/controlplane.yaml | ||
| image: $(params.apiserver-image) | ||
| EOF | ||
| fi | ||
| if [ -n "$(params.apiserver-parameters)" ]; then | ||
| cat <<EOF >> /tmp/controlplane.yaml | ||
| args: $(params.apiserver-parameters) | ||
| EOF | ||
| fi | ||
| if [ -n "$(params.kubernetes-version)" ]; then | ||
| cat <<EOF >> /tmp/controlplane.yaml | ||
| kubernetesVersion: "$(params.kubernetes-version)" | ||
| EOF | ||
| fi | ||
| kubectl apply -f /tmp/controlplane.yaml | ||
|
|
||
| echo "waiting for controlplane to be ready" | ||
| while true; do | ||
| status0=$(kubectl get -f /tmp/controlplane.yaml -o jsonpath='{.status.conditions[0].status}') | ||
| status1=$(kubectl get -f /tmp/controlplane.yaml -o jsonpath='{.status.conditions[1].status}') | ||
| if [[ "$status0" == "True" ]] && [[ "$status1" == "True" ]]; then | ||
| echo "controlplane is ready" | ||
| break | ||
| fi | ||
| done | ||
| kubectl get -f /tmp/controlplane.yaml -o yaml | ||
| - name: retrieve-kubeconfig | ||
| image: bitnami/kubectl | ||
| script: | | ||
| echo $(workspaces.config.path) | ||
| ls -ld $(workspaces.config.path) | ||
| kubectl get secret -n $(params.cluster-name) $(params.cluster-name)-kube-admin-config -ojsonpath='{.data.config}' | base64 -d > $(workspaces.config.path)/kubeconfig | ||
| # TODO: remove debug | ||
| cat $(workspaces.config.path)/kubeconfig | ||
| # sanity check to ensure it's live. | ||
| kubectl --kubeconfig $(workspaces.config.path)/kubeconfig version | ||
| - name: install-cni | ||
| image: bitnami/kubectl | ||
|
|
||
| script: | | ||
| kubectl --kubeconfig=$(workspaces.config.path)/kubeconfig apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/master/config/$(params.cni-version)/aws-k8s-cni.yaml | ||
| - name: install-csi-driver | ||
| image: bitnami/kubectl | ||
| script: | | ||
| kubectl --kubeconfig=$(workspaces.config.path)/kubeconfig apply -k "github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=$(params.aws-ebs-csi-driver-version)" | ||
|
|
||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Keep this pipelinerun here will be automatically applied to Infra cluster due to FLUX sync. Can we move this example outside of tests dir ?