Skip to content

Commit 5cc0b17

Browse files
hakmanhakman
committed
azure: Add e2e testing
1 parent 6ac7fcc commit 5cc0b17

File tree

7 files changed

+81
-20
lines changed

7 files changed

+81
-20
lines changed

‎tests/e2e/e2e.mk‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,3 @@ test-e2e-install:
2020
cd $(KOPS_ROOT)/tests/e2e && \
2121
go install ./kubetest2-tester-kops && \
2222
go install ./kubetest2-kops
23-
24-
.PHONY: test-e2e-aws-simple-1-20
25-
test-e2e-aws-simple-1-20: test-e2e-install
26-
kubetest2 kops \
27-
-v 2 \
28-
--build --up --down \
29-
--cloud-provider=aws \
30-
--kops-version-marker=https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci-updown-green.txt \
31-
--kubernetes-version=https://dl.k8s.io/release/stable-1.20.txt \
32-
--template-path=tests/e2e/templates/simple.yaml.tmpl \
33-
--test=kops \
34-
-- \
35-
--ginkgo-args="--debug" \
36-
--test-package-marker=stable-1.20.txt \
37-
--parallel 25 \
38-
--skip-regex="\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[HPA\]|Dashboard|RuntimeClass|RuntimeHandler"

‎tests/e2e/kubetest2-kops/azure/zones.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package azure
18+
19+
import (
20+
"math/rand"
21+
)
22+
23+
var allZones = []string{
24+
"uksouth-1",
25+
"uksouth-2",
26+
"uksouth-3",
27+
}
28+
29+
// RandomZones returns a random set of availability zones within a region
30+
func RandomZones(count int) ([]string, error) {
31+
n := rand.Intn(1000) % len(allZones)
32+
chosenZone := allZones[n]
33+
34+
chosenZones := make([]string, 0)
35+
chosenZones = append(chosenZones, chosenZone)
36+
37+
return chosenZones, nil
38+
}

‎tests/e2e/kubetest2-kops/deployer/common.go‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,15 @@ func (d *deployer) initialize() error {
7373
d.SSHPublicKeyPath = publicKeyPath
7474
d.SSHPrivateKeyPath = privateKeyPath
7575
}
76+
case "azure":
77+
publicKeyPath, privateKeyPath, err := util.CreateSSHKeyPair(d.ClusterName)
78+
if err != nil {
79+
return err
80+
}
81+
d.SSHPublicKeyPath = publicKeyPath
82+
d.SSHPrivateKeyPath = privateKeyPath
83+
// TODO: Check if we can use "kops" as SSH user
84+
d.SSHUser = "ubuntu"
7685
case "digitalocean":
7786
if d.SSHPrivateKeyPath == "" {
7887
d.SSHPrivateKeyPath = os.Getenv("DO_SSH_PRIVATE_KEY_FILE")
@@ -180,6 +189,7 @@ func (d *deployer) verifyKopsFlags() error {
180189

181190
switch d.CloudProvider {
182191
case "aws":
192+
case "azure":
183193
case "gce":
184194
case "digitalocean":
185195
default:
@@ -222,14 +232,24 @@ func (d *deployer) env() []string {
222232
// Recognized by the e2e framework
223233
// https://github.com/kubernetes/kubernetes/blob/a750d8054a6cb3167f495829ce3e77ab0ccca48e/test/e2e/framework/ssh/ssh.go#L59-L62
224234
vars = append(vars, fmt.Sprintf("KUBE_SSH_KEY_PATH=%v", d.SSHPrivateKeyPath))
235+
} else if d.CloudProvider == "azure" {
236+
// Pass through some env vars if set
237+
for _, k := range []string{"AZURE_TENANT_ID", "AZURE_SUBSCRIPTION_ID", "AZURE_CLIENT_ID", "AZURE_FEDERATED_TOKEN_FILE", "AZURE_STORAGE_ACCOUNT"} {
238+
v := os.Getenv(k)
239+
if v != "" {
240+
vars = append(vars, k+"="+v)
241+
} else {
242+
klog.Warningf("Azure env var %s is empty.", k)
243+
}
244+
}
225245
} else if d.CloudProvider == "digitalocean" {
226246
// Pass through some env vars if set
227247
for _, k := range []string{"DIGITALOCEAN_ACCESS_TOKEN", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"} {
228248
v := os.Getenv(k)
229249
if v != "" {
230250
vars = append(vars, k+"="+v)
231251
} else {
232-
klog.Warningf("DO env var %s is empty..", k)
252+
klog.Warningf("DO env var %s is empty.", k)
233253
}
234254
}
235255
}
@@ -339,6 +359,9 @@ func (d *deployer) stateStore() string {
339359
}
340360
d.createBucket = true
341361
ss = "s3://" + bucketName
362+
case "azure":
363+
// TODO: Use dynamic container name
364+
ss = "azureblob://cluster-state"
342365
case "gce":
343366
d.createBucket = true
344367
ss = "gs://" + gce.GCSBucketName(d.GCPProject, "state")

‎tests/e2e/kubetest2-kops/deployer/up.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"errors"
2222
"fmt"
23+
"os"
2324
osexec "os/exec"
2425
"path"
2526
"strconv"
@@ -31,6 +32,7 @@ import (
3132

3233
"k8s.io/klog/v2"
3334
"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
35+
"k8s.io/kops/tests/e2e/kubetest2-kops/azure"
3436
"k8s.io/kops/tests/e2e/kubetest2-kops/do"
3537
"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
3638
"k8s.io/kops/tests/e2e/pkg/kops"
@@ -190,6 +192,15 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e
190192
} else {
191193
args = appendIfUnset(args, "--master-size", "c5.large")
192194
}
195+
case "azure":
196+
// TODO: Check why Azure requires --network-cidr
197+
args = appendIfUnset(args, "--network-cidr", "10.0.0.0/16")
198+
args = appendIfUnset(args, "--cloud-labels", "DO-NOT-DELETE=kOps")
199+
args = appendIfUnset(args, "--control-plane-size", "Standard_D4s_v3")
200+
args = appendIfUnset(args, "--node-size", "Standard_D2s_v3")
201+
// TODO: Check if we can use "kops" as SSH user
202+
args = appendIfUnset(args, "--azure-admin-user", "ubuntu")
203+
args = appendIfUnset(args, "--azure-subscription-id", os.Getenv("AZURE_SUBSCRIPTION_ID"))
193204
case "gce":
194205
if isArm {
195206
args = appendIfUnset(args, "--master-size", "t2a-standard-2")
@@ -346,6 +357,8 @@ func (d *deployer) zones() ([]string, error) {
346357
switch d.CloudProvider {
347358
case "aws":
348359
return aws.RandomZones(d.ControlPlaneCount)
360+
case "azure":
361+
return azure.RandomZones(1)
349362
case "gce":
350363
return gce.RandomZones(1)
351364
case "digitalocean":

‎tests/e2e/pkg/tester/tester.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ func (t *Tester) addProviderFlag() error {
148148

149149
provider := ""
150150
switch cluster.Spec.LegacyCloudProvider {
151-
case "aws", "gce":
151+
case "aws", "azure", "gce":
152152
provider = cluster.Spec.LegacyCloudProvider
153153
case "digitalocean":
154154
default:
@@ -253,6 +253,8 @@ func (t *Tester) addRegionFlag() error {
253253
case "aws":
254254
zone := cluster.Spec.Subnets[0].Zone
255255
region = zone[:len(zone)-1]
256+
case "azure":
257+
region = cluster.Spec.Subnets[0].Region
256258
case "gce":
257259
region = cluster.Spec.Subnets[0].Region
258260
default:

‎upup/pkg/fi/cloudup/utils.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,9 @@ func BuildCloud(cluster *kops.Cluster) (fi.Cloud, error) {
157157
}
158158

159159
cloudTags := map[string]string{azure.TagClusterName: cluster.ObjectMeta.Name}
160+
for k, v := range cluster.Spec.CloudLabels {
161+
cloudTags[k] = v
162+
}
160163

161164
subscriptionID := cluster.Spec.CloudProvider.Azure.SubscriptionID
162165
resourceGroupName := cluster.Spec.CloudProvider.Azure.ResourceGroupName

‎util/pkg/vfs/azureclient.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,9 @@ import (
2323

2424
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
2525
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
26-
"k8s.io/klog/v2"
2726
)
2827

2928
func newAzureClient(ctx context.Context) (*azblob.Client, error) {
30-
klog.Infof("New Azure Blob client")
3129
accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
3230
if accountName == "" {
3331
return nil, fmt.Errorf("AZURE_STORAGE_ACCOUNT must be set")

0 commit comments

Comments
 (0)