update the ips for Q4 2025 and expand to all AWS regions

Makefile

@@ -124,5 +124,9 @@ lint:
 # shell linter
 shellcheck:
 	hack/make-rules/shellcheck.sh
+
+# Run all the code generators
+codegen:
+	hack/make-rules/codegen.sh
 #################################################################################
 .PHONY: all archeio geranos build unit integration test e2e-test clean update gofmt verify verify-generated lint shellcheck

cmd/archeio/internal/app/buckets.go

@@ -33,36 +33,112 @@ func awsRegionToHostURL(region, defaultURL string) string {
 	// and then additional regions we're mapping to that bucket
 	// based roughly on physical adjacency (and therefore _presumed_ latency)
 	//
-	// if you add a bucket, add a case for the region it is in, and consider
-	// shifting other regions that do not have their own bucket
+	// As of late 2025, we don't have access to cn-northwest-1 or cn-north-1 regions as they are part of the aws-cn partition.
+	// So we are mapping them to ap-east-1(Hong Kong) for now.
+	// aws ec2 describe-regions --all-regions --query "Regions[].RegionName" --output json | jq .[] | awk '{print $0","}' | sort --version-sort
 
+	// Africa (Cape Town)
+	case "af-south-1":
+		return "https://prod-registry-k8s-io-af-south-1.s3.dualstack.af-south-1.amazonaws.com"
+	// Asia Pacific (Hong Kong) and China Regions
+	case "ap-east-1", "cn-northwest-1", "cn-north-1":
+		return "https://prod-registry-k8s-io-ap-east-1.s3.dualstack.ap-east-1.amazonaws.com"
+	// Asia Pacific (Taipei)
+	case "ap-east-2":
+		return "https://prod-registry-k8s-io-ap-east-1.s3.dualstack.ap-east-1.amazonaws.com"
+	// Asia Pacific (Tokyo)
+	case "ap-northeast-1":
+		return "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com"
+	// Asia Pacific (Seoul)
+	case "ap-northeast-2":
+		return "https://prod-registry-k8s-io-ap-northeast-2.s3.dualstack.ap-northeast-2.amazonaws.com"
+	// Asia Pacific (Osaka)
+	case "ap-northeast-3":
+		return "https://prod-registry-k8s-io-ap-northeast-3.s3.dualstack.ap-northeast-3.amazonaws.com"
+	// Asia Pacific (Singapore)
+	case "ap-southeast-1":
+		return "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com"
+	// Asia Pacific (Sydney)
+	case "ap-southeast-2":
+		return "https://prod-registry-k8s-io-ap-southeast-2.s3.dualstack.ap-southeast-2.amazonaws.com"
+	// Asia Pacific (Jakarta)
+	case "ap-southeast-3":
+		return "https://prod-registry-k8s-io-ap-southeast-3.s3.dualstack.ap-southeast-3.amazonaws.com"
+	// Asia Pacific (Melbourne)
+	case "ap-southeast-4":
+		return "https://prod-registry-k8s-io-ap-southeast-4.s3.dualstack.ap-southeast-4.amazonaws.com"
+	// Asia Pacific (Singapore)
+	case "ap-southeast-5":
+		return "https://prod-registry-k8s-io-ap-southeast-5.s3.dualstack.ap-southeast-5.amazonaws.com"
+	// Asia Pacific (New Zealand)
+	case "ap-southeast-6":
+		return "https://prod-registry-k8s-io-ap-southeast-6.s3.dualstack.ap-southeast-6.amazonaws.com"
+	// Asia Pacific (Thailand)
+	case "ap-southeast-7":
+		return "https://prod-registry-k8s-io-ap-southeast-7.s3.dualstack.ap-southeast-7.amazonaws.com"
+	// Asia Pacific (Mumbai)
+	case "ap-south-1":
+		return "https://prod-registry-k8s-io-ap-south-1.s3.dualstack.ap-south-1.amazonaws.com"
+	// Asia Pacific (Hyderabad)
+	case "ap-south-2":
+		return "https://prod-registry-k8s-io-ap-south-2.s3.dualstack.ap-south-2.amazonaws.com"
+	// Canada (Central)
+	case "ca-central-1":
+		return "https://prod-registry-k8s-io-ca-central-1.s3.dualstack.ca-central-1.amazonaws.com"
+	// Canada (Calgary)
+	case "ca-west-1":
+		return "https://prod-registry-k8s-io-ca-west-1.s3.dualstack.ca-west-1.amazonaws.com"
+	// Europe (Frankfurt)
+	case "eu-central-1":
+		return "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com"
+	// Europe (Zurich)
+	case "eu-central-2":
+		return "https://prod-registry-k8s-io-eu-central-2.s3.dualstack.eu-central-2.amazonaws.com"
+	// Europe (Stockholm)
+	case "eu-north-1":
+		return "https://prod-registry-k8s-io-eu-north-1.s3.dualstack.eu-north-1.amazonaws.com"
+	// Europe (Milan)
+	case "eu-south-1":
+		return "https://prod-registry-k8s-io-eu-south-1.s3.dualstack.eu-south-1.amazonaws.com"
+	// Europe (Spain)
+	case "eu-south-2":
+		return "https://prod-registry-k8s-io-eu-south-2.s3.dualstack.eu-south-2.amazonaws.com"
+	// Europe (Ireland)
+	case "eu-west-1":
+		return "https://prod-registry-k8s-io-eu-west-1.s3.dualstack.eu-west-1.amazonaws.com"
+	// Europe (London)
+	case "eu-west-2":
+		return "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com"
+	// Europe (Paris)
+	case "eu-west-3":
+		return "https://prod-registry-k8s-io-eu-west-3.s3.dualstack.eu-west-3.amazonaws.com"
+	// Israel (Tel Aviv)
+	case "il-central-1":
+		return "https://prod-registry-k8s-io-il-central-1.s3.dualstack.il-central-1.amazonaws.com"
+	// Middle East (UAE)
+	case "me-central-1":
+		return "https://prod-registry-k8s-io-me-central-1.s3.dualstack.me-central-1.amazonaws.com"
+	// Middle East (Bahrain)
+	case "me-south-1":
+		return "https://prod-registry-k8s-io-me-south-1.s3.dualstack.me-south-1.amazonaws.com"
+	// Mexico (Central)
+	case "mx-central-1":
+		return "https://prod-registry-k8s-io-mx-central-1.s3.dualstack.mx-central-1.amazonaws.com"
+	// South America (São Paulo)
+	case "sa-east-1":
+		return "https://prod-registry-k8s-io-sa-east-1.s3.dualstack.sa-east-1.amazonaws.com"
 	// US East (N. Virginia)
-	case "us-east-1", "sa-east-1", "mx-central-1":
+	case "us-east-1":
 		return "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com"
 	// US East (Ohio)
-	case "us-east-2", "ca-central-1":
+	case "us-east-2":
 		return "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com"
 	// US West (N. California)
 	case "us-west-1":
 		return "https://prod-registry-k8s-io-us-west-1.s3.dualstack.us-west-1.amazonaws.com"
 	// US West (Oregon)
-	case "us-west-2", "ca-west-1":
+	case "us-west-2":
 		return "https://prod-registry-k8s-io-us-west-2.s3.dualstack.us-west-2.amazonaws.com"
-	// Asia Pacific (Mumbai)
-	case "ap-south-1", "ap-south-2", "me-south-1", "me-central-1", "me-west-1":
-		return "https://prod-registry-k8s-io-ap-south-1.s3.dualstack.ap-south-1.amazonaws.com"
-	// Asia Pacific (Tokyo)
-	case "ap-northeast-1", "ap-northeast-2", "ap-northeast-3":
-		return "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com"
-	// Asia Pacific (Singapore)
-	case "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "ap-east-1", "ap-east-2", "cn-northwest-1", "cn-north-1":
-		return "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com"
-	// Europe (Frankfurt)
-	case "eu-central-1", "eu-central-2", "eu-south-1", "eu-south-2", "il-central-1":
-		return "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com"
-	// Europe (Ireland)
-	case "eu-west-1", "af-south-1", "eu-west-2", "eu-west-3", "eu-north-1":
-		return "https://prod-registry-k8s-io-eu-west-1.s3.dualstack.eu-west-1.amazonaws.com"
 	default:
 		return defaultURL
 	}

cmd/archeio/internal/app/handlers_test.go

@@ -26,8 +26,8 @@ import (
 func TestMakeHandler(t *testing.T) {
 	registryConfig := RegistryConfig{
 		// the v2 test below tests being redirected to k8s.gcr.io as that one doesn't have UpstreamRegistryPath
-		UpstreamRegistryEndpoint: "https://us.gcr.io",
-		UpstreamRegistryPath:     "k8s-artifacts-prod",
+		UpstreamRegistryEndpoint: "https://us-central1-docker.pkg.dev",
+		UpstreamRegistryPath:     "k8s-artifacts-prod/images",
 		InfoURL:                  "https://github.com/kubernetes/k8s.io/tree/main/registry.k8s.io",
 		PrivacyURL:               "https://www.linuxfoundation.org/privacy-policy/",
 	}
@@ -79,23 +79,23 @@ func TestMakeHandler(t *testing.T) {
 			Name:           "/v2/pause/manifests/latest",
 			Request:        httptest.NewRequest("GET", "http://localhost:8080/v2/pause/manifests/latest", nil),
 			ExpectedStatus: http.StatusTemporaryRedirect,
-			ExpectedURL:    "https://us.gcr.io/v2/k8s-artifacts-prod/pause/manifests/latest",
+			ExpectedURL:    "https://us-central1-docker.pkg.dev/v2/k8s-artifacts-prod/images/pause/manifests/latest",
 		},
 		{
 			Name:           "/v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
 			Request:        httptest.NewRequest("GET", "http://localhost:8080/v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e", nil),
 			ExpectedStatus: http.StatusTemporaryRedirect,
-			ExpectedURL:    "https://us.gcr.io/v2/k8s-artifacts-prod/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
+			ExpectedURL:    "https://us-central1-docker.pkg.dev/v2/k8s-artifacts-prod/images/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
 		},
 		{
-			Name: "AWS IP, /v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
+			Name: "AWS eu-west-3 IP, /v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
 			Request: func() *http.Request {
 				r := httptest.NewRequest("GET", "http://localhost:8080/v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e", nil)
 				r.RemoteAddr = "35.180.1.1:888"
 				return r
 			}(),
 			ExpectedStatus: http.StatusTemporaryRedirect,
-			ExpectedURL:    "https://prod-registry-k8s-io-eu-west-1.s3.dualstack.eu-west-1.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
+			ExpectedURL:    "https://prod-registry-k8s-io-eu-west-3.s3.dualstack.eu-west-3.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
 		},
 		{
 			Name: "GCP IP, /v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
@@ -105,7 +105,7 @@ func TestMakeHandler(t *testing.T) {
 				return r
 			}(),
 			ExpectedStatus: http.StatusTemporaryRedirect,
-			ExpectedURL:    "https://us.gcr.io/v2/k8s-artifacts-prod/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
+			ExpectedURL:    "https://us-central1-docker.pkg.dev/v2/k8s-artifacts-prod/images/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
 		},
 	}
 	for i := range testCases {
@@ -164,6 +164,7 @@ func TestMakeV2Handler(t *testing.T) {
 			"https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e": true,
 			"https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e":     true,
 			"https://prod-registry-k8s-io-eu-west-1.s3.dualstack.eu-west-1.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e":           true,
+			"https://prod-registry-k8s-io-eu-west-3.s3.dualstack.eu-west-3.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e":           true,
 			"https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-2.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e":           true,
 			"https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e":           true,
 			"https://prod-registry-k8s-io-us-west-1.s3.dualstack.us-west-1.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e":           true,
@@ -210,23 +211,23 @@ func TestMakeV2Handler(t *testing.T) {
 			ExpectedStatus: http.StatusNotFound,
 		},
 		{
-			Name: "AWS IP, /v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
+			Name: "AWS eu-west-3 IP, /v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
 			Request: func() *http.Request {
 				r := httptest.NewRequest("GET", "http://localhost:8080/v2/pause/blobs/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e", nil)
 				r.RemoteAddr = "35.180.1.1:888"
 				return r
 			}(),
 			ExpectedStatus: http.StatusTemporaryRedirect,
-			ExpectedURL:    "https://prod-registry-k8s-io-eu-west-1.s3.dualstack.eu-west-1.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
+			ExpectedURL:    "https://prod-registry-k8s-io-eu-west-3.s3.dualstack.eu-west-3.amazonaws.com/containers/images/sha256:da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e",
 		},
 		{
-			Name:           "AWS IP, /v2/pause/manifests/latest",
+			Name:           "Fetching image manifest, /v2/pause/manifests/latest",
 			Request:        httptest.NewRequest("GET", "http://localhost:8080/v2/pause/manifests/latest", nil),
 			ExpectedStatus: http.StatusTemporaryRedirect,
 			ExpectedURL:    "https://k8s.gcr.io/v2/pause/manifests/latest",
 		},
 		{
-			Name: "AWS IP, /v2/pause/blobs/sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1234567",
+			Name: "AWS eu-west-3 IP, /v2/pause/blobs/sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1234567",
 			Request: func() *http.Request {
 				r := httptest.NewRequest("GET", "http://localhost:8080/v2/pause/blobs/sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1234567", nil)
 				r.RemoteAddr = "35.180.1.1:888"

hack/make-rules/codegen.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# script to run all codegen
+set -o errexit -o nounset -o pipefail
+
+# cd to the repo root
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." &> /dev/null && pwd -P)"
+cd "${REPO_ROOT}"
+
+source hack/tools/setup-go.sh
+
+echo "Downloading AWS & GCP IP ranges data..."
+curl -fLo 'pkg/net/cloudcidrs/internal/ranges2go/data/aws-ip-ranges.json' 'https://ip-ranges.amazonaws.com/ip-ranges.json'
+curl -fLo 'pkg/net/cloudcidrs/internal/ranges2go/data/gcp-cloud.json' 'https://www.gstatic.com/ipranges/cloud.json'
+
+# AWS adds IP ranges for unreleased regions which we want to exclude
+EXCLUDED_AWS_REGIONS="me-west-1,sa-west-1" \
+OUT_FILE=pkg/net/cloudcidrs/zz_generated_range_data.go \
+DATA_DIR=pkg/net/cloudcidrs/internal/ranges2go/data \
+    go run ./pkg/net/cloudcidrs/internal/ranges2go

hack/make-rules/verify-generated.sh

@@ -28,12 +28,13 @@ trap 'rm -rf ${tmpdir?}' EXIT
 # generate and compare
 OUT_FILE="${tmpdir}"/zz_generated_range_data.go
 export OUT_FILE
-DATA_DIR="${REPO_ROOT}"/pkg/net/cloudcidrs/internal/ranges2go/data
-export DATA_DIR
-./pkg/net/cloudcidrs/internal/ranges2go/run.sh
+# keep excluded list in sync with hack/make-rules/codegen.sh
+EXCLUDED_AWS_REGIONS="me-west-1,sa-west-1" \
+DATA_DIR="${REPO_ROOT}"/pkg/net/cloudcidrs/internal/ranges2go/data \
+    go run ./pkg/net/cloudcidrs/internal/ranges2go
 
 if ! diff "${OUT_FILE}" ./pkg/net/cloudcidrs/zz_generated_range_data.go; then
     >&2 echo ""
-    >&2 echo "generated file is out of date, please run 'go generate ./...' to regenerate"
+    >&2 echo "generated file is out of date, please run 'make codegen' to regenerate"
     exit 1
 fi

pkg/net/cloudcidrs/doc.go

@@ -17,4 +17,4 @@ limitations under the License.
 // cloudcidrs contains Cloud CIDR matching
 package cloudcidrs
 
-//go:generate ./internal/ranges2go/run.sh
+//go:generate go run ./internal/ranges2go