feat: Add E2E Endpoint Checks

test/e2e/e2e_test.go

@@ -33,6 +33,7 @@ var _ = BeforeSuite(func() {
 	GinkgoWriter.Printf("Namespace %q for e2e tests\n", namespaceName)
 
 	kaitoNamespace := os.Getenv("KAITO_NAMESPACE")
+	Expect(createCurlDebugPod(kaitoNamespace)).To(Succeed(), "Failed to create curl debug pod")
 
 	if nodeProvisionerName == "azkarpenter" {
 		karpenterNamespace := os.Getenv("KARPENTER_NAMESPACE")

test/e2e/preset_test.go

@@ -4,14 +4,10 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
-	"log"
-	"math/rand"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
+	kaitov1alpha1 "github.com/kaito-project/kaito/api/v1alpha1"
+	"github.com/kaito-project/kaito/test/e2e/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
@@ -20,10 +16,13 @@
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"log"
+	"math/rand"
+	"os"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	kaitov1alpha1 "github.com/kaito-project/kaito/api/v1alpha1"
-	"github.com/kaito-project/kaito/test/e2e/utils"
+	"strconv"
+	"strings"
+	"time"
 )
 
 const (
@@ -42,6 +41,8 @@
 	WorkspaceRevisionAnnotation = "workspace.kaito.io/revision"
 )
 
+const curlPodName = "curl-debug-pod"
+
 var (
 	datasetImageName1     = "e2e-dataset"
 	fullDatasetImageName1 = utils.GetEnv("E2E_ACR_REGISTRY") + "/" + datasetImageName1 + ":0.0.1"
@@ -608,6 +609,141 @@
 	})
 }
 
+// Create a temporary debug pod with curl if it doesn't already exist
+func createCurlDebugPod(namespace string) error {
+	By("Creating a temporary curl debug pod")
+
+	existingPod := &v1.Pod{}
+	err := utils.TestingCluster.KubeClient.Get(context.TODO(), client.ObjectKey{
+		Namespace: namespace,
+		Name:      curlPodName,
+	}, existingPod)
+
+	if err == nil {
+		By(fmt.Sprintf("Debug pod %s already exists", curlPodName))
+		return nil
+	} else if !errors.IsNotFound(err) {
+		return fmt.Errorf("failed to check existing pod: %v", err)
+	}
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      curlPodName,
+			Namespace: namespace,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "curl-container",
+					Image: "curlimages/curl",
+					Command: []string{
+						"sleep", "3600", // Keeps the pod running for long enough
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+		},
+	}
+
+	// Create the pod
+	err = utils.TestingCluster.KubeClient.Create(context.TODO(), pod)
+	if err != nil {
+		return fmt.Errorf("failed to create curl debug pod: %v", err)
+	}
+
+	// Wait for the pod to be Running
+	Eventually(func() bool {
+		fetchedPod := &v1.Pod{}
+		err := utils.TestingCluster.KubeClient.Get(context.TODO(), client.ObjectKey{
+			Namespace: namespace,
+			Name:      curlPodName,
+		}, fetchedPod)
+		if err != nil {
+			return false
+		}
+		return fetchedPod.Status.Phase == v1.PodRunning
+	}, 60*time.Second, 2*time.Second).Should(BeTrue(), "Curl debug pod did not reach Running state")
+
+	return nil
+}
+
+// Execute a curl command inside the debug pod
+func execCurlInPod(namespace string, cmd []string) (string, error) {
+	By(fmt.Sprintf("Executing curl command in %s", curlPodName))
+
+	// Get Kubernetes clientset and config
+	coreClient, err := utils.GetK8sClientset()
+	if err != nil {
+		return "", fmt.Errorf("failed to get k8s clientset: %v", err)
+	}
+
+	// Get REST config for creating the SPDYExecutor
+	k8sConfig, err := utils.GetK8sConfig()
+	if err != nil {
+		return "", fmt.Errorf("failed to get k8s config: %v", err)
+	}
+
+	// Use the existing ExecSync function
+	output, err := utils.ExecSync(context.TODO(), k8sConfig, coreClient, namespace, curlPodName, v1.PodExecOptions{
+		Container: "curl-container",
+		Command:   cmd,
+		Stdout:    true,
+		Stderr:    true,
+		TTY:       false,
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("failed to execute curl command: %w", err)
+	}
+
+	return output, nil
+}
+
+func validateModelsEndpoint(workspaceObj *kaitov1alpha1.Workspace) {
+	By("Validating the /v1/models endpoint using a curl debug pod")
+
+	namespace := workspaceObj.Namespace
+	serviceURL := fmt.Sprintf("http://%s.%s.svc.cluster.local:80/v1/models",
+		workspaceObj.Name, namespace)
+
+	curlCmd := []string{"curl", "-s", "-X", "GET", serviceURL}
+
+	response, err := execCurlInPod(namespace, curlCmd)
+	Expect(err).ToNot(HaveOccurred(), "Failed to execute curl GET in debug pod")
+
+	fmt.Printf("Response from /v1/models: %s\n", response)
+
+	modelName := workspaceObj.Inference.Preset.Name
+	expectedModelID := fmt.Sprintf(`"id":"%s"`, modelName)
+	Expect(response).To(ContainSubstring(expectedModelID), "Expected model ID '%s' not found in response", modelName)
+}
+
+func validateCompletionsEndpoint(workspaceObj *kaitov1alpha1.Workspace) {
+	By("Validating the /v1/completions endpoint using a curl debug pod")
+
+	namespace := workspaceObj.Namespace
+	serviceURL := fmt.Sprintf("http://%s.%s.svc.cluster.local:80/v1/completions",
+		workspaceObj.Name, namespace)
+
+	modelName := workspaceObj.Inference.Preset.Name
+	requestBody := fmt.Sprintf(`{
+            "model": "%s",
+            "prompt": "What is Kubernetes?",
+            "max_tokens": 7,
+            "temperature": 0
+        }`, modelName)
+
+	// Explicitly pass the curl command for POST request
+	curlCmd := []string{"curl", "-s", "-X", "POST", "-H", "Content-Type: application/json", "-d", requestBody, serviceURL}
+
+	response, err := execCurlInPod(namespace, curlCmd)
+	Expect(err).ToNot(HaveOccurred(), "Failed to execute curl POST in debug pod")
+
+	fmt.Printf("Response from /v1/completions: %s\n", response)
+
+	Expect(response).To(ContainSubstring(`"object":"text_completion"`), "Expected 'text_completion' object not found in response")
+}
+
 func cleanupResources(workspaceObj *kaitov1alpha1.Workspace) {
 	By("Cleaning up resources", func() {
 		if !CurrentSpecReport().Failed() {

test/e2e/preset_vllm_test.go

@@ -45,6 +45,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
 	})
 
 	It("should create a deepseek-distilled-qwen-14b workspace with preset public mode successfully", func() {
@@ -68,6 +70,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
 	})
 
 	It("should create a falcon workspace with preset public mode successfully", func() {
@@ -88,6 +92,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
 	})
 
 	It("should create a mistral workspace with preset public mode successfully", func() {
@@ -108,6 +114,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
 	})
 
 	It("should create a Phi-2 workspace with preset public mode successfully", func() {
@@ -128,6 +136,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
 	})
 
 	It("should create a Phi-3-mini-128k-instruct workspace with preset public mode successfully", func() {
@@ -148,6 +158,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
 	})
 
 	It("should create a qwen2.5 coder workspace with preset public mode and 2 gpu successfully", func() {
@@ -169,6 +181,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
 	})
 
 	It("should create a falcon workspace with adapter successfully", func() {
@@ -189,6 +203,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
 
 		validateInitContainers(workspaceObj, expectedInitContainers2)