WIP some cleanup for dev envs

deploy/components/inference-gateway/deployments.yaml

@@ -21,12 +21,10 @@ spec:
         image: quay.io/vllm-d/gateway-api-inference-extension/epp:latest
         imagePullPolicy: IfNotPresent
         args:
-        - -refreshMetricsInterval
-        - "500ms"
         - -poolName
         - "vllm-llama3-8b-instruct"
         - -v
-        - "4"
+        - "5"
         - --zap-encoder
         - "json"
         - -grpcPort

deploy/components/inference-gateway/httproutes.yaml

@@ -15,3 +15,5 @@ spec:
       kind: InferencePool
       name: vllm-llama3-8b-instruct
       port: 8000
+    timeouts:
+      request: 30s

deploy/components/inference-gateway/inference-models.yaml

@@ -7,3 +7,6 @@ spec:
   criticality: Critical
   poolRef:
     name: vllm-llama3-8b-instruct
+  targetModels:
+  - name: food-review-1
+    weight: 100

deploy/components/inference-gateway/kustomization.yaml

@@ -23,6 +23,7 @@ resources:
 - destination-rules.yaml
 - inference-pools.yaml
 - inference-models.yaml
+- services.yaml
 - deployments.yaml
 - gateways.yaml
 - httproutes.yaml

deploy/components/inference-gateway/rbac.yaml

@@ -6,6 +6,7 @@ rules:
 - apiGroups:
   - "inference.networking.x-k8s.io"
   resources:
+  - "inferencepools"
   - "inferencemodels"
   verbs:
   - "get"
@@ -19,14 +20,6 @@ rules:
   - "get"
   - "watch"
   - "list"
-- apiGroups:
-  - "inference.networking.x-k8s.io"
-  resources:
-  - "inferencepools"
-  verbs:
-  - "get"
-  - "watch"
-  - "list"
 - apiGroups:
   - "discovery.k8s.io"
   resources:

deploy/components/inference-gateway/services.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: endpoint-picker
+spec:
+  selector:
+    app: endpoint-picker
+  ports:
+  - protocol: TCP
+    port: 9002
+    targetPort: 9002
+    appProtocol: http2
+  type: ClusterIP

deploy/components/istio-control-plane/kustomization.yaml

@@ -23,3 +23,4 @@ resources:
 - webhooks.yaml
 - deployments.yaml
 - hpa.yaml
+- telemetry.yaml

deploy/components/istio-control-plane/telemetry.yaml

@@ -0,0 +1,10 @@
+# Enables debug logging for Gateways
+apiVersion: telemetry.istio.io/v1
+kind: Telemetry
+metadata:
+  name: mesh-default
+  namespace: istio-system
+spec:
+  accessLogging:
+  - providers:
+    - name: envoy

deploy/components/vllm-sim/deployments.yaml

@@ -22,7 +22,10 @@ spec:
         args:
         - "--port=8000"
         - "--model=food-review"
-        # - "--lora=lora10,lora20,lora30"
-        # - "--time-to-first-token=500"
         ports:
-          - containerPort: 8000
+        - name: http
+          containerPort: 8000
+          protocol: TCP
+        env:
+        - name: PORT
+          value: "8000"

deploy/environments/dev/kind/kustomization.yaml

@@ -19,4 +19,5 @@ resources:
 - ../../../components/inference-gateway/
 
 patches:
-- path: gateway.yaml
+- path: patch-deployments.yaml
+- path: patch-gateways.yaml

deploy/environments/dev/kind/patch-deployments.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: endpoint-picker
+spec:
+  template:
+    spec:
+      containers:
+      - name: epp
+        args:
+        - -poolName
+        - "vllm-llama3-8b-instruct"
+        - -poolNamespace
+        - "default"
+        - -v
+        - "5"
+        - --zap-encoder
+        - "json"
+        - -grpcPort
+        - "9002"
+        - -grpcHealthPort
+        - "9003"

deploy/environments/dev/kind/gateway.yaml → deploy/environments/dev/kind/patch-gateways.yaml

@@ -3,4 +3,4 @@ kind: Gateway
 metadata:
   name: inference-gateway
   annotations:
-    networking.istio.io/service-type: NodePort
+    networking.istio.io/service-type: NodePort

deploy/environments/dev/kubernetes/patch-deployments.yaml

@@ -10,14 +10,12 @@ spec:
       containers:
       - name: epp
         args:
-        - -poolNamespace
-        - ${NAMESPACE}
-        - -refreshMetricsInterval
-        - "500ms"
         - -poolName
         - "vllm-llama3-8b-instruct"
+        - -poolNamespace
+        - ${NAMESPACE}
         - -v
-        - "4"
+        - "5"
         - --zap-encoder
         - "json"
         - -grpcPort

pkg/epp/handlers/request.go

@@ -134,6 +134,8 @@ func (s *StreamingServer) HandleRequestBody(
 
 func (s *StreamingServer) HandleRequestHeaders(ctx context.Context, reqCtx *RequestContext, req *extProcPb.ProcessingRequest_RequestHeaders) error {
 	reqCtx.RequestReceivedTimestamp = time.Now()
+	logger := log.FromContext(ctx)
+	logger.V(logutil.TRACE).Info("Headers Handler", "request", req)
 
 	for _, header := range req.RequestHeaders.Headers.GetHeaders() {
 		value := string(header.RawValue)
@@ -159,5 +161,8 @@ func (s *StreamingServer) HandleRequestHeaders(ctx context.Context, reqCtx *Requ
 		endpoint := pod.Address + ":" + strconv.Itoa(int(pool.Spec.TargetPortNumber))
 		s.populateRequestHeaderResponse(reqCtx, endpoint, 0)
 	}
+
+	logger.V(logutil.TRACE).Info("Headers Handler", "handler", "complete")
+
 	return nil
 }

pkg/epp/handlers/response.go

@@ -81,6 +81,9 @@ func (s *StreamingServer) HandleResponseBody(
 			},
 		},
 	}
+
+	logger.V(logutil.TRACE).Info("Handle Response Body", "reqCtx", reqCtx)
+
 	return reqCtx, nil
 }
 

pkg/epp/handlers/server.go

@@ -322,6 +322,7 @@ func (s *StreamingServer) Process(srv extProcPb.ExternalProcessor_ProcessServer)
 // Order of requests matter in FULL_DUPLEX_STREAMING. For both request and response, the order of response sent back MUST be: Header->Body->Trailer, with trailer being optional.
 func (r *RequestContext) updateStateAndSendIfNeeded(srv extProcPb.ExternalProcessor_ProcessServer, logger logr.Logger) error {
 	loggerTrace := logger.V(logutil.TRACE)
+	loggerTrace.Info("updateStateAndSendIfNeeded", "step", "started")
 	// No switch statement as we could send multiple responses in one pass.
 	if r.RequestState == RequestReceived && r.reqHeaderResp != nil {
 		loggerTrace.Info("Sending request header response", "obj", r.reqHeaderResp)
@@ -334,6 +335,7 @@ func (r *RequestContext) updateStateAndSendIfNeeded(srv extProcPb.ExternalProces
 	if r.RequestState == HeaderRequestResponseComplete && r.reqBodyResp != nil {
 		loggerTrace.Info("Sending request body response")
 		if err := srv.Send(r.reqBodyResp); err != nil {
+			loggerTrace.Info("failed to send response back to Envoy", "err", err)
 			return status.Errorf(codes.Unknown, "failed to send response back to Envoy: %v", err)
 		}
 		r.RequestState = BodyRequestResponsesComplete
@@ -345,15 +347,18 @@ func (r *RequestContext) updateStateAndSendIfNeeded(srv extProcPb.ExternalProces
 	if r.RequestState == BodyRequestResponsesComplete && r.reqTrailerResp != nil {
 		// Trailers in requests are not guaranteed
 		if err := srv.Send(r.reqTrailerResp); err != nil {
+			loggerTrace.Info("failed to send response back to Envoy", "err", err)
 			return status.Errorf(codes.Unknown, "failed to send response back to Envoy: %v", err)
 		}
+		loggerTrace.Info("sent reqTrailerResp back to Envoy", "reqTrailerResp", r.reqTrailerResp)
 	}
 	if r.RequestState == ResponseRecieved && r.respHeaderResp != nil {
 		loggerTrace.Info("Sending response header response", "obj", r.respHeaderResp)
 		if err := srv.Send(r.respHeaderResp); err != nil {
 			return status.Errorf(codes.Unknown, "failed to send response back to Envoy: %v", err)
 		}
 		r.RequestState = HeaderResponseResponseComplete
+		loggerTrace.Info("send respHeaderResp", "reqHeaderResp", r.reqHeaderResp)
 	}
 	if r.RequestState == HeaderResponseResponseComplete && r.respBodyResp != nil {
 		loggerTrace.Info("Sending response body response")
@@ -365,15 +370,19 @@ func (r *RequestContext) updateStateAndSendIfNeeded(srv extProcPb.ExternalProces
 		if body.ResponseBody.Response.GetBodyMutation().GetStreamedResponse().GetEndOfStream() {
 			r.RequestState = BodyResponseResponsesComplete
 		}
+		loggerTrace.Info("dumping the response so a new stream message can begin")
 		// Dump the response so a new stream message can begin
 		r.respBodyResp = nil
 	}
 	if r.RequestState == BodyResponseResponsesComplete && r.respTrailerResp != nil {
 		// Trailers in requests are not guaranteed
 		if err := srv.Send(r.respTrailerResp); err != nil {
+			loggerTrace.Info("failed to send response back to Envoy", "err", err)
 			return status.Errorf(codes.Unknown, "failed to send response back to Envoy: %v", err)
 		}
+		loggerTrace.Info("sent respTrailerResp", "respTrailerResp", r.respTrailerResp)
 	}
+	loggerTrace.Info("updateStateAndSendIfNeeded", "step", "complete")
 	return nil
 }