fix(worker): give the worker a real health endpoint and a mode-aware HEALTHCHECK

Dockerfile

@@ -393,7 +393,12 @@ RUN go install github.com/mikefarah/yq/v4@latest
 # If you cannot find a more suitable place for an addition, this layer is a suitable place for it.
 FROM requirements-drivers
 
-ENV HEALTHCHECK_ENDPOINT=http://localhost:8080/readyz
+# Optional override for the HEALTHCHECK target. Left empty so healthcheck.sh
+# derives the endpoint from the mode the container is actually running — the
+# same image runs `local-ai run` (HTTP on 8080) and `local-ai worker` (HTTP on
+# the gRPC base port minus one), and a hardcoded default marked every worker
+# permanently unhealthy (#10987). Set it to pin an explicit URL.
+ENV HEALTHCHECK_ENDPOINT=""
 
 ARG CUDA_MAJOR_VERSION=12
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
@@ -403,6 +408,7 @@ ENV NVIDIA_VISIBLE_DEVICES=all
 WORKDIR /
 
 COPY ./entrypoint.sh .
+COPY ./scripts/build/healthcheck.sh .
 
 # Copy the binary
 COPY --from=builder /build/local-ai ./
@@ -413,9 +419,22 @@ RUN --mount=from=builder,src=/build/,dst=/mnt/build \
 # Make sure the models directory exists
 RUN mkdir -p /models /backends /data
 
-# Define the health check command
-HEALTHCHECK --interval=1m --timeout=10m --retries=10 \
-  CMD curl -f ${HEALTHCHECK_ENDPOINT} || exit 1
+# Define the health check command.
+#
+# --start-period is the knob for slow starts, not --timeout/--retries. Since
+# #10949 a frontend's startup preload materializes HuggingFace artifacts before
+# the HTTP server binds (31 GB observed on a live cluster), so a healthy replica
+# can legitimately fail probes for a long time. Failures inside the start period
+# leave the container `starting` instead of burning retries, and the period ends
+# early on the first success — so a generous value costs a fast-starting
+# container nothing. A process that actually died is handled by the restart
+# policy, not by health.
+#
+# --timeout is a per-probe deadline: 10m meant a wedged probe could hang for ten
+# minutes and stretch detection without bound. A localhost curl that has not
+# answered in 10s is itself the fault being detected.
+HEALTHCHECK --start-period=60m --interval=1m --timeout=10s --retries=3 \
+  CMD /healthcheck.sh
 
 VOLUME /models /backends /configuration /data
 EXPOSE 8080

core/services/nodes/file_transfer_server.go

@@ -46,17 +46,26 @@ const (
 // It provides PUT/GET/POST endpoints for uploading, downloading, and allocating temp files,
 // as well as backend log REST and WebSocket endpoints when logStore is non-nil.
 // Auth is via Bearer token (registration token), using constant-time comparison.
-func StartFileTransferServer(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, logStore ...*model.BackendLogStore) (*http.Server, error) {
+// A nil readiness fails open, keeping /readyz's historical always-200 answer.
+func StartFileTransferServer(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, logStore ...*model.BackendLogStore) (*http.Server, error) {
 	listener, err := net.Listen("tcp", addr)
 	if err != nil {
 		return nil, fmt.Errorf("listen %s: %w", addr, err)
 	}
-	return StartFileTransferServerWithListener(listener, stagingDir, modelsDir, dataDir, token, maxUploadSize, logStore...)
+	return StartFileTransferServerWithReadiness(listener, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, logStore...)
 }
 
 // StartFileTransferServerWithListener starts the server on an existing listener.
 // This avoids the TOCTOU race of closing a listener and re-binding to the same port.
 func StartFileTransferServerWithListener(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, logStore ...*model.BackendLogStore) (*http.Server, error) {
+	return StartFileTransferServerWithReadiness(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, nil, logStore...)
+}
+
+// StartFileTransferServerWithReadiness is StartFileTransferServerWithListener
+// plus a readiness gate for the /readyz probe. A nil readiness fails open, so
+// the probe keeps its historical always-200 behaviour for callers that have no
+// meaningful readiness signal to report.
+func StartFileTransferServerWithReadiness(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, logStore ...*model.BackendLogStore) (*http.Server, error) {
 	if err := os.MkdirAll(stagingDir, 0750); err != nil {
 		return nil, fmt.Errorf("creating staging dir %s: %w", stagingDir, err)
 	}
@@ -131,18 +140,30 @@ func StartFileTransferServerWithListener(lis net.Listener, stagingDir, modelsDir
 
 	// Liveness/readiness probes — unauthenticated so container orchestrators
 	// (Docker HEALTHCHECK, k8s probes) can hit them without the bearer token.
-	// Reaching this point means the listener is bound and the mux is serving.
-	healthHandler := func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != http.MethodGet && r.Method != http.MethodHead {
-			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
-			return
+	probe := func(check func() error) http.HandlerFunc {
+		return func(w http.ResponseWriter, r *http.Request) {
+			if r.Method != http.MethodGet && r.Method != http.MethodHead {
+				http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+				return
+			}
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			if err := check(); err != nil {
+				w.WriteHeader(http.StatusServiceUnavailable)
+				_, _ = w.Write([]byte("not ready: " + err.Error()))
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte("ok"))
 		}
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		w.WriteHeader(http.StatusOK)
-		_, _ = w.Write([]byte("ok"))
 	}
-	mux.HandleFunc("/readyz", healthHandler)
-	mux.HandleFunc("/healthz", healthHandler)
+
+	// Liveness: reaching this point means the listener is bound and the mux is
+	// serving. Deliberately independent of readiness — a worker whose NATS link
+	// is momentarily down must not be restarted, or a NATS blip becomes a
+	// cluster-wide restart storm.
+	mux.HandleFunc("/healthz", probe(func() error { return nil }))
+	// Readiness: "can this worker actually accept work?" See WorkerReadiness.
+	mux.HandleFunc("/readyz", probe(readiness.Check))
 
 	addr := lis.Addr().String()
 	server := &http.Server{

core/services/nodes/worker_readiness.go

@@ -0,0 +1,75 @@
+package nodes
+
+import (
+	"errors"
+	"sync/atomic"
+)
+
+// WorkerReadiness is the gate behind a worker's /readyz probe.
+//
+// It exists because the worker's HTTP file-transfer server is started before
+// the worker has connected to NATS, and must keep serving after NATS drops.
+// The probe is therefore installed after the fact rather than passed as a
+// value, and must be safe to read from HTTP handler goroutines while the
+// startup goroutine is still installing it.
+type WorkerReadiness struct {
+	probe atomic.Pointer[func() error]
+}
+
+// Set installs (or replaces) the readiness probe.
+func (r *WorkerReadiness) Set(fn func() error) {
+	if r == nil {
+		return
+	}
+	r.probe.Store(&fn)
+}
+
+// Check reports whether the worker can accept work. A nil receiver, or one with
+// no probe installed, fails open: callers that never wire readiness (the
+// frontend's own file-transfer server, tests, embedders) keep the historical
+// always-ready behaviour rather than being wedged out of rotation forever.
+func (r *WorkerReadiness) Check() error {
+	if r == nil {
+		return nil
+	}
+	fn := r.probe.Load()
+	if fn == nil || *fn == nil {
+		return nil
+	}
+	return (*fn)()
+}
+
+// natsConn is the slice of *messaging.Client the readiness probe needs. Kept
+// as a local interface so this package does not import messaging (which would
+// be an import cycle) and so tests can supply a fake.
+type natsConn interface {
+	IsConnected() bool
+}
+
+// ErrNATSDisconnected is reported by NATSReadiness when the worker has lost its
+// NATS connection.
+var ErrNATSDisconnected = errors.New("NATS connection is down: worker cannot receive work")
+
+// NATSReadiness builds the worker's readiness probe.
+//
+// A worker's real health is not "a port is open" — that is precisely the
+// failure mode of issue #10987, where a process that serves nothing still
+// answered 200. All of a worker's actual work (backend install/start/stop
+// events, inference dispatch, file-staging notifications) arrives over NATS, so
+// a worker with a dead NATS link is up and useless. Registration is already
+// implied by the probe being reachable at all: the file-transfer server is only
+// started after the worker has successfully registered with the frontend.
+//
+// This is deliberately something the controller cannot already see. The node
+// registry's status/last_heartbeat is fed by an HTTP heartbeat to the frontend,
+// a completely different network path — a worker can keep heartbeating happily
+// while its NATS connection is dead, and look healthy in the registry. The
+// local probe closes that gap.
+func NATSReadiness(conn natsConn) func() error {
+	return func() error {
+		if conn == nil || !conn.IsConnected() {
+			return ErrNATSDisconnected
+		}
+		return nil
+	}
+}

core/services/nodes/worker_readiness_test.go

@@ -0,0 +1,101 @@
+package nodes
+
+import (
+	"errors"
+	"net"
+	"net/http"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// fakeConn stands in for *messaging.Client, which cannot be constructed without
+// a live NATS server. Only IsConnected() is consulted by the readiness probe.
+type fakeConn struct{ connected bool }
+
+func (f *fakeConn) IsConnected() bool { return f.connected }
+
+var _ = Describe("WorkerReadiness", func() {
+	Describe("the gate itself", func() {
+		It("reports ready when no probe has been installed yet", func() {
+			// Fail open: an embedder (or the frontend) that never wires a probe
+			// keeps the historical always-ready behaviour.
+			r := &WorkerReadiness{}
+			Expect(r.Check()).To(Succeed())
+		})
+
+		It("surfaces the installed probe's error", func() {
+			r := &WorkerReadiness{}
+			r.Set(func() error { return errors.New("boom") })
+			Expect(r.Check()).To(MatchError(ContainSubstring("boom")))
+		})
+
+		It("lets a later Set replace an earlier probe", func() {
+			r := &WorkerReadiness{}
+			r.Set(func() error { return errors.New("boom") })
+			r.Set(func() error { return nil })
+			Expect(r.Check()).To(Succeed())
+		})
+	})
+
+	Describe("NATSReadiness", func() {
+		It("reports ready while the NATS connection is up", func() {
+			Expect(NATSReadiness(&fakeConn{connected: true})()).To(Succeed())
+		})
+
+		It("reports not-ready once the NATS connection drops", func() {
+			// This is the failure mode issue #10987 is about: the process is
+			// up and the port is bound, but the worker can receive no work.
+			err := NATSReadiness(&fakeConn{connected: false})()
+			Expect(err).To(MatchError(ContainSubstring("NATS")))
+		})
+	})
+
+	Describe("the file transfer server probes", func() {
+		var (
+			srv     *http.Server
+			baseURL string
+			ready   *WorkerReadiness
+		)
+
+		BeforeEach(func() {
+			lis, err := net.Listen("tcp", "127.0.0.1:0")
+			Expect(err).ToNot(HaveOccurred())
+			ready = &WorkerReadiness{}
+			srv, err = StartFileTransferServerWithReadiness(
+				lis, GinkgoT().TempDir(), GinkgoT().TempDir(), GinkgoT().TempDir(),
+				"tok", 1024, ready, nil,
+			)
+			Expect(err).ToNot(HaveOccurred())
+			baseURL = "http://" + lis.Addr().String()
+			DeferCleanup(func() { ShutdownFileTransferServer(srv) })
+		})
+
+		get := func(path string) int {
+			client := &http.Client{Timeout: 5 * time.Second}
+			resp, err := client.Get(baseURL + path)
+			Expect(err).ToNot(HaveOccurred())
+			defer func() { _ = resp.Body.Close() }()
+			return resp.StatusCode
+		}
+
+		It("serves /readyz 200 while the probe reports ready", func() {
+			ready.Set(func() error { return nil })
+			Expect(get("/readyz")).To(Equal(http.StatusOK))
+		})
+
+		It("serves /readyz 503 once the probe reports not-ready", func() {
+			ready.Set(func() error { return errors.New("NATS disconnected") })
+			Expect(get("/readyz")).To(Equal(http.StatusServiceUnavailable))
+		})
+
+		It("keeps /healthz at 200 even when readiness fails", func() {
+			// Liveness is deliberately independent of readiness: a worker whose
+			// NATS link is briefly down must not be killed and restarted, or a
+			// NATS outage turns into a restart storm across every worker.
+			ready.Set(func() error { return errors.New("NATS disconnected") })
+			Expect(get("/healthz")).To(Equal(http.StatusOK))
+		})
+	})
+})

core/services/worker/worker.go

@@ -149,7 +149,12 @@ func Run(ctx *cliContext.Context, cfg *Config) error {
 	httpAddr := cfg.resolveHTTPAddr()
 	stagingDir := filepath.Join(cfg.ModelsPath, "..", "staging")
 	dataDir := filepath.Join(cfg.ModelsPath, "..", "data")
-	httpServer, err := nodes.StartFileTransferServer(httpAddr, stagingDir, cfg.ModelsPath, dataDir, cfg.RegistrationToken, config.DefaultMaxUploadSize, ml.BackendLogs())
+	// The readiness gate is created here but only armed once NATS is up, below.
+	// Until then /readyz reports ready, which is correct: reaching this line
+	// means the worker has already registered with the frontend, so it is
+	// mid-startup rather than broken.
+	readiness := &nodes.WorkerReadiness{}
+	httpServer, err := nodes.StartFileTransferServer(httpAddr, stagingDir, cfg.ModelsPath, dataDir, cfg.RegistrationToken, config.DefaultMaxUploadSize, readiness, ml.BackendLogs())
 	if err != nil {
 		return fmt.Errorf("starting HTTP file transfer server: %w", err)
 	}
@@ -163,6 +168,11 @@ func Run(ctx *cliContext.Context, cfg *Config) error {
 	}
 	defer natsClient.Close()
 
+	// Arm the readiness gate now that the worker can actually receive work.
+	// From here /readyz tracks the live NATS link, so a worker that is up but
+	// cut off from the bus reports 503 instead of a meaningless 200 (#10987).
+	readiness.Set(nodes.NATSReadiness(natsClient))
+
 	// Start heartbeat goroutine (after NATS is connected so IsConnected check works)
 	go func() {
 		ticker := time.NewTicker(heartbeatInterval)

docker-compose.distributed.yaml

@@ -101,12 +101,12 @@ services:
         - BASE_IMAGE=ubuntu:24.04
     command:
       - worker
-    # The image's default HEALTHCHECK targets the server's /readyz on 8080.
-    # Workers don't run the OpenAI API server — their HTTP file transfer
-    # server runs on the gRPC base port - 1 (50050 here) and exposes /readyz.
-    # Override the env var so the inherited HEALTHCHECK probes the right port.
+    # No HEALTHCHECK_ENDPOINT override is needed: the image's healthcheck
+    # detects worker mode and derives the port from LOCALAI_SERVE_ADDR below
+    # (gRPC base port - 1 = 50050). The worker's /readyz reports 503 while its
+    # NATS connection is down, so `unhealthy` here means the worker genuinely
+    # cannot receive work.
     environment:
-      HEALTHCHECK_ENDPOINT: "http://localhost:50050/readyz"
       LOCALAI_SERVE_ADDR: "0.0.0.0:50051"
       LOCALAI_ADVERTISE_ADDR: "worker-1:50051"
       LOCALAI_ADVERTISE_HTTP_ADDR: "worker-1:50050"
@@ -200,10 +200,9 @@ services:
       - |
         apt-get update -qq && apt-get install -y -qq docker.io >/dev/null 2>&1
         exec /entrypoint.sh agent-worker
-    # The agent worker is NATS-only — no HTTP server to probe. Disable the
-    # image's inherited HEALTHCHECK so the container doesn't show unhealthy.
-    healthcheck:
-      disable: true
+    # The agent worker is NATS-only — no HTTP server to probe. The image's
+    # healthcheck detects that mode and reports healthy rather than probing a
+    # port that will never bind, so no override is needed here.
     environment:
       LOCALAI_NATS_URL: "nats://nats:4222"
       LOCALAI_REGISTER_TO: "http://localai:8080"

docs/content/features/distributed-mode.md

@@ -232,6 +232,19 @@ local-ai worker \
 **HTTP file transfer:** Each worker also runs a small HTTP server for file transfer (model files, configs). By default it listens on the gRPC base port - 1 (e.g., if gRPC base is 50051, HTTP is on 50050). gRPC ports grow upward from the base port as additional models are loaded. Set `--advertise-http-addr` if the auto-detected address is not routable from the frontend.
 {{% /notice %}}
 
+### Worker Health Probes
+
+The worker's HTTP server (base port - 1, default 50050) exposes two unauthenticated probes:
+
+| Endpoint | Meaning |
+|----------|---------|
+| `/healthz` | **Liveness.** 200 whenever the process is up and serving. Deliberately independent of readiness, so a brief NATS outage does not trigger a restart storm across every worker. |
+| `/readyz` | **Readiness.** 200 only when the worker is registered *and* its NATS connection is live; 503 otherwise. |
+
+`/readyz` reports something the frontend cannot see on its own. The node registry's `status` and `last_heartbeat` are driven by an HTTP heartbeat to the frontend, which is a different network path from NATS — a worker can keep heartbeating while its NATS link is dead, and so appear `healthy` in the registry while being unable to receive any work. The local probe closes that gap.
+
+The container image's `HEALTHCHECK` detects worker mode and probes this endpoint automatically; no `HEALTHCHECK_ENDPOINT` override is needed. Set `HEALTHCHECK_ENDPOINT` only to pin an explicit URL.
+
 ### Worker Address Configuration
 
 The simplest way to configure a worker's network address is with a single variable: