[AgentX] DeepSeek-v4-pro llm-d NVL72-B200

benchmarks/llm-d/binaries-b200-v0.10.0.env

@@ -0,0 +1,33 @@
+# B200-only binary pins for the agentX-ported EPP config used by
+# dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml (disagg-profile-handler
+# `deciders:` shape + inflight-load-producer / approx-prefix-cache-producer /
+# prefix-cache-affinity-filter / token-load-scorer plugins), which require
+# the v0.10.0 EPP/pd-sidecar router build. Kept as a sibling file (not a
+# change to binaries.env) so GB200's shared v0.9.0 pins and bin path are
+# completely untouched.
+#
+# Pull with extract-binaries.sh instead of binaries.env's defaults:
+#   BINARIES_ENV_FILE=binaries-b200-v0.10.0.env \
+#     benchmarks/llm-d/extract-binaries.sh
+# Run once (or whenever these pins change) on a host with docker + registry
+# access and the /home/sa-shared filesystem mounted.
+
+# --- Endpoint Picker (EPP / inference scheduler) ---
+EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0"
+EPP_BIN_PATH="/app/epp"
+
+# --- P/D routing sidecar (pd-sidecar) ---
+ROUTING_SIDECAR_IMAGE="ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0"
+ROUTING_SIDECAR_BIN_PATH="/app/pd-sidecar"
+
+# --- Envoy front proxy ---
+# Unchanged from binaries.env; re-declared here so this file is
+# self-contained (extract-binaries.sh always extracts all three).
+ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2"
+ENVOY_BIN_PATH="/usr/local/bin/envoy"
+
+# B200 is x86_64 (unlike GB200's Grace/arm64, binaries.env's default).
+LLMD_BIN_PLATFORM="${LLMD_BIN_PLATFORM:-linux/amd64}"
+
+# Dedicated path so this never collides with GB200's shared bin dir.
+LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}"

benchmarks/llm-d/extract-binaries.sh

@@ -13,12 +13,19 @@
 #   ./extract-binaries.sh                 # uses binaries.env defaults
 #   LLMD_BIN_DIR=/some/dir ./extract-binaries.sh
 #   LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh   # for an x86 test
+#   BINARIES_ENV_FILE=binaries-b200-v0.10.0.env ./extract-binaries.sh
+#     # pull a different set of image pins (e.g. a router-version bump
+#     # scoped to one cluster) without touching the shared binaries.env
 
 set -euo pipefail
 
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARIES_ENV_FILE="${BINARIES_ENV_FILE:-$HERE/binaries.env}"
+# Resolve a bare filename against this script's directory, so
+# BINARIES_ENV_FILE=foo.env works regardless of the caller's cwd.
+[[ "$BINARIES_ENV_FILE" != /* ]] && BINARIES_ENV_FILE="$HERE/$BINARIES_ENV_FILE"
 # shellcheck source=/dev/null
-source "$HERE/binaries.env"
+source "$BINARIES_ENV_FILE"
 
 echo "Extracting llm-d binaries -> $LLMD_BIN_DIR (platform $LLMD_BIN_PLATFORM)"
 mkdir -p "$LLMD_BIN_DIR"

benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm aggregated benchmark
+# (agentX-flavored TP8 / DEP8, one engine does both prefill and decode).
+# Sibling of dsv4_fp4_b200_llmd-vllm-disagg.sh - same shape, but always
+# submits with DECODE_NODES=0 (no decode role at all; see the aggregated
+# recipes' header comments and server.sh's IS_AGGREGATED handling). The
+# runner resolves this script via
+#   SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh"
+# from launch_b200-dgxc-slurm.sh when DISAGG=false.
+
+set -euo pipefail
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    MODEL_PATH \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+if [[ "$DECODE_NODES" != "0" ]]; then
+    echo "Error: dsv4_fp4_b200_llmd-vllm-agg.sh requires DECODE_NODES=0 (got $DECODE_NODES); aggregated mode has no decode role" >&2
+    exit 1
+fi
+
+set -x
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1
+
+# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity).
+export GPUS_PER_NODE="${GPUS_PER_NODE:-8}"
+
+export TIME_LIMIT="${TIME_LIMIT:-08:00:00}"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+# Aggregated is always a single engine (no multi-engine high-tpt split), so
+# PREFILL_WORKERS is always 1. DECODE_WORKERS is unused (DECODE_NODES=0) but
+# still exported since submit.sh/server.sh read it unconditionally.
+export PREFILL_WORKERS="${PREFILL_WORKERS:-1}"
+export DECODE_WORKERS="${DECODE_WORKERS:-1}"
+
+JOB_ID=$(bash ./submit.sh \
+    "$PREFILL_NODES" \
+    "$DECODE_NODES" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf \
+    "$RANDOM_RANGE_RATIO")
+
+if [[ -z "$JOB_ID" ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm P/D disagg benchmark
+# (agentX-flavored 1P-DEP8/1D-DEP8). Sibling of
+# dsv4_fp4_gb200_llmd-vllm-disagg.sh - same shape, different topology
+# (B200 = 8 GPUs/node, so each DEP8 role fits on ONE node; GB200 = 4
+# GPUs/node, role spans 2 nodes). The runner resolves this script via
+#   SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh"
+# from launch_b200-dgxc-slurm.sh.
+
+set -euo pipefail
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    MODEL_PATH \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+set -x
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1
+
+# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity).
+export GPUS_PER_NODE="${GPUS_PER_NODE:-8}"
+
+export TIME_LIMIT="${TIME_LIMIT:-08:00:00}"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+# Worker count per role (Option B multi-engine). Prefer an explicit
+# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else
+# fall back to the matrix num-worker fields (PREFILL_NUM_WORKERS/
+# DECODE_NUM_WORKERS); else 1 (single engine = unchanged 1P+1D). submit.sh
+# reads these.
+export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}"
+export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}"
+
+JOB_ID=$(bash ./submit.sh \
+    "$PREFILL_NODES" \
+    "$DECODE_NODES" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf \
+    "$RANDOM_RANGE_RATIO")
+
+if [[ -z "$JOB_ID" ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml

@@ -0,0 +1,181 @@
+# DeepSeek-V4-Pro (FP4) on B200, 1P-DEP8/1D-DEP8 P/D disagg via llmd-vllm.
+#
+# This is the agentX-flavored sibling of dsv4-fp4-gb200-mid-curve-megamoe.yaml:
+# same topology shape (1 prefill DEP8 engine + 1 decode DEP8 engine), but the
+# EPP scheduling config and per-role vLLM tuning are ported from the agentX
+# (llm-manifesto/agentx-mvp) DeepSeek-V4-Pro K8s deployment instead of being
+# InferenceX-native, per direct request. Selected via
+# CONFIG_FILE=dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml in the
+# dsv4-fp4-b200-llmd-vllm-agentx master-config key.
+#
+# Topology: B200 = 8 GPUs/node, so each DEP8 role fits on exactly ONE node
+# (unlike GB200's 4 GPUs/node, where the same DEP8 role needs 2 nodes).
+# PREFILL_NODES=1 DECODE_NODES=1 GPUS_PER_NODE=8 -> 2 nodes / 16 GPUs total.
+# Because neither role spans >1 node, server.sh's LWS_GROUP_SIZE is always 1
+# here, so the cross-node --data-parallel-hybrid-lb / NVSHMEM-ibgda code paths
+# never activate - the GB200 recipes' cross-node NIC/NVSHMEM tuning (e.g.
+# NVSHMEM_HCA_PE_MAPPING) simply does not apply and is intentionally omitted.
+#
+# Source (agentX): llm-manifesto/models/deepseek-v4-ix/ix-disagg-base.yaml +
+# 1P-EP8-1D-EP8.yaml. agentX's Mooncake/MultiConnector KV chain and CPU
+# offload are Kubernetes/manifesto-specific and are NOT ported - server.sh
+# hardcodes plain NixlConnector for every llmd-vllm disagg run (no per-recipe
+# override point), matching what the existing GB200 llm-d recipes already do.
+# agentX's kv_cache_metrics/kv_cache_metrics_sample (observability-only, not
+# present in any InferenceX llm-d recipe or confirmed supported by this
+# image's vLLM build) are also dropped as an unnecessary compatibility risk.
+# max_model_len is set to 9280 (ISL 8192 + OSL 1024 + headroom, matching the
+# existing InferenceX GB200 recipes) instead of agentX's general-purpose
+# 1048576, since this recipe only ever serves the fixed 8k/1k benchmark.
+#
+# ---- EPP scheduling config ----
+# Ported EXACTLY (plugin list, parameters, weights) from the agentx-mvp
+# results snapshot manifest.yaml
+# (results/dsv4-pro-dspark/results_ilmarkov-ix-2p-ep8-1d-ep8-dspark-1308-long-v2),
+# which is a real EPP config that ran a 1P-EP8-1D-EP8 DeepSeek-V4-Pro K8s
+# deployment. Only the file-discovery plugin + dataLayer wiring is added on
+# top (InferenceX's own no-Kubernetes addition - agentX's K8s deployment used
+# InferencePool pod discovery instead, which doesn't apply here).
+#
+# This config uses disagg-profile-handler's `deciders: {prefill: ...}`
+# parameter shape (not InferenceX's existing GB200 recipes' `deciderPluginName`
+# shape) and additional plugins (inflight-load-producer,
+# approx-prefix-cache-producer, prefix-cache-affinity-filter,
+# token-load-scorer) not present in the current InferenceX EPP image. This
+# requires the EPP/pd-sidecar binaries to be bumped to v0.10.0 for this
+# recipe - see the LLMD_BIN_DIR override in runners/launch_b200-dgxc-slurm.sh
+# (B200-only; GB200's shared v0.9.0 binaries/path are untouched).
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+
+plugins:
+  - name: file-disc
+    type: file-discovery
+    parameters:
+      path: /tmp/endpoints.yaml
+      watchFile: false
+
+  - type: prefill-filter
+  - type: decode-filter
+  - type: inflight-load-producer
+  - type: approx-prefix-cache-producer
+    parameters:
+      autoTune: false
+      blockSizeTokens: 256
+      maxPrefixTokensToMatch: 1048576
+      maxPrefixBlocksToMatch: 4096
+      lruCapacityPerServer: 45272
+  - type: prefix-cache-affinity-filter
+    parameters:
+      peakPrefillThroughput: 4783
+      maxTTFTPenaltyMs: 30000
+  - type: prefix-cache-scorer
+  - type: token-load-scorer
+    parameters:
+      queueThresholdTokens: 3000000
+  - type: active-request-scorer
+  - type: queue-scorer
+  - type: always-disagg-pd-decider
+  - type: disagg-profile-handler
+    parameters:
+      deciders:
+        prefill: always-disagg-pd-decider
+  - type: max-score-picker
+    name: prefill-picker
+  - type: max-score-picker
+    name: decode-picker
+
+schedulingProfiles:
+  - name: prefill
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: prefix-cache-affinity-filter
+      - pluginRef: prefix-cache-scorer
+        weight: 6
+      - pluginRef: token-load-scorer
+        weight: 3
+      - pluginRef: queue-scorer
+        weight: 3
+      - pluginRef: prefill-picker
+  - name: decode
+    plugins:
+      - pluginRef: decode-filter
+      - pluginRef: active-request-scorer
+      - pluginRef: decode-picker
+
+dataLayer:
+  discovery:
+    pluginRef: file-disc
+
+# ---- Per-role vLLM flags ----
+# Ported from agentX's ix-disagg-base.yaml (prefill/decode roles) +
+# 1P-EP8-1D-EP8.yaml overrides. Both roles are DEP8 (TP=1, DP=8, EP on),
+# FLASHINFER_MLA_SPARSE_DSV4 attention, deep_gemm_mega_moe MoE backend,
+# EP weight filter (keeps only served experts resident - required for DEP8
+# to avoid OOM). See file header for what was intentionally dropped/adapted.
+prefill:
+  tp: 1
+  enable-expert-parallel: true
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --enforce-eager
+    --gpu-memory-utilization 0.97
+    --max-model-len 9280
+    --max-num-seqs 64
+    --max-num-batched-tokens 8192
+    --long-prefill-token-threshold 1024
+    --enable-cumem-allocator
+    --block-size 256
+    --tokenizer-mode deepseek_v4
+    --moe-backend deep_gemm_mega_moe
+    --enable-ep-weight-filter
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --numa-bind
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    PYTHONHASHSEED: "0"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
+
+decode:
+  tp: 1
+  enable-expert-parallel: true
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --max-num-seqs 64
+    --max-num-batched-tokens 256
+    --max-cudagraph-capture-size 64
+    --gpu-memory-utilization 0.95
+    --max-model-len 9280
+    --enable-cumem-allocator
+    --block-size 256
+    --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
+    --tokenizer-mode deepseek_v4
+    --moe-backend deep_gemm_mega_moe
+    --enable-ep-weight-filter
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --numa-bind
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    PYTHONHASHSEED: "0"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
+
+# ---- SLURM resource directives ----
+slurm:
+  time_limit: "08:00:00"