@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
yaml 121 lines 5.33 KB
Raw
1 # =============================================================================
2 # Model catalog for the vLLM disaggregated (P/D) inference CI.
3 # -----------------------------------------------------------------------------
4 # Loaded by vllm_disagg.sh for the selected MODEL_NAME. WIDE_EP_MODE selects the
5 # flag set (0 -> tp keys, 1 -> ep keys):
6 # tp -> base_flags + prefill.tp / decode.tp
7 # ep -> base_flags + prefill.ep / decode.ep
8 #
9 # Keep ONLY model-specific perf flags here. The launcher owns everything
10 # topological / mode-structural:
11 # - tp: --host/--port, --tensor-parallel-size, --kv-transfer-config
12 # - ep: --tp 1, --data-parallel-size[-local], --data-parallel-address/-rpc-port,
13 # --data-parallel-start-rank/--headless (children), --enable-expert-parallel,
14 # --all2all-backend mori, --no-enable-prefix-caching,
15 # --api-server-count + --kv-transfer-config (masters only)
16 # Do NOT put any of those in this file.
17 #
18 # Format: a top-level `models:` list. The launcher selects the entry whose
19 # `model:` matches MODEL_NAME. Each entry:
20 # model : name (must match MODEL_NAME; also the dir under MODEL_DIR)
21 # env : model/arch-specific environment variables. The launcher
22 # exports these BEFORE `vllm serve`, but only if they are
23 # not already set, so precedence is:
24 # caller/inline env > models.yaml env:
25 # (Transport/fabric/engine env stays in cluster.sh,
26 # e.g. MORI_RDMA_DEVICES, NCCL_IB_HCA, VLLM_USE_V1.)
27 # base_flags : always applied (both roles, both modes)
28 # prefill/decode.<tp|ep> : role + mode specific perf flags
29 # experimental_flags : optional extra flags (omit if empty)
30 # =============================================================================
31
32 models:
33 - model: DeepSeek-V3
34 env:
35 VLLM_ROCM_USE_AITER: "1"
36 VLLM_ROCM_USE_AITER_MLA: "1"
37 VLLM_ROCM_USE_AITER_MOE: "1"
38 VLLM_ROCM_USE_AITER_RMSNORM: "1"
39 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "0"
40 VLLM_ROCM_USE_AITER_FP8BMM: "false"
41 base_flags: "--trust-remote-code --kv-cache-dtype fp8"
42 prefill:
43 tp: "--gpu-memory-utilization 0.85"
44 ep: "--gpu-memory-utilization 0.85 --enforce-eager"
45 decode:
46 tp: "--gpu-memory-utilization 0.85"
47 ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'
48
49 - model: MiniMax-M3-MXFP8
50 env:
51 VLLM_ROCM_USE_AITER: "1"
52 VLLM_ROCM_USE_AITER_MOE: "1"
53 VLLM_ROCM_USE_AITER_RMSNORM: "1"
54 VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
55 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT6"
56 VLLM_KV_CACHE_LAYOUT: "HND"
57 base_flags: "--trust-remote-code --attention-backend TRITON_ATTN --block-size 128 --language-model-only --kv-cache-dtype fp8"
58 prefill:
59 tp: "--gpu-memory-utilization 0.85 --enforce-eager"
60 ep: "--gpu-memory-utilization 0.85 --enforce-eager"
61 decode:
62 tp: "--gpu-memory-utilization 0.85"
63 ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'
64
65 - model: DeepSeek-R1-MXFP4
66 env:
67 VLLM_ROCM_USE_AITER: "1"
68 VLLM_ROCM_USE_AITER_MLA: "1"
69 VLLM_ROCM_USE_AITER_MOE: "1"
70 VLLM_ROCM_USE_AITER_RMSNORM: "1"
71 VLLM_ROCM_USE_AITER_FP8BMM: "false"
72 base_flags: "--trust-remote-code --kv-cache-dtype fp8"
73 prefill:
74 tp: "--gpu-memory-utilization 0.85"
75 ep: "--gpu-memory-utilization 0.85 --enforce-eager"
76 decode:
77 tp: "--gpu-memory-utilization 0.85"
78 ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'
79
80 - model: Kimi-K2.5-MXFP4
81 env:
82 VLLM_ROCM_USE_AITER: "1"
83 VLLM_ROCM_USE_AITER_MOE: "1"
84 VLLM_ROCM_USE_AITER_RMSNORM: "1"
85 base_flags: "--trust-remote-code"
86 prefill:
87 tp: "--gpu-memory-utilization 0.85"
88 ep: "--gpu-memory-utilization 0.85 --enforce-eager"
89 decode:
90 tp: "--gpu-memory-utilization 0.85"
91 ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'
92
93 - model: Kimi-K2.6-MXFP4
94 env:
95 VLLM_ROCM_USE_AITER: "1"
96 AMDGCN_USE_BUFFER_OPS: "1"
97 VLLM_ROCM_USE_AITER_MLA: "1"
98 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT4"
99 VLLM_ROCM_USE_SKINNY_GEMM: "0"
100 base_flags: "--trust-remote-code --kv-cache-dtype fp8 --mm-encoder-tp-mode data --block-size 1 --attention-backend ROCM_AITER_MLA"
101 prefill:
102 tp: "--gpu-memory-utilization 0.9"
103 ep: "--gpu-memory-utilization 0.85 --enforce-eager"
104 decode:
105 tp: "--gpu-memory-utilization 0.9"
106 ep: '--gpu-memory-utilization 0.75 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8"]}'
107
108 - model: GLM-5.2-FP8
109 env:
110 VLLM_ROCM_USE_AITER: "1"
111 VLLM_ROCM_USE_AITER_MLA: "1"
112 VLLM_ROCM_USE_AITER_MOE: "1"
113 VLLM_ROCM_USE_AITER_RMSNORM: "1"
114 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "0"
115 base_flags: "--trust-remote-code --kv-cache-dtype fp8"
116 prefill:
117 tp: "--gpu-memory-utilization 0.85"
118 ep: "--gpu-memory-utilization 0.85 --enforce-eager"
119 decode:
120 tp: "--gpu-memory-utilization 0.85"
121 ep: '--gpu-memory-utilization 0.85 --compilation-config {"cudagraph_mode":"PIECEWISE"}'