envs.py · tduncan/Nvidia-Optimization-Stack-VLLM

@tduncan / Nvidia-Optimization-Stack-VLLM

Code Commits Issues Pulls Cloud Agent CI/CD

main

py 2,457 lines 114 KB

Raw

1	# SPDX-License-Identifier: Apache-2.0
2	# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4	import functools
5	import json
6	import logging
7	import os
8	import sys
9	import tempfile
10	import uuid
11	from collections.abc import Callable
12	from typing import TYPE_CHECKING, Any, Literal
13
14	if TYPE_CHECKING:
15	VLLM_HOST_IP: str = ""
16	VLLM_PORT: int \| None = None
17	VLLM_RPC_BASE_PATH: str = tempfile.gettempdir()
18	VLLM_USE_MODELSCOPE: bool = False
19	VLLM_USE_FASTOKENS: bool = False
20	VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
21	VLLM_NCCL_SO_PATH: str \| None = None
22	LD_LIBRARY_PATH: str \| None = None
23	VLLM_ROCM_SLEEP_MEM_CHUNK_SIZE: int = 256
24	LOCAL_RANK: int = 0
25	CUDA_VISIBLE_DEVICES: str \| None = None
26	VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60
27	VLLM_ENGINE_READY_TIMEOUT_S: int = 600
28	VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: float = 30.0
29	VLLM_API_KEY: str \| None = None
30	VLLM_DEBUG_LOG_API_SERVER_RESPONSE: bool = False
31	S3_ACCESS_KEY_ID: str \| None = None
32	S3_SECRET_ACCESS_KEY: str \| None = None
33	S3_ENDPOINT_URL: str \| None = None
34	VLLM_MODEL_REDIRECT_PATH: str \| None = None
35	VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm")
36	VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm")
37	VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai"
38	VLLM_NO_USAGE_STATS: bool = False
39	VLLM_DO_NOT_TRACK: bool = False
40	VLLM_USAGE_SOURCE: str = "production"
41	VLLM_CONFIGURE_LOGGING: bool = True
42	VLLM_LOGGING_LEVEL: str = "INFO"
43	VLLM_LOGGING_PREFIX: str = ""
44	VLLM_LOGGING_STREAM: str = "ext://sys.stdout"
45	VLLM_LOGGING_CONFIG_PATH: str \| None = None
46	VLLM_LOGGING_COLOR: str = "auto"
47	NO_COLOR: bool = False
48	FORCE_COLOR: bool = False
49	VLLM_LOG_STATS_INTERVAL: float = 10.0
50	VLLM_TRACE_FUNCTION: int = 0
51	VLLM_USE_FLASHINFER_SAMPLER: bool = True
52	VLLM_PP_LAYER_PARTITION: str \| None = None
53	VLLM_CPU_KVCACHE_SPACE: int \| None = 0
54	VLLM_CPU_OMP_THREADS_BIND: str = "auto"
55	VLLM_CPU_NUM_OF_RESERVED_CPU: int \| None = None
56	VLLM_CPU_CI_ENV: bool = False
57	VLLM_CPU_ATTN_SPLIT_KV: bool = True
58	VLLM_ZENTORCH_WEIGHT_PREPACK: bool = True
59	VLLM_CPU_INT4_W4A8: bool = True
60	VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
61	VLLM_XLA_CHECK_RECOMPILATION: bool = False
62	VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: int = 512
63	VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN: int = 8192
64	VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE: Literal["auto", "nccl", "shm"] = "auto"
65	VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False
66	VLLM_USE_RAY_WRAPPED_PP_COMM: bool = True
67	VLLM_XPU_PP_MICROBATCH: bool = True
68	VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = False
69	VLLM_DISTRIBUTED_USE_SPLIT_GROUP: bool = False
70	VLLM_XLA_USE_SPMD: bool = False
71	VLLM_WORKER_MULTIPROC_METHOD: Literal["fork", "spawn"] = "fork"
72	VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
73	VLLM_ASSETS_CACHE_MODEL_CLEAN: bool = False
74	VLLM_IMAGE_FETCH_TIMEOUT: int = 5
75	VLLM_VIDEO_FETCH_TIMEOUT: int = 30
76	VLLM_AUDIO_FETCH_TIMEOUT: int = 10
77	VLLM_MEDIA_CACHE: str = ""
78	VLLM_MEDIA_CACHE_MAX_SIZE_MB: int = 5120
79	VLLM_MEDIA_CACHE_TTL_HOURS: float = 24
80	VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3
81	VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256
82	VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
83	VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
84	VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
85	VLLM_MAX_AUDIO_DECODE_DURATION_S: int = 600
86	VLLM_MAX_AUDIO_DECODE_BYTES: int = 268_435_456
87	VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2))
88	VLLM_MAX_EMBED_DECODE_BYTES: int = 2_147_483_648
89	VLLM_MAX_IMAGE_PIXELS: int = 178_956_970
90	VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
91	VLLM_MEDIA_CONNECTOR: str = "http"
92	VLLM_TARGET_DEVICE: str = "cuda"
93	VLLM_MAIN_CUDA_VERSION: str = "13.0"
94	VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest"
95	VLLM_BATCH_INVARIANT: bool = False
96	VLLM_TRITON_USE_TD: bool \| None = None
97	VLLM_GPU_SYNC_CHECK: Literal["warn", "error"] \| None = None
98	MAX_JOBS: str \| None = None
99	NVCC_THREADS: str \| None = None
100	VLLM_USE_PRECOMPILED: bool = False
101	VLLM_USE_PRECOMPILED_RUST: bool = False
102	VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
103	VLLM_DOCKER_BUILD_CONTEXT: bool = False
104	VLLM_BUILD_COMMIT: str = "unknown"
105	VLLM_BUILD_PIPELINE: str = "local"
106	VLLM_BUILD_URL: str = ""
107	VLLM_IMAGE_TAG: str = ""
108	VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
109	CMAKE_BUILD_TYPE: Literal["Debug", "Release", "RelWithDebInfo"] \| None = None
110	VERBOSE: bool = False
111	VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
112	VLLM_HTTP_TIMEOUT_KEEP_ALIVE: int = 5 # seconds
113	VLLM_MAX_N_SEQUENCES: int = 16384
114	VLLM_MAX_COMPLETION_PROMPTS: int = 1024
115	VLLM_MAX_STOP_STRINGS: int = 4
116	VLLM_MAX_NUM_BAD_WORDS: int = 128
117	VLLM_MAX_BAD_WORDS_TOTAL_TOKENS: int = 1024
118	VLLM_PLUGINS: list[str] \| None = None
119	VLLM_LORA_RESOLVER_CACHE_DIR: str \| None = None
120	VLLM_LORA_RESOLVER_HF_REPO_LIST: str \| None = None
121	VLLM_USE_AOT_COMPILE: bool = False
122	VLLM_USE_BYTECODE_HOOK: bool = True
123	VLLM_FORCE_AOT_LOAD: bool = False
124	VLLM_USE_MEGA_AOT_ARTIFACT: bool = False
125	VLLM_USE_TRITON_AWQ: bool = False
126	VLLM_FASTSAFETENSORS_QUEUE_SIZE: int = 0
127	VLLM_TRITON_FORCE_FIRST_CONFIG: bool = False
128	VLLM_TRITON_JIT_WARMUP_NUM_THREADS: int = 4
129	VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
130	VLLM_SKIP_P2P_CHECK: bool = False
131	VLLM_DISABLED_KERNELS: list[str] = []
132	VLLM_USE_HW_AGNOSTIC: bool = False
133	VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
134	VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda"
135	VLLM_DISABLE_PYNCCL: bool = False
136	VLLM_USE_OINK_OPS: bool = False
137	VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD: bool = False
138	VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD: bool = True
139	VLLM_ROCM_USE_AITER: bool = False
140	VLLM_ROCM_USE_AITER_CUSTOM_AR: bool = True
141	VLLM_ROCM_USE_AITER_LINEAR: bool = True
142	VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
143	VLLM_ROCM_USE_AITER_MOE: bool = True
144	VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
145	VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4: bool \| None = None
146	VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
147	VLLM_ROCM_USE_AITER_RMSNORM: bool = True
148	VLLM_ROCM_USE_AITER_MLA: bool = True
149	VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
150	VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["asm", "segmented"] = "segmented"
151	VLLM_ROCM_USE_AITER_MHA: bool = True
152	VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
153	VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: bool = False
154	VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
155	VLLM_ROCM_USE_AITER_FP8BMM: bool = True
156	VLLM_ROCM_USE_AITER_FP4BMM: bool = True
157	VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: bool = False
158	VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: bool = False
159	VLLM_ROCM_USE_AITER_TRITON_GEMM: bool = True
160	VLLM_ROCM_USE_SKINNY_GEMM: bool = True
161	VLLM_ROCM_FP8_PADDING: bool = True
162	VLLM_ROCM_MOE_PADDING: bool = True
163	VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT: bool = False
164	VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
165	VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
166	VLLM_DISABLE_COMPILE_CACHE: bool = False
167	VLLM_REPLICATE_EMBED: bool = False
168	VLLM_USE_LAYERNAME: bool = True
169	Q_SCALE_CONSTANT: int = 200
170	K_SCALE_CONSTANT: int = 200
171	V_SCALE_CONSTANT: int = 100
172	VLLM_USE_RUST_FRONTEND: bool = False
173	VLLM_USE_RUST_BENCH: bool = False
174	VLLM_RUST_FRONTEND_PATH: str \| None = "auto"
175	VLLM_SERVER_DEV_MODE: bool = False
176	VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
177	VLLM_MLA_DISABLE: bool = False
178	VLLM_RAY_PER_WORKER_GPUS: float = 1.0
179	VLLM_RAY_BUNDLE_INDICES: str = ""
180	VLLM_CUDART_SO_PATH: str \| None = None
181	VLLM_DP_RANK: int = 0
182	VLLM_DP_RANK_LOCAL: int = -1
183	VLLM_DP_SIZE: int = 1
184	VLLM_USE_STANDALONE_COMPILE: bool = True
185	VLLM_ENABLE_PREGRAD_PASSES: bool = True
186	VLLM_USE_BREAKABLE_CUDAGRAPH: bool = False
187	VLLM_DP_MASTER_IP: str = ""
188	VLLM_DP_MASTER_PORT: int = 0
189	VLLM_RANDOMIZE_DP_DUMMY_INPUTS: bool = False
190	VLLM_RAY_DP_PACK_STRATEGY: Literal["strict", "fill", "span"] = "strict"
191	VLLM_RAY_DP_PLACEMENT_NODE_IPS: str = ""
192	VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY: str = ""
193	VLLM_RAY_EXTRA_ENV_VARS_TO_COPY: str = ""
194	VLLM_MARLIN_USE_ATOMIC_ADD: bool = False
195	VLLM_MARLIN_INPUT_DTYPE: Literal["int8", "fp8"] \| None = None
196	VLLM_HUMMING_ONLINE_QUANT_CONFIG: dict[str, Any] \| None = None
197	VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] \| None = None
198	VLLM_HUMMING_USE_F16_ACCUM: bool = False
199	VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] \| None = None
200	VLLM_B12X_MOE_FP4_FORCE_A16: bool = False
201	VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
202	VLLM_V1_USE_OUTLINES_CACHE: bool = False
203	VLLM_TPU_USING_PATHWAYS: bool = False
204	VLLM_USE_DEEP_GEMM: bool = True
205	VLLM_MOE_USE_DEEP_GEMM: bool = True
206	VLLM_USE_DEEP_GEMM_E8M0: bool = True
207	VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True
208	VLLM_DCP_Q_REPLICATE: bool = False
209	VLLM_USE_DIRECT_DCP_A2A: bool \| None = None
210	VLLM_USE_DIRECT_DCP_Q_GATHER: bool \| None = None
211	VLLM_USE_DIRECT_DCP_KV_GATHER: bool \| None = None
212	VLLM_DEEP_GEMM_WARMUP: Literal[
213	"skip",
214	"full",
215	"relax",
216	] = "relax"
217	VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
218	VLLM_MOE_SKIP_PADDING: bool = True
219	VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
220	VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False
221	VLLM_KIMI_K3_GEMM_AR: bool = True
222	VLLM_ENABLE_GEMM_RS: bool = False
223	VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
224	VLLM_USE_FLASHINFER_MOE_INT4: bool = False
225	VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str \| None = None
226	VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS: list[str] \| None = None
227	VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
228	VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
229	VLLM_XGRAMMAR_CACHE_MB: int = 0
230	VLLM_REGEX_COMPILATION_TIMEOUT_S: int = 5
231	VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256
232	VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False
233	VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False
234	VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
235	VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
236	VLLM_P2P_SIDE_CHANNEL_HOST: str = "localhost"
237	VLLM_P2P_SIDE_CHANNEL_PORT: int = 5710
238	VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost"
239	VLLM_EC_SIDE_CHANNEL_PORT: int = 5601
240	VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
241	VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
242	VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
243	VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
244	MOONCAKE_PREFERRED_SEGMENT: str \| None = None
245	MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str \| None = None
246	VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
247	VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
248	VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
249	VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
250	VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
251	VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
252	VLLM_KV_CACHE_LAYOUT: (
253	Literal["LBNHC", "LBHNC", "LHBNC", "NHD", "HND", "BLHNC", "BLNHC", "BHLNC"]
254	\| None
255	) = None
256	VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] \| None = None
257	VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
258	VLLM_RAISE_ON_LOGIT_NANS: bool = False
259	VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
260	"FP", "INT8", "INT6", "INT4", "INT3", "NONE"
261	] = "NONE"
262	VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16: bool = True
263	VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: int \| None = None
264	VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int \| None = None
265	VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int \| None = None
266	VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480
267	VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0
268	VLLM_ENABLE_CUDAGRAPH_GC: bool = False
269	VLLM_LOOPBACK_IP: str = ""
270	VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
271	VLLM_ENABLE_RESPONSES_API_STORE: bool = False
272	VLLM_ENABLE_COHERE_API: bool = False
273	VLLM_HAS_FLASHINFER_CUBIN: bool = False
274	VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
275	VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
276	VLLM_ALLREDUCE_USE_FLASHINFER: bool = True
277	VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC: bool = False
278	VLLM_TUNED_CONFIG_FOLDER: str \| None = None
279	VLLM_ENABLE_STARTUP_PLAN: bool = False
280	VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
281	VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: bool = False
282	VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
283	VLLM_SYSTEM_START_DATE: str \| None = None
284	VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False
285	VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
286	VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False
287	VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True
288	VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME: str = "VLLM_OBJECT_STORAGE_SHM_BUFFER"
289	VLLM_DEEPEP_BUFFER_SIZE_MB: int = 1024
290	VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE: bool = False
291	VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL: bool = False
292	VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE: bool = True
293	VLLM_DEEPEP_V2_PREFER_OVERLAP: bool = False
294	VLLM_DEEPEP_V2_ALLOW_MULTIPLE_REDUCTION: bool = False
295	VLLM_DBO_COMM_SMS: int = 20
296	VLLM_PATTERN_MATCH_DEBUG: str \| None = None
297	VLLM_DEBUG_DUMP_PATH: str \| None = None
298	VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE: bool = True
299	VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True
300	VLLM_USE_NCCL_SYMM_MEM: bool = False
301	VLLM_NCCL_INCLUDE_PATH: str \| None = None
302	VLLM_GC_DEBUG: str = ""
303	VLLM_DEBUG_WORKSPACE: bool = False
304	VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
305	VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION: bool = False
306	VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
307	VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
308	VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
309	VLLM_USE_V2_MODEL_RUNNER: bool \| None = None
310	VLLM_LOG_MODEL_INSPECTION: bool = False
311	VLLM_DEBUG_MFU_METRICS: bool = False
312	VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
313	VLLM_WEIGHT_OFFLOADING_DISABLE_UVA: bool = False
314	VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS: int = 0
315	VLLM_WSL2_ENABLE_PIN_MEMORY: bool = False
316	VLLM_DISABLE_LOG_LOGO: bool = False
317	VLLM_LORA_DISABLE_PDL: bool = False
318	VLLM_ENABLE_CUDA_COMPATIBILITY: bool = False
319	VLLM_CUDA_COMPATIBILITY_PATH: str \| None = None
320	VLLM_SKIP_MODEL_NAME_VALIDATION: bool = False
321	"""If set, vLLM will skip model name validation in API requests.
322	This allows any model name to be accepted in the 'model' field of requests,
323	making the server model-name agnostic. Useful for proxy/gateway scenarios."""
324	VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False
325	VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False
326	VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True
327	VLLM_XPU_FORCE_N_CONTIG_WEIGHT: bool = False
328	VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
329	VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
330	VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
331	VLLM_GPU_NIC_PCIE_MAPPING: str = ""
332	VLLM_NIC_SELECTION_VARS: str = ""
333	VLLM_ENABLE_HPC_OPS: bool = False
334
335
336	def get_default_cache_root():
337	return os.getenv(
338	"XDG_CACHE_HOME",
339	os.path.join(os.path.expanduser("~"), ".cache"),
340	)
341
342
343	def get_default_config_root():
344	return os.getenv(
345	"XDG_CONFIG_HOME",
346	os.path.join(os.path.expanduser("~"), ".config"),
347	)
348
349
350	def maybe_convert_int(value: str \| None) -> int \| None:
351	if value is None:
352	return None
353	return int(value)
354
355
356	def maybe_convert_bool(value: str \| None) -> bool \| None:
357	if value is None:
358	return None
359	return bool(int(value))
360
361
362	def maybe_convert_json_str_or_file(value: str \| None) -> dict[str, Any] \| None:
363	if value is None:
364	return None
365	if os.path.exists(value):
366	with open(value) as f:
367	return json.load(f)
368	return json.loads(value)
369
370
371	def disable_compile_cache() -> bool:
372	return bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0")))
373
374
375	def use_aot_compile() -> bool:
376	from vllm.utils.torch_utils import is_torch_equal_or_newer
377
378	default_value = (
379	"1"
380	if is_torch_equal_or_newer("2.10.0") and not disable_compile_cache()
381	else "0"
382	)
383
384	return os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1"
385
386
387	def use_mega_aot_artifact():
388	from vllm.utils.torch_utils import is_torch_equal_or_newer
389
390	default_value = (
391	"1" if is_torch_equal_or_newer("2.12.0.dev") and use_aot_compile() else "0"
392	)
393
394	return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1"
395
396
397	def env_with_choices(
398	env_name: str,
399	default: str \| None,
400	choices: list[str] \| Callable[[], list[str]],
401	case_sensitive: bool = True,
402	) -> Callable[[], str \| None]:
403	"""Create a lambda that validates environment variable against allowed choices.
404
405	Args:
406	env_name: Name of the environment variable
407	default: Default value if not set (can be None)
408	choices: List of valid string options or callable that returns list
409	case_sensitive: Whether validation should be case sensitive
410
411	Returns:
412	Lambda function for environment_variables dict
413
414	"""
415
416	def _get_validated_env() -> str \| None:
417	value = os.getenv(env_name)
418	if value is None:
419	return default
420
421	# Resolve choices if it's a callable (for lazy loading)
422	actual_choices = choices() if callable(choices) else choices
423
424	if not case_sensitive:
425	check_value = value.lower()
426	check_choices = [choice.lower() for choice in actual_choices]
427	else:
428	check_value = value
429	check_choices = actual_choices
430
431	if check_value not in check_choices:
432	raise ValueError(
433	f"Invalid value '{value}' for {env_name}. "
434	f"Valid options: {actual_choices}."
435	)
436
437	return value
438
439	return _get_validated_env
440
441
442	def env_list_with_choices(
443	env_name: str,
444	default: list[str],
445	choices: list[str] \| Callable[[], list[str]],
446	case_sensitive: bool = True,
447	) -> Callable[[], list[str]]:
448	"""Create a lambda that validates environment variable
449	containing comma-separated values against allowed choices
450
451	Args:
452	env_name: Name of the environment variable
453	default: Default list of values if not set
454	choices: List of valid string options or callable that returns list
455	case_sensitive: Whether validation should be case sensitive
456
457	Returns:
458	Lambda function for environment_variables
459	dict that returns list of strings
460
461	"""
462
463	def _get_validated_env_list() -> list[str]:
464	value = os.getenv(env_name)
465	if value is None:
466	return default
467
468	# Split comma-separated values and strip whitespace
469	values = [v.strip() for v in value.split(",") if v.strip()]
470
471	if not values:
472	return default
473
474	# Resolve choices if it's a callable (for lazy loading)
475	actual_choices = choices() if callable(choices) else choices
476
477	# Validate each value
478	for val in values:
479	if not case_sensitive:
480	check_value = val.lower()
481	check_choices = [choice.lower() for choice in actual_choices]
482	else:
483	check_value = val
484	check_choices = actual_choices
485
486	if check_value not in check_choices:
487	raise ValueError(
488	f"Invalid value '{val}' in {env_name}. "
489	f"Valid options: {actual_choices}."
490	)
491
492	return values
493
494	return _get_validated_env_list
495
496
497	def env_set_with_choices(
498	env_name: str,
499	default: list[str],
500	choices: list[str] \| Callable[[], list[str]],
501	case_sensitive: bool = True,
502	) -> Callable[[], set[str]]:
503	"""Creates a lambda which that validates environment variable
504	containing comma-separated values against allowed choices which
505	returns choices as a set.
506	"""
507
508	def _get_validated_env_set() -> set[str]:
509	return set(env_list_with_choices(env_name, default, choices, case_sensitive)())
510
511	return _get_validated_env_set
512
513
514	def get_vllm_port() -> int \| None:
515	"""Get the port from VLLM_PORT environment variable.
516
517	Returns:
518	The port number as an integer if VLLM_PORT is set, None otherwise.
519
520	Raises:
521	ValueError: If VLLM_PORT is a URI, suggest k8s service discovery issue.
522
523	"""
524	if "VLLM_PORT" not in os.environ:
525	return None
526
527	port = os.getenv("VLLM_PORT", "0")
528
529	try:
530	return int(port)
531	except ValueError as err:
532	from urllib3.util import parse_url
533
534	parsed = parse_url(port)
535	if parsed.scheme:
536	raise ValueError(
537	f"VLLM_PORT '{port}' appears to be a URI. "
538	"This may be caused by a Kubernetes service discovery issue,"
539	"check the warning in: https://docs.vllm.ai/en/latest/configuration/env_vars.html"
540	) from None
541	raise ValueError(f"VLLM_PORT '{port}' must be a valid integer") from err
542
543
544	def _generate_shm_name() -> str:
545	import pybase64
546
547	# Fit macOS's 30-character limit without truncating the UUID.
548	encoded_uuid = pybase64.urlsafe_b64encode(uuid.uuid4().bytes).decode("ascii")
549	return "vllm_mm_" + encoded_uuid.rstrip("=")
550
551
552	def get_env_or_set_default(
553	env_name: str,
554	default_factory: Callable[[], str],
555	) -> Callable[[], str]:
556	"""Create a lambda that returns an environment variable value if set,
557	or generates and sets a default value using the provided factory function.
558	"""
559
560	def _get_or_set_default() -> str:
561	value = os.getenv(env_name)
562	if value is not None:
563	return value
564
565	default_value = default_factory()
566	os.environ[env_name] = default_value
567	return default_value
568
569	return _get_or_set_default
570
571
572	# The start-* and end* here are used by the documentation generator
573	# to extract the used env vars.
574
575	# --8<-- [start:env-vars-definition]
576
577	logger = logging.getLogger(__name__)
578
579
580	def _resolve_rust_cli_path() -> str \| None:
581	"""Resolve the vllm-rs binary path.
582
583	Returns None unless VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH is enabled.
584	When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default)
585	to the actual binary path.
586	"""
587	use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0"))) or bool(
588	int(os.environ.get("VLLM_USE_RUST_BENCH", "0"))
589	)
590	raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto")
591
592	if not use_rust:
593	if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None:
594	logger.warning(
595	"VLLM_RUST_FRONTEND_PATH is set without enabling "
596	"VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH. "
597	"Set one of them to 1 to use the vllm-rs binary."
598	)
599	return None
600
601	if raw.lower() in ("auto", "1", "true"):
602	pkg_dir = os.path.dirname(os.path.abspath(__file__))
603	candidate = os.path.join(pkg_dir, "vllm-rs")
604	if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
605	return candidate
606
607	raise FileNotFoundError(
608	"VLLM_RUST_FRONTEND_PATH=auto but the vllm-rs binary was "
609	f"not found at {candidate}. "
610	"Build with setuptools-rust or set the path explicitly."
611	)
612	return raw
613
614
615	environment_variables: dict[str, Callable[[], Any]] = {
616	# ================== Installation Time Env Vars ==================
617	# Target device of vLLM, supporting [cuda (by default),
618	# rocm, cpu]
619	"VLLM_TARGET_DEVICE": lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda").lower(),
620	# Main CUDA version of vLLM. This follows PyTorch but can be overridden.
621	"VLLM_MAIN_CUDA_VERSION": lambda: (
622	os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower() or "13.0"
623	),
624	# Controls PyTorch float32 matmul precision mode within vLLM workers.
625	# Valid options mirror torch.set_float32_matmul_precision
626	"VLLM_FLOAT32_MATMUL_PRECISION": env_with_choices(
627	"VLLM_FLOAT32_MATMUL_PRECISION",
628	"highest",
629	["highest", "high", "medium"],
630	case_sensitive=False,
631	),
632	# Enable batch-invariant mode: deterministic results regardless of
633	# batch composition. Requires NVIDIA GPU with compute capability >= 9.0.
634	"VLLM_BATCH_INVARIANT": lambda: bool(int(os.getenv("VLLM_BATCH_INVARIANT", "0"))),
635	"VLLM_REPLICATE_EMBED": lambda: (
636	os.getenv("VLLM_REPLICATE_EMBED", "0").strip().lower() in ("1", "true")
637	),
638	# Use tensor descriptors for Q/K/V loads and output stores in the
639	# Triton unified-attention kernel. Enables HW 2D block reads on
640	# Intel XPU; the non-TD branch is dead-code-eliminated at Triton
641	# compile time so other platforms see no overhead. Tri-state override:
642	# unset (default) lets the `triton_attn` backend auto-select per
643	# platform (currently auto-enabled on XPU only); ``1`` forces TD on;
644	# ``0`` forces TD off. Useful for A/B benchmarking the TD path.
645	"VLLM_TRITON_USE_TD": lambda: {"1": True, "0": False}.get(
646	os.getenv("VLLM_TRITON_USE_TD", "").strip()
647	),
648	# If set, enable GPU<->CPU synchronization checking around the worker's
649	# `execute_model` and `sample_tokens` calls, via PyTorch's sync debug mode
650	# plus wrappers flagging `non_blocking` CPU<->CUDA copies that silently
651	# block the host (CPU tensors that are not pinned or not densely laid out).
652	# Valid values are "warn" (warn on each sync) or "error" (raise on sync).
653	# Unset disables the check. See `torch.cuda.set_sync_debug_mode`.
654	"VLLM_GPU_SYNC_CHECK": env_with_choices(
655	"VLLM_GPU_SYNC_CHECK", None, ["warn", "error"]
656	),
657	# Maximum number of compilation jobs to run in parallel.
658	# By default this is the number of CPUs
659	"MAX_JOBS": lambda: os.getenv("MAX_JOBS", None),
660	# Number of threads to use for nvcc
661	# By default this is 1.
662	# If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
663	"NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None),
664	# If set, vllm will use precompiled native binaries (*.so and vllm-rs).
665	"VLLM_USE_PRECOMPILED": lambda: (
666	os.environ.get("VLLM_USE_PRECOMPILED", "").strip().lower() in ("1", "true")
667	or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION"))
668	),
669	# If set, vllm will use the precompiled Rust frontend binary (vllm-rs).
670	"VLLM_USE_PRECOMPILED_RUST": lambda: (
671	os.environ.get("VLLM_USE_PRECOMPILED_RUST", "").strip().lower() in ("1", "true")
672	),
673	# If set, skip adding +precompiled suffix to version string
674	"VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX": lambda: bool(
675	int(os.environ.get("VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX", "0"))
676	),
677	# Used to mark that setup.py is running in a Docker build context,
678	# in order to force the use of precompiled binaries.
679	"VLLM_DOCKER_BUILD_CONTEXT": lambda: (
680	os.environ.get("VLLM_DOCKER_BUILD_CONTEXT", "").strip().lower() in ("1", "true")
681	),
682	# Build provenance metadata embedded in official vllm-openai images.
683	# Set via Docker ENV at image build time; informational only.
684	"VLLM_BUILD_COMMIT": lambda: os.environ.get("VLLM_BUILD_COMMIT", "unknown"),
685	"VLLM_BUILD_PIPELINE": lambda: os.environ.get("VLLM_BUILD_PIPELINE", "local"),
686	"VLLM_BUILD_URL": lambda: os.environ.get("VLLM_BUILD_URL", ""),
687	"VLLM_IMAGE_TAG": lambda: os.environ.get("VLLM_IMAGE_TAG", ""),
688	# CMake build type
689	# If not set, defaults to "Debug" or "RelWithDebInfo"
690	# Available options: "Debug", "Release", "RelWithDebInfo"
691	"CMAKE_BUILD_TYPE": env_with_choices(
692	"CMAKE_BUILD_TYPE", None, ["Debug", "Release", "RelWithDebInfo"]
693	),
694	# If set, vllm will print verbose logs during installation
695	"VERBOSE": lambda: bool(int(os.getenv("VERBOSE", "0"))),
696	# Root directory for vLLM configuration files
697	# Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set
698	# Note that this not only affects how vllm finds its configuration files
699	# during runtime, but also affects how vllm installs its configuration
700	# files during installation.
701	"VLLM_CONFIG_ROOT": lambda: os.path.expanduser(
702	os.getenv(
703	"VLLM_CONFIG_ROOT",
704	os.path.join(get_default_config_root(), "vllm"),
705	)
706	),
707	# ================== Runtime Env Vars ==================
708	# Root directory for vLLM cache files
709	# Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set
710	"VLLM_CACHE_ROOT": lambda: os.path.expanduser(
711	os.getenv(
712	"VLLM_CACHE_ROOT",
713	os.path.join(get_default_cache_root(), "vllm"),
714	)
715	),
716	# used in distributed environment to determine the ip address
717	# of the current node, when the node has multiple network interfaces.
718	# If you are using multi-node inference, you should set this differently
719	# on each node.
720	"VLLM_HOST_IP": lambda: os.getenv("VLLM_HOST_IP", ""),
721	# used in distributed environment to manually set the communication port
722	# Note: if VLLM_PORT is set, and some code asks for multiple ports, the
723	# VLLM_PORT will be used as the first port, and the rest will be generated
724	# by incrementing the VLLM_PORT value.
725	"VLLM_PORT": get_vllm_port,
726	# path used for ipc when the frontend api server is running in
727	# multi-processing mode to communicate with the backend engine process.
728	"VLLM_RPC_BASE_PATH": lambda: os.getenv(
729	"VLLM_RPC_BASE_PATH", tempfile.gettempdir()
730	),
731	# If true, will load models from ModelScope instead of Hugging Face Hub.
732	"VLLM_USE_MODELSCOPE": lambda: (
733	os.environ.get("VLLM_USE_MODELSCOPE", "False").strip().lower() in ("1", "true")
734	),
735	# If true, replace the Rust BPE backend that powers HF fast tokenizers
736	# with the `fastokens` (https://github.com/crusoecloud/fastokens) shim.
737	# Applies to any tokenizer mode that loads an HF fast tokenizer
738	# (`hf`, `deepseek_v32`, `deepseek_v4`, …). The `fastokens`
739	# Python package must be installed.
740	"VLLM_USE_FASTOKENS": lambda: bool(int(os.getenv("VLLM_USE_FASTOKENS", "0"))),
741	# Interval in seconds to log a warning message when the ring buffer is full
742	"VLLM_RINGBUFFER_WARNING_INTERVAL": lambda: int(
743	os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60")
744	),
745	# path to cudatoolkit home directory, under which should be bin, include,
746	# and lib directories.
747	"CUDA_HOME": lambda: os.environ.get("CUDA_HOME", None),
748	# Path to the NCCL library file. It is needed because nccl>=2.19 brought
749	# by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234
750	"VLLM_NCCL_SO_PATH": lambda: os.environ.get("VLLM_NCCL_SO_PATH", None),
751	# when `VLLM_NCCL_SO_PATH` is not set, vllm will try to find the nccl
752	# library file in the locations specified by `LD_LIBRARY_PATH`
753	"LD_LIBRARY_PATH": lambda: os.environ.get("LD_LIBRARY_PATH", None),
754	# flag to control the chunk size (in MB) for sleeping memory allocations under ROCm
755	"VLLM_ROCM_SLEEP_MEM_CHUNK_SIZE": lambda: int(
756	os.environ.get("VLLM_ROCM_SLEEP_MEM_CHUNK_SIZE", "256")
757	),
758	# Feature flag to enable/disable Inductor standalone compile.
759	# In torch <= 2.7 we ignore this flag; in torch >= 2.9 this is
760	# enabled by default.
761	"VLLM_USE_STANDALONE_COMPILE": lambda: (
762	os.environ.get("VLLM_USE_STANDALONE_COMPILE", "1") == "1"
763	),
764	# Inductor's pre-grad passes don't do anything for vLLM.
765	# The pre-grad passes get run even on cache-hit and negatively impact
766	# vllm cold compile times by O(1s)
767	# Can remove this after the following issue gets fixed
768	# TODO(luka): maybe_inplace requires this
769	# https://github.com/pytorch/pytorch/issues/174502
770	"VLLM_ENABLE_PREGRAD_PASSES": lambda: (
771	os.environ.get("VLLM_ENABLE_PREGRAD_PASSES", "1") == "1"
772	),
773	# Experimental: breakable cudagraph does not rely on torch.compile
774	"VLLM_USE_BREAKABLE_CUDAGRAPH": lambda: (
775	os.environ.get("VLLM_USE_BREAKABLE_CUDAGRAPH", "0") == "1"
776	),
777	# Debug pattern matching inside custom passes.
778	# Should be set to the fx.Node name (e.g. 'getitem_34' or 'scaled_mm_3').
779	"VLLM_PATTERN_MATCH_DEBUG": lambda: os.environ.get(
780	"VLLM_PATTERN_MATCH_DEBUG", None
781	),
782	# Dump fx graphs to the given directory.
783	# It will override CompilationConfig.debug_dump_path if set.
784	"VLLM_DEBUG_DUMP_PATH": lambda: os.environ.get("VLLM_DEBUG_DUMP_PATH", None),
785	# Feature flag to enable/disable AOT compilation. This will ensure
786	# compilation is done in warmup phase and the compilation will be
787	# reused in subsequent calls.
788	"VLLM_USE_AOT_COMPILE": use_aot_compile,
789	# Feature flag to enable/disable bytecode in
790	# TorchCompileWithNoGuardsWrapper.
791	"VLLM_USE_BYTECODE_HOOK": lambda: bool(
792	int(os.environ.get("VLLM_USE_BYTECODE_HOOK", "1"))
793	),
794	# Force vllm to always load AOT compiled models from disk. Failure
795	# to load will result in a hard error when this is enabled.
796	# Will be ignored when VLLM_USE_AOT_COMPILE is disabled.
797	"VLLM_FORCE_AOT_LOAD": lambda: os.environ.get("VLLM_FORCE_AOT_LOAD", "0") == "1",
798	# Enable loading compiled models directly from cached standalone compile artifacts
799	# without re-splitting graph modules. This reduces overhead during model
800	# loading by using reconstruct_serializable_fn_from_mega_artifact.
801	"VLLM_USE_MEGA_AOT_ARTIFACT": use_mega_aot_artifact,
802	# local rank of the process in the distributed setting, used to determine
803	# the GPU device id
804	"LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")),
805	# used to control the visible devices in the distributed setting
806	"CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
807	# timeout for each iteration in the engine
808	"VLLM_ENGINE_ITERATION_TIMEOUT_S": lambda: int(
809	os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60")
810	),
811	# Timeout in seconds for waiting for engine cores to become ready
812	# during startup. Default is 600 seconds (10 minutes).
813	"VLLM_ENGINE_READY_TIMEOUT_S": lambda: int(
814	os.environ.get("VLLM_ENGINE_READY_TIMEOUT_S", "600")
815	),
816	# Maximum wall-clock seconds allowed for a single chat template render.
817	# Set to 0 to disable the timeout.
818	"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT": lambda: float(
819	os.environ.get("VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT", "30")
820	),
821	# API key for vLLM API server
822	"VLLM_API_KEY": lambda: os.environ.get("VLLM_API_KEY", None),
823	# Whether to log responses from API Server for debugging
824	"VLLM_DEBUG_LOG_API_SERVER_RESPONSE": lambda: (
825	os.environ.get("VLLM_DEBUG_LOG_API_SERVER_RESPONSE", "False").lower() == "true"
826	),
827	# S3 access information, used for tensorizer to load model from S3
828	"S3_ACCESS_KEY_ID": lambda: os.environ.get("S3_ACCESS_KEY_ID", None),
829	"S3_SECRET_ACCESS_KEY": lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None),
830	"S3_ENDPOINT_URL": lambda: os.environ.get("S3_ENDPOINT_URL", None),
831	# Usage stats collection
832	"VLLM_USAGE_STATS_SERVER": lambda: os.environ.get(
833	"VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai"
834	),
835	"VLLM_NO_USAGE_STATS": lambda: os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1",
836	"VLLM_DO_NOT_TRACK": lambda: (
837	(
838	os.environ.get("VLLM_DO_NOT_TRACK", None)
839	or os.environ.get("DO_NOT_TRACK", None)
840	or "0"
841	)
842	== "1"
843	),
844	"VLLM_USAGE_SOURCE": lambda: os.environ.get("VLLM_USAGE_SOURCE", "production"),
845	# Logging configuration
846	# If set to 0, vllm will not configure logging
847	# If set to 1, vllm will configure logging using the default configuration
848	# or the configuration file specified by VLLM_LOGGING_CONFIG_PATH
849	"VLLM_CONFIGURE_LOGGING": lambda: bool(
850	int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
851	),
852	"VLLM_LOGGING_CONFIG_PATH": lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"),
853	# this is used for configuring the default logging level
854	"VLLM_LOGGING_LEVEL": lambda: os.getenv("VLLM_LOGGING_LEVEL", "INFO").upper(),
855	# this is used for configuring the default logging stream
856	"VLLM_LOGGING_STREAM": lambda: os.getenv("VLLM_LOGGING_STREAM", "ext://sys.stdout"),
857	# if set, VLLM_LOGGING_PREFIX will be prepended to all log messages
858	"VLLM_LOGGING_PREFIX": lambda: os.getenv("VLLM_LOGGING_PREFIX", ""),
859	# Controls colored logging output. Options: "auto" (default, colors when terminal),
860	# "1" (always use colors), "0" (never use colors)
861	"VLLM_LOGGING_COLOR": lambda: os.getenv("VLLM_LOGGING_COLOR", "auto"),
862	# Standard unix flag for disabling ANSI color codes
863	"NO_COLOR": lambda: os.getenv("NO_COLOR", "0") != "0",
864	# De-facto standard flag for forcing ANSI color codes (e.g. non-tty case)
865	"FORCE_COLOR": lambda: os.getenv("FORCE_COLOR", "0") != "0",
866	# If set, vllm will log stats at this interval in seconds
867	# If not set, vllm will log stats every 10 seconds.
868	"VLLM_LOG_STATS_INTERVAL": lambda: (
869	val
870	if (val := float(os.getenv("VLLM_LOG_STATS_INTERVAL", "10."))) > 0.0
871	else 10.0
872	),
873	# Trace function calls
874	# If set to 1, vllm will trace function calls
875	# Useful for debugging
876	"VLLM_TRACE_FUNCTION": lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")),
877	# Whether to use the FlashInfer top-k / top-p sampler on CUDA. Enabled
878	# by default when the hardware supports it — set to 0 to opt out
879	# explicitly, which forces the PyTorch-native (Triton for bs>=8) path.
880	"VLLM_USE_FLASHINFER_SAMPLER": lambda: (
881	bool(int(os.environ["VLLM_USE_FLASHINFER_SAMPLER"]))
882	if "VLLM_USE_FLASHINFER_SAMPLER" in os.environ
883	else True
884	),
885	# Pipeline stage partition strategy
886	"VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
887	# (CPU backend only) CPU key-value cache space.
888	# default is None and will be set as 4 GB
889	"VLLM_CPU_KVCACHE_SPACE": lambda: (
890	int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0"))
891	if "VLLM_CPU_KVCACHE_SPACE" in os.environ
892	else None
893	),
894	# (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
895	# "0,1,2", "0-31,33". CPU cores of different ranks are separated by '\|'.
896	"VLLM_CPU_OMP_THREADS_BIND": lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "auto"),
897	# (CPU backend only) CPU cores not used by OMP threads .
898	# Those CPU cores will not be used by OMP threads of a rank.
899	"VLLM_CPU_NUM_OF_RESERVED_CPU": lambda: (
900	int(os.getenv("VLLM_CPU_NUM_OF_RESERVED_CPU", "0"))
901	if "VLLM_CPU_NUM_OF_RESERVED_CPU" in os.environ
902	else None
903	),
904	# (CPU backend only) whether vLLM is running in a CI environment.
905	"VLLM_CPU_CI_ENV": lambda: bool(int(os.getenv("VLLM_CPU_CI_ENV", "0"))),
906	# (CPU backend only) whether to enable attention spilt KV.
907	"VLLM_CPU_ATTN_SPLIT_KV": lambda: bool(
908	int(os.getenv("VLLM_CPU_ATTN_SPLIT_KV", "1"))
909	),
910	# (Zen CPU backend) eagerly prepack weights into ZenDNN blocked layout
911	# at model load time. Eliminates per-inference layout conversion overhead.
912	"VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool(
913	int(os.getenv("VLLM_ZENTORCH_WEIGHT_PREPACK", "1"))
914	),
915	# (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ, and
916	# on Zen CPUs whether to serve int4 checkpoints as DA8W4 rather than W4A16.
917	"VLLM_CPU_INT4_W4A8": lambda: bool(int(os.getenv("VLLM_CPU_INT4_W4A8", "1"))),
918	# If the env var is set, Ray Compiled Graph uses the specified
919	# channel type to communicate between workers belonging to
920	# different pipeline-parallel stages.
921	# Available options:
922	# - "auto": use the default channel type
923	# - "nccl": use NCCL for communication
924	# - "shm": use shared memory and gRPC for communication
925	"VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE": env_with_choices(
926	"VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE", "auto", ["auto", "nccl", "shm"]
927	),
928	# If the env var is set, it enables GPU communication overlap
929	# (experimental feature) in Ray's Compiled Graph.
930	"VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM": lambda: bool(
931	int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "0"))
932	),
933	# If the env var is set, it uses a Ray Communicator wrapping
934	# vLLM's pipeline parallelism communicator to interact with Ray's
935	# Compiled Graph. Otherwise, it uses Ray's NCCL communicator.
936	"VLLM_USE_RAY_WRAPPED_PP_COMM": lambda: bool(
937	int(os.getenv("VLLM_USE_RAY_WRAPPED_PP_COMM", "1"))
938	),
939	# Using a flag to control microbatch on XPU device, users on XPU device can
940	# decide to disable it when they needed.
941	"VLLM_XPU_PP_MICROBATCH": lambda: bool(
942	int(os.getenv("VLLM_XPU_PP_MICROBATCH", "1"))
943	),
944	# When True and distributed_executor_backend="ray", use RayExecutorV2
945	# (MQ-based) instead of RayDistributedExecutor (compiled-graph backend).
946	"VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool(
947	int(os.getenv("VLLM_USE_RAY_V2_EXECUTOR_BACKEND", "1"))
948	),
949	# When True, GroupCoordinator constructs its CPU/device subgroups via
950	# ``torch.distributed.split_group(backend=...)``
951	# and ``init_distributed_environment`` initializes the default PG with
952	# mixed ``cpu:gloo,cuda:nccl`` backend + eager ``device_id`` binding.
953	"VLLM_DISTRIBUTED_USE_SPLIT_GROUP": lambda: bool(
954	int(os.getenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0"))
955	),
956	# Use dedicated multiprocess context for workers.
957	# Both spawn and fork work
958	"VLLM_WORKER_MULTIPROC_METHOD": env_with_choices(
959	"VLLM_WORKER_MULTIPROC_METHOD", "fork", ["spawn", "fork"]
960	),
961	# Path to the cache for storing downloaded assets
962	"VLLM_ASSETS_CACHE": lambda: os.path.expanduser(
963	os.getenv(
964	"VLLM_ASSETS_CACHE",
965	os.path.join(get_default_cache_root(), "vllm", "assets"),
966	)
967	),
968	# If the env var is set, we will clean model file in
969	# this path $VLLM_ASSETS_CACHE/model_streamer/$model_name
970	"VLLM_ASSETS_CACHE_MODEL_CLEAN": lambda: bool(
971	int(os.getenv("VLLM_ASSETS_CACHE_MODEL_CLEAN", "0"))
972	),
973	# Timeout for fetching images when serving multimodal models
974	# Default is 5 seconds
975	"VLLM_IMAGE_FETCH_TIMEOUT": lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
976	# Timeout for fetching videos when serving multimodal models
977	# Default is 30 seconds
978	"VLLM_VIDEO_FETCH_TIMEOUT": lambda: int(
979	os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "30")
980	),
981	# Timeout for fetching audio when serving multimodal models
982	# Default is 10 seconds
983	"VLLM_AUDIO_FETCH_TIMEOUT": lambda: int(
984	os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")
985	),
986	# Directory for caching media downloads (images, video, audio fetched
987	# from URLs during inference). Empty string disables caching.
988	"VLLM_MEDIA_CACHE": lambda: os.getenv("VLLM_MEDIA_CACHE", ""),
989	# Maximum cache size in MB. When exceeded, least-recently-used entries
990	# are evicted. Default is 5120 (5 GB).
991	"VLLM_MEDIA_CACHE_MAX_SIZE_MB": lambda: int(
992	os.getenv("VLLM_MEDIA_CACHE_MAX_SIZE_MB", "5120")
993	),
994	# Time-to-live in hours for cached media files. Entries older than this
995	# are evicted regardless of cache size. Default is 24 hours.
996	"VLLM_MEDIA_CACHE_TTL_HOURS": lambda: float(
997	os.getenv("VLLM_MEDIA_CACHE_TTL_HOURS", "24")
998	),
999	# Maximum number of retries for fetching media (images, audio, video)
1000	# from URLs. Each retry quadruples the timeout. Default is 3.
1001	"VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int(
1002	os.getenv("VLLM_MEDIA_FETCH_MAX_RETRIES", "3")
1003	),
1004	# Maximum size in MB for a single remote media download. The limit is
1005	# enforced while streaming the response body so oversized or infinite
1006	# responses cannot grow the API server heap without bound. Default is 256.
1007	"VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB": lambda: int(
1008	os.getenv("VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", "256")
1009	),
1010	# Whether to allow HTTP redirects when fetching from media URLs.
1011	# Default to True
1012	"VLLM_MEDIA_URL_ALLOW_REDIRECTS": lambda: bool(
1013	int(os.getenv("VLLM_MEDIA_URL_ALLOW_REDIRECTS", "1"))
1014	),
1015	# Max number of workers for the thread pool handling
1016	# media bytes loading. Set to 1 to disable parallel processing.
1017	# Default is 8
1018	"VLLM_MEDIA_LOADING_THREAD_COUNT": lambda: int(
1019	os.getenv("VLLM_MEDIA_LOADING_THREAD_COUNT", "8")
1020	),
1021	# Maximum filesize in MB for a single audio file. Enforced on all
1022	# audio inputs (multimodal chat, speech-to-text uploads, data: URLs,
1023	# and local file:// paths). Files larger than this will be rejected
1024	# before decoding. Default is 25 MB.
1025	"VLLM_MAX_AUDIO_CLIP_FILESIZE_MB": lambda: int(
1026	os.getenv("VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", "25")
1027	),
1028	# Maximum decoded audio duration in seconds. Compressed audio files
1029	# (e.g. OPUS at very low bitrate) can expand into gigabytes of float32
1030	# PCM. This limit is enforced during decoding so the memory is never
1031	# allocated. Default is 600s (10 minutes).
1032	"VLLM_MAX_AUDIO_DECODE_DURATION_S": lambda: int(
1033	os.getenv("VLLM_MAX_AUDIO_DECODE_DURATION_S", "600")
1034	),
1035	# Maximum float32 PCM bytes that audio decoding may allocate.
1036	# Guards against sample-rate forgery where an attacker inflates
1037	# the header sample rate to bypass the duration guard while the
1038	# actual frame count still causes a multi-GiB allocation.
1039	# Default is 256 MiB (sufficient for 600s mono 48 kHz float32).
1040	"VLLM_MAX_AUDIO_DECODE_BYTES": lambda: int(
1041	os.getenv("VLLM_MAX_AUDIO_DECODE_BYTES", "268435456")
1042	),
1043	# Maximum bytes a client-supplied embedding payload (`prompt_embeds`,
1044	# `image_embeds`, `audio_embeds`, `video_embeds`) may allocate once it is
1045	# densified. A sparse tensor carries its own declared shape, so a payload
1046	# of a few hundred bytes can expand into hundreds of GiB. The limit is
1047	# checked before `to_dense()` so the memory is never allocated. Set to 0
1048	# to disable. Default is 2 GiB, which covers a 128K-token float32
1049	# embedding at hidden_size 4096.
1050	"VLLM_MAX_EMBED_DECODE_BYTES": lambda: int(
1051	os.getenv("VLLM_MAX_EMBED_DECODE_BYTES", "2147483648")
1052	),
1053	# Maximum number of worker threads used for STT preprocessing. The default
1054	# intentionally caps at 2 because that performed best in profiling.
1055	# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
1056	"VLLM_MAX_AUDIO_PREPROCESS_WORKERS": lambda: int(
1057	os.getenv(
1058	"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
1059	str(max(1, min(os.cpu_count() or 1, 2))),
1060	)
1061	),
1062	# Maximum decoded image size in pixels. Small compressed images can
1063	# expand into gigabytes of raster memory. This limit is enforced before
1064	# decoding so the memory is never allocated. Default matches PIL's
1065	# built-in 2x decompression-bomb threshold (~179M pixels, ~680 MB RGB).
1066	"VLLM_MAX_IMAGE_PIXELS": lambda: int(
1067	os.getenv("VLLM_MAX_IMAGE_PIXELS", "178956970")
1068	),
1069	# Backend for Video IO — selects the frame-sampling algorithm.
1070	# - "opencv": uniform sampling.
1071	# - "opencv_dynamic": duration-aware dynamic sampling.
1072	#
1073	# Custom backend implementations can be registered
1074	# via `@VIDEO_LOADER_REGISTRY.register("my_custom_video_loader")` and
1075	# imported at runtime.
1076	# If a non-existing backend is used, an AssertionError will be thrown.
1077	"VLLM_VIDEO_LOADER_BACKEND": lambda: os.getenv(
1078	"VLLM_VIDEO_LOADER_BACKEND", "opencv"
1079	),
1080	# Media connector implementation.
1081	# - "http": Default connector that supports fetching media via HTTP.
1082	#
1083	# Custom implementations can be registered
1084	# via `@MEDIA_CONNECTOR_REGISTRY.register("my_custom_media_connector")` and
1085	# imported at runtime.
1086	# If a non-existing backend is used, an AssertionError will be thrown.
1087	"VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"),
1088	# Path to the XLA persistent cache directory.
1089	# Only used for XLA devices such as TPUs.
1090	"VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser(
1091	os.getenv(
1092	"VLLM_XLA_CACHE_PATH",
1093	os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
1094	)
1095	),
1096	# If set, assert on XLA recompilation after each execution step.
1097	"VLLM_XLA_CHECK_RECOMPILATION": lambda: bool(
1098	int(os.getenv("VLLM_XLA_CHECK_RECOMPILATION", "0"))
1099	),
1100	# Enable SPMD mode for TPU backend.
1101	"VLLM_XLA_USE_SPMD": lambda: bool(int(os.getenv("VLLM_XLA_USE_SPMD", "0"))),
1102	# Maximum size (in MB) for logits tensor in sparse MLA indexer prefill chunks.
1103	# Bounds the [M, N] float32 logits tensor to prevent CUDA OOM.
1104	# Default: 512 MB
1105	"VLLM_SPARSE_INDEXER_MAX_LOGITS_MB": lambda: int(
1106	os.getenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512")
1107	),
1108	# KV context length each adaptive-verification profiling request pretends to
1109	# carry, so the profiled step reads a realistic amount of cache.
1110	# Raise it for long-context deployments, where step cost is dominated by
1111	# attention over a much larger KV cache than the default assumes.
1112	# Clamped to max_model_len - query_len. Default: 8192 tokens
1113	"VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN": lambda: int(
1114	os.getenv("VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN", "8192")
1115	),
1116	# If set, the OpenAI API server will stay alive even after the underlying
1117	# AsyncLLMEngine errors and stops serving requests
1118	"VLLM_KEEP_ALIVE_ON_ENGINE_DEATH": lambda: bool(
1119	int(os.getenv("VLLM_KEEP_ALIVE_ON_ENGINE_DEATH", "0"))
1120	),
1121	# If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows
1122	# the user to specify a max sequence length greater than
1123	# the max length derived from the model's config.json.
1124	# To enable this, set VLLM_ALLOW_LONG_MAX_MODEL_LEN=1.
1125	"VLLM_ALLOW_LONG_MAX_MODEL_LEN": lambda: (
1126	os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower()
1127	in ("1", "true")
1128	),
1129	"VLLM_TEST_FORCE_LOAD_FORMAT": lambda: os.getenv(
1130	"VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"
1131	),
1132	# Queue size for fastsafetensors ParallelLoader pipelined weight
1133	# loading. Peak load-time VRAM is roughly
1134	# model_weights + (1 + queue_size) * shard_size.
1135	# Default 0 preserves the non-pipelined memory footprint so this
1136	# change does not shrink the loadable-model envelope. Set to 1
1137	# (or higher) to overlap producing the next shard's device buffer
1138	# with the consumer copying the current shard into model params,
1139	# at the cost of `queue_size` extra shard-sized buffers resident
1140	# at peak during loading.
1141	"VLLM_FASTSAFETENSORS_QUEUE_SIZE": lambda: int(
1142	os.getenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", "0")
1143	),
1144	# Timeout in seconds for keeping HTTP connections alive in API server
1145	"VLLM_HTTP_TIMEOUT_KEEP_ALIVE": lambda: int(
1146	os.environ.get("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5")
1147	),
1148	# Maximum allowed value for the `n` sampling parameter (number of output
1149	# sequences per request). Limits resource consumption to prevent
1150	# denial-of-service via excessively large fan-out. Default: 16384.
1151	"VLLM_MAX_N_SEQUENCES": lambda: int(
1152	os.environ.get("VLLM_MAX_N_SEQUENCES", "16384")
1153	),
1154	# Maximum number of prompts allowed in a single /v1/completions request
1155	# when the prompt field is a list. Prevents unbounded fan-out of engine
1156	# requests from a single API call. Default: 1024.
1157	"VLLM_MAX_COMPLETION_PROMPTS": lambda: int(
1158	os.environ.get("VLLM_MAX_COMPLETION_PROMPTS", "1024")
1159	),
1160	# Maximum number of stop strings allowed in a single request.
1161	"VLLM_MAX_STOP_STRINGS": lambda: int(os.environ.get("VLLM_MAX_STOP_STRINGS", "4")),
1162	# Maximum number of bad-word token sequences generated per request.
1163	"VLLM_MAX_NUM_BAD_WORDS": lambda: int(
1164	os.environ.get("VLLM_MAX_NUM_BAD_WORDS", "128")
1165	),
1166	# Maximum total number of bad-word tokens (summed across all bad words)
1167	# allowed per request. Bounds the per-request GPU buffer width.
1168	"VLLM_MAX_BAD_WORDS_TOTAL_TOKENS": lambda: int(
1169	os.environ.get("VLLM_MAX_BAD_WORDS_TOTAL_TOKENS", "1024")
1170	),
1171	# a list of plugin names to load, separated by commas.
1172	# if this is not set, it means all plugins will be loaded
1173	# if this is set to an empty string, no plugins will be loaded
1174	"VLLM_PLUGINS": lambda: (
1175	None
1176	if "VLLM_PLUGINS" not in os.environ
1177	else os.environ["VLLM_PLUGINS"].split(",")
1178	),
1179	# a local directory to look in for unrecognized LoRA adapters.
1180	# only works if plugins are enabled and
1181	# VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled.
1182	"VLLM_LORA_RESOLVER_CACHE_DIR": lambda: os.getenv(
1183	"VLLM_LORA_RESOLVER_CACHE_DIR", None
1184	),
1185	# A remote HF repo(s) containing one or more LoRA adapters, which
1186	# may be downloaded and leveraged as needed. Only works if plugins
1187	# are enabled and VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled.
1188	# Values should be comma separated.
1189	"VLLM_LORA_RESOLVER_HF_REPO_LIST": lambda: os.getenv(
1190	"VLLM_LORA_RESOLVER_HF_REPO_LIST", None
1191	),
1192	# If set, vLLM will use Triton implementations of AWQ.
1193	"VLLM_USE_TRITON_AWQ": lambda: bool(int(os.getenv("VLLM_USE_TRITON_AWQ", "0"))),
1194	# If set, monkey-patch triton.runtime.autotuner.Autotuner.run to skip
1195	# benchmarking and select the first valid config (walking past invalid
1196	# ones). Used to eliminate autotuning variability when measuring kernel
1197	# performance and applied before running any kernel.
1198	"VLLM_TRITON_FORCE_FIRST_CONFIG": lambda: (
1199	os.environ.get("VLLM_TRITON_FORCE_FIRST_CONFIG", "0").strip().lower()
1200	in ("1", "true")
1201	),
1202	# Maximum compiler threads per worker for registered CUDA Triton JIT warmup.
1203	# Set to 1 for serial warmup. Does not affect runtime JIT or autotuning.
1204	"VLLM_TRITON_JIT_WARMUP_NUM_THREADS": lambda: int(
1205	os.getenv("VLLM_TRITON_JIT_WARMUP_NUM_THREADS", "4")
1206	),
1207	# If set, allow loading or unloading lora adapters in runtime,
1208	"VLLM_ALLOW_RUNTIME_LORA_UPDATING": lambda: (
1209	os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower()
1210	in ("1", "true")
1211	),
1212	# We assume drivers can report p2p status correctly.
1213	# If the program hangs when using custom allreduce,
1214	# potantially caused by a bug in the driver (535 series),
1215	# if might be helpful to set VLLM_SKIP_P2P_CHECK=0
1216	# so that vLLM can verify if p2p is actually working.
1217	# See https://github.com/vllm-project/vllm/blob/a9b15c606fea67a072416ea0ea115261a2756058/vllm/distributed/device_communicators/custom_all_reduce_utils.py#L101-L108 for details. # noqa
1218	"VLLM_SKIP_P2P_CHECK": lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "1") == "1",
1219	# List of quantization kernels that should be disabled, used for testing
1220	# and performance comparisons. Currently only affects MPLinearKernel
1221	# selection
1222	# (kernels: MacheteLinearKernel, MarlinLinearKernel, ExllamaLinearKernel)
1223	"VLLM_DISABLED_KERNELS": lambda: (
1224	[]
1225	if "VLLM_DISABLED_KERNELS" not in os.environ
1226	else os.environ["VLLM_DISABLED_KERNELS"].split(",")
1227	),
1228	# Selects hw-agnostic layers for HF transformer backend
1229	"VLLM_USE_HW_AGNOSTIC": lambda: (
1230	os.getenv("VLLM_USE_HW_AGNOSTIC", "False").lower() in ("true", "1")
1231	),
1232	"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
1233	int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
1234	),
1235	# Select the GDN MTP decode implementation. "cuda" uses the fused decode
1236	# kernel where supported and falls back to "triton" otherwise; setting it
1237	# explicitly to "cuda" raises when unsupported.
1238	"VLLM_GDN_DECODE_KERNEL": env_with_choices(
1239	"VLLM_GDN_DECODE_KERNEL",
1240	"cuda",
1241	["cuda", "triton"],
1242	case_sensitive=False,
1243	),
1244	# Disable pynccl (using torch.distributed instead)
1245	"VLLM_DISABLE_PYNCCL": lambda: (
1246	os.getenv("VLLM_DISABLE_PYNCCL", "False").lower() in ("true", "1")
1247	),
1248	# Optional: enable external Oink custom ops (e.g., Blackwell RMSNorm).
1249	# Disabled by default.
1250	"VLLM_USE_OINK_OPS": lambda: (
1251	os.getenv("VLLM_USE_OINK_OPS", "False").lower() in ("true", "1")
1252	),
1253	# Disable aiter ops unless specifically enabled.
1254	# Acts as a parent switch to enable the rest of the other operations.
1255	# Set to 1 to dequantize MXFP4 weights to BF16 once at load time and run as
1256	# a BF16 checkpoint (no per-step weight dequant). This improves emulation
1257	# latency at the cost of additional device memory. Default off.
1258	"VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD": lambda: (
1259	os.getenv("VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", "False").lower()
1260	in ("true", "1")
1261	),
1262	# On hardware without a native MXFP8 kernel (e.g. ROCm gfx942 / MI300), the
1263	# MXFP8 emulation path dequantizes weights MXFP8->BF16 once at load time and
1264	# runs as a BF16 checkpoint (no per-step dequant). Set to 0 to fall back to
1265	# per-step dequant: keeps the 1-byte MXFP8 weights (~half the weight memory)
1266	# at the cost of dequantizing every forward step (much slower). Default on.
1267	"VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD": lambda: (
1268	os.getenv("VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD", "True").lower()
1269	in ("true", "1")
1270	),
1271	"VLLM_ROCM_USE_AITER": lambda: (
1272	os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1")
1273	),
1274	# Use AITER's CustomAllreduce as the custom-allreduce backend inside vLLM's
1275	# CudaCommunicator on ROCm. Also enables AITER AG/RS for DP communication.
1276	"VLLM_ROCM_USE_AITER_CUSTOM_AR": lambda: (
1277	os.getenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "True").lower() in ("true", "1")
1278	),
1279	# use aiter linear op if aiter ops are enabled
1280	# The following list of related ops
1281	# - scaled_mm (per-tensor / rowwise)
1282	# - use aiter tuned gemms for unquantized gemms
1283	"VLLM_ROCM_USE_AITER_LINEAR": lambda: (
1284	os.getenv("VLLM_ROCM_USE_AITER_LINEAR", "True").lower() in ("true", "1")
1285	),
1286	"VLLM_ROCM_USE_AITER_LINEAR_HIPBMM": lambda: (
1287	os.getenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", "False").lower() in ("true", "1")
1288	),
1289	# Whether to use aiter moe ops.
1290	# By default is enabled.
1291	"VLLM_ROCM_USE_AITER_MOE": lambda: (
1292	os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
1293	),
1294	# Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2):
1295	# auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4.
1296	"VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices(
1297	"VLLM_ROCM_USE_AITER_MOE_SITUV2",
1298	"auto",
1299	["auto", "a4w4", "a8w4", "a16w4", "0", "1"],
1300	case_sensitive=False,
1301	),
1302	# Opt-in switch for a4w4 (FP4 activation) MoE on DeepSeek V4.1, AITER
1303	# MXFP4 backend. Default is a8w4 (FP8); set to "1" to enable a4w4
1304	# ("true" is not accepted -- only "0"/"1"). Raises if set for other
1305	# models.
1306	"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4": lambda: maybe_convert_bool(
1307	os.getenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4")
1308	),
1309	# MoE sorting dispatch policy for AITER fused MoE kernels.
1310	# 0 = auto (default): single-pass for small batches, multi-pass
1311	# for large batches
1312	# 1 = always single-pass: one kernel launch, no workspace,
1313	# may be preferred for low-concurrency decode workloads
1314	# 2 = always multi-pass: can be faster for MoE-heavy models
1315	# (e.g., +2-5% on Qwen3-Next, +1.5% on DeepSeek-V3 at TP4,
1316	# see PR #39177 for benchmarks)
1317	"VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": lambda: int(
1318	os.getenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", "0")
1319	),
1320	# use aiter rms norm op if aiter ops are enabled.
1321	"VLLM_ROCM_USE_AITER_RMSNORM": lambda: (
1322	os.getenv("VLLM_ROCM_USE_AITER_RMSNORM", "True").lower() in ("true", "1")
1323	),
1324	# Whether to use aiter mla ops.
1325	# By default is enabled.
1326	"VLLM_ROCM_USE_AITER_MLA": lambda: (
1327	os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1")
1328	),
1329	# Kernel for causal multi-token (spec-decode) verify steps under decode
1330	# context parallelism on gfx950: "asm" uses AITER's round-robin ASM
1331	# decode, "segmented" the Triton segmented MLA path.
1332	"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
1333	"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
1334	"segmented",
1335	["asm", "segmented"],
1336	case_sensitive=False,
1337	),
1338	# Small-head (<16) AITER MLA decode kernel selection. Small head counts
1339	# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
1340	# the Gluon small-head kernel or through the padded persistent-scheduling
1341	# (PS) ASM kernel. "auto" (default) keeps Gluon for head counts that divide
1342	# 16 where a Gluon build exists (gfx950/CDNA4) and otherwise uses the padded
1343	# PS ASM decode; "gluon" forces the Gluon path wherever a build exists;
1344	# "asm" forces the padded PS ASM decode. On gfx942/CDNA3 there is no Gluon
1345	# build, so the ASM path is always used regardless of this setting.
1346	"VLLM_ROCM_AITER_MLA_ASM_PADDING": env_with_choices(
1347	"VLLM_ROCM_AITER_MLA_ASM_PADDING",
1348	"auto",
1349	["auto", "gluon", "asm"],
1350	case_sensitive=False,
1351	),
1352	# Whether to use aiter mha ops.
1353	# By default is enabled.
1354	"VLLM_ROCM_USE_AITER_MHA": lambda: (
1355	os.getenv("VLLM_ROCM_USE_AITER_MHA", "True").lower() in ("true", "1")
1356	),
1357	# Whether to use aiter fp4 gemm asm.
1358	# By default is disabled.
1359	"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM": lambda: (
1360	os.getenv("VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", "False").lower() in ("true", "1")
1361	),
1362	# Whether sparse MLA prefill and decode run on aiter's Triton kernel, which
1363	# reads the KV cache as stored (bf16 or fp8, paged or flat). Used by the
1364	# ROCM_AITER_MLA_SPARSE backend (DeepSeek V3.2, GLM-5.x) and DeepSeek
1365	# V4 / V4.1. gfx950 only. By default is disabled.
1366	"VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA": lambda: (
1367	os.getenv("VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA", "False").lower()
1368	in ("true", "1")
1369	),
1370	# Whether to use aiter rope.
1371	# By default is disabled.
1372	"VLLM_ROCM_USE_AITER_TRITON_ROPE": lambda: (
1373	os.getenv("VLLM_ROCM_USE_AITER_TRITON_ROPE", "False").lower() in ("true", "1")
1374	),
1375	# Whether to use aiter triton fp8 bmm kernel
1376	# By default is enabled.
1377	"VLLM_ROCM_USE_AITER_FP8BMM": lambda: (
1378	os.getenv("VLLM_ROCM_USE_AITER_FP8BMM", "True").lower() in ("true", "1")
1379	),
1380	# Whether to use aiter triton fp4 bmm kernel
1381	# By default is enabled.
1382	"VLLM_ROCM_USE_AITER_FP4BMM": lambda: (
1383	os.getenv("VLLM_ROCM_USE_AITER_FP4BMM", "True").lower() in ("true", "1")
1384	),
1385	# Use AITER triton unified attention for V1 attention
1386	"VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION": lambda: (
1387	os.getenv("VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION", "False").lower()
1388	in ("true", "1")
1389	),
1390	# Whether to use aiter fusion shared experts ops.
1391	# By default is disabled.
1392	"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": lambda: (
1393	os.getenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "False").lower()
1394	in ("true", "1")
1395	),
1396	# Whether to use aiter triton kernels for gemm ops.
1397	# By default is enabled.
1398	"VLLM_ROCM_USE_AITER_TRITON_GEMM": lambda: (
1399	os.getenv("VLLM_ROCM_USE_AITER_TRITON_GEMM", "True").lower() in ("true", "1")
1400	),
1401	# use rocm skinny gemms
1402	"VLLM_ROCM_USE_SKINNY_GEMM": lambda: (
1403	os.getenv("VLLM_ROCM_USE_SKINNY_GEMM", "True").lower() in ("true", "1")
1404	),
1405	# Pad the fp8 weights to 256 bytes for ROCm
1406	"VLLM_ROCM_FP8_PADDING": lambda: bool(int(os.getenv("VLLM_ROCM_FP8_PADDING", "1"))),
1407	# Pad the weights for the moe kernel
1408	"VLLM_ROCM_MOE_PADDING": lambda: bool(int(os.getenv("VLLM_ROCM_MOE_PADDING", "1"))),
1409	# Whether to use the shuffled kv cache layout
1410	"VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": lambda: (
1411	os.getenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", "False").lower() in ("true", "1")
1412	),
1413	# Custom quick allreduce kernel for MI3* cards
1414	# Choice of quantization level: FP, INT8, INT6, INT4, INT3 or NONE
1415	# Recommended for large models to get allreduce
1416	"VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": env_with_choices(
1417	"VLLM_ROCM_QUICK_REDUCE_QUANTIZATION",
1418	"NONE",
1419	["FP", "INT8", "INT6", "INT4", "INT3", "NONE"],
1420	),
1421	# Custom quick allreduce kernel for MI3* cards
1422	# Due to the lack of the bfloat16 asm instruction, bfloat16
1423	# kernels are slower than fp16,
1424	# If environment variable is set to 1, the input is converted to fp16
1425	"VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16": lambda: (
1426	os.getenv("VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", "True").lower()
1427	in ("true", "1")
1428	),
1429	# Custom quick allreduce kernel for MI3* cards.
1430	# Controls the maximum allowed number of data bytes(MB) for custom quick
1431	# allreduce communication.
1432	# Default: 2048 MB.
1433	# Data exceeding this size will use either custom allreduce or RCCL
1434	# communication.
1435	"VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB": lambda: maybe_convert_int(
1436	os.environ.get("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", None)
1437	),
1438	# Custom quick allreduce kernel for MI3* cards.
1439	# Controls the minimum allowed number of data bytes(MB) required to use
1440	# custom quick allreduce communication.
1441	# If unset, use the built-in threshold table.
1442	"VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB": lambda: maybe_convert_int(
1443	os.environ.get("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", None)
1444	),
1445	# Controls the minimum tensor size (KB, where 1 KB = 1024 bytes) required
1446	# to use the configured QuickReduce codec. Smaller tensors use FP
1447	# QuickReduce. This does not affect QuickReduce eligibility.
1448	"VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB": lambda: maybe_convert_int(
1449	os.environ.get("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", None)
1450	),
1451	# If set, enable multiprocessing in LLM for the V1 code path.
1452	"VLLM_ENABLE_V1_MULTIPROCESSING": lambda: bool(
1453	int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))
1454	),
1455	"VLLM_LOG_BATCHSIZE_INTERVAL": lambda: float(
1456	os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")
1457	),
1458	"VLLM_DISABLE_COMPILE_CACHE": disable_compile_cache,
1459	# If set to "0", disable LayerName opaque type for layer_name
1460	# parameters in custom ops. Defaults to enabled on torch >= 2.11.
1461	"VLLM_USE_LAYERNAME": lambda: bool(int(os.getenv("VLLM_USE_LAYERNAME", "1"))),
1462	# If set, use the Rust frontend binary instead of the Python API server
1463	# process(es).
1464	"VLLM_USE_RUST_FRONTEND": lambda: bool(
1465	int(os.getenv("VLLM_USE_RUST_FRONTEND", "0"))
1466	),
1467	# If set, use the packaged Rust client for `vllm bench serve`.
1468	"VLLM_USE_RUST_BENCH": lambda: bool(int(os.getenv("VLLM_USE_RUST_BENCH", "0"))),
1469	# Path to the vllm-rs binary. Defaults to "auto" which discovers the
1470	# binary installed with the vllm package. Used when VLLM_USE_RUST_FRONTEND=1
1471	# or VLLM_USE_RUST_BENCH=1.
1472	"VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_cli_path(),
1473	# If set, vllm will run in development mode, which will enable
1474	# some additional endpoints for developing and debugging,
1475	# e.g. `/reset_prefix_cache`
1476	"VLLM_SERVER_DEV_MODE": lambda: bool(int(os.getenv("VLLM_SERVER_DEV_MODE", "0"))),
1477	# Controls the maximum number of requests to handle in a
1478	# single asyncio task when processing per-token outputs in the
1479	# V1 AsyncLLM interface. It is applicable when handling a high
1480	# concurrency of streaming requests.
1481	# Setting this too high can result in a higher variance of
1482	# inter-message latencies. Setting it too low can negatively impact
1483	# TTFT and overall throughput.
1484	"VLLM_V1_OUTPUT_PROC_CHUNK_SIZE": lambda: int(
1485	os.getenv("VLLM_V1_OUTPUT_PROC_CHUNK_SIZE", "128")
1486	),
1487	# If set, vLLM will disable the MLA attention optimizations.
1488	"VLLM_MLA_DISABLE": lambda: bool(int(os.getenv("VLLM_MLA_DISABLE", "0"))),
1489	# If set, vLLM will pick up the provided Flash Attention MLA
1490	# Number of GPUs per worker in Ray, if it is set to be a fraction,
1491	# it allows ray to schedule multiple actors on a single GPU,
1492	# so that users can colocate other actors on the same GPUs as vLLM.
1493	"VLLM_RAY_PER_WORKER_GPUS": lambda: float(
1494	os.getenv("VLLM_RAY_PER_WORKER_GPUS", "1.0")
1495	),
1496	# Bundle indices for Ray, if it is set, it can control precisely
1497	# which indices are used for the Ray bundle, for every worker.
1498	# Format: comma-separated list of integers, e.g. "0,1,2,3"
1499	"VLLM_RAY_BUNDLE_INDICES": lambda: os.getenv("VLLM_RAY_BUNDLE_INDICES", ""),
1500	# In some system, find_loaded_library() may not work. So we allow users to
1501	# specify the path through environment variable VLLM_CUDART_SO_PATH.
1502	"VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None),
1503	# Rank of the process in the data parallel setting
1504	"VLLM_DP_RANK": lambda: int(os.getenv("VLLM_DP_RANK", "0")),
1505	# Rank of the process in the data parallel setting.
1506	# Defaults to VLLM_DP_RANK when not set.
1507	"VLLM_DP_RANK_LOCAL": lambda: int(
1508	os.getenv("VLLM_DP_RANK_LOCAL", sys.modules[__name__].VLLM_DP_RANK)
1509	),
1510	# World size of the data parallel setting
1511	"VLLM_DP_SIZE": lambda: int(os.getenv("VLLM_DP_SIZE", "1")),
1512	# IP address of the master node in the data parallel setting
1513	"VLLM_DP_MASTER_IP": lambda: os.getenv("VLLM_DP_MASTER_IP", "127.0.0.1"),
1514	# Port of the master node in the data parallel setting
1515	"VLLM_DP_MASTER_PORT": lambda: int(os.getenv("VLLM_DP_MASTER_PORT", "0")),
1516	# Randomize inputs during dummy runs when using Data Parallel
1517	"VLLM_RANDOMIZE_DP_DUMMY_INPUTS": lambda: (
1518	os.environ.get("VLLM_RANDOMIZE_DP_DUMMY_INPUTS", "0") == "1"
1519	),
1520	# Strategy to pack the data parallel ranks for Ray.
1521	# Available options:
1522	# - "fill":
1523	# for DP master node, allocate exactly data-parallel-size-local DP ranks,
1524	# for non-master nodes, allocate as many DP ranks as can fit;
1525	# - "strict":
1526	# allocate exactly data-parallel-size-local DP ranks to each picked node;
1527	# - "span":
1528	# Should be used only when a single DP rank requires multiple nodes.
1529	# allocate one DP rank over as many nodes as required for set world_size;
1530	# This environment variable is ignored if data-parallel-backend is not Ray.
1531	"VLLM_RAY_DP_PACK_STRATEGY": lambda: os.getenv(
1532	"VLLM_RAY_DP_PACK_STRATEGY", "strict"
1533	),
1534	# Optional comma-separated list of node IPs that Ray data-parallel
1535	# placement groups may use. When set, create_dp_placement_groups only
1536	# considers these nodes (the DP master node is always included).
1537	# This environment variable is ignored if data-parallel-backend is not Ray.
1538	"VLLM_RAY_DP_PLACEMENT_NODE_IPS": lambda: os.getenv(
1539	"VLLM_RAY_DP_PLACEMENT_NODE_IPS", ""
1540	),
1541	# Comma-separated additional prefixes of env vars to copy from the
1542	# driver to Ray workers. These are merged with the built-in defaults
1543	# defined in ``vllm.ray.ray_env`` (VLLM_, etc.). Example: "MYLIB_,OTHER_"
1544	"VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY": lambda: os.getenv(
1545	"VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY", ""
1546	),
1547	# Comma-separated additional individual env var names to copy from
1548	# the driver to Ray workers. Merged with the built-in defaults
1549	# defined in ``vllm.ray.ray_env`` (PYTHONHASHSEED).
1550	# Example: "MY_SECRET,MY_FLAG"
1551	"VLLM_RAY_EXTRA_ENV_VARS_TO_COPY": lambda: os.getenv(
1552	"VLLM_RAY_EXTRA_ENV_VARS_TO_COPY", ""
1553	),
1554	# Use model_redirect to redirect the model name to a local folder.
1555	# `model_redirect` can be a json file mapping the model between
1556	# repo_id and local folder:
1557	# {"meta-llama/Llama-3.2-1B": "/tmp/Llama-3.2-1B"}
1558	# or a space separated values table file:
1559	# meta-llama/Llama-3.2-1B /tmp/Llama-3.2-1B
1560	"VLLM_MODEL_REDIRECT_PATH": lambda: os.environ.get(
1561	"VLLM_MODEL_REDIRECT_PATH", None
1562	),
1563	# Whether to use atomicAdd reduce in gptq/awq marlin kernel.
1564	"VLLM_MARLIN_USE_ATOMIC_ADD": lambda: (
1565	os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1"
1566	),
1567	# The activation dtype for marlin kernel
1568	"VLLM_MARLIN_INPUT_DTYPE": env_with_choices(
1569	"VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"]
1570	),
1571	# The online quantization dtype for humming kernel
1572	"VLLM_HUMMING_ONLINE_QUANT_CONFIG": lambda: maybe_convert_json_str_or_file(
1573	os.environ.get("VLLM_HUMMING_ONLINE_QUANT_CONFIG", None)
1574	),
1575	# The activation dtype config for humming kernel. Explicit schemas disable
1576	# fallback unless the config includes "allow_fallback": true.
1577	"VLLM_HUMMING_INPUT_QUANT_CONFIG": lambda: maybe_convert_json_str_or_file(
1578	os.environ.get("VLLM_HUMMING_INPUT_QUANT_CONFIG", None)
1579	),
1580	# Whether to use fp16 accumulator mma
1581	"VLLM_HUMMING_USE_F16_ACCUM": lambda: maybe_convert_bool(
1582	os.environ.get("VLLM_HUMMING_USE_F16_ACCUM", "0")
1583	),
1584	# Whether to use indexed gemm for humming moe
1585	# if 1, force use indexed gemm
1586	# if 0, force use grouped gemm
1587	# if None, choose better gemm type automatically
1588	"VLLM_HUMMING_MOE_GEMM_TYPE": lambda: os.environ.get(
1589	"VLLM_HUMMING_MOE_GEMM_TYPE", None
1590	),
1591	# Whether to use DeepEPLL kernels for NVFP4 quantization and dispatch method
1592	# only supported on Blackwell GPUs and with
1593	# https://github.com/deepseek-ai/DeepEP/pull/341
1594	"VLLM_DEEPEPLL_NVFP4_DISPATCH": lambda: bool(
1595	int(os.getenv("VLLM_DEEPEPLL_NVFP4_DISPATCH", "0"))
1596	),
1597	# Whether to turn on the outlines cache for V1
1598	# This cache is unbounded and on disk, so it's not safe to use in
1599	# an environment with potentially malicious users.
1600	"VLLM_V1_USE_OUTLINES_CACHE": lambda: (
1601	os.environ.get("VLLM_V1_USE_OUTLINES_CACHE", "0") == "1"
1602	),
1603	# Whether using Pathways
1604	"VLLM_TPU_USING_PATHWAYS": lambda: bool(
1605	"proxy" in os.getenv("JAX_PLATFORMS", "").lower()
1606	),
1607	# Allow use of DeepGemm kernels for fused moe ops.
1608	"VLLM_USE_DEEP_GEMM": lambda: bool(int(os.getenv("VLLM_USE_DEEP_GEMM", "1"))),
1609	# Allow use of DeepGemm specifically for MoE fused ops (overrides only MoE).
1610	"VLLM_MOE_USE_DEEP_GEMM": lambda: bool(
1611	int(os.getenv("VLLM_MOE_USE_DEEP_GEMM", "1"))
1612	),
1613	# Whether to use E8M0 scaling when DeepGEMM is used on Blackwell GPUs.
1614	"VLLM_USE_DEEP_GEMM_E8M0": lambda: bool(
1615	int(os.getenv("VLLM_USE_DEEP_GEMM_E8M0", "1"))
1616	),
1617	# Whether to create TMA-aligned scale tensor when DeepGEMM is used.
1618	"VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES": lambda: bool(
1619	int(os.getenv("VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES", "1"))
1620	),
1621	# Opt-in MLA DCP query replication: skip the decode query all-gather.
1622	"VLLM_DCP_Q_REPLICATE": lambda: bool(int(os.getenv("VLLM_DCP_Q_REPLICATE", "0"))),
1623	# DeepGemm JITs the kernels on-demand. The warmup attempts to make DeepGemm
1624	# JIT all the required kernels before model execution so there is no
1625	# JIT'ing in the hot-path. However, this warmup increases the engine
1626	# startup time by a couple of minutes.
1627	# Available options:
1628	# - "skip" : Skip warmup.
1629	# - "full" : Warmup deepgemm by running all possible gemm shapes the
1630	# engine could encounter.
1631	# - "relax" : Select gemm shapes to run based on some heuristics. The
1632	# heuristic aims to have the same effect as running all possible gemm
1633	# shapes, but provides no guarantees.
1634	"VLLM_DEEP_GEMM_WARMUP": env_with_choices(
1635	"VLLM_DEEP_GEMM_WARMUP",
1636	"relax",
1637	[
1638	"skip",
1639	"full",
1640	"relax",
1641	],
1642	),
1643	# Whether to use fused grouped_topk used for MoE expert selection.
1644	"VLLM_USE_FUSED_MOE_GROUPED_TOPK": lambda: bool(
1645	int(os.getenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "1"))
1646	),
1647	# Skip cudagraph/DP padding tokens in the MoE path by forcing their expert
1648	# ids to -1 so the dispatch and experts drop them. Requires a MoE kernel that
1649	# treats topk_id == -1 as a skip sentinel
1650	"VLLM_MOE_SKIP_PADDING": lambda: bool(int(os.getenv("VLLM_MOE_SKIP_PADDING", "1"))),
1651	# Kimi-K3 only. Under sequence-parallel MoE the dense and shared-expert MLPs
1652	# are replicated on every rank, so each rank streams the whole weight to
1653	# serve its own token shard. Shard them across TP instead: the MLP then
1654	# all-gathers the full token set, computes this rank's partial, and
1655	# reduce-scatters (which both sums across TP and restores the sequence
1656	# sharding). Trades weight bandwidth and resident memory for two collectives
1657	# per layer, so it only wins at low token counts: intended for decode
1658	# instances in a P/D disaggregated deployment, not for prefill or unified
1659	# serving.
1660	"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
1661	int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
1662	),
1663	# Kimi K3 only, and unrelated to the MoE flags above. Tap the pre-norm
1664	# AttnRes mixture, rather than the post-mixture sum, as the auxiliary
1665	# hidden state handed to a DFlash drafter. This changes the numerics the
1666	# speculator sees, so it is off by default while the effect is measured.
1667	"VLLM_KIMI_K3_AUX_ATTN_RES_STREAM": lambda: bool(
1668	int(os.getenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "0"))
1669	),
1670	# Use the SM100 BF16 GEMM-AR kernel for eligible Kimi-K3 row-parallel
1671	# attention projections. All TP ranks must belong to one NVLink domain.
1672	"VLLM_KIMI_K3_GEMM_AR": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_AR", "1"))),
1673	# Fuse eligible sequence-parallel row-parallel projections with their TP
1674	# reduce-scatter using the SM100 BF16/MXFP8 GEMM-RS kernel (Kimi-K3
1675	# attention/shared-expert projections, DeepSeek-V4.1 ``wo_b``). All TP
1676	# ranks must belong to one NVLink domain.
1677	"VLLM_ENABLE_GEMM_RS": lambda: bool(int(os.getenv("VLLM_ENABLE_GEMM_RS", "0"))),
1678	# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
1679	# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
1680	"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
1681	int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1"))
1682	),
1683	# Force b12x FP4 MoE to use BF16 activations.
1684	"VLLM_B12X_MOE_FP4_FORCE_A16": lambda: bool(
1685	int(os.getenv("VLLM_B12X_MOE_FP4_FORCE_A16", "0"))
1686	),
1687	# Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops.
1688	"VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool(
1689	int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0"))
1690	),
1691	# Control the cache sized used by the xgrammar compiler. The default
1692	# of 512 MB should be enough for roughly 1000 JSON schemas.
1693	# It can be changed with this variable if needed for some reason.
1694	"VLLM_XGRAMMAR_CACHE_MB": lambda: int(os.getenv("VLLM_XGRAMMAR_CACHE_MB", "512")),
1695	# Maximum time in seconds allowed for regex compilation in structured
1696	# output backends (xgrammar, outlines). Prevents ReDoS attacks where
1697	# adversarial patterns cause exponential DFA state-space explosion.
1698	# Set to 0 to disable the timeout (not recommended in production).
1699	"VLLM_REGEX_COMPILATION_TIMEOUT_S": lambda: int(
1700	os.getenv("VLLM_REGEX_COMPILATION_TIMEOUT_S", "5")
1701	),
1702	# Control the threshold for msgspec to use 'zero copy' for
1703	# serialization/deserialization of tensors. Tensors below
1704	# this limit will be encoded into the msgpack buffer, and
1705	# tensors above will instead be sent via a separate message.
1706	# While the sending side still actually copies the tensor
1707	# in all cases, on the receiving side, tensors above this
1708	# limit will actually be zero-copy decoded. The unit is bytes.
1709	"VLLM_MSGPACK_ZERO_COPY_THRESHOLD": lambda: int(
1710	os.getenv("VLLM_MSGPACK_ZERO_COPY_THRESHOLD", "256")
1711	),
1712	# If set, allow insecure serialization using pickle.
1713	# This is useful for environments where it is deemed safe to use the
1714	# insecure method and it is needed for some reason.
1715	"VLLM_ALLOW_INSECURE_SERIALIZATION": lambda: bool(
1716	int(os.getenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0"))
1717	),
1718	# Temporary: skip adding random suffix to internal request IDs. May be
1719	# needed for KV connectors that match request IDs across instances.
1720	"VLLM_DISABLE_REQUEST_ID_RANDOMIZATION": lambda: bool(
1721	int(os.getenv("VLLM_DISABLE_REQUEST_ID_RANDOMIZATION", "0"))
1722	),
1723	# IP address used for NIXL handshake between remote agents.
1724	"VLLM_NIXL_SIDE_CHANNEL_HOST": lambda: os.getenv(
1725	"VLLM_NIXL_SIDE_CHANNEL_HOST", "localhost"
1726	),
1727	# Port used for NIXL handshake between remote agents.
1728	"VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
1729	os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")
1730	),
1731	# Address the P2P KV-offload control socket binds to. Defaults to
1732	# ``localhost`` (loopback only); must be set to the node IP for
1733	# cross-host P2P so remote peers can reach the socket.
1734	"VLLM_P2P_SIDE_CHANNEL_HOST": lambda: os.getenv(
1735	"VLLM_P2P_SIDE_CHANNEL_HOST", "localhost"
1736	),
1737	# Port the P2P KV-offload control socket binds to.
1738	"VLLM_P2P_SIDE_CHANNEL_PORT": lambda: int(
1739	os.getenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5710")
1740	),
1741	# IP address used for the EC connector's ZMQ side channel
1742	# (producer ROUTER bind, consumer DEALER dial).
1743	"VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv(
1744	"VLLM_EC_SIDE_CHANNEL_HOST", "localhost"
1745	),
1746	# Port for the EC connector's ZMQ side channel; advertised to peers
1747	# via `ec_transfer_params.peer_port` on the producer's response.
1748	"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
1749	os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
1750	),
1751	# Per-request timeout (seconds) when a prefiller worker registers with
1752	# the Mooncake bootstrap server.
1753	"VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float(
1754	os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0")
1755	),
1756	# Port used for Mooncake handshake between remote agents.
1757	"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
1758	os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
1759	),
1760	# Log per-batch memory/disk tier breakdown on external GETs.
1761	"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
1762	os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
1763	),
1764	# Number of parallel KV-load receive threads per worker rank. Lets the
1765	# per-request control overhead (Python prep + master key lookup) of one
1766	# request overlap with the RDMA transfer of another, keeping the transfer
1767	# engine's queue pairs busy. Helps when that overhead is significant or
1768	# per-request batches are too small to saturate the link on their own.
1769	"VLLM_MOONCAKE_LOAD_RECV_THREADS": lambda: int(
1770	os.getenv("VLLM_MOONCAKE_LOAD_RECV_THREADS", "1")
1771	),
1772	# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
1773	"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
1774	os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")
1775	),
1776	# Pin this rank to a specific owner segment ("host:port").
1777	"MOONCAKE_PREFERRED_SEGMENT": lambda: os.getenv("MOONCAKE_PREFERRED_SEGMENT"),
1778	# Override the hostname the rank registers as a Mooncake requester.
1779	"MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv(
1780	"MOONCAKE_REQUESTER_LOCAL_HOSTNAME"
1781	),
1782	# Override the directory for the FlashInfer autotune config cache.
1783	"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv(
1784	"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None
1785	),
1786	# Comma-separated FlashInfer op names to exclude from autotuning, using
1787	# the heuristic fallback tactic instead. Unset: skip "fp4_gemm" when the
1788	# CuTe-DSL NVFP4 linear kernel is selected. Empty: skip nothing.
1789	"VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS": lambda: (
1790	None
1791	if "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS" not in os.environ
1792	else [
1793	v.strip()
1794	for v in os.environ["VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS"].split(",")
1795	if v.strip()
1796	]
1797	),
1798	# Flashinfer fused allreduce backend.
1799	"VLLM_FLASHINFER_ALLREDUCE_BACKEND": env_with_choices(
1800	"VLLM_FLASHINFER_ALLREDUCE_BACKEND",
1801	"auto",
1802	["auto", "trtllm", "mnnvl"],
1803	),
1804	# Control the workspace buffer size for the FlashInfer backend.
1805	"VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int(
1806	os.getenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(394 * 1024 * 1024))
1807	),
1808	# Control the maximum number of tokens per expert supported by the
1809	# NVFP4 MoE CUTLASS Kernel. This value is used to create a buffer for
1810	# the blockscale tensor of activations NVFP4 Quantization.
1811	# This is used to prevent the kernel from running out of memory.
1812	"VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE": lambda: int(
1813	os.getenv("VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE", "163840")
1814	),
1815	# MoE routing strategy selector.
1816	# See `RoutingSimulator.get_available_strategies()` # for available
1817	# strategies.
1818	# Custom routing strategies can be registered by
1819	# RoutingSimulator.register_strategy()
1820	# Note: custom strategies may not produce correct model outputs
1821	"VLLM_MOE_ROUTING_SIMULATION_STRATEGY": lambda: os.environ.get(
1822	"VLLM_MOE_ROUTING_SIMULATION_STRATEGY", ""
1823	).lower(),
1824	# Regex timeout for use by the vLLM tool parsing plugins.
1825	"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
1826	os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
1827	),
1828	# Enforce function parameter schemas in structural-tag based tool calling.
1829	"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
1830	os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
1831	),
1832	# Control the max chunk bytes (in MB) for the rpc message queue.
1833	# Object larger than this threshold will be broadcast to worker
1834	# processes via zmq.
1835	"VLLM_MQ_MAX_CHUNK_BYTES_MB": lambda: int(
1836	os.getenv("VLLM_MQ_MAX_CHUNK_BYTES_MB", "16")
1837	),
1838	# Timeout in seconds for execute_model RPC calls in multiprocessing
1839	# executor (only applies when TP > 1).
1840	"VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": lambda: int(
1841	os.getenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "300")
1842	),
1843	# Timeout in seconds for engine and worker process shutdown
1844	"VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS": lambda: int(
1845	os.getenv("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", "5")
1846	),
1847	# KV Cache layout used throughout vllm.
1848	# Some common values are:
1849	# - LBNHC
1850	# - LBHNC
1851	# Where N=num_states, H=num_heads and C=state_content. The default value will
1852	# leave the layout choice to the backend. Mind that backends may only
1853	# implement and support a subset of all possible layouts.
1854	# LHBNC hoists the head dim outside the block dim; backends must opt in via
1855	# AttentionBackend.supported_kv_cache_layouts().
1856	"VLLM_KV_CACHE_LAYOUT": env_with_choices(
1857	"VLLM_KV_CACHE_LAYOUT",
1858	None,
1859	["LBNHC", "LBHNC", "LHBNC", "NHD", "HND", "BLHNC", "BLNHC", "BHLNC"],
1860	),
1861	# SSM conv state layout used for Mamba models.
1862	# - SD: (state_len, dim) — dim contiguous (default)
1863	# - DS: (dim, state_len) — TP-sharded dim on dim1,
1864	# consistent with SSM temporal state and LBHNC KV cache layout.
1865	"VLLM_SSM_CONV_STATE_LAYOUT": env_with_choices(
1866	"VLLM_SSM_CONV_STATE_LAYOUT", None, ["SD", "DS"]
1867	),
1868	# Enable checking whether the generated logits contain NaNs,
1869	# indicating corrupted output. Useful for debugging low level bugs
1870	# or bad hardware but it may add compute overhead.
1871	"VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
1872	int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
1873	or int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
1874	),
1875	# Raise an exception when generated logits contain NaNs. Enabling this
1876	# also enables the NaN computation required to detect them.
1877	"VLLM_RAISE_ON_LOGIT_NANS": lambda: bool(
1878	int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
1879	),
1880	# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
1881	"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
1882	os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
1883	),
1884	# If set, it means we pre-downloaded cubin files and flashinfer will
1885	# read the cubin files directly.
1886	"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
1887	int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0"))
1888	),
1889	# Controls garbage collection during CUDA graph capture.
1890	# If set to 0 (default), enables GC freezing to speed up capture time.
1891	# If set to 1, allows GC to run during capture.
1892	"VLLM_ENABLE_CUDAGRAPH_GC": lambda: bool(
1893	int(os.getenv("VLLM_ENABLE_CUDAGRAPH_GC", "0"))
1894	),
1895	# Used to force set up loopback IP
1896	"VLLM_LOOPBACK_IP": lambda: os.getenv("VLLM_LOOPBACK_IP", ""),
1897	# Used to set the process name prefix for vLLM processes.
1898	# This is useful for debugging and monitoring purposes.
1899	# The default value is "VLLM".
1900	"VLLM_PROCESS_NAME_PREFIX": lambda: os.getenv("VLLM_PROCESS_NAME_PREFIX", "VLLM"),
1901	# Allow chunked local attention with hybrid kv cache manager.
1902	# Currently using the Hybrid KV cache manager with chunked local attention
1903	# in the Llama4 models (the only models currently using chunked local attn)
1904	# causes a latency regression. For this reason, we disable it by default.
1905	# This flag is used to allow users to enable it if they want to (to save on
1906	# kv-cache memory usage and enable longer contexts)
1907	# TODO(lucas): Remove this flag once latency regression is resolved.
1908	"VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE": lambda: bool(
1909	int(os.getenv("VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE", "1"))
1910	),
1911	# Enables support for the "store" option in the OpenAI Responses API.
1912	# When set to 1, vLLM's OpenAI server will retain the input and output
1913	# messages for those requests in memory. By default, this is disabled (0),
1914	# and the "store" option is ignored.
1915	# NOTE/WARNING:
1916	# 1. Messages are kept in memory only (not persisted to disk) and will be
1917	# lost when the vLLM server shuts down.
1918	# 2. Enabling this option will cause a memory leak, as stored messages are
1919	# never removed from memory until the server terminates.
1920	"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
1921	int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
1922	),
1923	# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``
1924	# and its render endpoint at ``POST /cohere/v2/chat/render``.
1925	# Default off
1926	"VLLM_ENABLE_COHERE_API": lambda: bool(
1927	int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
1928	),
1929	# If set, use the fp8 mfma in rocm paged attention.
1930	"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
1931	int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
1932	),
1933	# Whether to use pytorch symmetric memory for allreduce
1934	"VLLM_ALLREDUCE_USE_SYMM_MEM": lambda: bool(
1935	int(os.getenv("VLLM_ALLREDUCE_USE_SYMM_MEM", "1"))
1936	),
1937	# Whether to use FlashInfer allreduce
1938	"VLLM_ALLREDUCE_USE_FLASHINFER": lambda: bool(
1939	int(os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER", "1"))
1940	),
1941	# Whether to use FlashInfer's single-node PCIe CUDA-IPC all-reduce.
1942	# This backend has a strict single-stream contract and is opt-in while its
1943	# integration is being qualified.
1944	"VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC": lambda: bool(
1945	int(os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC", "0"))
1946	),
1947	# Experimental: use this to enable MCP tool calling for non harmony models
1948	"VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": lambda: bool(
1949	int(os.getenv("VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", "0"))
1950	),
1951	# User override folder for tuned Triton-kernel configs. Shared by MoE,
1952	# Mamba SSU, and LoRA. Filenames are distinct so one folder can hold all.
1953	# Each component first checks this folder, then the configs shipped with
1954	# vLLM (if any). If no JSON matches, it uses a hard-coded heuristic.
1955	"VLLM_TUNED_CONFIG_FOLDER": lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None),
1956	# Opt-in persistence of the startup plan. When enabled, each worker
1957	# saves the memory-profiling result (the suggested --kv-cache-memory value
1958	# and the free-memory baseline) under VLLM_CACHE_ROOT/startup_plan/,
1959	# keyed by a hardware+config fingerprint, and later boots auto-apply it
1960	# -- skipping memory profiling -- when the fingerprint matches and
1961	# current free memory >= the recorded baseline.
1962	# See vllm/v1/worker/startup_plan.py.
1963	"VLLM_ENABLE_STARTUP_PLAN": lambda: bool(
1964	int(os.getenv("VLLM_ENABLE_STARTUP_PLAN", "0"))
1965	),
1966	# Valid values are container,code_interpreter,web_search_preview
1967	# ex VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS=container,code_interpreter
1968	# If the server_label of your mcp tool is not in this list it will
1969	# be completely ignored.
1970	"VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": env_set_with_choices(
1971	"VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS",
1972	default=[],
1973	choices=["container", "code_interpreter", "web_search_preview"],
1974	),
1975	# Allows harmony instructions to be injected on system messages
1976	"VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": lambda: bool(
1977	int(os.getenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "0"))
1978	),
1979	# Pin the conversation start date injected into the Harmony system
1980	# message. When unset the current date is used, which introduces
1981	# non-determinism (different tokens -> different model behaviour at
1982	# temperature=0). Set to an ISO date string, e.g. "2023-09-12",
1983	# for reproducible inference or testing.
1984	"VLLM_SYSTEM_START_DATE": lambda: os.getenv("VLLM_SYSTEM_START_DATE", None),
1985	# Enable automatic retry when tool call JSON parsing fails
1986	# If enabled, returns an error message to the model to retry
1987	# If disabled (default), raises an exception and fails the request
1988	"VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY": lambda: bool(
1989	int(os.getenv("VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY", "0"))
1990	),
1991	# Add optional custom scopes for profiling, disable to avoid overheads
1992	"VLLM_CUSTOM_SCOPES_FOR_PROFILING": lambda: bool(
1993	int(os.getenv("VLLM_CUSTOM_SCOPES_FOR_PROFILING", "0"))
1994	),
1995	# Add optional nvtx scopes for profiling, disable to avoid overheads
1996	"VLLM_NVTX_SCOPES_FOR_PROFILING": lambda: bool(
1997	int(os.getenv("VLLM_NVTX_SCOPES_FOR_PROFILING", "0"))
1998	),
1999	# Represent block hashes in KV cache events as 64-bit integers instead of
2000	# raw bytes. Defaults to True for backward compatibility.
2001	"VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES": lambda: bool(
2002	int(os.getenv("VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES", "1"))
2003	),
2004	# Name of the shared memory buffer used for object storage.
2005	# Only effective when mm_config.mm_processor_cache_type == "shm".
2006	# Automatically generates a unique UUID-based name per process tree
2007	# if not explicitly set.
2008	"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME": get_env_or_set_default(
2009	"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME",
2010	_generate_shm_name,
2011	),
2012	# The size in MB of the buffers (NVL and RDMA) used by DeepEP
2013	"VLLM_DEEPEP_BUFFER_SIZE_MB": lambda: int(
2014	os.getenv("VLLM_DEEPEP_BUFFER_SIZE_MB", "1024")
2015	),
2016	# Force DeepEP to use intranode kernel for inter-node communication in
2017	# high throughput mode. This is useful archive higher prefill throughput
2018	# on system supports multi-node nvlink (e.g GB200).
2019	"VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE": lambda: bool(
2020	int(os.getenv("VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE", "0"))
2021	),
2022	# Allow DeepEP to use MNNVL (multi-node nvlink) for internode_ll kernel,
2023	# turn this for better latency on GB200 like system
2024	"VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool(
2025	int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0"))
2026	),
2027	# DeepEP v2: use NVLink+RDMA hybrid communication across NVLink domains.
2028	# * hybrid mode = multi-node RDMA
2029	# * non hybrid mode = nvlink
2030	# Set to 1 (the default) to allow DeepEP to autoselect hybrid vs non-hybrid.
2031	# Set to 0 to disable hybrid mode.
2032	"VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE": lambda: bool(
2033	int(os.getenv("VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE", "1"))
2034	),
2035	# DeepEP v2: use fewer SMs at slight throughput cost
2036	"VLLM_DEEPEP_V2_PREFER_OVERLAP": lambda: bool(
2037	int(os.getenv("VLLM_DEEPEP_V2_PREFER_OVERLAP", "0"))
2038	),
2039	# DeepEP v2: trade precision for transfer size in combine
2040	"VLLM_DEEPEP_V2_ALLOW_MULTIPLE_REDUCTION": lambda: bool(
2041	int(os.getenv("VLLM_DEEPEP_V2_ALLOW_MULTIPLE_REDUCTION", "0"))
2042	),
2043	# The number of SMs/CUs to allocate for communication kernels when
2044	# running DBO; the rest will be allocated to compute.
2045	# Default: 20 on CUDA (SMs), 64 on ROCm (CUs).
2046	"VLLM_DBO_COMM_SMS": lambda: int(
2047	os.getenv(
2048	"VLLM_DBO_COMM_SMS",
2049	"64"
2050	if hasattr(__import__("torch").version, "hip")
2051	and __import__("torch").version.hip is not None
2052	else "20",
2053	)
2054	),
2055	# Enable max_autotune & coordinate_descent_tuning in inductor_config
2056	# to compile static shapes passed from compile_sizes in compilation_config
2057	# If set to 1, enable max_autotune; By default, this is enabled (1)
2058	"VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE": lambda: bool(
2059	int(os.getenv("VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE", "1"))
2060	),
2061	# If set to 1, enable coordinate_descent_tuning;
2062	# By default, this is enabled (1)
2063	"VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING": lambda: bool(
2064	int(os.getenv("VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING", "1"))
2065	),
2066	# Flag to enable NCCL symmetric memory allocation and registration
2067	"VLLM_USE_NCCL_SYMM_MEM": lambda: bool(
2068	int(os.getenv("VLLM_USE_NCCL_SYMM_MEM", "0"))
2069	),
2070	# NCCL header path
2071	"VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None),
2072	# GC debug config
2073	# - VLLM_GC_DEBUG=0: disable GC debugger
2074	# - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times
2075	# - VLLM_GC_DEBUG='{"top_objects":5}': enable GC debugger with
2076	# top 5 collected objects
2077	"VLLM_GC_DEBUG": lambda: os.getenv("VLLM_GC_DEBUG", ""),
2078	# Debug workspace allocations.
2079	# logging of workspace resize operations.
2080	"VLLM_DEBUG_WORKSPACE": lambda: bool(int(os.getenv("VLLM_DEBUG_WORKSPACE", "0"))),
2081	# Disables parallel execution of shared_experts via separate cuda stream
2082	"VLLM_DISABLE_SHARED_EXPERTS_STREAM": lambda: bool(
2083	int(os.getenv("VLLM_DISABLE_SHARED_EXPERTS_STREAM", "0"))
2084	),
2085	# Emergency rollback for the DeepSeek-V4 NVIDIA MegaMoE path. By default,
2086	# DeepGEMM computes replicated FP8 shared experts in the same persistent
2087	# SM100 kernel as the routed FP4 experts.
2088	"VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION": lambda: bool(
2089	int(os.getenv("VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION", "0"))
2090	),
2091	# Limits when we run shared_experts in a separate stream.
2092	# We found out that for large batch sizes, the separate stream
2093	# execution is not beneficial (most likely because of the input clone)
2094	# TODO(alexm-redhat): Tune to be more dynamic based on GPU type
2095	"VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD": lambda: int(
2096	int(os.getenv("VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD", 256))
2097	),
2098	# Token-count cutoff for multi-stream overlap of the attention input
2099	# GEMM with auxiliary GEMMs (e.g. fused_wqa_wkv overlapped with indexer
2100	# weights / kv-score projections in DeepSeek-V4). At or below this many
2101	# tokens the FP8 main GEMM has idle SMs to share with the bf16 aux GEMMs
2102	# and overlap is a 5-45% win; above it the FP8 GEMM saturates the device
2103	# and the cross-stream sync becomes pure overhead. Set to 0 to disable
2104	# the multi-stream path entirely. See #PR 41526 for the empirical result
2105	# for the default value of 1024 tokens.
2106	"VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int(
2107	os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024")
2108	),
2109	# Format for saving torch.compile cache artifacts
2110	# - "binary": saves as binary file
2111	# Safe for multiple vllm serve processes accessing the same torch compile cache.
2112	# - "unpacked": saves as directory structure (for inspection/debugging)
2113	# NOT multiprocess safe - race conditions may occur with multiple processes.
2114	# Allows viewing and setting breakpoints in Inductor's code output files.
2115	"VLLM_COMPILE_CACHE_SAVE_FORMAT": env_with_choices(
2116	"VLLM_COMPILE_CACHE_SAVE_FORMAT", "binary", ["binary", "unpacked"]
2117	),
2118	# Flag to control the v2 model runner. If unset, use config defaults.
2119	"VLLM_USE_V2_MODEL_RUNNER": lambda: maybe_convert_bool(
2120	os.getenv("VLLM_USE_V2_MODEL_RUNNER", None)
2121	),
2122	# Log model inspection after loading.
2123	# If enabled, logs a transformers-style hierarchical view of the model
2124	# with quantization methods and attention backends.
2125	"VLLM_LOG_MODEL_INSPECTION": lambda: bool(
2126	int(os.getenv("VLLM_LOG_MODEL_INSPECTION", "0"))
2127	),
2128	# Debug logging for --enable-mfu-metrics
2129	"VLLM_DEBUG_MFU_METRICS": lambda: bool(
2130	int(os.getenv("VLLM_DEBUG_MFU_METRICS", "0"))
2131	),
2132	# Disable using pytorch's pin memory for CPU offloading.
2133	"VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY": lambda: bool(
2134	int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY", "0"))
2135	),
2136	# Disable using UVA (Unified Virtual Addressing) for CPU offloading.
2137	"VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": lambda: bool(
2138	int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0"))
2139	),
2140	# Max descriptors per CPU-KV-offload batch-memcpy call. 0 = platform default
2141	# (ROCm chunks at 8192, since hipMemcpyBatchAsync faults above that on
2142	# rocm 7.14/7.15 when batch copy is optimized; CUDA uncapped). Set >0 to override.
2143	"VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS": lambda: int(
2144	os.getenv("VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS", "0")
2145	),
2146	# On WSL2 with a compatible kernel (>= 4.19.121), pinned memory is
2147	# supported but disabled by default due to a small performance regression.
2148	# Set to 1 when pinned memory or UVA is required (e.g. CPU offloading
2149	# or v2 model runner).
2150	"VLLM_WSL2_ENABLE_PIN_MEMORY": lambda: bool(
2151	int(os.getenv("VLLM_WSL2_ENABLE_PIN_MEMORY", "0"))
2152	),
2153	# Disable logging of vLLM logo at server startup time.
2154	"VLLM_DISABLE_LOG_LOGO": lambda: bool(int(os.getenv("VLLM_DISABLE_LOG_LOGO", "0"))),
2155	# Disable PDL for LoRA, as enabling PDL with LoRA on SM100 causes
2156	# Triton compilation to fail.
2157	"VLLM_LORA_DISABLE_PDL": lambda: bool(int(os.getenv("VLLM_LORA_DISABLE_PDL", "0"))),
2158	# Enable CUDA compatibility mode for datacenter GPUs with older
2159	# driver versions than the CUDA toolkit major version of vLLM.
2160	"VLLM_ENABLE_CUDA_COMPATIBILITY": lambda: (
2161	os.environ.get("VLLM_ENABLE_CUDA_COMPATIBILITY", "0").strip().lower()
2162	in ("1", "true")
2163	),
2164	# Path to the CUDA compatibility libraries when CUDA compatibility is enabled.
2165	"VLLM_CUDA_COMPATIBILITY_PATH": lambda: os.environ.get(
2166	"VLLM_CUDA_COMPATIBILITY_PATH", None
2167	),
2168	# Skip model name validation in OpenAI API requests.
2169	# When set to 1, any model name will be accepted in the 'model' field
2170	# of API requests. This is useful for proxy/gateway scenarios where
2171	# the actual model is served but different names may be used in requests.
2172	"VLLM_SKIP_MODEL_NAME_VALIDATION": lambda: (
2173	os.getenv("VLLM_SKIP_MODEL_NAME_VALIDATION", "0").strip().lower()
2174	in ("1", "true")
2175	),
2176	# Whether it is a scale up launch engine for elastic EP,
2177	# Should only be set by EngineCoreClient.
2178	"VLLM_ELASTIC_EP_SCALE_UP_LAUNCH": lambda: bool(
2179	int(os.getenv("VLLM_ELASTIC_EP_SCALE_UP_LAUNCH", "0"))
2180	),
2181	# Whether to wait for all requests to drain before sending the
2182	# scaling command in elastic EP.
2183	"VLLM_ELASTIC_EP_DRAIN_REQUESTS": lambda: bool(
2184	int(os.getenv("VLLM_ELASTIC_EP_DRAIN_REQUESTS", "0"))
2185	),
2186	# If set to 1, enable CUDA graph memory estimation during memory profiling.
2187	# This profiles CUDA graph memory usage to provide more accurate KV cache
2188	# memory allocation. Enabled by default as of v0.21.0
2189	"VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": lambda: bool(
2190	int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "1"))
2191	),
2192	# Force N-contiguous weight layout for all XPU unquantized linears.
2193	"VLLM_XPU_FORCE_N_CONTIG_WEIGHT": lambda: bool(
2194	int(os.getenv("VLLM_XPU_FORCE_N_CONTIG_WEIGHT", "0"))
2195	),
2196	# whether use xpu specific sample kernel
2197	"VLLM_XPU_USE_SAMPLER_KERNEL": lambda: bool(
2198	int(os.getenv("VLLM_XPU_USE_SAMPLER_KERNEL", "1"))
2199	),
2200	# Kernel backend for INC weight-only intN (WNA16) linear layers on XPU.
2201	# "auto" keeps the default preference order (ARK when importable, else the
2202	# oneDNN w4a16 path). "ark" forces the auto_round_kernel backend, "w4a16"
2203	# forces oneDNN int4_gemm_w4a16, and "w4a8" additionally quantizes
2204	# activations to per-token int8 (int4_gemm_w4a8) for large token counts.
2205	# The two oneDNN backends are int4-only; ARK also serves int2.
2206	# Which one is fastest is device-dependent, so this is left as an opt-in.
2207	"VLLM_XPU_INC_WNA16_BACKEND": env_with_choices(
2208	"VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"]
2209	),
2210	# Enable simple KV offload.
2211	"VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool(
2212	int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0"))
2213	),
2214	# Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable.
2215	"VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool(
2216	os.getenv("VLLM_USE_DIRECT_DCP_A2A")
2217	),
2218	"VLLM_USE_DIRECT_DCP_Q_GATHER": lambda: maybe_convert_bool(
2219	os.getenv("VLLM_USE_DIRECT_DCP_Q_GATHER")
2220	),
2221	"VLLM_USE_DIRECT_DCP_KV_GATHER": lambda: maybe_convert_bool(
2222	os.getenv("VLLM_USE_DIRECT_DCP_KV_GATHER")
2223	),
2224	# Whether to enable dual cuda streams for LoRA computation
2225	# (used by both BaseLinearLayerWithLoRA and FusedMoEWithLoRA to
2226	# overlap the base layer compute with the LoRA fast path).
2227	"VLLM_LORA_ENABLE_DUAL_STREAM": lambda: bool(
2228	int(os.getenv("VLLM_LORA_ENABLE_DUAL_STREAM", "0"))
2229	),
2230	# If set to 1, use Python spinloop extension to poll in a more efficient
2231	# way when using the mp backend.
2232	"VLLM_USE_SPINLOOP_EXT": lambda: bool(int(os.getenv("VLLM_USE_SPINLOOP_EXT", "0"))),
2233	# Comma-separated GPU_BDF=NIC_BDF pairs for RDMA NIC selection.
2234	# Must be set together with VLLM_NIC_SELECTION_VARS.
2235	"VLLM_GPU_NIC_PCIE_MAPPING": lambda: os.getenv("VLLM_GPU_NIC_PCIE_MAPPING", ""),
2236	# Comma-separated list of env vars to set from the GPU-NIC mapping.
2237	# Each entry is VAR_NAME or VAR_NAME:<suffix> (suffix appended to
2238	# RDMA device name). Must be set together with VLLM_GPU_NIC_PCIE_MAPPING.
2239	"VLLM_NIC_SELECTION_VARS": lambda: os.getenv("VLLM_NIC_SELECTION_VARS", ""),
2240	# If set to 1, enable the HPC fused kernels (requires the hpc package
2241	# (.so) and an sm100/sm103 device). Covers:
2242	# * the HY V4 iHC ops -- each of the eager HYV4HCPreLayer /
2243	# HYV4HCPostLayer / HYV4HCHeadLayer bodies becomes one kernel launch;
2244	# * the gated-MLA output gating (attn_out * sigmoid(gate projection)),
2245	# fused into gated_mla_gemm; elementwise gating only.
2246	# Each op additionally checks its own shape / dtype constraints and falls
2247	# back to the eager path when they do not hold.
2248	"VLLM_ENABLE_HPC_OPS": lambda: bool(int(os.getenv("VLLM_ENABLE_HPC_OPS", "0"))),
2249	# Whether to skip version suffix when building package
2250	"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
2251	int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
2252	),
2253	}
2254
2255
2256	# --8<-- [end:env-vars-definition]
2257
2258
2259	def __getattr__(name: str):
2260	"""Gets environment variables lazily.
2261
2262	NOTE: After enable_envs_cache() invocation (which triggered after service
2263	initialization), all environment variables will be cached.
2264	"""
2265	if name in environment_variables:
2266	return environment_variables[name]()
2267	raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
2268
2269
2270	def _is_envs_cache_enabled() -> bool:
2271	"""Checked if __getattr__ is wrapped with functools.cache."""
2272	global __getattr__
2273	return hasattr(__getattr__, "cache_clear")
2274
2275
2276	def enable_envs_cache() -> None:
2277	"""Enables caching of environment variables. This is useful for performance
2278	reasons, as it avoids the need to re-evaluate environment variables on
2279	every call.
2280
2281	NOTE: Currently, it's invoked after service initialization to reduce
2282	runtime overhead. This also means that environment variables should NOT
2283	be updated after the service is initialized.
2284	"""
2285	if _is_envs_cache_enabled():
2286	# Avoid wrapping functools.cache multiple times
2287	return
2288	# Tag __getattr__ with functools.cache
2289	global __getattr__
2290	__getattr__ = functools.cache(__getattr__)
2291
2292	# Cache all environment variables
2293	for key in environment_variables:
2294	__getattr__(key)
2295
2296
2297	def disable_envs_cache() -> None:
2298	"""Resets the environment variables cache. It could be used to isolate environments
2299	between unit tests.
2300	"""
2301	global __getattr__
2302	# If __getattr__ is wrapped by functions.cache, unwrap the caching layer.
2303	if _is_envs_cache_enabled():
2304	assert hasattr(__getattr__, "__wrapped__")
2305	__getattr__ = __getattr__.__wrapped__
2306
2307
2308	def __dir__():
2309	return list(environment_variables.keys())
2310
2311
2312	def is_set(name: str):
2313	"""Check if an environment variable is explicitly set."""
2314	if name in environment_variables:
2315	return name in os.environ
2316	raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
2317
2318
2319	def compile_factors() -> dict[str, object]:
2320	"""Return env vars used for torch.compile cache keys.
2321
2322	Start with every known vLLM env var; drop entries in `ignored_factors`;
2323	hash everything else. This keeps the cache key aligned across workers."""
2324	ignored_factors: set[str] = {
2325	"MAX_JOBS",
2326	"VLLM_TRITON_JIT_WARMUP_NUM_THREADS",
2327	"VLLM_RPC_BASE_PATH",
2328	"VLLM_USE_MODELSCOPE",
2329	"VLLM_RINGBUFFER_WARNING_INTERVAL",
2330	"VLLM_DEBUG_DUMP_PATH",
2331	"VLLM_PORT",
2332	"VLLM_CACHE_ROOT",
2333	# Runtime memory-plan persistence; does not affect compiled graphs.
2334	"VLLM_ENABLE_STARTUP_PLAN",
2335	# Location-only derived paths: where a cache/config directory lives
2336	# cannot affect compiled artifacts, and hashing them means relocating
2337	# HOME or the XDG roots silently invalidates every compile cache
2338	# (VLLM_CACHE_ROOT above and VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR below
2339	# are already ignored for the same reason).
2340	"VLLM_XLA_CACHE_PATH",
2341	"VLLM_CONFIG_ROOT",
2342	"LD_LIBRARY_PATH",
2343	"VLLM_SERVER_DEV_MODE",
2344	"VLLM_DP_MASTER_IP",
2345	"VLLM_DP_MASTER_PORT",
2346	"VLLM_NIXL_SIDE_CHANNEL_HOST",
2347	"VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
2348	"VLLM_MODEL_REDIRECT_PATH",
2349	"VLLM_HOST_IP",
2350	"VLLM_ELASTIC_EP_SCALE_UP_LAUNCH",
2351	"VLLM_FORCE_AOT_LOAD",
2352	"S3_ACCESS_KEY_ID",
2353	"S3_SECRET_ACCESS_KEY",
2354	"S3_ENDPOINT_URL",
2355	# Credential; never affects compiled artifacts and must not be
2356	# persisted in cache_key_factors.json.
2357	"VLLM_API_KEY",
2358	"VLLM_USAGE_STATS_SERVER",
2359	"VLLM_NO_USAGE_STATS",
2360	"VLLM_DO_NOT_TRACK",
2361	"VLLM_LOGGING_LEVEL",
2362	"VLLM_LOGGING_PREFIX",
2363	"VLLM_LOGGING_STREAM",
2364	"VLLM_LOGGING_CONFIG_PATH",
2365	"VLLM_LOGGING_COLOR",
2366	"VLLM_LOG_STATS_INTERVAL",
2367	"VLLM_DEBUG_LOG_API_SERVER_RESPONSE",
2368	"VLLM_TUNED_CONFIG_FOLDER",
2369	"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR",
2370	"VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS",
2371	"VLLM_ENGINE_ITERATION_TIMEOUT_S",
2372	"VLLM_HTTP_TIMEOUT_KEEP_ALIVE",
2373	"VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS",
2374	"VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS",
2375	"VLLM_KEEP_ALIVE_ON_ENGINE_DEATH",
2376	"VLLM_IMAGE_FETCH_TIMEOUT",
2377	"VLLM_VIDEO_FETCH_TIMEOUT",
2378	"VLLM_AUDIO_FETCH_TIMEOUT",
2379	"VLLM_MEDIA_CACHE",
2380	"VLLM_MEDIA_CACHE_MAX_SIZE_MB",
2381	"VLLM_MEDIA_CACHE_TTL_HOURS",
2382	"VLLM_MEDIA_FETCH_MAX_RETRIES",
2383	"VLLM_MEDIA_URL_ALLOW_REDIRECTS",
2384	"VLLM_MEDIA_LOADING_THREAD_COUNT",
2385	"VLLM_MAX_AUDIO_CLIP_FILESIZE_MB",
2386	"VLLM_MAX_AUDIO_DECODE_DURATION_S",
2387	"VLLM_MAX_AUDIO_DECODE_BYTES",
2388	"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
2389	"VLLM_MAX_EMBED_DECODE_BYTES",
2390	"VLLM_MAX_IMAGE_PIXELS",
2391	"VLLM_VIDEO_LOADER_BACKEND",
2392	"VLLM_MEDIA_CONNECTOR",
2393	"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME",
2394	"VLLM_ASSETS_CACHE",
2395	"VLLM_ASSETS_CACHE_MODEL_CLEAN",
2396	"VLLM_WORKER_MULTIPROC_METHOD",
2397	"VLLM_ENABLE_V1_MULTIPROCESSING",
2398	"VLLM_V1_OUTPUT_PROC_CHUNK_SIZE",
2399	"VLLM_CPU_KVCACHE_SPACE",
2400	"VLLM_CPU_MOE_PREPACK",
2401	"VLLM_ZENTORCH_WEIGHT_PREPACK",
2402	"VLLM_TEST_FORCE_LOAD_FORMAT",
2403	"VLLM_ENABLE_CUDA_COMPATIBILITY",
2404	"VLLM_CUDA_COMPATIBILITY_PATH",
2405	"VLLM_SKIP_MODEL_NAME_VALIDATION",
2406	"LOCAL_RANK",
2407	"CUDA_VISIBLE_DEVICES",
2408	"NO_COLOR",
2409	"VLLM_SKIP_VERSION_SUFFIX",
2410	"FORCE_COLOR",
2411	}
2412
2413	from vllm.config.utils import normalize_value
2414
2415	factors: dict[str, object] = {}
2416	for factor, getter in environment_variables.items():
2417	if factor in ignored_factors:
2418	continue
2419
2420	try:
2421	raw = getter()
2422	except Exception as exc: # pragma: no cover - defensive logging
2423	logger.warning(
2424	"Skipping environment variable %s while hashing compile factors: %s",
2425	factor,
2426	exc,
2427	)
2428	continue
2429
2430	factors[factor] = normalize_value(raw)
2431
2432	ray_noset_env_vars = [
2433	# Refer to
2434	# https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/nvidia_gpu.py#L11
2435	# https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/amd_gpu.py#L11
2436	# https://github.com/ray-project/ray/blob/b97d21dab233c2bd8ed7db749a82a1e594222b5c/python/ray/_private/accelerators/amd_gpu.py#L10
2437	# https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/npu.py#L12
2438	# https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/hpu.py#L12
2439	# https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/neuron.py#L14
2440	# https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/tpu.py#L38
2441	# https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/intel_gpu.py#L10
2442	# https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/rbln.py#L10
2443	"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES",
2444	"RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES",
2445	"RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES",
2446	"RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES",
2447	"RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES",
2448	"RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES",
2449	"RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS",
2450	"RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR",
2451	"RAY_EXPERIMENTAL_NOSET_RBLN_RT_VISIBLE_DEVICES",
2452	]
2453
2454	for var in ray_noset_env_vars:
2455	factors[var] = normalize_value(os.getenv(var))
2456
2457	return factors