A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 16:15 UTC
[Bugfix] fix confusing OOM errors during v1 init (#28051)
Signed-off-by: Shivam <shivamprasad91@gmail.com> Signed-off-by: shivampr <shivampr.dev@gmail.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com>
shivampr committed
Dec 10, 2025 at 15:17 UTC
8580919ac36b9ada425668264437c70935943e05
3 files changed
+138
-65
tests/v1/engine/test_init_error_messaging.py
new
+54
@@ -0,0 +1,54 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+
6
+from vllm.v1.core.kv_cache_utils import check_enough_kv_cache_memory
7
+from vllm.v1.kv_cache_interface import FullAttentionSpec
8
+
9
+
10
+def test_kv_cache_oom_no_memory():
11
+ from unittest.mock import MagicMock
12
+
13
+ config = MagicMock()
14
+ config.model_config.max_model_len = 2048
15
+
16
+ spec = {
17
+ "layer_0": FullAttentionSpec(
18
+ block_size=16,
19
+ num_kv_heads=8,
20
+ head_size=128,
21
+ dtype="float16",
22
+ )
23
+ }
24
+
25
+ with pytest.raises(ValueError):
26
+ check_enough_kv_cache_memory(config, spec, 0)
27
+
28
+
29
+def test_kv_cache_oom_insufficient_memory(monkeypatch):
30
+ from unittest.mock import MagicMock
31
+
32
+ config = MagicMock()
33
+ config.model_config.max_model_len = 2048
34
+ config.cache_config.block_size = 16
35
+ config.parallel_config.tensor_parallel_size = 1
36
+ config.parallel_config.pipeline_parallel_size = 1
37
+ config.parallel_config.decode_context_parallel_size = 1
38
+
39
+ monkeypatch.setattr(
40
+ "vllm.v1.core.kv_cache_utils.max_memory_usage_bytes",
41
+ lambda c, s: 100 * 1024**3, # 100 GiB
42
+ )
43
+
44
+ spec = {
45
+ "layer_0": FullAttentionSpec(
46
+ block_size=16,
47
+ num_kv_heads=8,
48
+ head_size=128,
49
+ dtype="float16",
50
+ )
51
+ }
52
+
53
+ with pytest.raises(ValueError):
54
+ check_enough_kv_cache_memory(config, spec, 1024**3) # 1 GiB
vllm/v1/core/kv_cache_utils.py
+7
-3
@@ -687,7 +687,9 @@ def check_enough_kv_cache_memory(
687
raise ValueError(
688
"No available memory for the cache blocks. "
689
"Try increasing `gpu_memory_utilization` when "
690
- "initializing the engine."
690
+ "initializing the engine. "
691
+ "See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ "
692
+ "for more details."
693
)
694
695
max_model_len = vllm_config.model_config.max_model_len
@@ -711,8 +713,10 @@ def check_enough_kv_cache_memory(
713
f"cache is needed, which is larger than the available KV cache "
714
f"memory ({available_memory / GiB_bytes:.2f} GiB). "
715
f"{estimated_msg} "
714
- f"Try increasing `gpu_memory_utilization` or decreasing "
715
- f"`max_model_len` when initializing the engine."
716
+ f"Try increasing `gpu_memory_utilization` or decreasing `max_model_len` "
717
+ f"when initializing the engine. "
718
+ f"See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ "
719
+ f"for more details."
720
)
721
722
vllm/v1/worker/gpu_model_runner.py
+77
-62
@@ -3571,74 +3571,89 @@ class GPUModelRunner(
3571
if self.parallel_config.enable_eplb:
3572
self.eplb_state = EplbState(self.parallel_config, self.device)
3573
eplb_models = 0
3574
- with DeviceMemoryProfiler() as m:
3575
- time_before_load = time.perf_counter()
3576
- model_loader = get_model_loader(self.load_config)
3577
- self.model = model_loader.load_model(
3578
- vllm_config=self.vllm_config, model_config=self.model_config
3579
- )
3580
- if self.lora_config:
3581
- self.model = self.load_lora_model(
3582
- self.model, self.vllm_config, self.device
3574
+
3575
+ try:
3576
+ with DeviceMemoryProfiler() as m:
3577
+ time_before_load = time.perf_counter()
3578
+ model_loader = get_model_loader(self.load_config)
3579
+ self.model = model_loader.load_model(
3580
+ vllm_config=self.vllm_config, model_config=self.model_config
3581
)
3584
- if hasattr(self, "drafter"):
3585
- logger.info_once("Loading drafter model...")
3586
- self.drafter.load_model(self.model)
3587
- if (
3588
- hasattr(self.drafter, "model")
3589
- and is_mixture_of_experts(self.drafter.model)
3590
- and self.parallel_config.enable_eplb
3591
- ):
3592
- spec_config = self.vllm_config.speculative_config
3593
- assert spec_config is not None
3594
- assert spec_config.draft_model_config is not None
3595
- logger.info_once(
3596
- "EPLB is enabled for drafter model %s.",
3597
- spec_config.draft_model_config.model,
3582
+ if self.lora_config:
3583
+ self.model = self.load_lora_model(
3584
+ self.model, self.vllm_config, self.device
3585
)
3586
+ if hasattr(self, "drafter"):
3587
+ logger.info_once("Loading drafter model...")
3588
+ self.drafter.load_model(self.model)
3589
+ if (
3590
+ hasattr(self.drafter, "model")
3591
+ and is_mixture_of_experts(self.drafter.model)
3592
+ and self.parallel_config.enable_eplb
3593
+ ):
3594
+ spec_config = self.vllm_config.speculative_config
3595
+ assert spec_config is not None
3596
+ assert spec_config.draft_model_config is not None
3597
+ logger.info_once(
3598
+ "EPLB is enabled for drafter model %s.",
3599
+ spec_config.draft_model_config.model,
3600
+ )
3601
3600
- global_expert_load = (
3601
- global_expert_loads[eplb_models]
3602
- if global_expert_loads
3603
- else None
3604
- )
3605
- old_global_expert_indices = (
3606
- old_global_expert_indices_per_model[eplb_models]
3607
- if old_global_expert_indices_per_model
3608
- else None
3609
- )
3610
- if self.eplb_state is None:
3611
- self.eplb_state = EplbState(self.parallel_config, self.device)
3612
- self.eplb_state.add_model(
3613
- self.drafter.model,
3614
- spec_config.draft_model_config,
3615
- global_expert_load,
3616
- old_global_expert_indices,
3617
- rank_mapping,
3618
- )
3619
- eplb_models += 1
3602
+ global_expert_load = (
3603
+ global_expert_loads[eplb_models]
3604
+ if global_expert_loads
3605
+ else None
3606
+ )
3607
+ old_global_expert_indices = (
3608
+ old_global_expert_indices_per_model[eplb_models]
3609
+ if old_global_expert_indices_per_model
3610
+ else None
3611
+ )
3612
+ if self.eplb_state is None:
3613
+ self.eplb_state = EplbState(
3614
+ self.parallel_config, self.device
3615
+ )
3616
+ self.eplb_state.add_model(
3617
+ self.drafter.model,
3618
+ spec_config.draft_model_config,
3619
+ global_expert_load,
3620
+ old_global_expert_indices,
3621
+ rank_mapping,
3622
+ )
3623
+ eplb_models += 1
3624
3621
- if self.use_aux_hidden_state_outputs:
3622
- if not supports_eagle3(self.get_model()):
3623
- raise RuntimeError(
3624
- "Model does not support EAGLE3 interface but "
3625
- "aux_hidden_state_outputs was requested"
3626
- )
3625
+ if self.use_aux_hidden_state_outputs:
3626
+ if not supports_eagle3(self.get_model()):
3627
+ raise RuntimeError(
3628
+ "Model does not support EAGLE3 interface but "
3629
+ "aux_hidden_state_outputs was requested"
3630
+ )
3631
3628
- # Try to get auxiliary layers from speculative config,
3629
- # otherwise use model's default layers
3630
- aux_layers = self._get_eagle3_aux_layers_from_config()
3631
- if aux_layers:
3632
- logger.info(
3633
- "Using auxiliary layers from speculative config: %s",
3634
- aux_layers,
3635
- )
3636
- else:
3637
- aux_layers = self.model.get_eagle3_aux_hidden_state_layers()
3632
+ # Try to get auxiliary layers from speculative config,
3633
+ # otherwise use model's default layers
3634
+ aux_layers = self._get_eagle3_aux_layers_from_config()
3635
+ if aux_layers:
3636
+ logger.info(
3637
+ "Using auxiliary layers from speculative config: %s",
3638
+ aux_layers,
3639
+ )
3640
+ else:
3641
+ aux_layers = self.model.get_eagle3_aux_hidden_state_layers()
3642
3639
- self.model.set_aux_hidden_state_layers(aux_layers)
3640
- time_after_load = time.perf_counter()
3641
- self.model_memory_usage = m.consumed_memory
3643
+ self.model.set_aux_hidden_state_layers(aux_layers)
3644
+ time_after_load = time.perf_counter()
3645
+ self.model_memory_usage = m.consumed_memory
3646
+ except torch.cuda.OutOfMemoryError as e:
3647
+ msg = (
3648
+ "Failed to load model - not enough GPU memory. "
3649
+ "Try lowering --gpu-memory-utilization to free memory for weights, "
3650
+ "increasing --tensor-parallel-size, or using --quantization. "
3651
+ "See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ "
3652
+ "for more tips."
3653
+ )
3654
+ combined_msg = f"{msg} (original error: {e})"
3655
+ logger.error(combined_msg)
3656
+ raise e
3657
logger.info_once(
3658
"Model loading took %.4f GiB memory and %.6f seconds",
3659
self.model_memory_usage / GiB_bytes,