A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 23:00 UTC
[Bugfix] Fix DeepSeek V3.2 OOM during CG memory profiling (#36691)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Matthew Bonanni committed
Mar 10, 2026 at 23:01 UTC
8ab3d7427cf54f2505c934344c7c7ecd2ce32c99
1 file changed
+5
-7
vllm/v1/worker/gpu_model_runner.py
+5
-7
@@ -5550,16 +5550,14 @@ class GPUModelRunner(
5550
kv_cache_spec = self.get_kv_cache_spec()
5551
kv_cache_groups = get_kv_cache_groups(self.vllm_config, kv_cache_spec)
5552
min_blocks = self.compilation_config.max_cudagraph_capture_size or 1
5553
- if kv_cache_groups:
5554
- page_size = kv_cache_groups[0].kv_cache_spec.page_size_bytes
5555
- group_size = max(len(g.layer_names) for g in kv_cache_groups)
5556
- available_memory = min_blocks * page_size * group_size
5557
- else:
5558
- available_memory = 1 # Attention-free model
5553
5554
+ # Temporarily change num_gpu_blocks_override to allocate a minimal KV cache
5555
+ saved_override = self.cache_config.num_gpu_blocks_override
5556
+ self.cache_config.num_gpu_blocks_override = min_blocks
5557
minimal_config = get_kv_cache_config_from_groups(
5561
- self.vllm_config, kv_cache_groups, available_memory=available_memory
5558
+ self.vllm_config, kv_cache_groups, available_memory=0
5559
)
5560
+ self.cache_config.num_gpu_blocks_override = saved_override
5561
5562
self.initialize_kv_cache(minimal_config)
5563
self.cache_config.num_gpu_blocks = minimal_config.num_blocks