A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[XPU] fix xpu to set cudagraph batch sizes (#23044)
Signed-off-by: calvin chen <wen.chen@dynamia.ai>
Calvin Chen committed
Aug 18, 2025 at 05:45 UTC
21e39436c8062ebbf4a160eebf56d7d303896e68
1 file changed
+4
-2
vllm/v1/worker/gpu_model_runner.py
+4
-2
@@ -232,8 +232,10 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
232
# The convention is different.
233
# self.cudagraph_batch_sizes sorts in ascending order.
234
# The batch sizes in the config are in descending order.
235
- self.cudagraph_batch_sizes = list(
236
- reversed(self.compilation_config.cudagraph_capture_sizes))
235
+ if self.compilation_config.cudagraph_capture_sizes and \
236
+ self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
237
+ self.cudagraph_batch_sizes = list(
238
+ reversed(self.compilation_config.cudagraph_capture_sizes))
239
240
# Cache the device properties.
241
self._init_device_properties()