A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[v1] fix use compile sizes (#11000)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 9, 2024 at 13:47 UTC
1a2f8fb828f0444705db319786b2e901159f184e
2 files changed
+4
vllm/config.py
+1
@@ -2522,6 +2522,7 @@ class VllmConfig:
2522
self.compilation_config.custom_ops = ["none"]
2523
self.compilation_config.use_cudagraph = True
2524
self.compilation_config.use_inductor = True
2525
+ self.compilation_config.cudagraph_num_of_warmups = 1
2526
self.compilation_config.pass_config.enable_fusion = False
2527
self.compilation_config.pass_config.enable_reshape = False
2528
self.compilation_config.level = CompilationLevel.PIECEWISE
vllm/v1/worker/gpu_model_runner.py
+3
@@ -582,6 +582,9 @@ class GPUModelRunner:
582
# can reuse the memory pool allocated for the large shapes.
583
with graph_capture():
584
for num_tokens in reversed(self.cudagraph_batch_sizes):
585
+ for _ in range(self.vllm_config.compilation_config.
586
+ cudagraph_num_of_warmups):
587
+ self._dummy_run(self.model, num_tokens, self.kv_caches)
588
self._dummy_run(self.model, num_tokens, self.kv_caches)
589
590
end_time = time.perf_counter()