@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1a2f8fb828
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[v1] fix use compile sizes (#11000)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 9, 2024 at 13:47 UTC 1a2f8fb828f0444705db319786b2e901159f184e
2 files changed +4
vllm/config.py
+1
@@ -2522,6 +2522,7 @@ class VllmConfig:
2522 self.compilation_config.custom_ops = ["none"]
2523 self.compilation_config.use_cudagraph = True
2524 self.compilation_config.use_inductor = True
2525 + self.compilation_config.cudagraph_num_of_warmups = 1
2526 self.compilation_config.pass_config.enable_fusion = False
2527 self.compilation_config.pass_config.enable_reshape = False
2528 self.compilation_config.level = CompilationLevel.PIECEWISE
vllm/v1/worker/gpu_model_runner.py
+3
@@ -582,6 +582,9 @@ class GPUModelRunner:
582 # can reuse the memory pool allocated for the large shapes.
583 with graph_capture():
584 for num_tokens in reversed(self.cudagraph_batch_sizes):
585 + for _ in range(self.vllm_config.compilation_config.
586 + cudagraph_num_of_warmups):
587 + self._dummy_run(self.model, num_tokens, self.kv_caches)
588 self._dummy_run(self.model, num_tokens, self.kv_caches)
589
590 end_time = time.perf_counter()