@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15713e3b75
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[BugFix] Update draft model TP size check to allow matching target TP size (#9394)

Co-authored-by: Baoyuan Qi <qibaoyuan@126.com>

Nick Hill committed Oct 21, 2024 at 22:14 UTC 15713e3b7579d56758fab1150c99dd49633b5669
1 file changed +3 -3
vllm/config.py
+3 -3
@@ -1408,11 +1408,11 @@ class SpeculativeConfig:
1408 else:
1409 speculative_draft_tensor_parallel_size = \
1410 target_parallel_config.tensor_parallel_size
1411 - elif speculative_draft_tensor_parallel_size != 1:
1412 - # TODO(wooyeon): allow tp values larger than 1
1411 + elif speculative_draft_tensor_parallel_size not in (
1412 + 1, target_parallel_config.tensor_parallel_size):
1413 raise ValueError(
1414 f"{speculative_draft_tensor_parallel_size=} cannot be "
1415 - f"other value than 1")
1415 + f"other value than 1 or target model tensor_parallel_size")
1416
1417 draft_parallel_config = ParallelConfig(
1418 pipeline_parallel_size=target_parallel_config.