A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[BugFix] Update draft model TP size check to allow matching target TP size (#9394)
Co-authored-by: Baoyuan Qi <qibaoyuan@126.com>
Nick Hill committed
Oct 21, 2024 at 22:14 UTC
15713e3b7579d56758fab1150c99dd49633b5669
1 file changed
+3
-3
vllm/config.py
+3
-3
@@ -1408,11 +1408,11 @@ class SpeculativeConfig:
1408
else:
1409
speculative_draft_tensor_parallel_size = \
1410
target_parallel_config.tensor_parallel_size
1411
- elif speculative_draft_tensor_parallel_size != 1:
1412
- # TODO(wooyeon): allow tp values larger than 1
1411
+ elif speculative_draft_tensor_parallel_size not in (
1412
+ 1, target_parallel_config.tensor_parallel_size):
1413
raise ValueError(
1414
f"{speculative_draft_tensor_parallel_size=} cannot be "
1415
- f"other value than 1")
1415
+ f"other value than 1 or target model tensor_parallel_size")
1416
1417
draft_parallel_config = ParallelConfig(
1418
pipeline_parallel_size=target_parallel_config.