A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] [SpecDecode] Default speculative_draft_tensor_parallel_size to 1 when using MLPSpeculator (#7105)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Aug 4, 2024 at 16:13 UTC
b1c9aa3daa7dcd981f0f77231b46883624b72dd0
1 file changed
+12
-4
vllm/config.py
+12
-4
@@ -1068,7 +1068,7 @@ class SpeculativeConfig:
1068
draft_parallel_config = (
1069
SpeculativeConfig.create_draft_parallel_config(
1070
target_parallel_config,
1071
- speculative_draft_tensor_parallel_size))
1071
+ speculative_draft_tensor_parallel_size, draft_hf_config))
1072
1073
if num_speculative_tokens is None:
1074
raise ValueError(
@@ -1136,15 +1136,23 @@ class SpeculativeConfig:
1136
@staticmethod
1137
def create_draft_parallel_config(
1138
target_parallel_config: ParallelConfig,
1139
- speculative_draft_tensor_parallel_size: Optional[int]
1139
+ speculative_draft_tensor_parallel_size: Optional[int],
1140
+ draft_hf_config: PretrainedConfig,
1141
) -> ParallelConfig:
1142
"""Create a parallel config for use by the draft worker.
1143
1144
This is mostly a copy of the target parallel config, except the tp_size.
1145
"""
1146
if speculative_draft_tensor_parallel_size is None:
1146
- speculative_draft_tensor_parallel_size = \
1147
- target_parallel_config.tensor_parallel_size
1147
+ if draft_hf_config.model_type == "mlp_speculator":
1148
+ speculative_draft_tensor_parallel_size = 1
1149
+ if target_parallel_config.tensor_parallel_size > 1:
1150
+ logger.warning(
1151
+ "MLPSpeculator cannot currently be run with tp>1; "
1152
+ "setting speculative_draft_tensor_parallel_size=1")
1153
+ else:
1154
+ speculative_draft_tensor_parallel_size = \
1155
+ target_parallel_config.tensor_parallel_size
1156
elif speculative_draft_tensor_parallel_size != 1:
1157
# TODO(wooyeon): allow tp values larger than 1
1158
raise ValueError(