@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b1c9aa3daa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] [SpecDecode] Default speculative_draft_tensor_parallel_size to 1 when using MLPSpeculator (#7105)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Aug 4, 2024 at 16:13 UTC b1c9aa3daa7dcd981f0f77231b46883624b72dd0
1 file changed +12 -4
vllm/config.py
+12 -4
@@ -1068,7 +1068,7 @@ class SpeculativeConfig:
1068 draft_parallel_config = (
1069 SpeculativeConfig.create_draft_parallel_config(
1070 target_parallel_config,
1071 - speculative_draft_tensor_parallel_size))
1071 + speculative_draft_tensor_parallel_size, draft_hf_config))
1072
1073 if num_speculative_tokens is None:
1074 raise ValueError(
@@ -1136,15 +1136,23 @@ class SpeculativeConfig:
1136 @staticmethod
1137 def create_draft_parallel_config(
1138 target_parallel_config: ParallelConfig,
1139 - speculative_draft_tensor_parallel_size: Optional[int]
1139 + speculative_draft_tensor_parallel_size: Optional[int],
1140 + draft_hf_config: PretrainedConfig,
1141 ) -> ParallelConfig:
1142 """Create a parallel config for use by the draft worker.
1143
1144 This is mostly a copy of the target parallel config, except the tp_size.
1145 """
1146 if speculative_draft_tensor_parallel_size is None:
1146 - speculative_draft_tensor_parallel_size = \
1147 - target_parallel_config.tensor_parallel_size
1147 + if draft_hf_config.model_type == "mlp_speculator":
1148 + speculative_draft_tensor_parallel_size = 1
1149 + if target_parallel_config.tensor_parallel_size > 1:
1150 + logger.warning(
1151 + "MLPSpeculator cannot currently be run with tp>1; "
1152 + "setting speculative_draft_tensor_parallel_size=1")
1153 + else:
1154 + speculative_draft_tensor_parallel_size = \
1155 + target_parallel_config.tensor_parallel_size
1156 elif speculative_draft_tensor_parallel_size != 1:
1157 # TODO(wooyeon): allow tp values larger than 1
1158 raise ValueError(