@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 263d8ee150
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix missing task for speculative decoding (#9524)

Cyrus Leung committed Oct 19, 2024 at 14:49 UTC 263d8ee150a737ddb8b2d49254bf712d8bb08a0b
1 file changed +14 -9
vllm/config.py
+14 -9
@@ -33,8 +33,10 @@ logger = init_logger(__name__)
33 _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
34 _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
35
36 -Task = Literal["generate", "embedding"]
37 -TaskOption = Literal["auto", Task]
36 +TaskOption = Literal["auto", "generate", "embedding"]
37 +
38 +# "draft" is only used internally for speculative decoding
39 +_Task = Literal["generate", "embedding", "draft"]
40
41
42 class ModelConfig:
@@ -115,7 +117,7 @@ class ModelConfig:
117
118 def __init__(self,
119 model: str,
118 - task: TaskOption,
120 + task: Union[TaskOption, _Task],
121 tokenizer: str,
122 tokenizer_mode: str,
123 trust_remote_code: bool,
@@ -255,18 +257,21 @@ class ModelConfig:
257
258 def _resolve_task(
259 self,
258 - task_option: TaskOption,
260 + task_option: Union[TaskOption, _Task],
261 hf_config: PretrainedConfig,
260 - ) -> Tuple[Set[Task], Task]:
262 + ) -> Tuple[Set[_Task], _Task]:
263 + if task_option == "draft":
264 + return {"draft"}, "draft"
265 +
266 architectures = getattr(hf_config, "architectures", [])
267
263 - task_support: Dict[Task, bool] = {
268 + task_support: Dict[_Task, bool] = {
269 # NOTE: Listed from highest to lowest priority,
270 # in case the model supports multiple of them
271 "generate": ModelRegistry.is_text_generation_model(architectures),
272 "embedding": ModelRegistry.is_embedding_model(architectures),
273 }
269 - supported_tasks_lst: List[Task] = [
274 + supported_tasks_lst: List[_Task] = [
275 task for task, is_supported in task_support.items() if is_supported
276 ]
277 supported_tasks = set(supported_tasks_lst)
@@ -1002,7 +1007,7 @@ class SchedulerConfig:
1007 """
1008
1009 def __init__(self,
1005 - task: Task,
1010 + task: _Task,
1011 max_num_batched_tokens: Optional[int],
1012 max_num_seqs: int,
1013 max_model_len: int,
@@ -1269,7 +1274,7 @@ class SpeculativeConfig:
1274 ngram_prompt_lookup_min = 0
1275 draft_model_config = ModelConfig(
1276 model=speculative_model,
1272 - task=target_model_config.task,
1277 + task="draft",
1278 tokenizer=target_model_config.tokenizer,
1279 tokenizer_mode=target_model_config.tokenizer_mode,
1280 trust_remote_code=target_model_config.trust_remote_code,