A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix missing task for speculative decoding (#9524)
Cyrus Leung committed
Oct 19, 2024 at 14:49 UTC
263d8ee150a737ddb8b2d49254bf712d8bb08a0b
1 file changed
+14
-9
vllm/config.py
+14
-9
@@ -33,8 +33,10 @@ logger = init_logger(__name__)
33
_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
34
_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
35
36
-Task = Literal["generate", "embedding"]
37
-TaskOption = Literal["auto", Task]
36
+TaskOption = Literal["auto", "generate", "embedding"]
37
+
38
+# "draft" is only used internally for speculative decoding
39
+_Task = Literal["generate", "embedding", "draft"]
40
41
42
class ModelConfig:
@@ -115,7 +117,7 @@ class ModelConfig:
117
118
def __init__(self,
119
model: str,
118
- task: TaskOption,
120
+ task: Union[TaskOption, _Task],
121
tokenizer: str,
122
tokenizer_mode: str,
123
trust_remote_code: bool,
@@ -255,18 +257,21 @@ class ModelConfig:
257
258
def _resolve_task(
259
self,
258
- task_option: TaskOption,
260
+ task_option: Union[TaskOption, _Task],
261
hf_config: PretrainedConfig,
260
- ) -> Tuple[Set[Task], Task]:
262
+ ) -> Tuple[Set[_Task], _Task]:
263
+ if task_option == "draft":
264
+ return {"draft"}, "draft"
265
+
266
architectures = getattr(hf_config, "architectures", [])
267
263
- task_support: Dict[Task, bool] = {
268
+ task_support: Dict[_Task, bool] = {
269
# NOTE: Listed from highest to lowest priority,
270
# in case the model supports multiple of them
271
"generate": ModelRegistry.is_text_generation_model(architectures),
272
"embedding": ModelRegistry.is_embedding_model(architectures),
273
}
269
- supported_tasks_lst: List[Task] = [
274
+ supported_tasks_lst: List[_Task] = [
275
task for task, is_supported in task_support.items() if is_supported
276
]
277
supported_tasks = set(supported_tasks_lst)
@@ -1002,7 +1007,7 @@ class SchedulerConfig:
1007
"""
1008
1009
def __init__(self,
1005
- task: Task,
1010
+ task: _Task,
1011
max_num_batched_tokens: Optional[int],
1012
max_num_seqs: int,
1013
max_model_len: int,
@@ -1269,7 +1274,7 @@ class SpeculativeConfig:
1274
ngram_prompt_lookup_min = 0
1275
draft_model_config = ModelConfig(
1276
model=speculative_model,
1272
- task=target_model_config.task,
1277
+ task="draft",
1278
tokenizer=target_model_config.tokenizer,
1279
tokenizer_mode=target_model_config.tokenizer_mode,
1280
trust_remote_code=target_model_config.trust_remote_code,