A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] chat method add_generation_prompt param (#7734)
Brian Li committed
Aug 22, 2024 at 01:33 UTC
d3c002eadcbaabe1cc2e5fe94321cdc6383cd4e3
1 file changed
+3
-3
vllm/entrypoints/llm.py
+3
-3
@@ -353,7 +353,7 @@ class LLM:
353
use_tqdm: bool = True,
354
lora_request: Optional[LoRARequest] = None,
355
chat_template: Optional[str] = None,
356
- add_generation_template: bool = True,
356
+ add_generation_prompt: bool = True,
357
) -> List[RequestOutput]:
358
"""
359
Generates responses for chat messages.
@@ -374,7 +374,7 @@ class LLM:
374
lora_request: LoRA request to use for generation, if any.
375
chat_template: The template to use for structuring the chat.
376
If not provided, the model's default chat template will be used.
377
- add_generation_template: If True, adds a generation template
377
+ add_generation_prompt: If True, adds a generation template
378
to each message.
379
380
Returns:
@@ -392,7 +392,7 @@ class LLM:
392
tokenizer,
393
conversations,
394
chat_template=chat_template,
395
- add_generation_template=add_generation_template)
395
+ add_generation_prompt=add_generation_prompt)
396
397
return self.generate(
398
prompts,