@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d3c002eadc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] chat method add_generation_prompt param (#7734)

Brian Li committed Aug 22, 2024 at 01:33 UTC d3c002eadcbaabe1cc2e5fe94321cdc6383cd4e3
1 file changed +3 -3
vllm/entrypoints/llm.py
+3 -3
@@ -353,7 +353,7 @@ class LLM:
353 use_tqdm: bool = True,
354 lora_request: Optional[LoRARequest] = None,
355 chat_template: Optional[str] = None,
356 - add_generation_template: bool = True,
356 + add_generation_prompt: bool = True,
357 ) -> List[RequestOutput]:
358 """
359 Generates responses for chat messages.
@@ -374,7 +374,7 @@ class LLM:
374 lora_request: LoRA request to use for generation, if any.
375 chat_template: The template to use for structuring the chat.
376 If not provided, the model's default chat template will be used.
377 - add_generation_template: If True, adds a generation template
377 + add_generation_prompt: If True, adds a generation template
378 to each message.
379
380 Returns:
@@ -392,7 +392,7 @@ class LLM:
392 tokenizer,
393 conversations,
394 chat_template=chat_template,
395 - add_generation_template=add_generation_template)
395 + add_generation_prompt=add_generation_prompt)
396
397 return self.generate(
398 prompts,