A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
Chat method for offline llm (#5049)
Co-authored-by: nunjunj <ray@g-3ff9f30f2ed650001.c.vllm-405802.internal> Co-authored-by: nunjunj <ray@g-1df6075697c3f0001.c.vllm-405802.internal> Co-authored-by: nunjunj <ray@g-c5a2c23abc49e0001.c.vllm-405802.internal> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
nunjunj committed
Aug 16, 2024 at 09:41 UTC
3b19e39dc59975c4240082a6b45f7c2a123b7164
4 files changed
+168
-29
.buildkite/test-pipeline.yaml
+1
@@ -147,6 +147,7 @@ steps:
147
- pip install awscli tensorizer # for llava example and tensorizer test
148
- python3 offline_inference.py
149
- python3 cpu_offload.py
150
+ - python3 offline_inference_chat.py
151
- python3 offline_inference_with_prefix.py
152
- python3 llm_engine_example.py
153
- python3 offline_inference_vision_language.py
examples/offline_inference_chat.py
new
+53
@@ -0,0 +1,53 @@
1
+from vllm import LLM, SamplingParams
2
+
3
+llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
4
+sampling_params = SamplingParams(temperature=0.5)
5
+
6
+
7
+def print_outputs(outputs):
8
+ for output in outputs:
9
+ prompt = output.prompt
10
+ generated_text = output.outputs[0].text
11
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
12
+ print("-" * 80)
13
+
14
+
15
+print("=" * 80)
16
+
17
+# In this script, we demonstrate how to pass input to the chat method:
18
+
19
+conversation = [
20
+ {
21
+ "role": "system",
22
+ "content": "You are a helpful assistant"
23
+ },
24
+ {
25
+ "role": "user",
26
+ "content": "Hello"
27
+ },
28
+ {
29
+ "role": "assistant",
30
+ "content": "Hello! How can I assist you today?"
31
+ },
32
+ {
33
+ "role": "user",
34
+ "content": "Write an essay about the importance of higher education.",
35
+ },
36
+]
37
+outputs = llm.chat(conversation,
38
+ sampling_params=sampling_params,
39
+ use_tqdm=False)
40
+print_outputs(outputs)
41
+
42
+# A chat template can be optionally supplied.
43
+# If not, the model will use its default chat template.
44
+
45
+# with open('template_falcon_180b.jinja', "r") as f:
46
+# chat_template = f.read()
47
+
48
+# outputs = llm.chat(
49
+# conversations,
50
+# sampling_params=sampling_params,
51
+# use_tqdm=False,
52
+# chat_template=chat_template,
53
+# )
tests/entrypoints/llm/test_generate.py
+19
@@ -140,3 +140,22 @@ def test_multiple_sampling_params(llm: LLM):
140
# sampling_params is None, default params should be applied
141
outputs = llm.generate(PROMPTS, sampling_params=None)
142
assert len(PROMPTS) == len(outputs)
143
+
144
+
145
+def test_chat():
146
+
147
+ llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
148
+
149
+ prompt1 = "Explain the concept of entropy."
150
+ messages = [
151
+ {
152
+ "role": "system",
153
+ "content": "You are a helpful assistant"
154
+ },
155
+ {
156
+ "role": "user",
157
+ "content": prompt1
158
+ },
159
+ ]
160
+ outputs = llm.chat(messages)
161
+ assert len(outputs) == 1
vllm/entrypoints/llm.py
+95
-29
@@ -6,6 +6,9 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.engine.llm_engine import LLMEngine
9
+from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
10
+ apply_chat_template,
11
+ parse_chat_messages)
12
from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
13
from vllm.inputs.parse import parse_and_batch_prompt
14
from vllm.logger import init_logger
@@ -87,7 +90,7 @@ class LLM:
90
disable_custom_all_reduce: See ParallelConfig
91
**kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
92
:ref:`engine_args`)
90
-
93
+
94
Note:
95
This class is intended to be used for offline inference. For online
96
serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
@@ -138,8 +141,12 @@ class LLM:
141
142
if "disable_log_stats" not in kwargs:
143
kwargs["disable_log_stats"] = True
141
- removed_vision_keys = ("image_token_id", "image_feature_size",
142
- "image_input_shape", "image_input_type")
144
+ removed_vision_keys = (
145
+ "image_token_id",
146
+ "image_feature_size",
147
+ "image_input_shape",
148
+ "image_input_type",
149
+ )
150
if any(k in kwargs for k in removed_vision_keys):
151
raise TypeError(
152
"There is no need to pass vision-related arguments anymore.")
@@ -259,11 +266,12 @@ class LLM:
266
) -> List[RequestOutput]:
267
...
268
262
- @deprecate_kwargs("prompts",
263
- "prompt_token_ids",
264
- is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
265
- additional_message="Please use the 'inputs' parameter "
266
- "instead.")
269
+ @deprecate_kwargs(
270
+ "prompts",
271
+ "prompt_token_ids",
272
+ is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
273
+ additional_message="Please use the 'inputs' parameter instead.",
274
+ )
275
def generate(
276
self,
277
prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
@@ -286,17 +294,17 @@ class LLM:
294
Args:
295
inputs: A list of inputs to generate completions for.
296
sampling_params: The sampling parameters for text generation. If
289
- None, we use the default sampling parameters.
290
- When it is a single value, it is applied to every prompt.
291
- When it is a list, the list must have the same length as the
297
+ None, we use the default sampling parameters.
298
+ When it is a single value, it is applied to every prompt.
299
+ When it is a list, the list must have the same length as the
300
prompts and it is paired one by one with the prompt.
301
use_tqdm: Whether to use tqdm to display the progress bar.
302
lora_request: LoRA request to use for generation, if any.
295
- prompt_adapter_request: Prompt Adapter request to use for
303
+ prompt_adapter_request: Prompt Adapter request to use for
304
generation, if any.
305
306
Returns:
299
- A list of `RequestOutput` objects containing the
307
+ A list of ``RequestOutput`` objects containing the
308
generated completions in the same order as the input prompts.
309
310
Note:
@@ -339,6 +347,62 @@ class LLM:
347
outputs = self._run_engine(use_tqdm=use_tqdm)
348
return LLMEngine.validate_outputs(outputs, RequestOutput)
349
350
+ def chat(
351
+ self,
352
+ messages: List[ChatCompletionMessageParam],
353
+ sampling_params: Optional[Union[SamplingParams,
354
+ List[SamplingParams]]] = None,
355
+ use_tqdm: bool = True,
356
+ lora_request: Optional[LoRARequest] = None,
357
+ chat_template: Optional[str] = None,
358
+ add_generation_template: bool = True,
359
+ ) -> List[RequestOutput]:
360
+ """
361
+ Generates responses for chat messages.
362
+
363
+ Converts the messages to prompts using the tokenizer and calls
364
+ the :meth:`generate` method to generate the responses.
365
+
366
+ Args:
367
+ messages: A list of messages to generate responses for. Each
368
+ message is a list of dictionaries with 'role' and 'content'
369
+ keys.
370
+ sampling_params: The sampling parameters for text generation.
371
+ If None, we use the default sampling parameters. When it
372
+ is a single value, it is applied to every prompt. When it
373
+ is a list, the list must have the same length as the
374
+ prompts and it is paired one by one with the prompt.
375
+ use_tqdm: Whether to use tqdm to display the progress bar.
376
+ lora_request: LoRA request to use for generation, if any.
377
+ chat_template: The template to use for structuring the chat.
378
+ If not provided, the model's default chat template will be used.
379
+ add_generation_template: If True, adds a generation template
380
+ to each message.
381
+
382
+ Returns:
383
+ A list of ``RequestOutput`` objects containing the generated
384
+ responses in the same order as the input messages.
385
+ """
386
+
387
+ tokenizer = self.get_tokenizer()
388
+ model_config = self.llm_engine.get_model_config()
389
+
390
+ conversations, _ = parse_chat_messages(messages, model_config,
391
+ tokenizer)
392
+
393
+ prompts = apply_chat_template(
394
+ tokenizer,
395
+ conversations,
396
+ chat_template=chat_template,
397
+ add_generation_template=add_generation_template)
398
+
399
+ return self.generate(
400
+ prompts,
401
+ sampling_params,
402
+ use_tqdm=use_tqdm,
403
+ lora_request=lora_request,
404
+ )
405
+
406
@overload # LEGACY: single (prompt + optional token ids)
407
def encode(
408
self,
@@ -413,11 +477,12 @@ class LLM:
477
) -> List[EmbeddingRequestOutput]:
478
...
479
416
- @deprecate_kwargs("prompts",
417
- "prompt_token_ids",
418
- is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
419
- additional_message="Please use the 'inputs' parameter "
420
- "instead.")
480
+ @deprecate_kwargs(
481
+ "prompts",
482
+ "prompt_token_ids",
483
+ is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
484
+ additional_message="Please use the 'inputs' parameter instead.",
485
+ )
486
def encode(
487
self,
488
prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
@@ -443,7 +508,7 @@ class LLM:
508
use the default pooling parameters.
509
use_tqdm: Whether to use tqdm to display the progress bar.
510
lora_request: LoRA request to use for generation, if any.
446
- prompt_adapter_request: Prompt Adapter request to use for
511
+ prompt_adapter_request: Prompt Adapter request to use for
512
generation, if any.
513
514
Returns:
@@ -563,15 +628,15 @@ class LLM:
628
params[i] if isinstance(params, Sequence) else params,
629
lora_request=lora_request[i] if isinstance(
630
lora_request, Sequence) else lora_request,
566
- prompt_adapter_request=prompt_adapter_request)
631
+ prompt_adapter_request=prompt_adapter_request,
632
+ )
633
634
def _add_request(
569
- self,
570
- inputs: PromptInputs,
571
- params: Union[SamplingParams, PoolingParams],
572
- lora_request: Optional[Union[List[LoRARequest],
573
- LoRARequest]] = None,
574
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
635
+ self,
636
+ inputs: PromptInputs,
637
+ params: Union[SamplingParams, PoolingParams],
638
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
639
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
640
) -> None:
641
request_id = str(next(self.request_counter))
642
self.llm_engine.add_request(
@@ -579,7 +644,8 @@ class LLM:
644
inputs,
645
params,
646
lora_request=lora_request,
582
- prompt_adapter_request=prompt_adapter_request)
647
+ prompt_adapter_request=prompt_adapter_request,
648
+ )
649
650
def _add_guided_processor(
651
self,
@@ -628,8 +694,8 @@ class LLM:
694
in_spd = total_in_toks / pbar.format_dict["elapsed"]
695
total_out_toks += sum(
696
len(stp.token_ids) for stp in output.outputs)
631
- out_spd = total_out_toks / pbar.format_dict[
632
- "elapsed"]
697
+ out_spd = (total_out_toks /
698
+ pbar.format_dict["elapsed"])
699
pbar.postfix = (
700
f"est. speed input: {in_spd:.2f} toks/s, "
701
f"output: {out_spd:.2f} toks/s")