@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b19e39dc5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

Chat method for offline llm (#5049)

Co-authored-by: nunjunj <ray@g-3ff9f30f2ed650001.c.vllm-405802.internal> Co-authored-by: nunjunj <ray@g-1df6075697c3f0001.c.vllm-405802.internal> Co-authored-by: nunjunj <ray@g-c5a2c23abc49e0001.c.vllm-405802.internal> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

nunjunj committed Aug 16, 2024 at 09:41 UTC 3b19e39dc59975c4240082a6b45f7c2a123b7164
4 files changed +168 -29
.buildkite/test-pipeline.yaml
+1
@@ -147,6 +147,7 @@ steps:
147 - pip install awscli tensorizer # for llava example and tensorizer test
148 - python3 offline_inference.py
149 - python3 cpu_offload.py
150 + - python3 offline_inference_chat.py
151 - python3 offline_inference_with_prefix.py
152 - python3 llm_engine_example.py
153 - python3 offline_inference_vision_language.py
examples/offline_inference_chat.py new
+53
@@ -0,0 +1,53 @@
1 +from vllm import LLM, SamplingParams
2 +
3 +llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
4 +sampling_params = SamplingParams(temperature=0.5)
5 +
6 +
7 +def print_outputs(outputs):
8 + for output in outputs:
9 + prompt = output.prompt
10 + generated_text = output.outputs[0].text
11 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
12 + print("-" * 80)
13 +
14 +
15 +print("=" * 80)
16 +
17 +# In this script, we demonstrate how to pass input to the chat method:
18 +
19 +conversation = [
20 + {
21 + "role": "system",
22 + "content": "You are a helpful assistant"
23 + },
24 + {
25 + "role": "user",
26 + "content": "Hello"
27 + },
28 + {
29 + "role": "assistant",
30 + "content": "Hello! How can I assist you today?"
31 + },
32 + {
33 + "role": "user",
34 + "content": "Write an essay about the importance of higher education.",
35 + },
36 +]
37 +outputs = llm.chat(conversation,
38 + sampling_params=sampling_params,
39 + use_tqdm=False)
40 +print_outputs(outputs)
41 +
42 +# A chat template can be optionally supplied.
43 +# If not, the model will use its default chat template.
44 +
45 +# with open('template_falcon_180b.jinja', "r") as f:
46 +# chat_template = f.read()
47 +
48 +# outputs = llm.chat(
49 +# conversations,
50 +# sampling_params=sampling_params,
51 +# use_tqdm=False,
52 +# chat_template=chat_template,
53 +# )
tests/entrypoints/llm/test_generate.py
+19
@@ -140,3 +140,22 @@ def test_multiple_sampling_params(llm: LLM):
140 # sampling_params is None, default params should be applied
141 outputs = llm.generate(PROMPTS, sampling_params=None)
142 assert len(PROMPTS) == len(outputs)
143 +
144 +
145 +def test_chat():
146 +
147 + llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
148 +
149 + prompt1 = "Explain the concept of entropy."
150 + messages = [
151 + {
152 + "role": "system",
153 + "content": "You are a helpful assistant"
154 + },
155 + {
156 + "role": "user",
157 + "content": prompt1
158 + },
159 + ]
160 + outputs = llm.chat(messages)
161 + assert len(outputs) == 1
vllm/entrypoints/llm.py
+95 -29
@@ -6,6 +6,9 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.llm_engine import LLMEngine
9 +from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
10 + apply_chat_template,
11 + parse_chat_messages)
12 from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
13 from vllm.inputs.parse import parse_and_batch_prompt
14 from vllm.logger import init_logger
@@ -87,7 +90,7 @@ class LLM:
90 disable_custom_all_reduce: See ParallelConfig
91 **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
92 :ref:`engine_args`)
90 -
93 +
94 Note:
95 This class is intended to be used for offline inference. For online
96 serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
@@ -138,8 +141,12 @@ class LLM:
141
142 if "disable_log_stats" not in kwargs:
143 kwargs["disable_log_stats"] = True
141 - removed_vision_keys = ("image_token_id", "image_feature_size",
142 - "image_input_shape", "image_input_type")
144 + removed_vision_keys = (
145 + "image_token_id",
146 + "image_feature_size",
147 + "image_input_shape",
148 + "image_input_type",
149 + )
150 if any(k in kwargs for k in removed_vision_keys):
151 raise TypeError(
152 "There is no need to pass vision-related arguments anymore.")
@@ -259,11 +266,12 @@ class LLM:
266 ) -> List[RequestOutput]:
267 ...
268
262 - @deprecate_kwargs("prompts",
263 - "prompt_token_ids",
264 - is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
265 - additional_message="Please use the 'inputs' parameter "
266 - "instead.")
269 + @deprecate_kwargs(
270 + "prompts",
271 + "prompt_token_ids",
272 + is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
273 + additional_message="Please use the 'inputs' parameter instead.",
274 + )
275 def generate(
276 self,
277 prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
@@ -286,17 +294,17 @@ class LLM:
294 Args:
295 inputs: A list of inputs to generate completions for.
296 sampling_params: The sampling parameters for text generation. If
289 - None, we use the default sampling parameters.
290 - When it is a single value, it is applied to every prompt.
291 - When it is a list, the list must have the same length as the
297 + None, we use the default sampling parameters.
298 + When it is a single value, it is applied to every prompt.
299 + When it is a list, the list must have the same length as the
300 prompts and it is paired one by one with the prompt.
301 use_tqdm: Whether to use tqdm to display the progress bar.
302 lora_request: LoRA request to use for generation, if any.
295 - prompt_adapter_request: Prompt Adapter request to use for
303 + prompt_adapter_request: Prompt Adapter request to use for
304 generation, if any.
305
306 Returns:
299 - A list of `RequestOutput` objects containing the
307 + A list of ``RequestOutput`` objects containing the
308 generated completions in the same order as the input prompts.
309
310 Note:
@@ -339,6 +347,62 @@ class LLM:
347 outputs = self._run_engine(use_tqdm=use_tqdm)
348 return LLMEngine.validate_outputs(outputs, RequestOutput)
349
350 + def chat(
351 + self,
352 + messages: List[ChatCompletionMessageParam],
353 + sampling_params: Optional[Union[SamplingParams,
354 + List[SamplingParams]]] = None,
355 + use_tqdm: bool = True,
356 + lora_request: Optional[LoRARequest] = None,
357 + chat_template: Optional[str] = None,
358 + add_generation_template: bool = True,
359 + ) -> List[RequestOutput]:
360 + """
361 + Generates responses for chat messages.
362 +
363 + Converts the messages to prompts using the tokenizer and calls
364 + the :meth:`generate` method to generate the responses.
365 +
366 + Args:
367 + messages: A list of messages to generate responses for. Each
368 + message is a list of dictionaries with 'role' and 'content'
369 + keys.
370 + sampling_params: The sampling parameters for text generation.
371 + If None, we use the default sampling parameters. When it
372 + is a single value, it is applied to every prompt. When it
373 + is a list, the list must have the same length as the
374 + prompts and it is paired one by one with the prompt.
375 + use_tqdm: Whether to use tqdm to display the progress bar.
376 + lora_request: LoRA request to use for generation, if any.
377 + chat_template: The template to use for structuring the chat.
378 + If not provided, the model's default chat template will be used.
379 + add_generation_template: If True, adds a generation template
380 + to each message.
381 +
382 + Returns:
383 + A list of ``RequestOutput`` objects containing the generated
384 + responses in the same order as the input messages.
385 + """
386 +
387 + tokenizer = self.get_tokenizer()
388 + model_config = self.llm_engine.get_model_config()
389 +
390 + conversations, _ = parse_chat_messages(messages, model_config,
391 + tokenizer)
392 +
393 + prompts = apply_chat_template(
394 + tokenizer,
395 + conversations,
396 + chat_template=chat_template,
397 + add_generation_template=add_generation_template)
398 +
399 + return self.generate(
400 + prompts,
401 + sampling_params,
402 + use_tqdm=use_tqdm,
403 + lora_request=lora_request,
404 + )
405 +
406 @overload # LEGACY: single (prompt + optional token ids)
407 def encode(
408 self,
@@ -413,11 +477,12 @@ class LLM:
477 ) -> List[EmbeddingRequestOutput]:
478 ...
479
416 - @deprecate_kwargs("prompts",
417 - "prompt_token_ids",
418 - is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
419 - additional_message="Please use the 'inputs' parameter "
420 - "instead.")
480 + @deprecate_kwargs(
481 + "prompts",
482 + "prompt_token_ids",
483 + is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
484 + additional_message="Please use the 'inputs' parameter instead.",
485 + )
486 def encode(
487 self,
488 prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
@@ -443,7 +508,7 @@ class LLM:
508 use the default pooling parameters.
509 use_tqdm: Whether to use tqdm to display the progress bar.
510 lora_request: LoRA request to use for generation, if any.
446 - prompt_adapter_request: Prompt Adapter request to use for
511 + prompt_adapter_request: Prompt Adapter request to use for
512 generation, if any.
513
514 Returns:
@@ -563,15 +628,15 @@ class LLM:
628 params[i] if isinstance(params, Sequence) else params,
629 lora_request=lora_request[i] if isinstance(
630 lora_request, Sequence) else lora_request,
566 - prompt_adapter_request=prompt_adapter_request)
631 + prompt_adapter_request=prompt_adapter_request,
632 + )
633
634 def _add_request(
569 - self,
570 - inputs: PromptInputs,
571 - params: Union[SamplingParams, PoolingParams],
572 - lora_request: Optional[Union[List[LoRARequest],
573 - LoRARequest]] = None,
574 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
635 + self,
636 + inputs: PromptInputs,
637 + params: Union[SamplingParams, PoolingParams],
638 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
639 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
640 ) -> None:
641 request_id = str(next(self.request_counter))
642 self.llm_engine.add_request(
@@ -579,7 +644,8 @@ class LLM:
644 inputs,
645 params,
646 lora_request=lora_request,
582 - prompt_adapter_request=prompt_adapter_request)
647 + prompt_adapter_request=prompt_adapter_request,
648 + )
649
650 def _add_guided_processor(
651 self,
@@ -628,8 +694,8 @@ class LLM:
694 in_spd = total_in_toks / pbar.format_dict["elapsed"]
695 total_out_toks += sum(
696 len(stp.token_ids) for stp in output.outputs)
631 - out_spd = total_out_toks / pbar.format_dict[
632 - "elapsed"]
697 + out_spd = (total_out_toks /
698 + pbar.format_dict["elapsed"])
699 pbar.postfix = (
700 f"est. speed input: {in_spd:.2f} toks/s, "
701 f"output: {out_spd:.2f} toks/s")