@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2529d09b5a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Frontend] Batch inference for llm.chat() API (#8648)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

Andy committed Sep 24, 2024 at 12:44 UTC 2529d09b5a4a124a316b6976e7d782f54e0bddde
3 files changed +111 -33
examples/offline_inference_chat.py
+27
@@ -39,6 +39,33 @@ outputs = llm.chat(conversation,
39 use_tqdm=False)
40 print_outputs(outputs)
41
42 +# You can run batch inference with llm.chat API
43 +conversation = [
44 + {
45 + "role": "system",
46 + "content": "You are a helpful assistant"
47 + },
48 + {
49 + "role": "user",
50 + "content": "Hello"
51 + },
52 + {
53 + "role": "assistant",
54 + "content": "Hello! How can I assist you today?"
55 + },
56 + {
57 + "role": "user",
58 + "content": "Write an essay about the importance of higher education.",
59 + },
60 +]
61 +conversations = [conversation for _ in range(10)]
62 +
63 +# We turn on tqdm progress bar to verify it's indeed running batch inference
64 +outputs = llm.chat(messages=conversations,
65 + sampling_params=sampling_params,
66 + use_tqdm=True)
67 +print_outputs(outputs)
68 +
69 # A chat template can be optionally supplied.
70 # If not, the model will use its default chat template.
71
tests/entrypoints/llm/test_generate.py
+35
@@ -162,6 +162,41 @@ def test_chat():
162 assert len(outputs) == 1
163
164
165 +def test_multi_chat():
166 +
167 + llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
168 +
169 + prompt1 = "Explain the concept of entropy."
170 + prompt2 = "Explain what among us is."
171 +
172 + conversation1 = [
173 + {
174 + "role": "system",
175 + "content": "You are a helpful assistant"
176 + },
177 + {
178 + "role": "user",
179 + "content": prompt1
180 + },
181 + ]
182 +
183 + conversation2 = [
184 + {
185 + "role": "system",
186 + "content": "You are a helpful assistant"
187 + },
188 + {
189 + "role": "user",
190 + "content": prompt2
191 + },
192 + ]
193 +
194 + messages = [conversation1, conversation2]
195 +
196 + outputs = llm.chat(messages)
197 + assert len(outputs) == 2
198 +
199 +
200 @pytest.mark.parametrize("image_urls",
201 [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
202 def test_chat_multi_image(image_urls: List[str]):
vllm/entrypoints/llm.py
+49 -33
@@ -485,7 +485,8 @@ class LLM:
485
486 def chat(
487 self,
488 - messages: List[ChatCompletionMessageParam],
488 + messages: Union[List[ChatCompletionMessageParam],
489 + List[List[ChatCompletionMessageParam]]],
490 sampling_params: Optional[Union[SamplingParams,
491 List[SamplingParams]]] = None,
492 use_tqdm: bool = True,
@@ -505,8 +506,9 @@ class LLM:
506 to the OpenAI API.
507
508 Args:
508 - messages: A single conversation represented as a list of messages.
509 - Each message is a dictionary with 'role' and 'content' keys.
509 + messages: A list of conversations or a single conversation.
510 + - Each conversation is represented as a list of messages.
511 + - Each message is a dictionary with 'role' and 'content' keys.
512 sampling_params: The sampling parameters for text generation.
513 If None, we use the default sampling parameters. When it
514 is a single value, it is applied to every prompt. When it
@@ -523,42 +525,56 @@ class LLM:
525 A list of ``RequestOutput`` objects containing the generated
526 responses in the same order as the input messages.
527 """
528 + list_of_messages: List[List[ChatCompletionMessageParam]]
529
527 - tokenizer = self.get_tokenizer()
528 - model_config = self.llm_engine.get_model_config()
529 -
530 - conversation, mm_data = parse_chat_messages(messages, model_config,
531 - tokenizer)
532 -
533 - prompt: Union[str, List[int]]
534 - if isinstance(tokenizer, MistralTokenizer):
535 - prompt = apply_mistral_chat_template(
536 - tokenizer,
537 - messages=messages,
538 - chat_template=chat_template,
539 - add_generation_prompt=add_generation_prompt,
540 - tools=tools,
541 - )
530 + # Handle multi and single conversations
531 + if is_list_of(messages, list):
532 + # messages is List[List[...]]
533 + list_of_messages = messages
534 else:
543 - prompt = apply_hf_chat_template(
544 - tokenizer,
545 - conversation=conversation,
546 - chat_template=chat_template,
547 - add_generation_prompt=add_generation_prompt,
548 - tools=tools,
549 - )
535 + # messages is List[...]
536 + list_of_messages = [messages]
537 +
538 + prompts: List[Union[TokensPrompt, TextPrompt]] = []
539 +
540 + for msgs in list_of_messages:
541 + tokenizer = self.get_tokenizer()
542 + model_config = self.llm_engine.get_model_config()
543 +
544 + conversation, mm_data = parse_chat_messages(
545 + msgs, model_config, tokenizer)
546 +
547 + prompt_data: Union[str, List[int]]
548 + if isinstance(tokenizer, MistralTokenizer):
549 + prompt_data = apply_mistral_chat_template(
550 + tokenizer,
551 + messages=msgs,
552 + chat_template=chat_template,
553 + add_generation_prompt=add_generation_prompt,
554 + tools=tools,
555 + )
556 + else:
557 + prompt_data = apply_hf_chat_template(
558 + tokenizer,
559 + conversation=conversation,
560 + chat_template=chat_template,
561 + add_generation_prompt=add_generation_prompt,
562 + tools=tools,
563 + )
564 +
565 + prompt: Union[TokensPrompt, TextPrompt]
566 + if is_list_of(prompt_data, int):
567 + prompt = TokensPrompt(prompt_token_ids=prompt_data)
568 + else:
569 + prompt = TextPrompt(prompt=prompt_data)
570
551 - inputs: PromptInputs
552 - if is_list_of(prompt, int):
553 - inputs = TokensPrompt(prompt_token_ids=prompt)
554 - else:
555 - inputs = TextPrompt(prompt=prompt)
571 + if mm_data is not None:
572 + prompt["multi_modal_data"] = mm_data
573
557 - if mm_data is not None:
558 - inputs["multi_modal_data"] = mm_data
574 + prompts.append(prompt)
575
576 return self.generate(
561 - inputs,
577 + prompts,
578 sampling_params=sampling_params,
579 use_tqdm=use_tqdm,
580 lora_request=lora_request,