@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c054b7a62
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend] Clean up type annotations for mistral tokenizer (#8314)

Cyrus Leung committed Sep 11, 2024 at 00:49 UTC 8c054b7a6290551c868451dfd449d40cf37d8b62
6 files changed +114 -59
tests/async_engine/test_chat_template.py
+3 -2
@@ -1,6 +1,7 @@
1 import pytest
2
3 -from vllm.entrypoints.chat_utils import apply_chat_template, load_chat_template
3 +from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
4 + load_chat_template)
5 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
6 from vllm.transformers_utils.tokenizer import get_tokenizer
7
@@ -87,7 +88,7 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
88 add_generation_prompt=add_generation_prompt)
89
90 # Call the function and get the result
90 - result = apply_chat_template(
91 + result = apply_hf_chat_template(
92 tokenizer,
93 conversation=mock_request.messages,
94 chat_template=mock_request.chat_template or template_content,
vllm/entrypoints/chat_utils.py
+41 -20
@@ -23,6 +23,7 @@ from openai.types.chat import (ChatCompletionMessageToolCallParam,
23 # yapf: enable
24 # pydantic needs the TypedDict from typing_extensions
25 from pydantic import ConfigDict
26 +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
27 from typing_extensions import Required, TypeAlias, TypedDict
28
29 from vllm.config import ModelConfig
@@ -31,7 +32,7 @@ from vllm.multimodal import MultiModalDataDict
32 from vllm.multimodal.utils import (async_get_and_parse_audio,
33 async_get_and_parse_image,
34 get_and_parse_audio, get_and_parse_image)
34 -from vllm.transformers_utils.tokenizer import AnyTokenizer
35 +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
36
37 logger = init_logger(__name__)
38
@@ -379,6 +380,9 @@ def _parse_chat_message_content_parts(
380 audio_url = _AudioParser(part)["audio_url"]
381
382 mm_parser.parse_audio(audio_url["url"])
383 + elif part_type == "refusal":
384 + text = _RefusalParser(part)["refusal"]
385 + texts.append(text)
386 else:
387 raise NotImplementedError(f"Unknown part type: {part_type}")
388
@@ -433,6 +437,21 @@ def _parse_chat_message_content(
437 return result
438
439
440 +def _postprocess_messages(messages: List[ConversationMessage]) -> None:
441 + # per the Transformers docs & maintainers, tool call arguments in
442 + # assistant-role messages with tool_calls need to be dicts not JSON str -
443 + # this is how tool-use chat templates will expect them moving forwards
444 + # so, for messages that have tool_calls, parse the string (which we get
445 + # from openAI format) to dict
446 + for message in messages:
447 + if (message["role"] == "assistant" and "tool_calls" in message
448 + and isinstance(message["tool_calls"], list)):
449 +
450 + for item in message["tool_calls"]:
451 + item["function"]["arguments"] = json.loads(
452 + item["function"]["arguments"])
453 +
454 +
455 def parse_chat_messages(
456 messages: List[ChatCompletionMessageParam],
457 model_config: ModelConfig,
@@ -446,6 +465,8 @@ def parse_chat_messages(
465
466 conversation.extend(sub_messages)
467
468 + _postprocess_messages(conversation)
469 +
470 return conversation, mm_tracker.all_mm_data()
471
472
@@ -462,41 +483,41 @@ def parse_chat_messages_futures(
483
484 conversation.extend(sub_messages)
485
486 + _postprocess_messages(conversation)
487 +
488 return conversation, mm_tracker.all_mm_data()
489
490
468 -def apply_chat_template(
469 - tokenizer: AnyTokenizer,
491 +def apply_hf_chat_template(
492 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
493 conversation: List[ConversationMessage],
494 chat_template: Optional[str],
495 *,
496 tokenize: bool = False, # Different from HF's default
497 **kwargs: Any,
475 -) -> Union[str, List[int]]:
498 +) -> str:
499 if chat_template is None and tokenizer.chat_template is None:
500 raise ValueError(
501 "As of transformers v4.44, default chat template is no longer "
502 "allowed, so you must provide a chat template if the tokenizer "
503 "does not define one.")
504
482 - # per the Transformers docs & maintainers, tool call arguments in
483 - # assistant-role messages with tool_calls need to be dicts not JSON str -
484 - # this is how tool-use chat templates will expect them moving forwards
485 - # so, for messages that have tool_calls, parse the string (which we get
486 - # from openAI format) to dict
487 - for message in conversation:
488 - if (message["role"] == "assistant" and "tool_calls" in message
489 - and isinstance(message["tool_calls"], list)):
505 + return tokenizer.apply_chat_template(
506 + conversation=conversation, # type: ignore[arg-type]
507 + chat_template=chat_template,
508 + tokenize=tokenize,
509 + **kwargs,
510 + )
511
491 - for i in range(len(message["tool_calls"])):
492 - args: str = message["tool_calls"][i]["function"]["arguments"]
493 - parsed_args: Dict = json.loads(args)
494 - message["tool_calls"][i]["function"]["arguments"] = parsed_args
512
496 - prompt = tokenizer.apply_chat_template(
497 - conversation=conversation,
513 +def apply_mistral_chat_template(
514 + tokenizer: MistralTokenizer,
515 + messages: List[ChatCompletionMessageParam],
516 + chat_template: Optional[str],
517 + **kwargs: Any,
518 +) -> List[int]:
519 + return tokenizer.apply_chat_template(
520 + messages=messages,
521 chat_template=chat_template,
499 - tokenize=tokenize,
522 **kwargs,
523 )
502 - return prompt
vllm/entrypoints/llm.py
+18 -8
@@ -6,7 +6,8 @@ from tqdm import tqdm
6 from vllm.engine.arg_utils import EngineArgs
7 from vllm.engine.llm_engine import LLMEngine
8 from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
9 - apply_chat_template,
9 + apply_hf_chat_template,
10 + apply_mistral_chat_template,
11 parse_chat_messages)
12 from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
13 from vllm.inputs.parse import parse_and_batch_prompt
@@ -19,7 +20,7 @@ from vllm.outputs import EmbeddingRequestOutput, RequestOutput
20 from vllm.pooling_params import PoolingParams
21 from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sampling_params import SamplingParams
22 -from vllm.transformers_utils.tokenizer import (AnyTokenizer,
23 +from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
24 get_cached_tokenizer)
25 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
26 from vllm.usage.usage_lib import UsageContext
@@ -393,12 +394,21 @@ class LLM:
394 conversation, mm_data = parse_chat_messages(messages, model_config,
395 tokenizer)
396
396 - prompt = apply_chat_template(
397 - tokenizer,
398 - conversation,
399 - chat_template=chat_template,
400 - add_generation_prompt=add_generation_prompt,
401 - )
397 + prompt: Union[str, List[int]]
398 + if isinstance(tokenizer, MistralTokenizer):
399 + prompt = apply_mistral_chat_template(
400 + tokenizer,
401 + messages=messages,
402 + chat_template=chat_template,
403 + add_generation_prompt=add_generation_prompt,
404 + )
405 + else:
406 + prompt = apply_hf_chat_template(
407 + tokenizer,
408 + conversation=conversation,
409 + chat_template=chat_template,
410 + add_generation_prompt=add_generation_prompt,
411 + )
412
413 inputs: PromptInputs
414 if is_list_of(prompt, int):
vllm/entrypoints/openai/serving_chat.py
+30 -18
@@ -11,7 +11,8 @@ from fastapi import Request
11 from vllm.config import ModelConfig
12 from vllm.engine.protocol import AsyncEngineClient
13 from vllm.entrypoints.chat_utils import (ConversationMessage,
14 - apply_chat_template,
14 + apply_hf_chat_template,
15 + apply_mistral_chat_template,
16 load_chat_template,
17 parse_chat_messages_futures)
18 from vllm.entrypoints.logger import RequestLogger
@@ -35,7 +36,7 @@ from vllm.outputs import CompletionOutput, RequestOutput
36 from vllm.sequence import Logprob
37 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
38 log_tracing_disabled_warning)
38 -from vllm.transformers_utils.tokenizer import AnyTokenizer
39 +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
40 from vllm.utils import iterate_with_cancellation, random_uuid
41
42 logger = init_logger(__name__)
@@ -121,15 +122,27 @@ class OpenAIServingChat(OpenAIServing):
122 tool.model_dump() for tool in request.tools
123 ]
124
124 - prompt = apply_chat_template(
125 - tokenizer,
126 - conversation=conversation,
127 - chat_template=request.chat_template or self.chat_template,
128 - add_generation_prompt=request.add_generation_prompt,
129 - tools=tool_dicts,
130 - documents=request.documents,
131 - **(request.chat_template_kwargs or {}),
132 - )
125 + prompt: Union[str, List[int]]
126 + if isinstance(tokenizer, MistralTokenizer):
127 + prompt = apply_mistral_chat_template(
128 + tokenizer,
129 + messages=request.messages,
130 + chat_template=request.chat_template or self.chat_template,
131 + add_generation_prompt=request.add_generation_prompt,
132 + tools=tool_dicts,
133 + documents=request.documents,
134 + **(request.chat_template_kwargs or {}),
135 + )
136 + else:
137 + prompt = apply_hf_chat_template(
138 + tokenizer,
139 + conversation=conversation,
140 + chat_template=request.chat_template or self.chat_template,
141 + add_generation_prompt=request.add_generation_prompt,
142 + tools=tool_dicts,
143 + documents=request.documents,
144 + **(request.chat_template_kwargs or {}),
145 + )
146 except Exception as e:
147 logger.error("Error in applying chat template from request: %s", e)
148 return self.create_error_response(str(e))
@@ -307,11 +320,10 @@ class OpenAIServingChat(OpenAIServing):
320 # Send response to echo the input portion of the
321 # last message
322 if request.echo:
310 - last_msg_content: Optional[str] = ""
311 - if conversation and conversation[-1].get(
312 - "content") and conversation[-1].get(
313 - "role") == role:
314 - last_msg_content = conversation[-1]["content"]
323 + last_msg_content: str = ""
324 + if conversation and "content" in conversation[
325 + -1] and conversation[-1].get("role") == role:
326 + last_msg_content = conversation[-1]["content"] or ""
327
328 if last_msg_content:
329 for i in range(num_choices):
@@ -659,8 +671,8 @@ class OpenAIServingChat(OpenAIServing):
671
672 if request.echo:
673 last_msg_content = ""
662 - if conversation and conversation[-1].get(
663 - "content") and conversation[-1].get("role") == role:
674 + if conversation and "content" in conversation[-1] and conversation[
675 + -1].get("role") == role:
676 last_msg_content = conversation[-1]["content"] or ""
677
678 for choice in choices:
vllm/entrypoints/openai/serving_tokenization.py
+18 -7
@@ -2,7 +2,8 @@ from typing import List, Optional, Union
2
3 from vllm.config import ModelConfig
4 from vllm.engine.protocol import AsyncEngineClient
5 -from vllm.entrypoints.chat_utils import (apply_chat_template,
5 +from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
6 + apply_mistral_chat_template,
7 load_chat_template,
8 parse_chat_messages_futures)
9 from vllm.entrypoints.logger import RequestLogger
@@ -18,6 +19,7 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
19 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
20 OpenAIServing)
21 from vllm.logger import init_logger
22 +from vllm.transformers_utils.tokenizer import MistralTokenizer
23 from vllm.utils import random_uuid
24
25 logger = init_logger(__name__)
@@ -66,6 +68,7 @@ class OpenAIServingTokenization(OpenAIServing):
68
69 tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
70
71 + prompt: Union[str, List[int]]
72 if isinstance(request, TokenizeChatRequest):
73 model_config = self.model_config
74
@@ -77,12 +80,20 @@ class OpenAIServingTokenization(OpenAIServing):
80 logger.warning(
81 "Multi-modal inputs are ignored during tokenization")
82
80 - prompt = apply_chat_template(
81 - tokenizer,
82 - conversation=conversation,
83 - chat_template=self.chat_template,
84 - add_generation_prompt=request.add_generation_prompt,
85 - )
83 + if isinstance(tokenizer, MistralTokenizer):
84 + prompt = apply_mistral_chat_template(
85 + tokenizer,
86 + messages=request.messages,
87 + chat_template=self.chat_template,
88 + add_generation_prompt=request.add_generation_prompt,
89 + )
90 + else:
91 + prompt = apply_hf_chat_template(
92 + tokenizer,
93 + conversation=conversation,
94 + chat_template=self.chat_template,
95 + add_generation_prompt=request.add_generation_prompt,
96 + )
97 else:
98 prompt = request.prompt
99
vllm/transformers_utils/tokenizers/mistral.py
+4 -4
@@ -16,7 +16,7 @@ from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
16 Tekkenizer)
17
18 if TYPE_CHECKING:
19 - from vllm.entrypoints.chat_utils import ConversationMessage
19 + from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
20
21
22 @dataclass
@@ -122,19 +122,19 @@ class MistralTokenizer:
122 return []
123
124 def encode(self, prompt: str) -> List[int]:
125 - # `encode ` should only be used for prompt completion
125 + # `encode` should only be used for prompt completion
126 # it should never be used for chat_completion.
127 # For chat completion use `apply_chat_template`
128 return self.tokenizer.encode(prompt, bos=True, eos=False)
129
130 def apply_chat_template(self,
131 - conversation: List["ConversationMessage"],
131 + messages: List["ChatCompletionMessageParam"],
132 tools: Optional[Dict[str, Any]] = None,
133 **kwargs) -> List[int]:
134 assert tools is None, "`tools` are not yet supported."
135
136 request = ChatCompletionRequest(
137 - messages=conversation) # type: ignore[type-var]
137 + messages=messages) # type: ignore[type-var]
138 encoded = self.mistral.encode_chat_completion(request)
139
140 # encode-decode to get clean prompt