A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Frontend] Clean up type annotations for mistral tokenizer (#8314)
Cyrus Leung committed
Sep 11, 2024 at 00:49 UTC
8c054b7a6290551c868451dfd449d40cf37d8b62
6 files changed
+114
-59
tests/async_engine/test_chat_template.py
+3
-2
@@ -1,6 +1,7 @@
1
import pytest
2
3
-from vllm.entrypoints.chat_utils import apply_chat_template, load_chat_template
3
+from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
4
+ load_chat_template)
5
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
6
from vllm.transformers_utils.tokenizer import get_tokenizer
7
@@ -87,7 +88,7 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
88
add_generation_prompt=add_generation_prompt)
89
90
# Call the function and get the result
90
- result = apply_chat_template(
91
+ result = apply_hf_chat_template(
92
tokenizer,
93
conversation=mock_request.messages,
94
chat_template=mock_request.chat_template or template_content,
vllm/entrypoints/chat_utils.py
+41
-20
@@ -23,6 +23,7 @@ from openai.types.chat import (ChatCompletionMessageToolCallParam,
23
# yapf: enable
24
# pydantic needs the TypedDict from typing_extensions
25
from pydantic import ConfigDict
26
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
27
from typing_extensions import Required, TypeAlias, TypedDict
28
29
from vllm.config import ModelConfig
@@ -31,7 +32,7 @@ from vllm.multimodal import MultiModalDataDict
32
from vllm.multimodal.utils import (async_get_and_parse_audio,
33
async_get_and_parse_image,
34
get_and_parse_audio, get_and_parse_image)
34
-from vllm.transformers_utils.tokenizer import AnyTokenizer
35
+from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
36
37
logger = init_logger(__name__)
38
@@ -379,6 +380,9 @@ def _parse_chat_message_content_parts(
380
audio_url = _AudioParser(part)["audio_url"]
381
382
mm_parser.parse_audio(audio_url["url"])
383
+ elif part_type == "refusal":
384
+ text = _RefusalParser(part)["refusal"]
385
+ texts.append(text)
386
else:
387
raise NotImplementedError(f"Unknown part type: {part_type}")
388
@@ -433,6 +437,21 @@ def _parse_chat_message_content(
437
return result
438
439
440
+def _postprocess_messages(messages: List[ConversationMessage]) -> None:
441
+ # per the Transformers docs & maintainers, tool call arguments in
442
+ # assistant-role messages with tool_calls need to be dicts not JSON str -
443
+ # this is how tool-use chat templates will expect them moving forwards
444
+ # so, for messages that have tool_calls, parse the string (which we get
445
+ # from openAI format) to dict
446
+ for message in messages:
447
+ if (message["role"] == "assistant" and "tool_calls" in message
448
+ and isinstance(message["tool_calls"], list)):
449
+
450
+ for item in message["tool_calls"]:
451
+ item["function"]["arguments"] = json.loads(
452
+ item["function"]["arguments"])
453
+
454
+
455
def parse_chat_messages(
456
messages: List[ChatCompletionMessageParam],
457
model_config: ModelConfig,
@@ -446,6 +465,8 @@ def parse_chat_messages(
465
466
conversation.extend(sub_messages)
467
468
+ _postprocess_messages(conversation)
469
+
470
return conversation, mm_tracker.all_mm_data()
471
472
@@ -462,41 +483,41 @@ def parse_chat_messages_futures(
483
484
conversation.extend(sub_messages)
485
486
+ _postprocess_messages(conversation)
487
+
488
return conversation, mm_tracker.all_mm_data()
489
490
468
-def apply_chat_template(
469
- tokenizer: AnyTokenizer,
491
+def apply_hf_chat_template(
492
+ tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
493
conversation: List[ConversationMessage],
494
chat_template: Optional[str],
495
*,
496
tokenize: bool = False, # Different from HF's default
497
**kwargs: Any,
475
-) -> Union[str, List[int]]:
498
+) -> str:
499
if chat_template is None and tokenizer.chat_template is None:
500
raise ValueError(
501
"As of transformers v4.44, default chat template is no longer "
502
"allowed, so you must provide a chat template if the tokenizer "
503
"does not define one.")
504
482
- # per the Transformers docs & maintainers, tool call arguments in
483
- # assistant-role messages with tool_calls need to be dicts not JSON str -
484
- # this is how tool-use chat templates will expect them moving forwards
485
- # so, for messages that have tool_calls, parse the string (which we get
486
- # from openAI format) to dict
487
- for message in conversation:
488
- if (message["role"] == "assistant" and "tool_calls" in message
489
- and isinstance(message["tool_calls"], list)):
505
+ return tokenizer.apply_chat_template(
506
+ conversation=conversation, # type: ignore[arg-type]
507
+ chat_template=chat_template,
508
+ tokenize=tokenize,
509
+ **kwargs,
510
+ )
511
491
- for i in range(len(message["tool_calls"])):
492
- args: str = message["tool_calls"][i]["function"]["arguments"]
493
- parsed_args: Dict = json.loads(args)
494
- message["tool_calls"][i]["function"]["arguments"] = parsed_args
512
496
- prompt = tokenizer.apply_chat_template(
497
- conversation=conversation,
513
+def apply_mistral_chat_template(
514
+ tokenizer: MistralTokenizer,
515
+ messages: List[ChatCompletionMessageParam],
516
+ chat_template: Optional[str],
517
+ **kwargs: Any,
518
+) -> List[int]:
519
+ return tokenizer.apply_chat_template(
520
+ messages=messages,
521
chat_template=chat_template,
499
- tokenize=tokenize,
522
**kwargs,
523
)
502
- return prompt
vllm/entrypoints/llm.py
+18
-8
@@ -6,7 +6,8 @@ from tqdm import tqdm
6
from vllm.engine.arg_utils import EngineArgs
7
from vllm.engine.llm_engine import LLMEngine
8
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
9
- apply_chat_template,
9
+ apply_hf_chat_template,
10
+ apply_mistral_chat_template,
11
parse_chat_messages)
12
from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
13
from vllm.inputs.parse import parse_and_batch_prompt
@@ -19,7 +20,7 @@ from vllm.outputs import EmbeddingRequestOutput, RequestOutput
20
from vllm.pooling_params import PoolingParams
21
from vllm.prompt_adapter.request import PromptAdapterRequest
22
from vllm.sampling_params import SamplingParams
22
-from vllm.transformers_utils.tokenizer import (AnyTokenizer,
23
+from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
24
get_cached_tokenizer)
25
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
26
from vllm.usage.usage_lib import UsageContext
@@ -393,12 +394,21 @@ class LLM:
394
conversation, mm_data = parse_chat_messages(messages, model_config,
395
tokenizer)
396
396
- prompt = apply_chat_template(
397
- tokenizer,
398
- conversation,
399
- chat_template=chat_template,
400
- add_generation_prompt=add_generation_prompt,
401
- )
397
+ prompt: Union[str, List[int]]
398
+ if isinstance(tokenizer, MistralTokenizer):
399
+ prompt = apply_mistral_chat_template(
400
+ tokenizer,
401
+ messages=messages,
402
+ chat_template=chat_template,
403
+ add_generation_prompt=add_generation_prompt,
404
+ )
405
+ else:
406
+ prompt = apply_hf_chat_template(
407
+ tokenizer,
408
+ conversation=conversation,
409
+ chat_template=chat_template,
410
+ add_generation_prompt=add_generation_prompt,
411
+ )
412
413
inputs: PromptInputs
414
if is_list_of(prompt, int):
vllm/entrypoints/openai/serving_chat.py
+30
-18
@@ -11,7 +11,8 @@ from fastapi import Request
11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import AsyncEngineClient
13
from vllm.entrypoints.chat_utils import (ConversationMessage,
14
- apply_chat_template,
14
+ apply_hf_chat_template,
15
+ apply_mistral_chat_template,
16
load_chat_template,
17
parse_chat_messages_futures)
18
from vllm.entrypoints.logger import RequestLogger
@@ -35,7 +36,7 @@ from vllm.outputs import CompletionOutput, RequestOutput
36
from vllm.sequence import Logprob
37
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
38
log_tracing_disabled_warning)
38
-from vllm.transformers_utils.tokenizer import AnyTokenizer
39
+from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
40
from vllm.utils import iterate_with_cancellation, random_uuid
41
42
logger = init_logger(__name__)
@@ -121,15 +122,27 @@ class OpenAIServingChat(OpenAIServing):
122
tool.model_dump() for tool in request.tools
123
]
124
124
- prompt = apply_chat_template(
125
- tokenizer,
126
- conversation=conversation,
127
- chat_template=request.chat_template or self.chat_template,
128
- add_generation_prompt=request.add_generation_prompt,
129
- tools=tool_dicts,
130
- documents=request.documents,
131
- **(request.chat_template_kwargs or {}),
132
- )
125
+ prompt: Union[str, List[int]]
126
+ if isinstance(tokenizer, MistralTokenizer):
127
+ prompt = apply_mistral_chat_template(
128
+ tokenizer,
129
+ messages=request.messages,
130
+ chat_template=request.chat_template or self.chat_template,
131
+ add_generation_prompt=request.add_generation_prompt,
132
+ tools=tool_dicts,
133
+ documents=request.documents,
134
+ **(request.chat_template_kwargs or {}),
135
+ )
136
+ else:
137
+ prompt = apply_hf_chat_template(
138
+ tokenizer,
139
+ conversation=conversation,
140
+ chat_template=request.chat_template or self.chat_template,
141
+ add_generation_prompt=request.add_generation_prompt,
142
+ tools=tool_dicts,
143
+ documents=request.documents,
144
+ **(request.chat_template_kwargs or {}),
145
+ )
146
except Exception as e:
147
logger.error("Error in applying chat template from request: %s", e)
148
return self.create_error_response(str(e))
@@ -307,11 +320,10 @@ class OpenAIServingChat(OpenAIServing):
320
# Send response to echo the input portion of the
321
# last message
322
if request.echo:
310
- last_msg_content: Optional[str] = ""
311
- if conversation and conversation[-1].get(
312
- "content") and conversation[-1].get(
313
- "role") == role:
314
- last_msg_content = conversation[-1]["content"]
323
+ last_msg_content: str = ""
324
+ if conversation and "content" in conversation[
325
+ -1] and conversation[-1].get("role") == role:
326
+ last_msg_content = conversation[-1]["content"] or ""
327
328
if last_msg_content:
329
for i in range(num_choices):
@@ -659,8 +671,8 @@ class OpenAIServingChat(OpenAIServing):
671
672
if request.echo:
673
last_msg_content = ""
662
- if conversation and conversation[-1].get(
663
- "content") and conversation[-1].get("role") == role:
674
+ if conversation and "content" in conversation[-1] and conversation[
675
+ -1].get("role") == role:
676
last_msg_content = conversation[-1]["content"] or ""
677
678
for choice in choices:
vllm/entrypoints/openai/serving_tokenization.py
+18
-7
@@ -2,7 +2,8 @@ from typing import List, Optional, Union
2
3
from vllm.config import ModelConfig
4
from vllm.engine.protocol import AsyncEngineClient
5
-from vllm.entrypoints.chat_utils import (apply_chat_template,
5
+from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
6
+ apply_mistral_chat_template,
7
load_chat_template,
8
parse_chat_messages_futures)
9
from vllm.entrypoints.logger import RequestLogger
@@ -18,6 +19,7 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
19
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
20
OpenAIServing)
21
from vllm.logger import init_logger
22
+from vllm.transformers_utils.tokenizer import MistralTokenizer
23
from vllm.utils import random_uuid
24
25
logger = init_logger(__name__)
@@ -66,6 +68,7 @@ class OpenAIServingTokenization(OpenAIServing):
68
69
tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
70
71
+ prompt: Union[str, List[int]]
72
if isinstance(request, TokenizeChatRequest):
73
model_config = self.model_config
74
@@ -77,12 +80,20 @@ class OpenAIServingTokenization(OpenAIServing):
80
logger.warning(
81
"Multi-modal inputs are ignored during tokenization")
82
80
- prompt = apply_chat_template(
81
- tokenizer,
82
- conversation=conversation,
83
- chat_template=self.chat_template,
84
- add_generation_prompt=request.add_generation_prompt,
85
- )
83
+ if isinstance(tokenizer, MistralTokenizer):
84
+ prompt = apply_mistral_chat_template(
85
+ tokenizer,
86
+ messages=request.messages,
87
+ chat_template=self.chat_template,
88
+ add_generation_prompt=request.add_generation_prompt,
89
+ )
90
+ else:
91
+ prompt = apply_hf_chat_template(
92
+ tokenizer,
93
+ conversation=conversation,
94
+ chat_template=self.chat_template,
95
+ add_generation_prompt=request.add_generation_prompt,
96
+ )
97
else:
98
prompt = request.prompt
99
vllm/transformers_utils/tokenizers/mistral.py
+4
-4
@@ -16,7 +16,7 @@ from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
16
Tekkenizer)
17
18
if TYPE_CHECKING:
19
- from vllm.entrypoints.chat_utils import ConversationMessage
19
+ from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
20
21
22
@dataclass
@@ -122,19 +122,19 @@ class MistralTokenizer:
122
return []
123
124
def encode(self, prompt: str) -> List[int]:
125
- # `encode ` should only be used for prompt completion
125
+ # `encode` should only be used for prompt completion
126
# it should never be used for chat_completion.
127
# For chat completion use `apply_chat_template`
128
return self.tokenizer.encode(prompt, bos=True, eos=False)
129
130
def apply_chat_template(self,
131
- conversation: List["ConversationMessage"],
131
+ messages: List["ChatCompletionMessageParam"],
132
tools: Optional[Dict[str, Any]] = None,
133
**kwargs) -> List[int]:
134
assert tools is None, "`tools` are not yet supported."
135
136
request = ChatCompletionRequest(
137
- messages=conversation) # type: ignore[type-var]
137
+ messages=messages) # type: ignore[type-var]
138
encoded = self.mistral.encode_chat_completion(request)
139
140
# encode-decode to get clean prompt