| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import asyncio |
| 5 | import time |
| 6 | from dataclasses import dataclass |
| 7 | from typing import Any |
| 8 | from unittest.mock import Mock |
| 9 | |
| 10 | import pytest |
| 11 | from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy |
| 12 | |
| 13 | from vllm.renderers import ChatParams |
| 14 | from vllm.renderers.mistral import MistralRenderer, safe_apply_chat_template |
| 15 | from vllm.tokenizers.mistral import MistralTokenizer |
| 16 | |
| 17 | MODEL_NAME = "mistralai/Mistral-7B-Instruct-v0.3" |
| 18 | |
| 19 | |
| 20 | @dataclass |
| 21 | class MockHFConfig: |
| 22 | model_type: str = "any" |
| 23 | |
| 24 | |
| 25 | @dataclass |
| 26 | class MockModelConfig: |
| 27 | runner_type = "generate" |
| 28 | model: str = MODEL_NAME |
| 29 | tokenizer: str = MODEL_NAME |
| 30 | trust_remote_code: bool = False |
| 31 | max_model_len: int = 100 |
| 32 | tokenizer_revision = None |
| 33 | tokenizer_mode = "mistral" |
| 34 | hf_config = MockHFConfig() |
| 35 | encoder_config: dict[str, Any] | None = None |
| 36 | enable_prompt_embeds: bool = True |
| 37 | skip_tokenizer_init: bool = False |
| 38 | is_encoder_decoder: bool = False |
| 39 | is_multimodal_model: bool = False |
| 40 | supports_multimodal_inputs: bool = False |
| 41 | renderer_num_workers: int = 1 |
| 42 | |
| 43 | |
| 44 | @dataclass |
| 45 | class MockParallelConfig: |
| 46 | _api_process_rank: int = 0 |
| 47 | |
| 48 | |
| 49 | @dataclass |
| 50 | class MockVllmConfig: |
| 51 | model_config: MockModelConfig |
| 52 | parallel_config: MockParallelConfig |
| 53 | |
| 54 | |
| 55 | @pytest.mark.asyncio |
| 56 | async def test_async_mistral_tokenizer_does_not_block_event_loop(): |
| 57 | expected_tokens = [1, 2, 3] |
| 58 | |
| 59 | # Mock the blocking version to sleep |
| 60 | def mocked_apply_chat_template(*_args, **_kwargs): |
| 61 | time.sleep(2) |
| 62 | return expected_tokens |
| 63 | |
| 64 | mock_model_config = MockModelConfig(skip_tokenizer_init=True) |
| 65 | mock_tokenizer = Mock(spec=MistralTokenizer) |
| 66 | mock_tokenizer.apply_chat_template = mocked_apply_chat_template |
| 67 | mock_renderer = MistralRenderer( |
| 68 | MockVllmConfig(mock_model_config, parallel_config=MockParallelConfig()), |
| 69 | tokenizer=mock_tokenizer, |
| 70 | ) |
| 71 | |
| 72 | task = mock_renderer.render_messages_async([], ChatParams()) |
| 73 | |
| 74 | # Ensure the event loop is not blocked |
| 75 | blocked_count = 0 |
| 76 | for _i in range(20): # Check over ~2 seconds |
| 77 | start = time.perf_counter() |
| 78 | await asyncio.sleep(0) |
| 79 | elapsed = time.perf_counter() - start |
| 80 | |
| 81 | # an overly generous elapsed time for slow machines |
| 82 | if elapsed >= 0.5: |
| 83 | blocked_count += 1 |
| 84 | |
| 85 | await asyncio.sleep(0.1) |
| 86 | |
| 87 | # Ensure task completes |
| 88 | _, prompt = await task |
| 89 | assert prompt["prompt_token_ids"] == expected_tokens, ( |
| 90 | "Mocked blocking tokenizer was not called" |
| 91 | ) |
| 92 | assert blocked_count == 0, "Event loop blocked during tokenization" |
| 93 | |
| 94 | |
| 95 | def test_apply_mistral_chat_template_thinking_chunk(): |
| 96 | messages = [ |
| 97 | { |
| 98 | "role": "system", |
| 99 | "content": [ |
| 100 | {"type": "text", "text": "You are a helpful assistant."}, |
| 101 | { |
| 102 | "type": "thinking", |
| 103 | "closed": True, |
| 104 | "thinking": "Only return the answer when you are confident.", |
| 105 | }, |
| 106 | ], |
| 107 | }, |
| 108 | {"role": "user", "content": "What is 2+2?"}, |
| 109 | { |
| 110 | "role": "assistant", |
| 111 | "content": [ |
| 112 | {"type": "text", "text": "Let me think about it."}, |
| 113 | {"type": "thinking", "closed": True, "thinking": "2+2 = 4"}, |
| 114 | { |
| 115 | "type": "text", |
| 116 | "text": "The answer is 4.", |
| 117 | }, |
| 118 | ], |
| 119 | }, |
| 120 | {"role": "user", "content": "Thanks, what is 3+3?"}, |
| 121 | ] |
| 122 | mistral_tokenizer = MistralTokenizer.from_pretrained( |
| 123 | "mistralai/Magistral-Small-2509" |
| 124 | ) |
| 125 | |
| 126 | tokens_ids = safe_apply_chat_template( |
| 127 | mistral_tokenizer, messages, chat_template=None, tools=None |
| 128 | ) |
| 129 | |
| 130 | string_tokens = mistral_tokenizer.mistral.decode( |
| 131 | tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP |
| 132 | ) |
| 133 | |
| 134 | expected_tokens = ( |
| 135 | r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the" |
| 136 | r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]" |
| 137 | r"[INST]What is 2+2?[/INST]" |
| 138 | r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>" |
| 139 | r"[INST]Thanks, what is 3+3?[/INST]" |
| 140 | ) |
| 141 | |
| 142 | assert string_tokens == expected_tokens |