| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import asyncio |
| 4 | from dataclasses import dataclass |
| 5 | from typing import Any |
| 6 | |
| 7 | import pytest |
| 8 | import torch |
| 9 | |
| 10 | from vllm.renderers.base import _SwappableExecutor |
| 11 | from vllm.renderers.hf import HfRenderer |
| 12 | from vllm.renderers.params import TokenizeParams |
| 13 | from vllm.utils.async_utils import make_async |
| 14 | |
| 15 | MODEL_NAME = "openai-community/gpt2" |
| 16 | |
| 17 | |
| 18 | @dataclass |
| 19 | class MockHFConfig: |
| 20 | model_type: str = "any" |
| 21 | |
| 22 | |
| 23 | @dataclass |
| 24 | class MockModelConfig: |
| 25 | runner_type = "generate" |
| 26 | model: str = MODEL_NAME |
| 27 | tokenizer: str = MODEL_NAME |
| 28 | trust_remote_code: bool = False |
| 29 | tokenizer_revision = None |
| 30 | tokenizer_mode = "auto" |
| 31 | hf_config = MockHFConfig() |
| 32 | encoder_config: dict[str, Any] | None = None |
| 33 | enable_prompt_embeds: bool = False |
| 34 | skip_tokenizer_init: bool = False |
| 35 | is_encoder_decoder: bool = False |
| 36 | is_multimodal_model: bool = False |
| 37 | supports_multimodal_inputs: bool = False |
| 38 | renderer_num_workers: int = 1 |
| 39 | hidden_size: int = 768 |
| 40 | dtype: torch.dtype = torch.float32 |
| 41 | |
| 42 | def get_hidden_size(self) -> int: |
| 43 | return self.hidden_size |
| 44 | |
| 45 | |
| 46 | @dataclass |
| 47 | class MockParallelConfig: |
| 48 | _api_process_rank: int = 0 |
| 49 | |
| 50 | |
| 51 | @dataclass |
| 52 | class MockVllmConfig: |
| 53 | model_config: MockModelConfig |
| 54 | parallel_config: MockParallelConfig |
| 55 | |
| 56 | |
| 57 | @dataclass |
| 58 | class DummyTokenizer: |
| 59 | truncation_side: str = "left" |
| 60 | max_chars_per_token: int = 1 |
| 61 | is_fast: bool = False |
| 62 | |
| 63 | def decode(self, tokens: list[int], **kwargs): |
| 64 | return str(tokens) |
| 65 | |
| 66 | def encode(self, text: str, **kwargs): |
| 67 | return list(range(len(text))) |
| 68 | |
| 69 | def __call__(self, text: str, **kwargs): |
| 70 | return {"input_ids": self.encode(text, **kwargs)} |
| 71 | |
| 72 | |
| 73 | def _build_renderer() -> HfRenderer: |
| 74 | return HfRenderer( |
| 75 | MockVllmConfig(MockModelConfig(), parallel_config=MockParallelConfig()), |
| 76 | tokenizer=DummyTokenizer(), |
| 77 | ) |
| 78 | |
| 79 | |
| 80 | def test_swappable_executor_keeps_make_async_wrappers_alive(): |
| 81 | pool = _SwappableExecutor(max_workers=1) |
| 82 | async_add = make_async(lambda x: x + 1, executor=pool) |
| 83 | |
| 84 | async def _run(): |
| 85 | assert await async_add(1) == 2 |
| 86 | old_inner = pool._inner |
| 87 | pool.replace_inner() |
| 88 | |
| 89 | with pytest.raises(RuntimeError, match="cannot schedule new futures"): |
| 90 | old_inner.submit(lambda: None) |
| 91 | |
| 92 | assert await async_add(40) == 41 |
| 93 | |
| 94 | try: |
| 95 | asyncio.run(_run()) |
| 96 | finally: |
| 97 | pool.shutdown(wait=False) |
| 98 | |
| 99 | |
| 100 | def test_replace_executor_does_not_break_tokenize_or_decode(): |
| 101 | renderer = _build_renderer() |
| 102 | executor = renderer._executor |
| 103 | old_inner = executor._inner |
| 104 | |
| 105 | async def _run(): |
| 106 | assert await renderer._tokenize_prompt_async( |
| 107 | {"prompt": "ab"}, |
| 108 | TokenizeParams(max_total_tokens=100), |
| 109 | ) |
| 110 | renderer._replace_executor() |
| 111 | assert renderer._executor is executor |
| 112 | assert executor._inner is not old_inner |
| 113 | |
| 114 | with pytest.raises(RuntimeError, match="cannot schedule new futures"): |
| 115 | old_inner.submit(lambda: None) |
| 116 | |
| 117 | tokenized = await renderer._tokenize_prompt_async( |
| 118 | {"prompt": "abc"}, |
| 119 | TokenizeParams(max_total_tokens=100), |
| 120 | ) |
| 121 | assert tokenized["prompt_token_ids"] == [0, 1, 2] |
| 122 | assert await renderer._async_tokenizer_decode([1, 2]) == "[1, 2]" |
| 123 | |
| 124 | try: |
| 125 | asyncio.run(_run()) |
| 126 | finally: |
| 127 | renderer.shutdown() |