| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import random |
| 4 | from typing import TYPE_CHECKING |
| 5 | from unittest.mock import MagicMock, call |
| 6 | |
| 7 | import pytest |
| 8 | |
| 9 | from vllm import LLM |
| 10 | from vllm.sampling_params import SamplingParams, StructuredOutputsParams |
| 11 | from vllm.v1.engine.llm_engine import LLMEngine |
| 12 | from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector |
| 13 | |
| 14 | if TYPE_CHECKING: |
| 15 | from tests.conftest import VllmRunner |
| 16 | else: |
| 17 | VllmRunner = object |
| 18 | |
| 19 | MODEL = "facebook/opt-125m" |
| 20 | DTYPE = "half" |
| 21 | |
| 22 | |
| 23 | @pytest.mark.skip_global_cleanup |
| 24 | def test_step_invalidates_mm_processor_cache_misses(): |
| 25 | engine = object.__new__(LLMEngine) |
| 26 | engine.should_execute_dummy_batch = False |
| 27 | engine.log_stats = False |
| 28 | engine.logger_manager = None |
| 29 | |
| 30 | mm_processor_cache = MagicMock() |
| 31 | engine.renderer = MagicMock(mm_processor_cache=mm_processor_cache) |
| 32 | |
| 33 | engine_core_output = MagicMock( |
| 34 | mm_cache_miss_hashes=["missing-hash-1", "missing-hash-2"] |
| 35 | ) |
| 36 | outputs = MagicMock( |
| 37 | outputs=[engine_core_output], |
| 38 | timestamp=0.0, |
| 39 | scheduler_stats=None, |
| 40 | ) |
| 41 | engine.engine_core = MagicMock() |
| 42 | engine.engine_core.get_output.return_value = outputs |
| 43 | |
| 44 | processed_outputs = MagicMock(reqs_to_abort=[], request_outputs=[]) |
| 45 | engine.output_processor = MagicMock() |
| 46 | engine.output_processor.process_outputs.return_value = processed_outputs |
| 47 | |
| 48 | assert engine.step() == [] |
| 49 | assert mm_processor_cache.invalidate.call_args_list == [ |
| 50 | call("missing-hash-1"), |
| 51 | call("missing-hash-2"), |
| 52 | ] |
| 53 | |
| 54 | |
| 55 | def _vllm_model( |
| 56 | apc: bool, |
| 57 | vllm_runner: type[VllmRunner], |
| 58 | *, |
| 59 | skip_tokenizer_init: bool = False, |
| 60 | ): |
| 61 | """Set up VllmRunner instance.""" |
| 62 | return vllm_runner( |
| 63 | MODEL, |
| 64 | dtype=DTYPE, |
| 65 | max_model_len=128, |
| 66 | enforce_eager=True, |
| 67 | enable_prefix_caching=apc, |
| 68 | gpu_memory_utilization=0.5, |
| 69 | skip_tokenizer_init=skip_tokenizer_init, |
| 70 | ) |
| 71 | |
| 72 | |
| 73 | @pytest.fixture( |
| 74 | # Function scope decouples tests & allows |
| 75 | # env var adjustment via monkeypatch |
| 76 | scope="function", |
| 77 | # Prefix caching |
| 78 | params=[False, True], |
| 79 | ) |
| 80 | def vllm_model(vllm_runner, request): |
| 81 | """VllmRunner test fixture parameterized by APC True/False.""" |
| 82 | with _vllm_model(request.param, vllm_runner) as vllm_model: |
| 83 | yield vllm_model |
| 84 | |
| 85 | |
| 86 | @pytest.fixture( |
| 87 | # Function scope decouples tests & allows |
| 88 | # env var adjustment via monkeypatch |
| 89 | scope="function", |
| 90 | # Prefix caching |
| 91 | params=[False, True], |
| 92 | ) |
| 93 | def vllm_model_skip_tokenizer_init(vllm_runner, request): |
| 94 | """VllmRunner test fixture with APC.""" |
| 95 | with _vllm_model( |
| 96 | request.param, |
| 97 | vllm_runner, |
| 98 | skip_tokenizer_init=True, |
| 99 | ) as vllm_model: |
| 100 | yield vllm_model |
| 101 | |
| 102 | |
| 103 | def _get_test_sampling_params( |
| 104 | prompt_list: list[str], |
| 105 | seed: int | None = 42, |
| 106 | structured_outputs: bool = False, |
| 107 | ) -> tuple[list[SamplingParams], list[int]]: |
| 108 | """Generate random sampling params for a batch.""" |
| 109 | rng = random.Random(seed) |
| 110 | |
| 111 | def get_mostly_n_gt1() -> int: |
| 112 | r"""Mostly n \in [2,20], ~1/3 n=1.""" |
| 113 | x = rng.randint(0, 28) |
| 114 | if x < 10: |
| 115 | return 1 |
| 116 | else: |
| 117 | return x - 8 |
| 118 | |
| 119 | n_list = [get_mostly_n_gt1() for _ in range(len(prompt_list))] |
| 120 | # High temperature to maximize the chance of unique completions |
| 121 | return [ |
| 122 | SamplingParams( |
| 123 | temperature=0.95, |
| 124 | top_p=0.95, |
| 125 | n=n, |
| 126 | seed=seed, |
| 127 | structured_outputs=StructuredOutputsParams(regex="[0-9]+") |
| 128 | if structured_outputs |
| 129 | else None, |
| 130 | ) |
| 131 | for n in n_list |
| 132 | ], n_list |
| 133 | |
| 134 | |
| 135 | def test_compatibility_with_skip_tokenizer_init( |
| 136 | vllm_model_skip_tokenizer_init: VllmRunner, |
| 137 | example_prompts: list[str], |
| 138 | ): |
| 139 | # Case 1: Structured output request should raise an error. |
| 140 | sampling_params_list, _ = _get_test_sampling_params( |
| 141 | example_prompts, |
| 142 | structured_outputs=True, |
| 143 | ) |
| 144 | llm: LLM = vllm_model_skip_tokenizer_init.llm |
| 145 | with pytest.raises(ValueError): |
| 146 | _ = llm.generate(example_prompts, sampling_params_list) |
| 147 | |
| 148 | |
| 149 | def test_parallel_sampling(vllm_model, example_prompts) -> None: |
| 150 | """Test passes if parallel sampling `n>1` yields `n` unique completions. |
| 151 | |
| 152 | Args: |
| 153 | vllm_model: VllmRunner instance under test. |
| 154 | example_prompt: test fixture providing prompts for testing. |
| 155 | |
| 156 | """ |
| 157 | sampling_params_list, n_list = _get_test_sampling_params(example_prompts) |
| 158 | llm: LLM = vllm_model.llm |
| 159 | outputs = llm.generate(example_prompts, sampling_params_list) |
| 160 | |
| 161 | # Validate each request response |
| 162 | for out, n in zip(outputs, n_list): |
| 163 | completion_counts: dict[str, int] = {} |
| 164 | # Assert correct number of completions |
| 165 | assert len(out.outputs) == n, f"{len(out.outputs)} completions; {n} expected." |
| 166 | for idx in range(n): |
| 167 | comp = out.outputs[idx] |
| 168 | # Assert correct completion indices |
| 169 | assert comp.index == idx, f"Index {comp.index}; expected {idx}." |
| 170 | text = comp.text |
| 171 | completion_counts[text] = completion_counts.get(text, 0) + 1 |
| 172 | # Assert unique completions |
| 173 | if len(completion_counts) != n: |
| 174 | repeats = {txt: num for (txt, num) in completion_counts.items() if num > 1} |
| 175 | raise AssertionError( |
| 176 | f"{len(completion_counts)} unique completions; expected" |
| 177 | f" {n}. Repeats: {repeats}" |
| 178 | ) |
| 179 | |
| 180 | |
| 181 | def test_engine_metrics(vllm_runner, example_prompts): |
| 182 | max_tokens = 100 |
| 183 | # Use spec decoding to test num_accepted_tokens_per_pos |
| 184 | speculative_config = { |
| 185 | "method": "ngram", |
| 186 | "prompt_lookup_max": 5, |
| 187 | "prompt_lookup_min": 3, |
| 188 | "num_speculative_tokens": 5, |
| 189 | } |
| 190 | |
| 191 | with vllm_runner( |
| 192 | MODEL, |
| 193 | speculative_config=speculative_config, |
| 194 | disable_log_stats=False, |
| 195 | ) as vllm_model: |
| 196 | llm: LLM = vllm_model.llm |
| 197 | sampling_params = SamplingParams(temperature=0.0, max_tokens=max_tokens) |
| 198 | outputs = llm.generate(example_prompts, sampling_params) |
| 199 | |
| 200 | n_prompts = len(example_prompts) |
| 201 | assert len(outputs) == n_prompts |
| 202 | |
| 203 | total_tokens = 0 |
| 204 | for out in outputs: |
| 205 | assert len(out.outputs) == 1 |
| 206 | total_tokens += len(out.outputs[0].token_ids) |
| 207 | assert total_tokens == max_tokens * n_prompts |
| 208 | |
| 209 | metrics = llm.get_metrics() |
| 210 | |
| 211 | def find_metric(name) -> list[Metric]: |
| 212 | found = [] |
| 213 | for metric in metrics: |
| 214 | if metric.name == name: |
| 215 | found.append(metric) |
| 216 | return found |
| 217 | |
| 218 | num_requests_running = find_metric("vllm:num_requests_running") |
| 219 | assert len(num_requests_running) == 1 |
| 220 | assert isinstance(num_requests_running[0], Gauge) |
| 221 | assert num_requests_running[0].value == 0.0 |
| 222 | |
| 223 | generation_tokens = find_metric("vllm:generation_tokens") |
| 224 | assert len(generation_tokens) == 1 |
| 225 | assert isinstance(generation_tokens[0], Counter) |
| 226 | assert generation_tokens[0].value == total_tokens |
| 227 | |
| 228 | request_generation_tokens = find_metric("vllm:request_generation_tokens") |
| 229 | assert len(request_generation_tokens) == 1 |
| 230 | assert isinstance(request_generation_tokens[0], Histogram) |
| 231 | assert "+Inf" in request_generation_tokens[0].buckets |
| 232 | assert request_generation_tokens[0].buckets["+Inf"] == n_prompts |
| 233 | assert request_generation_tokens[0].count == n_prompts |
| 234 | assert request_generation_tokens[0].sum == total_tokens |
| 235 | |
| 236 | num_accepted_tokens_per_pos = find_metric( |
| 237 | "vllm:spec_decode_num_accepted_tokens_per_pos" |
| 238 | ) |
| 239 | assert len(num_accepted_tokens_per_pos) == 1 |
| 240 | assert isinstance(num_accepted_tokens_per_pos[0], Vector) |
| 241 | assert len(num_accepted_tokens_per_pos[0].values) == 5 |
| 242 | |
| 243 | |
| 244 | @pytest.mark.parametrize("model", ["meta-llama/Llama-3.2-1B-Instruct"]) |
| 245 | def test_skip_tokenizer_initialization(model: str): |
| 246 | # This test checks if the flag skip_tokenizer_init skips the initialization |
| 247 | # of tokenizer and detokenizer. The generated output is expected to contain |
| 248 | # token ids. |
| 249 | llm = LLM( |
| 250 | model=model, |
| 251 | skip_tokenizer_init=True, |
| 252 | enforce_eager=True, |
| 253 | ) |
| 254 | sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True) |
| 255 | |
| 256 | with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"): |
| 257 | llm.generate("abc", sampling_params) |
| 258 | |
| 259 | outputs = llm.generate( |
| 260 | {"prompt_token_ids": [1, 2, 3]}, sampling_params=sampling_params |
| 261 | ) |
| 262 | assert len(outputs) > 0 |
| 263 | completions = outputs[0].outputs |
| 264 | assert len(completions) > 0 |
| 265 | assert completions[0].text == "" |
| 266 | assert completions[0].token_ids |