| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | """Unit tests for MRv2 GPUModelRunner.add_requests streaming input support.""" |
| 5 | |
| 6 | from unittest.mock import Mock |
| 7 | |
| 8 | import pytest |
| 9 | import torch |
| 10 | |
| 11 | from vllm.v1.core.sched.output import ( |
| 12 | CachedRequestData, |
| 13 | NewRequestData, |
| 14 | SchedulerOutput, |
| 15 | ) |
| 16 | from vllm.v1.worker.gpu.model_runner import GPUModelRunner |
| 17 | from vllm.v1.worker.gpu.states import RequestState |
| 18 | |
| 19 | # Not cpu_test: RequestState allocates pinned (UVA) memory, which requires a |
| 20 | # CUDA device even though the request state itself lives on the CPU. |
| 21 | |
| 22 | |
| 23 | @pytest.fixture |
| 24 | def mock_model_runner_with_req_states(): |
| 25 | """Create a mock MRv2 GPUModelRunner with a real RequestState.""" |
| 26 | runner = Mock(spec=GPUModelRunner) |
| 27 | runner.req_states = RequestState( |
| 28 | max_num_reqs=10, |
| 29 | max_model_len=1024, |
| 30 | max_num_batched_tokens=1024, |
| 31 | num_speculative_steps=0, |
| 32 | vocab_size=32000, |
| 33 | device=torch.device("cpu"), |
| 34 | ) |
| 35 | runner.encoder_cache = None |
| 36 | runner.model_state = Mock() |
| 37 | runner.block_tables = Mock() |
| 38 | runner.lora_state = Mock() |
| 39 | runner.pp_handler = None |
| 40 | runner.sampler = None |
| 41 | runner.prompt_logprobs_worker = None |
| 42 | runner.is_last_pp_rank = False |
| 43 | runner.pooling_runner = None |
| 44 | runner.adaptive_verification = None |
| 45 | |
| 46 | # Mock staged writes — they use Triton kernels that require GPU |
| 47 | runner.req_states.apply_staged_writes = Mock() |
| 48 | |
| 49 | # Bind the real methods to our mock |
| 50 | runner._remove_request = GPUModelRunner._remove_request.__get__(runner) |
| 51 | runner.add_requests = GPUModelRunner.add_requests.__get__(runner) |
| 52 | return runner |
| 53 | |
| 54 | |
| 55 | def _make_scheduler_output(new_reqs): |
| 56 | return SchedulerOutput( |
| 57 | scheduled_new_reqs=new_reqs, |
| 58 | scheduled_cached_reqs=CachedRequestData.make_empty(), |
| 59 | num_scheduled_tokens={}, |
| 60 | total_num_scheduled_tokens=0, |
| 61 | scheduled_spec_decode_tokens={}, |
| 62 | scheduled_encoder_inputs={}, |
| 63 | num_common_prefix_blocks=[], |
| 64 | finished_req_ids=set(), |
| 65 | free_encoder_mm_hashes=[], |
| 66 | ) |
| 67 | |
| 68 | |
| 69 | def test_e2e_streaming_request_update_basic_flow( |
| 70 | mock_model_runner_with_req_states, |
| 71 | ): |
| 72 | """Test that streaming sessions are updated correctly. |
| 73 | |
| 74 | This test validates that when a streaming session is updated with new |
| 75 | prompt tokens: |
| 76 | 1. The old request state is removed (no free_indices leak) |
| 77 | 2. The new state is written with updated prefill_token_ids |
| 78 | 3. model_state and block_tables are re-registered for the new state |
| 79 | """ |
| 80 | runner = mock_model_runner_with_req_states |
| 81 | req_states = runner.req_states |
| 82 | req_id = "streaming_req_0" |
| 83 | initial_free = len(req_states.free_indices) |
| 84 | |
| 85 | # Step 1: Add initial request with 3 prompt tokens, all computed |
| 86 | initial_req_data = NewRequestData( |
| 87 | req_id=req_id, |
| 88 | prompt_token_ids=[1, 2, 3], |
| 89 | prefill_token_ids=[1, 2, 3], |
| 90 | mm_features=[], |
| 91 | sampling_params=None, |
| 92 | pooling_params=None, |
| 93 | block_ids=([0],), |
| 94 | num_computed_tokens=3, |
| 95 | lora_request=None, |
| 96 | ) |
| 97 | runner.add_requests(_make_scheduler_output([initial_req_data])) |
| 98 | assert req_id in req_states.req_id_to_index |
| 99 | assert len(req_states.free_indices) == initial_free - 1 |
| 100 | |
| 101 | # Step 2: Create streaming update with extended prompt |
| 102 | # The scheduler has already set prefill_token_ids to the full sequence |
| 103 | # (original prompt + intermediate output + new prompt tokens) |
| 104 | updated_req_data = NewRequestData( |
| 105 | req_id=req_id, |
| 106 | prompt_token_ids=[1, 2, 3], |
| 107 | prefill_token_ids=[1, 2, 3, 10, 4, 5], |
| 108 | mm_features=[], |
| 109 | sampling_params=None, |
| 110 | pooling_params=None, |
| 111 | block_ids=([0, 1],), |
| 112 | num_computed_tokens=4, # 3 original prompt + 1 intermediate output |
| 113 | lora_request=None, |
| 114 | ) |
| 115 | runner.add_requests(_make_scheduler_output([updated_req_data])) |
| 116 | |
| 117 | # Step 3: Verify no free_indices leak (old slot recycled) |
| 118 | assert len(req_states.free_indices) == initial_free - 1 |
| 119 | |
| 120 | # Verify the request is still tracked with exactly one index |
| 121 | assert req_id in req_states.req_id_to_index |
| 122 | assert sum(1 for v in req_states.index_to_req_id.values() if v == req_id) == 1 |
| 123 | |
| 124 | # Verify state was updated with new values |
| 125 | new_idx = req_states.req_id_to_index[req_id] |
| 126 | assert req_states.prompt_len.np[new_idx] == 3 |
| 127 | assert req_states.prefill_len.np[new_idx] == 6 |
| 128 | assert req_states.num_computed_prefill_tokens[new_idx] == 4 |
| 129 | |
| 130 | # Verify model_state and block_tables were re-registered |
| 131 | runner.model_state.add_request.assert_called_with(new_idx, updated_req_data) |
| 132 | runner.block_tables.append_block_ids.assert_called_with( |
| 133 | new_idx, ([0, 1],), overwrite=True |
| 134 | ) |
| 135 | |
| 136 | |
| 137 | def test_e2e_streaming_with_multimodal_features( |
| 138 | mock_model_runner_with_req_states, |
| 139 | ): |
| 140 | """Test that streaming sessions with multimodal features are updated. |
| 141 | |
| 142 | This test validates that when a streaming session with mm features |
| 143 | is updated: |
| 144 | 1. The old request state is removed (no free_indices leak) |
| 145 | 2. encoder_cache is cleaned up and re-registered with new mm_features |
| 146 | 3. model_state is re-registered (recomputes M-RoPE positions etc.) |
| 147 | """ |
| 148 | runner = mock_model_runner_with_req_states |
| 149 | req_states = runner.req_states |
| 150 | req_id = "streaming_mm_req_0" |
| 151 | initial_free = len(req_states.free_indices) |
| 152 | |
| 153 | # Enable encoder_cache for multimodal |
| 154 | runner.encoder_cache = Mock() |
| 155 | |
| 156 | # Step 1: Add initial request with one audio feature |
| 157 | mm_feature_1 = Mock() |
| 158 | initial_req_data = NewRequestData( |
| 159 | req_id=req_id, |
| 160 | prompt_token_ids=[1, 2] + [0] * 10 + [3, 4], |
| 161 | prefill_token_ids=[1, 2] + [0] * 10 + [3, 4], |
| 162 | mm_features=[mm_feature_1], |
| 163 | sampling_params=None, |
| 164 | pooling_params=None, |
| 165 | block_ids=([0],), |
| 166 | num_computed_tokens=14, |
| 167 | lora_request=None, |
| 168 | ) |
| 169 | runner.add_requests(_make_scheduler_output([initial_req_data])) |
| 170 | assert req_id in req_states.req_id_to_index |
| 171 | |
| 172 | # Reset mocks to track only the streaming update calls |
| 173 | runner.encoder_cache.reset_mock() |
| 174 | runner.model_state.reset_mock() |
| 175 | |
| 176 | # Step 2: Create streaming update with additional multimodal feature |
| 177 | # The scheduler has folded the intermediate output (100) into |
| 178 | # prefill_token_ids and added a new audio chunk |
| 179 | mm_feature_2 = Mock() |
| 180 | updated_req_data = NewRequestData( |
| 181 | req_id=req_id, |
| 182 | prompt_token_ids=[1, 2] + [0] * 10 + [3, 4], |
| 183 | prefill_token_ids=[1, 2] + [0] * 10 + [3, 4, 100] + [0] * 5 + [5], |
| 184 | mm_features=[mm_feature_1, mm_feature_2], |
| 185 | sampling_params=None, |
| 186 | pooling_params=None, |
| 187 | block_ids=([0, 1],), |
| 188 | num_computed_tokens=14, |
| 189 | lora_request=None, |
| 190 | ) |
| 191 | runner.add_requests(_make_scheduler_output([updated_req_data])) |
| 192 | |
| 193 | # Step 3: Verify no free_indices leak |
| 194 | assert len(req_states.free_indices) == initial_free - 1 |
| 195 | assert sum(1 for v in req_states.index_to_req_id.values() if v == req_id) == 1 |
| 196 | |
| 197 | # Verify encoder_cache was cleaned up and re-registered |
| 198 | runner.encoder_cache.remove_request.assert_called_once_with(req_id) |
| 199 | runner.encoder_cache.add_request.assert_called_once_with( |
| 200 | req_id, [mm_feature_1, mm_feature_2] |
| 201 | ) |
| 202 | |
| 203 | # Verify model_state was re-registered with new data |
| 204 | new_idx = req_states.req_id_to_index[req_id] |
| 205 | runner.model_state.add_request.assert_called_once_with(new_idx, updated_req_data) |
| 206 | |
| 207 | # Verify updated prefill length |
| 208 | assert req_states.prefill_len.np[new_idx] == 21 |