| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import copy |
| 5 | import time |
| 6 | import uuid |
| 7 | from collections import deque |
| 8 | from concurrent.futures import Future, ThreadPoolExecutor |
| 9 | from unittest.mock import MagicMock, PropertyMock, patch |
| 10 | |
| 11 | import pytest |
| 12 | from transformers import AutoTokenizer |
| 13 | |
| 14 | from vllm import SamplingParams |
| 15 | from vllm.config import ( |
| 16 | CacheConfig, |
| 17 | ECTransferConfig, |
| 18 | KVTransferConfig, |
| 19 | ModelConfig, |
| 20 | ParallelConfig, |
| 21 | SchedulerConfig, |
| 22 | VllmConfig, |
| 23 | ) |
| 24 | from vllm.engine.arg_utils import EngineArgs |
| 25 | from vllm.platforms import current_platform |
| 26 | from vllm.utils.torch_utils import set_default_torch_num_threads |
| 27 | from vllm.v1.core.sched.interface import PauseState |
| 28 | from vllm.v1.engine import EngineCoreRequest |
| 29 | from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc |
| 30 | from vllm.v1.executor.abstract import Executor |
| 31 | from vllm.v1.executor.uniproc_executor import UniProcExecutor |
| 32 | from vllm.v1.kv_cache_interface import KVCacheConfig |
| 33 | from vllm.v1.outputs import ModelRunnerOutput |
| 34 | |
| 35 | from ...utils import create_new_process_for_each_test, multi_gpu_test |
| 36 | |
| 37 | if not current_platform.is_cuda(): |
| 38 | pytest.skip(reason="V1 currently only supported on CUDA.", allow_module_level=True) |
| 39 | |
| 40 | MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" |
| 41 | TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME) |
| 42 | # test_engine_core_concurrent_batches assumes exactly 12 tokens per prompt. |
| 43 | # Adjust prompt if changing model to maintain 12-token length. |
| 44 | PROMPT = "I am Gyoubu Masataka Oniwa" |
| 45 | PROMPT_TOKENS = TOKENIZER(PROMPT).input_ids |
| 46 | |
| 47 | _REQUEST_COUNTER = 0 |
| 48 | |
| 49 | |
| 50 | def make_request() -> EngineCoreRequest: |
| 51 | global _REQUEST_COUNTER |
| 52 | _REQUEST_COUNTER += 1 |
| 53 | request_id = f"request-{_REQUEST_COUNTER}" |
| 54 | return EngineCoreRequest( |
| 55 | request_id=request_id, |
| 56 | external_req_id=f"{request_id}-{uuid.uuid4()}", |
| 57 | prompt_token_ids=PROMPT_TOKENS, |
| 58 | mm_features=None, |
| 59 | sampling_params=SamplingParams(), |
| 60 | pooling_params=None, |
| 61 | arrival_time=time.time(), |
| 62 | lora_request=None, |
| 63 | cache_salt=None, |
| 64 | data_parallel_rank=None, |
| 65 | ) |
| 66 | |
| 67 | |
| 68 | @create_new_process_for_each_test() |
| 69 | def test_engine_core(): |
| 70 | """Setup the EngineCore.""" |
| 71 | engine_args = EngineArgs(model=MODEL_NAME) |
| 72 | vllm_config = engine_args.create_engine_config() |
| 73 | executor_class = Executor.get_class(vllm_config) |
| 74 | |
| 75 | with set_default_torch_num_threads(1): |
| 76 | engine_core = EngineCore( |
| 77 | vllm_config=vllm_config, executor_class=executor_class, log_stats=True |
| 78 | ) |
| 79 | """Test basic request lifecycle.""" |
| 80 | |
| 81 | # First request. |
| 82 | engine_core.add_request(*engine_core.preprocess_add_request(make_request())) |
| 83 | assert len(engine_core.scheduler.waiting) == 1 |
| 84 | assert len(engine_core.scheduler.running) == 0 |
| 85 | |
| 86 | _ = engine_core.step_fn() |
| 87 | assert len(engine_core.scheduler.waiting) == 0 |
| 88 | assert len(engine_core.scheduler.running) == 1 |
| 89 | |
| 90 | # Second request. |
| 91 | engine_core.add_request(*engine_core.preprocess_add_request(make_request())) |
| 92 | assert len(engine_core.scheduler.waiting) == 1 |
| 93 | assert len(engine_core.scheduler.running) == 1 |
| 94 | |
| 95 | _ = engine_core.step_fn() |
| 96 | assert len(engine_core.scheduler.waiting) == 0 |
| 97 | assert len(engine_core.scheduler.running) == 2 |
| 98 | |
| 99 | # Add two requests in a row. |
| 100 | engine_core.add_request(*engine_core.preprocess_add_request(make_request())) |
| 101 | engine_core.add_request(*engine_core.preprocess_add_request(make_request())) |
| 102 | assert len(engine_core.scheduler.waiting) == 2 |
| 103 | assert len(engine_core.scheduler.running) == 2 |
| 104 | |
| 105 | _ = engine_core.step_fn() |
| 106 | assert len(engine_core.scheduler.waiting) == 0 |
| 107 | assert len(engine_core.scheduler.running) == 4 |
| 108 | |
| 109 | # Loop through until they are all done. |
| 110 | while (outs := engine_core.step_fn()[0].get(0)) and outs.outputs: |
| 111 | pass |
| 112 | |
| 113 | assert len(engine_core.scheduler.waiting) == 0 |
| 114 | assert len(engine_core.scheduler.running) == 0 |
| 115 | """Test abort cycle.""" |
| 116 | |
| 117 | # Basic abort. |
| 118 | req = make_request() |
| 119 | request_id = req.request_id |
| 120 | |
| 121 | engine_core.add_request(*engine_core.preprocess_add_request(req)) |
| 122 | assert len(engine_core.scheduler.waiting) == 1 |
| 123 | assert len(engine_core.scheduler.running) == 0 |
| 124 | assert engine_core.scheduler.has_unfinished_requests() |
| 125 | assert not engine_core.scheduler.has_finished_requests() |
| 126 | |
| 127 | _ = engine_core.step_fn() |
| 128 | assert len(engine_core.scheduler.waiting) == 0 |
| 129 | assert len(engine_core.scheduler.running) == 1 |
| 130 | assert engine_core.scheduler.has_unfinished_requests() |
| 131 | assert not engine_core.scheduler.has_finished_requests() |
| 132 | |
| 133 | engine_core.abort_requests([request_id]) |
| 134 | assert len(engine_core.scheduler.waiting) == 0 |
| 135 | assert len(engine_core.scheduler.running) == 0 |
| 136 | assert not engine_core.scheduler.has_unfinished_requests() |
| 137 | assert engine_core.scheduler.has_finished_requests() |
| 138 | |
| 139 | _ = engine_core.step_fn() |
| 140 | assert not engine_core.scheduler.has_unfinished_requests() |
| 141 | assert not engine_core.scheduler.has_finished_requests() |
| 142 | |
| 143 | # Add, step, abort 1 of the 3. |
| 144 | req0 = make_request() |
| 145 | req1 = make_request() |
| 146 | req2 = make_request() |
| 147 | |
| 148 | engine_core.add_request(*engine_core.preprocess_add_request(req0)) |
| 149 | engine_core.add_request(*engine_core.preprocess_add_request(req1)) |
| 150 | assert len(engine_core.scheduler.waiting) == 2 |
| 151 | assert len(engine_core.scheduler.running) == 0 |
| 152 | |
| 153 | _ = engine_core.step_fn() |
| 154 | assert len(engine_core.scheduler.waiting) == 0 |
| 155 | assert len(engine_core.scheduler.running) == 2 |
| 156 | |
| 157 | engine_core.add_request(*engine_core.preprocess_add_request(req2)) |
| 158 | assert len(engine_core.scheduler.waiting) == 1 |
| 159 | assert len(engine_core.scheduler.running) == 2 |
| 160 | |
| 161 | _ = engine_core.step_fn() |
| 162 | assert len(engine_core.scheduler.waiting) == 0 |
| 163 | assert len(engine_core.scheduler.running) == 3 |
| 164 | |
| 165 | # Abort just one. |
| 166 | engine_core.abort_requests([req1.request_id]) |
| 167 | assert len(engine_core.scheduler.waiting) == 0 |
| 168 | assert len(engine_core.scheduler.running) == 2 |
| 169 | |
| 170 | _ = engine_core.step_fn() |
| 171 | assert len(engine_core.scheduler.waiting) == 0 |
| 172 | assert len(engine_core.scheduler.running) == 2 |
| 173 | |
| 174 | # Abort the other requests at the same time. |
| 175 | engine_core.abort_requests([req2.request_id, req0.request_id]) |
| 176 | assert len(engine_core.scheduler.waiting) == 0 |
| 177 | assert len(engine_core.scheduler.running) == 0 |
| 178 | |
| 179 | # Sending duplicate requests with same request_id |
| 180 | req0 = make_request() |
| 181 | req1 = make_request() |
| 182 | req0.request_id = req1.request_id = "test" |
| 183 | engine_core.add_request(*engine_core.preprocess_add_request(req0)) |
| 184 | |
| 185 | while engine_core.scheduler.has_requests(): |
| 186 | engine_core.step_fn() |
| 187 | |
| 188 | engine_core.add_request(*engine_core.preprocess_add_request(req1)) |
| 189 | while engine_core.scheduler.has_requests(): |
| 190 | engine_core.step_fn() |
| 191 | |
| 192 | assert len(engine_core.scheduler.waiting) == 0 |
| 193 | assert len(engine_core.scheduler.running) == 0 |
| 194 | |
| 195 | |
| 196 | @create_new_process_for_each_test() |
| 197 | def test_engine_core_advanced_sampling(): |
| 198 | """A basic end-to-end test to verify that the engine functions correctly |
| 199 | when additional sampling parameters, such as top_p, min_tokens, and |
| 200 | presence_penalty, are set. |
| 201 | """ |
| 202 | """Setup the EngineCore.""" |
| 203 | engine_args = EngineArgs(model=MODEL_NAME) |
| 204 | vllm_config = engine_args.create_engine_config() |
| 205 | executor_class = Executor.get_class(vllm_config) |
| 206 | |
| 207 | with set_default_torch_num_threads(1): |
| 208 | engine_core = EngineCore( |
| 209 | vllm_config=vllm_config, executor_class=executor_class, log_stats=True |
| 210 | ) |
| 211 | """Test basic request lifecycle.""" |
| 212 | # First request. |
| 213 | request: EngineCoreRequest = make_request() |
| 214 | request.sampling_params = SamplingParams( |
| 215 | min_tokens=4, |
| 216 | presence_penalty=1.0, |
| 217 | frequency_penalty=1.0, |
| 218 | repetition_penalty=0.1, |
| 219 | stop_token_ids=[1001, 1002], |
| 220 | ) |
| 221 | engine_core.add_request(*engine_core.preprocess_add_request(request)) |
| 222 | |
| 223 | def _check_engine_state(): |
| 224 | assert len(engine_core.scheduler.waiting) == 1 |
| 225 | assert len(engine_core.scheduler.running) == 0 |
| 226 | # Loop through until they are all done. |
| 227 | while engine_core.scheduler.has_requests(): |
| 228 | engine_core.step_fn() |
| 229 | assert len(engine_core.scheduler.waiting) == 0 |
| 230 | assert len(engine_core.scheduler.running) == 0 |
| 231 | |
| 232 | _check_engine_state() |
| 233 | |
| 234 | # Second request. |
| 235 | request2 = make_request() |
| 236 | request2.sampling_params = SamplingParams( |
| 237 | top_p=0.99, |
| 238 | top_k=50, |
| 239 | ) |
| 240 | engine_core.add_request(*engine_core.preprocess_add_request(request2)) |
| 241 | _check_engine_state() |
| 242 | |
| 243 | |
| 244 | @create_new_process_for_each_test() |
| 245 | def test_engine_core_concurrent_batches(): |
| 246 | """Test that the engine can handle multiple concurrent batches.""" |
| 247 | |
| 248 | def make_request_with_max_tokens(req_id: str, max_tokens: int) -> EngineCoreRequest: |
| 249 | request = make_request() |
| 250 | request.request_id = req_id |
| 251 | request.sampling_params.max_tokens = max_tokens |
| 252 | return request |
| 253 | |
| 254 | class DummyExecutor(UniProcExecutor): |
| 255 | def initialize_from_config(self, kv_cache_configs: list[KVCacheConfig]) -> None: |
| 256 | super().initialize_from_config(kv_cache_configs) |
| 257 | |
| 258 | # Create a thread pool with a single worker |
| 259 | self.thread_pool = ThreadPoolExecutor(max_workers=1) |
| 260 | |
| 261 | def execute_model( |
| 262 | self, |
| 263 | scheduler_output, |
| 264 | non_block=False, |
| 265 | ) -> Future[ModelRunnerOutput | None]: |
| 266 | """Make execute_model non-blocking.""" |
| 267 | # DummyExecutor used only for testing async case. |
| 268 | assert non_block |
| 269 | |
| 270 | def _execute(): |
| 271 | output = self.collective_rpc("execute_model", args=(scheduler_output,)) |
| 272 | # Make a copy because output[0] may be reused |
| 273 | # by the next batch. |
| 274 | return copy.deepcopy(output[0]) |
| 275 | |
| 276 | # Use the thread pool instead of creating a new thread |
| 277 | return self.thread_pool.submit(_execute) |
| 278 | |
| 279 | def sample_tokens( |
| 280 | self, grammar_output, non_block=False |
| 281 | ) -> Future[ModelRunnerOutput]: |
| 282 | """Make sample_tokens non-blocking.""" |
| 283 | # DummyExecutor used only for testing async case. |
| 284 | assert non_block |
| 285 | |
| 286 | def _execute(): |
| 287 | output = self.collective_rpc("sample_tokens", args=(grammar_output,)) |
| 288 | # Make a copy because output[0] may be reused |
| 289 | # by the next batch. |
| 290 | return copy.deepcopy(output[0]) |
| 291 | |
| 292 | # Use the thread pool instead of creating a new thread |
| 293 | return self.thread_pool.submit(_execute) |
| 294 | |
| 295 | def shutdown(self): |
| 296 | if hasattr(self, "thread_pool"): |
| 297 | self.thread_pool.shutdown(wait=False) |
| 298 | |
| 299 | engine_args = EngineArgs( |
| 300 | model=MODEL_NAME, |
| 301 | # To test concurrent batches. |
| 302 | max_num_seqs=2, |
| 303 | # Avoid all requests being scheduled once. |
| 304 | enable_prefix_caching=False, |
| 305 | max_num_batched_tokens=10, |
| 306 | # Reduce startup time. |
| 307 | enforce_eager=True, |
| 308 | # Test concurrent batch behaviour independently of async scheduling. |
| 309 | async_scheduling=False, |
| 310 | ) |
| 311 | vllm_config = engine_args.create_engine_config() |
| 312 | # Force two concurrent batches to exercise the batch queue independently |
| 313 | # of async scheduling (which is disabled above). |
| 314 | with ( |
| 315 | set_default_torch_num_threads(1), |
| 316 | patch.object( |
| 317 | VllmConfig, |
| 318 | "max_concurrent_batches", |
| 319 | new_callable=PropertyMock, |
| 320 | return_value=2, |
| 321 | ), |
| 322 | ): |
| 323 | engine_core = EngineCore( |
| 324 | vllm_config=vllm_config, log_stats=False, executor_class=DummyExecutor |
| 325 | ) |
| 326 | assert engine_core.batch_queue is not None |
| 327 | |
| 328 | # Add two requests in a row. Each request have 12 prompt tokens. |
| 329 | req0 = make_request_with_max_tokens("0", 5) |
| 330 | engine_core.add_request(*engine_core.preprocess_add_request(req0)) |
| 331 | req1 = make_request_with_max_tokens("1", 5) |
| 332 | engine_core.add_request(*engine_core.preprocess_add_request(req1)) |
| 333 | |
| 334 | # Schedule Batch 1: (10, req0) |
| 335 | assert engine_core.step_with_batch_queue()[0] is None |
| 336 | assert len(engine_core.batch_queue) == 1 |
| 337 | scheduler_output = engine_core.batch_queue[-1][1] |
| 338 | assert scheduler_output.num_scheduled_tokens["0"] == 10 |
| 339 | # num_computed_tokens should have been updated immediately. |
| 340 | assert engine_core.scheduler.requests[req0.request_id].num_computed_tokens == 10 |
| 341 | |
| 342 | # Schedule Batch 2: (2, req0), (8, req1) |
| 343 | assert engine_core.step_with_batch_queue()[0] == {} |
| 344 | assert len(engine_core.batch_queue) == 1 |
| 345 | scheduler_output = engine_core.batch_queue[-1][1] |
| 346 | assert scheduler_output.num_scheduled_tokens["0"] == 2 |
| 347 | assert scheduler_output.num_scheduled_tokens["1"] == 8 |
| 348 | # num_computed_tokens should have been updated immediately. |
| 349 | assert engine_core.scheduler.requests["0"].num_computed_tokens == 12 |
| 350 | assert engine_core.scheduler.requests["1"].num_computed_tokens == 8 |
| 351 | |
| 352 | assert engine_core.scheduler.get_num_unfinished_requests() == 2 |
| 353 | |
| 354 | # Finish Batch 1 and schedule Batch 3: (4, req1). |
| 355 | # Note that req0 cannot be scheduled |
| 356 | # because it is in the decoding stage now. |
| 357 | engine_core.step_with_batch_queue() |
| 358 | assert len(engine_core.batch_queue) == 1 |
| 359 | scheduler_output = engine_core.batch_queue[-1][1] |
| 360 | assert scheduler_output.num_scheduled_tokens["1"] == 4 |
| 361 | |
| 362 | # Finish Batch 2. Get first token of req0. |
| 363 | # Schedule Batch 4: (1, req0). |
| 364 | output = engine_core.step_with_batch_queue()[0].get(0) |
| 365 | assert output is not None |
| 366 | assert len(output.outputs) == 1 |
| 367 | assert engine_core.scheduler.requests[req0.request_id].num_tokens == 13 |
| 368 | scheduler_output = engine_core.batch_queue[-1][1] |
| 369 | assert scheduler_output.num_scheduled_tokens["0"] == 1 |
| 370 | |
| 371 | # Finish Batch 3. Get first token of req1. Schedule Batch 5: (1, req1). |
| 372 | output = engine_core.step_with_batch_queue()[0].get(0) |
| 373 | assert output is not None |
| 374 | assert len(output.outputs) == 1 |
| 375 | assert engine_core.scheduler.requests[req1.request_id].num_tokens == 13 |
| 376 | scheduler_output = engine_core.batch_queue[-1][1] |
| 377 | assert scheduler_output.num_scheduled_tokens["1"] == 1 |
| 378 | |
| 379 | # Loop until req0 is finished. |
| 380 | req_id = 0 |
| 381 | expected_num_tokens = [ |
| 382 | engine_core.scheduler.requests["0"].num_tokens + 1, |
| 383 | engine_core.scheduler.requests["1"].num_tokens + 1, |
| 384 | ] |
| 385 | while engine_core.scheduler.get_num_unfinished_requests() == 2: |
| 386 | output = engine_core.step_with_batch_queue()[0] |
| 387 | # Every step consumes an output. |
| 388 | assert output is not None |
| 389 | assert len(output[0].outputs) == 1 |
| 390 | if req_id in engine_core.scheduler.requests: |
| 391 | assert ( |
| 392 | engine_core.scheduler.requests[req_id].num_tokens |
| 393 | == expected_num_tokens[req_id] |
| 394 | ) |
| 395 | expected_num_tokens[req_id] += 1 |
| 396 | req_id = (req_id + 1) % 2 |
| 397 | |
| 398 | |
| 399 | @pytest.mark.parametrize("encoder_only", [True, False]) |
| 400 | def test_async_step_skips_sampling_for_mm_encoder_only(encoder_only): |
| 401 | """Encoder outputs are complete; only language-model steps need sampling.""" |
| 402 | core = MagicMock() |
| 403 | core.batch_queue = deque() |
| 404 | core.batch_queue_size = 2 |
| 405 | core.is_mm_encoder_only = encoder_only |
| 406 | core.is_pooling_model = False |
| 407 | core.scheduler.has_requests.side_effect = [True, False, False] |
| 408 | scheduled = core.scheduler.schedule.return_value |
| 409 | scheduled.total_num_scheduled_tokens = 16 |
| 410 | scheduled.pending_structured_output_tokens = False |
| 411 | |
| 412 | output = MagicMock() |
| 413 | executed: Future[ModelRunnerOutput | None] = Future() |
| 414 | executed.set_result(output if encoder_only else None) |
| 415 | sampled: Future[ModelRunnerOutput | None] = Future() |
| 416 | sampled.set_result(None if encoder_only else output) |
| 417 | core.model_executor.execute_model.return_value = executed |
| 418 | core.model_executor.sample_tokens.return_value = sampled |
| 419 | |
| 420 | EngineCore.step_with_batch_queue(core) |
| 421 | if core.batch_queue: |
| 422 | EngineCore.step_with_batch_queue(core) |
| 423 | |
| 424 | core.scheduler.update_from_output.assert_called_once_with(scheduled, output) |
| 425 | if encoder_only: |
| 426 | core.model_executor.sample_tokens.assert_not_called() |
| 427 | else: |
| 428 | core.model_executor.sample_tokens.assert_called_once() |
| 429 | |
| 430 | |
| 431 | @multi_gpu_test(num_gpus=2) |
| 432 | def test_engine_core_tp(): |
| 433 | """Test engine can initialize worker in tp properly.""" |
| 434 | """Setup the EngineCore.""" |
| 435 | engine_args = EngineArgs( |
| 436 | model=MODEL_NAME, |
| 437 | tensor_parallel_size=2, |
| 438 | # Reduce startup time. |
| 439 | enforce_eager=True, |
| 440 | ) |
| 441 | vllm_config = engine_args.create_engine_config() |
| 442 | executor_class = Executor.get_class(vllm_config) |
| 443 | |
| 444 | with set_default_torch_num_threads(1): |
| 445 | engine_core = EngineCore( |
| 446 | vllm_config=vllm_config, executor_class=executor_class, log_stats=True |
| 447 | ) |
| 448 | |
| 449 | def get_worker_cache_config_field(worker, key: str): |
| 450 | return getattr(worker.cache_config, key) |
| 451 | |
| 452 | num_gpu_blocks = engine_core.collective_rpc( |
| 453 | get_worker_cache_config_field, args=("num_gpu_blocks",) |
| 454 | ) |
| 455 | num_cpu_blocks = engine_core.collective_rpc( |
| 456 | get_worker_cache_config_field, args=("num_cpu_blocks",) |
| 457 | ) |
| 458 | assert all(x is not None for x in num_gpu_blocks) |
| 459 | assert all(x is not None for x in num_cpu_blocks) |
| 460 | |
| 461 | |
| 462 | @create_new_process_for_each_test() |
| 463 | def test_engine_core_invalid_request_id_type(): |
| 464 | """Test that engine raises TypeError for non-string request_id.""" |
| 465 | engine_args = EngineArgs(model=MODEL_NAME) |
| 466 | vllm_config = engine_args.create_engine_config() |
| 467 | executor_class = Executor.get_class(vllm_config) |
| 468 | |
| 469 | with set_default_torch_num_threads(1): |
| 470 | engine_core = EngineCore( |
| 471 | vllm_config=vllm_config, executor_class=executor_class, log_stats=True |
| 472 | ) |
| 473 | |
| 474 | # Test with UUID object (common mistake) |
| 475 | uuid_request = make_request() |
| 476 | uuid_request.request_id = uuid.uuid4() # UUID object instead of string |
| 477 | |
| 478 | with pytest.raises(TypeError, match="request_id must be a string, got.*UUID"): |
| 479 | engine_core.add_request(*engine_core.preprocess_add_request(uuid_request)) |
| 480 | |
| 481 | # Test with integer |
| 482 | int_request = make_request() |
| 483 | int_request.request_id = 12345 |
| 484 | |
| 485 | with pytest.raises(TypeError, match="request_id must be a string, got.*int"): |
| 486 | engine_core.add_request(*engine_core.preprocess_add_request(int_request)) |
| 487 | |
| 488 | # Test with None |
| 489 | none_request = make_request() |
| 490 | none_request.request_id = None |
| 491 | |
| 492 | with pytest.raises(TypeError, match="request_id must be a string, got.*NoneType"): |
| 493 | engine_core.add_request(*engine_core.preprocess_add_request(none_request)) |
| 494 | |
| 495 | # Verify engine is still functional after errors |
| 496 | valid_request = make_request() |
| 497 | engine_core.add_request(*engine_core.preprocess_add_request(valid_request)) |
| 498 | assert len(engine_core.scheduler.waiting) == 1 |
| 499 | assert len(engine_core.scheduler.running) == 0 |
| 500 | |
| 501 | |
| 502 | @create_new_process_for_each_test() |
| 503 | @pytest.mark.parametrize( |
| 504 | ("ec_role", "gpu_memory_utilization", "enable_prefix_caching"), |
| 505 | [ |
| 506 | ("ec_producer", 0.01, False), |
| 507 | # NOTE: ec_producer never allows prefix caching |
| 508 | ("ec_consumer", 0.7, True), |
| 509 | ("ec_consumer", 0.7, False), |
| 510 | ], |
| 511 | ) |
| 512 | @pytest.mark.parametrize("use_kv_connector", [False, True]) |
| 513 | def test_encoder_instance_zero_kv_cache( |
| 514 | ec_role: str, |
| 515 | gpu_memory_utilization: float, |
| 516 | enable_prefix_caching: bool, |
| 517 | use_kv_connector: bool, |
| 518 | ): |
| 519 | """EPD (Encoder-Prefill-Decode) Encoder-cache-specific tests. |
| 520 | |
| 521 | This test verifies encoder-only instance initializes with 0 KV cache blocks. |
| 522 | Under EPD disagg mode, Encoder instances (EC producer role) only execute |
| 523 | vision encoder, so they don't need KV cache for text generation. |
| 524 | """ |
| 525 | # Form vllm config |
| 526 | model_config = ModelConfig( |
| 527 | model="llava-hf/llava-1.5-7b-hf", # Multimodal model |
| 528 | enforce_eager=True, |
| 529 | trust_remote_code=True, |
| 530 | dtype="float16", |
| 531 | seed=42, |
| 532 | ) |
| 533 | scheduler_config = SchedulerConfig( |
| 534 | max_num_seqs=10, |
| 535 | max_num_batched_tokens=512, |
| 536 | max_model_len=512, |
| 537 | disable_hybrid_kv_cache_manager=True, |
| 538 | is_encoder_decoder=model_config.is_encoder_decoder, |
| 539 | ) |
| 540 | cache_config = CacheConfig( |
| 541 | block_size=16, |
| 542 | gpu_memory_utilization=gpu_memory_utilization, |
| 543 | cache_dtype="auto", |
| 544 | enable_prefix_caching=enable_prefix_caching, |
| 545 | ) |
| 546 | kv_transfer_config = ( |
| 547 | KVTransferConfig( |
| 548 | kv_connector="ExampleConnector", |
| 549 | kv_role="kv_both", |
| 550 | kv_connector_extra_config={"shared_storage_path": "local_storage"}, |
| 551 | ) |
| 552 | if use_kv_connector |
| 553 | else None |
| 554 | ) |
| 555 | ec_transfer_config = ECTransferConfig( |
| 556 | ec_connector="ECExampleConnector", |
| 557 | ec_role=ec_role, |
| 558 | ec_connector_extra_config={"shared_storage_path": "/tmp/ec_test_encoder"}, |
| 559 | ) |
| 560 | |
| 561 | vllm_config = VllmConfig( |
| 562 | model_config=model_config, |
| 563 | cache_config=cache_config, |
| 564 | scheduler_config=scheduler_config, |
| 565 | kv_transfer_config=kv_transfer_config, |
| 566 | ec_transfer_config=ec_transfer_config, |
| 567 | ) |
| 568 | |
| 569 | executor_class = Executor.get_class(vllm_config) |
| 570 | print(f"executor_class: {executor_class}") |
| 571 | |
| 572 | with set_default_torch_num_threads(1): |
| 573 | engine_core = EngineCore( |
| 574 | vllm_config=vllm_config, executor_class=executor_class, log_stats=True |
| 575 | ) |
| 576 | |
| 577 | # Check encoder cache manager exists |
| 578 | assert engine_core.scheduler.encoder_cache_manager is not None, ( |
| 579 | "encoder_cache_manager should exist" |
| 580 | ) |
| 581 | |
| 582 | if ec_role == "ec_producer": |
| 583 | # Check 1: num_blocks should be 0 |
| 584 | # NOTE: num_blocks=1 as BlockPool always needs a null_block. |
| 585 | kv_cache_config = engine_core.scheduler.kv_cache_manager.kv_cache_config |
| 586 | print(f"kv_cache_config: {kv_cache_config}") |
| 587 | assert kv_cache_config.num_blocks == 1, ( |
| 588 | f"ec_producer should only have 1 KV blocks, " |
| 589 | f"got {kv_cache_config.num_blocks}" |
| 590 | ) |
| 591 | |
| 592 | # Check 2: kv_cache_groups should be empty |
| 593 | assert len(kv_cache_config.kv_cache_groups) == 0, ( |
| 594 | f"ec_producer should have 0 KV cache groups, " |
| 595 | f"got {len(kv_cache_config.kv_cache_groups)}" |
| 596 | ) |
| 597 | |
| 598 | # Check 3: kv_cache_tensors should be empty |
| 599 | assert len(kv_cache_config.kv_cache_tensors) == 0, ( |
| 600 | f"Encoder instance should have 0 KV cache tensors, " |
| 601 | f"got {len(kv_cache_config.kv_cache_tensors)}" |
| 602 | ) |
| 603 | |
| 604 | # Check 4: Verify EC connector is initialized and is producer |
| 605 | assert engine_core.scheduler.ec_connector is not None, ( |
| 606 | "Encoder instance should have EC connector" |
| 607 | ) |
| 608 | assert engine_core.scheduler.ec_connector.is_producer, ( |
| 609 | "Encoder instance EC connector should be producer" |
| 610 | ) |
| 611 | |
| 612 | # Check 5: Verify chunked prefill is disabled |
| 613 | assert not vllm_config.scheduler_config.enable_chunked_prefill, ( |
| 614 | "Encoder instance should disable chunked prefill (no KV cache)" |
| 615 | ) |
| 616 | |
| 617 | elif ec_role == "ec_consumer": |
| 618 | # Check 1: num_blocks should be > 1 |
| 619 | kv_cache_config = engine_core.scheduler.kv_cache_manager.kv_cache_config |
| 620 | print(f"kv_cache_config: {kv_cache_config}") |
| 621 | assert kv_cache_config.num_blocks > 1, ( |
| 622 | f"ec_consumer should have >1 KV blocks, got {kv_cache_config.num_blocks}" |
| 623 | ) |
| 624 | |
| 625 | # Check 2: kv_cache_groups should NOT be empty |
| 626 | assert len(kv_cache_config.kv_cache_groups) > 0, ( |
| 627 | f"ec_consumer should have KV cache groups, " |
| 628 | f"got {len(kv_cache_config.kv_cache_groups)}" |
| 629 | ) |
| 630 | |
| 631 | # Check 3: Verify EC connector is consumer |
| 632 | assert engine_core.scheduler.ec_connector is not None, ( |
| 633 | "Consumer instance should have EC connector" |
| 634 | ) |
| 635 | assert not engine_core.scheduler.ec_connector.is_producer, ( |
| 636 | "Consumer instance EC connector should be consumer" |
| 637 | ) |
| 638 | |
| 639 | |
| 640 | def _cadenced_dp_engine_core( |
| 641 | monkeypatch, results: list[tuple[bool, bool]], dp_sync_interval: int = 32 |
| 642 | ): |
| 643 | """A bare DPEngineCoreProc whose all-reduce is scripted by `results`; |
| 644 | returns the core and the step numbers at which the all-reduce ran.""" |
| 645 | core = object.__new__(DPEngineCoreProc) |
| 646 | core.dp_group = object() |
| 647 | core.dp_sync_interval = dp_sync_interval |
| 648 | core.step_counter = 0 |
| 649 | core.pending_pause = False |
| 650 | core.ignore_start_dp_wave = False |
| 651 | synced: list[int] = [] |
| 652 | |
| 653 | def scripted_sync(dp_group, has_unfinished, pending_pause): |
| 654 | synced.append(core.step_counter) |
| 655 | return results.pop(0) |
| 656 | |
| 657 | monkeypatch.setattr(ParallelConfig, "sync_dp_state", staticmethod(scripted_sync)) |
| 658 | return core, synced |
| 659 | |
| 660 | |
| 661 | def test_dp_sync_interval_default_is_16(): |
| 662 | """Regression pin: lowering the default narrows the mid-wave pause tail |
| 663 | for async RL, where the engine is rarely idle when pause lands.""" |
| 664 | assert ParallelConfig.dp_sync_interval == 16 |
| 665 | |
| 666 | |
| 667 | @pytest.mark.parametrize("dp_sync_interval", [32, 16]) |
| 668 | def test_dp_sync_interval_normal_wave(monkeypatch, dp_sync_interval: int): |
| 669 | """First all-reduce of a wave after one step, then every N.""" |
| 670 | core, synced = _cadenced_dp_engine_core( |
| 671 | monkeypatch, [(True, False)] * 3, dp_sync_interval=dp_sync_interval |
| 672 | ) |
| 673 | for _ in range(dp_sync_interval * 2 + 6): |
| 674 | assert DPEngineCoreProc._has_global_unfinished_reqs(core, True) |
| 675 | assert synced == [1, dp_sync_interval, dp_sync_interval * 2] |
| 676 | |
| 677 | |
| 678 | def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch): |
| 679 | """A pause of an idle engine arms every rank before its kick-started |
| 680 | first step, so the step-1 sync reaches consensus after one dummy batch |
| 681 | regardless of the configured cadence.""" |
| 682 | core, synced = _cadenced_dp_engine_core( |
| 683 | monkeypatch, [(False, True)], dp_sync_interval=32 |
| 684 | ) |
| 685 | core.pending_pause = True |
| 686 | assert DPEngineCoreProc._has_global_unfinished_reqs(core, False) is False |
| 687 | assert synced == [1] |
| 688 | assert core.ignore_start_dp_wave |
| 689 | assert not core.pending_pause |
| 690 | |
| 691 | |
| 692 | def _pausable_engine_core_proc() -> EngineCoreProc: |
| 693 | """A bare EngineCoreProc holding just the state pause_scheduler touches.""" |
| 694 | core = object.__new__(EngineCoreProc) |
| 695 | core.model_executor = MagicMock() |
| 696 | core.scheduler = MagicMock() |
| 697 | core.scheduler.has_requests.return_value = False |
| 698 | core.batch_queue = None |
| 699 | core.engines_running = False |
| 700 | core._idle_state_callbacks = [] |
| 701 | return core |
| 702 | |
| 703 | |
| 704 | @pytest.mark.parametrize( |
| 705 | "pause_state,has_requests,has_batches", |
| 706 | [ |
| 707 | pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"), |
| 708 | pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"), |
| 709 | pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"), |
| 710 | ], |
| 711 | ) |
| 712 | def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches): |
| 713 | """Reject release before touching caches or memory if work can still use KV.""" |
| 714 | core = _pausable_engine_core_proc() |
| 715 | core.scheduler.pause_state = pause_state |
| 716 | core.scheduler.has_requests.return_value = has_requests |
| 717 | core.batch_queue = [object()] if has_batches else None |
| 718 | core._reset_caches = MagicMock() |
| 719 | |
| 720 | with pytest.raises(RuntimeError, match="requires a completed pause"): |
| 721 | core.release_kv_cache_memory() |
| 722 | |
| 723 | core._reset_caches.assert_not_called() |
| 724 | core.model_executor.discard.assert_not_called() |
| 725 | |
| 726 | |
| 727 | @pytest.mark.parametrize("deferred", [False, True]) |
| 728 | def test_pause_synchronizes_device_before_cache_reset(deferred: bool): |
| 729 | """A resolved pause promises an idle device: the barrier must run before |
| 730 | caches are cleared and before the caller is unblocked.""" |
| 731 | core = _pausable_engine_core_proc() |
| 732 | core.engines_running = deferred |
| 733 | order: list[str] = [] |
| 734 | core.model_executor.collective_rpc.side_effect = lambda method: order.append(method) |
| 735 | core._reset_caches = lambda: order.append("reset_caches") |
| 736 | |
| 737 | result = EngineCoreProc.pause_scheduler(core, mode="keep", clear_cache=True) |
| 738 | if deferred: |
| 739 | assert isinstance(result, Future) |
| 740 | assert not result.done() and order == [] |
| 741 | core.engines_running = False |
| 742 | core._notify_idle_state_callbacks() |
| 743 | assert result.result(timeout=0) is None |
| 744 | else: |
| 745 | assert result is None |
| 746 | assert order == ["synchronize_device", "reset_caches"] |