| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Test that we handle a startup Error and shutdown.""" |
| 4 | |
| 5 | import inspect |
| 6 | |
| 7 | import pytest |
| 8 | |
| 9 | from tests.utils import wait_for_gpu_memory_to_clear |
| 10 | from tests.v1.shutdown.utils import ( |
| 11 | SHUTDOWN_TEST_THRESHOLD_BYTES, |
| 12 | SHUTDOWN_TEST_TIMEOUT_SEC, |
| 13 | ) |
| 14 | from vllm import LLM |
| 15 | from vllm.distributed import get_tensor_model_parallel_rank |
| 16 | from vllm.engine.arg_utils import AsyncEngineArgs |
| 17 | from vllm.model_executor.models.llama import LlamaForCausalLM |
| 18 | from vllm.platforms import current_platform |
| 19 | from vllm.v1.engine.async_llm import AsyncLLM |
| 20 | |
| 21 | MODELS = ["hmellor/tiny-random-LlamaForCausalLM"] |
| 22 | |
| 23 | |
| 24 | def evil_method(self, *args, **kwargs): |
| 25 | """Evil method that raises an exception.""" |
| 26 | if get_tensor_model_parallel_rank() == 0: |
| 27 | raise Exception("Simulated Error in startup!") |
| 28 | |
| 29 | return self.model(*args, **kwargs, intermediate_tensors=None) |
| 30 | |
| 31 | |
| 32 | @pytest.fixture |
| 33 | def rocm_evil_method(rocm_sitecustomize_factory, request): |
| 34 | failing_method = request.getfixturevalue("failing_method") |
| 35 | lines = [ |
| 36 | "from vllm.distributed import get_tensor_model_parallel_rank", |
| 37 | "from vllm.model_executor.models.llama import LlamaForCausalLM", |
| 38 | inspect.getsource(evil_method), |
| 39 | f"LlamaForCausalLM.{failing_method} = {evil_method.__name__}", |
| 40 | ] |
| 41 | rocm_sitecustomize_factory(lines) |
| 42 | |
| 43 | |
| 44 | @pytest.mark.timeout(SHUTDOWN_TEST_TIMEOUT_SEC) |
| 45 | @pytest.mark.parametrize("model", MODELS) |
| 46 | @pytest.mark.parametrize("tensor_parallel_size", [2, 1]) |
| 47 | @pytest.mark.parametrize("failing_method", ["forward", "load_weights"]) |
| 48 | def test_async_llm_startup_error( |
| 49 | monkeypatch, |
| 50 | rocm_evil_method, |
| 51 | model: str, |
| 52 | tensor_parallel_size: int, |
| 53 | failing_method: str, |
| 54 | ) -> None: |
| 55 | """Test that AsyncLLM propagates an __init__ error & frees memory. |
| 56 | Test profiling (forward()) and load weights failures. |
| 57 | AsyncLLM always uses an MP client. |
| 58 | """ |
| 59 | if current_platform.device_count() < tensor_parallel_size: |
| 60 | pytest.skip(reason="Not enough CUDA devices") |
| 61 | |
| 62 | # Monkeypatch an error in the model. |
| 63 | monkeypatch.setattr(LlamaForCausalLM, failing_method, evil_method) |
| 64 | |
| 65 | engine_args = AsyncEngineArgs( |
| 66 | model=model, enforce_eager=True, tensor_parallel_size=tensor_parallel_size |
| 67 | ) |
| 68 | |
| 69 | # Confirm we get an exception. |
| 70 | with pytest.raises(Exception, match=r"initialization fail(ed|ure)"): |
| 71 | _ = AsyncLLM.from_engine_args(engine_args) |
| 72 | |
| 73 | # Confirm all the processes are cleaned up. |
| 74 | wait_for_gpu_memory_to_clear( |
| 75 | devices=list(range(tensor_parallel_size)), |
| 76 | threshold_bytes=SHUTDOWN_TEST_THRESHOLD_BYTES, |
| 77 | ) |
| 78 | |
| 79 | |
| 80 | @pytest.mark.timeout(SHUTDOWN_TEST_TIMEOUT_SEC) |
| 81 | @pytest.mark.parametrize("model", MODELS) |
| 82 | @pytest.mark.parametrize("tensor_parallel_size", [2, 1]) |
| 83 | @pytest.mark.parametrize("enable_multiprocessing", [True]) |
| 84 | @pytest.mark.parametrize("failing_method", ["forward", "load_weights"]) |
| 85 | def test_llm_startup_error( |
| 86 | monkeypatch, |
| 87 | rocm_evil_method, |
| 88 | model: str, |
| 89 | tensor_parallel_size: int, |
| 90 | enable_multiprocessing: bool, |
| 91 | failing_method: str, |
| 92 | ) -> None: |
| 93 | """Test that LLM propagates an __init__ error and frees memory. |
| 94 | Test profiling (forward()) and load weights failures. |
| 95 | TODO(andy) - LLM without multiprocessing. |
| 96 | """ |
| 97 | # Skip non-Llama models since we monkeypatch LlamaForCausalLM specifically. |
| 98 | # If MODELS list grows, each architecture needs its own test variant. |
| 99 | if model != "JackFram/llama-68m": |
| 100 | pytest.skip(reason="Only test JackFram/llama-68m") |
| 101 | if current_platform.device_count() < tensor_parallel_size: |
| 102 | pytest.skip(reason="Not enough CUDA devices") |
| 103 | |
| 104 | with monkeypatch.context() as m: |
| 105 | MP_VALUE = "1" if enable_multiprocessing else "0" |
| 106 | m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", MP_VALUE) |
| 107 | |
| 108 | # Monkeypatch an error in the model. |
| 109 | monkeypatch.setattr(LlamaForCausalLM, failing_method, evil_method) |
| 110 | |
| 111 | with pytest.raises( |
| 112 | Exception, |
| 113 | match=r"initialization fail(ed|ure)" |
| 114 | if enable_multiprocessing |
| 115 | else "Simulated Error in startup!", |
| 116 | ): |
| 117 | _ = LLM( |
| 118 | model=model, |
| 119 | enforce_eager=True, |
| 120 | tensor_parallel_size=tensor_parallel_size, |
| 121 | ) |
| 122 | |
| 123 | # Confirm all the processes are cleaned up. |
| 124 | wait_for_gpu_memory_to_clear( |
| 125 | devices=list(range(tensor_parallel_size)), |
| 126 | threshold_bytes=SHUTDOWN_TEST_THRESHOLD_BYTES, |
| 127 | ) |