| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Containing tests that check for regressions in vLLM's behavior. |
| 4 | |
| 5 | It should include tests that are reported by users and making sure they |
| 6 | will never happen again. |
| 7 | |
| 8 | """ |
| 9 | |
| 10 | import gc |
| 11 | |
| 12 | import pytest |
| 13 | import torch |
| 14 | |
| 15 | from tests.utils import large_gpu_mark |
| 16 | from vllm import LLM, SamplingParams |
| 17 | from vllm.platforms import current_platform |
| 18 | |
| 19 | |
| 20 | @pytest.mark.parametrize( |
| 21 | "model", |
| 22 | [ |
| 23 | pytest.param( |
| 24 | "distilbert/distilgpt2", |
| 25 | marks=[ |
| 26 | *([large_gpu_mark(min_gb=80)] if current_platform.is_rocm() else []), |
| 27 | ], |
| 28 | ), |
| 29 | ], |
| 30 | ) |
| 31 | def test_max_tokens_none(model): |
| 32 | sampling_params = SamplingParams(temperature=0.01, top_p=0.1, max_tokens=None) |
| 33 | llm = LLM( |
| 34 | model=model, |
| 35 | max_num_batched_tokens=4096, |
| 36 | tensor_parallel_size=1, |
| 37 | ) |
| 38 | prompts = ["Just say hello!"] |
| 39 | outputs = llm.generate(prompts, sampling_params=sampling_params) |
| 40 | |
| 41 | assert len(prompts) == len(outputs) |
| 42 | |
| 43 | |
| 44 | def test_gc(): |
| 45 | llm = LLM(model="distilbert/distilgpt2", enforce_eager=True) |
| 46 | del llm |
| 47 | |
| 48 | gc.collect() |
| 49 | torch.accelerator.empty_cache() |
| 50 | |
| 51 | # The memory allocated for model and KV cache should be released. |
| 52 | # The memory allocated for PyTorch and others should be less than 50MB. |
| 53 | # Usually, it's around 10MB. |
| 54 | allocated = torch.accelerator.memory_allocated() |
| 55 | assert allocated < 50 * 1024 * 1024 |
| 56 | |
| 57 | |
| 58 | def test_model_from_modelscope(monkeypatch: pytest.MonkeyPatch): |
| 59 | # model: https://www.modelscope.ai/models/qwen/Qwen1.5-0.5B-Chat |
| 60 | with monkeypatch.context() as m: |
| 61 | m.setenv("VLLM_USE_MODELSCOPE", "True") |
| 62 | m.setenv("MODELSCOPE_DOMAIN", "www.modelscope.ai") |
| 63 | # Don't use HF_TOKEN for ModelScope repos, otherwise it will fail |
| 64 | # with 400 Client Error: Bad Request. |
| 65 | m.setenv("HF_TOKEN", "") |
| 66 | attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto" |
| 67 | llm = LLM(model="qwen/Qwen1.5-0.5B-Chat", attention_backend=attn_backend) |
| 68 | |
| 69 | prompts = [ |
| 70 | "Hello, my name is", |
| 71 | "The president of the United States is", |
| 72 | "The capital of France is", |
| 73 | "The future of AI is", |
| 74 | ] |
| 75 | sampling_params = SamplingParams(temperature=0.8, top_p=0.95) |
| 76 | |
| 77 | outputs = llm.generate(prompts, sampling_params) |
| 78 | assert len(outputs) == 4 |