| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import contextlib |
| 5 | import os |
| 6 | from typing import Any, NamedTuple |
| 7 | |
| 8 | import openai # use the official client for correctness check |
| 9 | import pytest |
| 10 | |
| 11 | from tests.utils import RemoteOpenAIServer |
| 12 | |
| 13 | # # any model with a chat template should work here |
| 14 | MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct" |
| 15 | API_KEY = "abc-123" |
| 16 | ERROR_API_KEY = "abc" |
| 17 | ROOT_PATH = "llm" |
| 18 | |
| 19 | |
| 20 | @pytest.fixture(scope="module") |
| 21 | def server(): |
| 22 | args = [ |
| 23 | # use half precision for speed and memory savings in CI environment |
| 24 | "--dtype", |
| 25 | "float16", |
| 26 | "--enforce-eager", |
| 27 | "--max-model-len", |
| 28 | "4080", |
| 29 | "--root-path", # use --root-path=/llm for testing |
| 30 | "/" + ROOT_PATH, |
| 31 | ] |
| 32 | envs = os.environ.copy() |
| 33 | |
| 34 | envs["VLLM_API_KEY"] = API_KEY |
| 35 | with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server: |
| 36 | yield remote_server |
| 37 | |
| 38 | |
| 39 | class TestCase(NamedTuple): |
| 40 | model_name: str |
| 41 | base_url: list[str] |
| 42 | api_key: str |
| 43 | expected_error: Any |
| 44 | |
| 45 | |
| 46 | @pytest.mark.asyncio |
| 47 | @pytest.mark.parametrize( |
| 48 | "test_case", |
| 49 | [ |
| 50 | TestCase( |
| 51 | model_name=MODEL_NAME, |
| 52 | base_url=["v1"], # http://localhost:8000/v1 |
| 53 | api_key=ERROR_API_KEY, |
| 54 | expected_error=openai.AuthenticationError, |
| 55 | ), |
| 56 | TestCase( |
| 57 | model_name=MODEL_NAME, |
| 58 | base_url=[ROOT_PATH, "v1"], # http://localhost:8000/llm/v1 |
| 59 | api_key=ERROR_API_KEY, |
| 60 | expected_error=openai.AuthenticationError, |
| 61 | ), |
| 62 | TestCase( |
| 63 | model_name=MODEL_NAME, |
| 64 | base_url=["v1"], # http://localhost:8000/v1 |
| 65 | api_key=API_KEY, |
| 66 | expected_error=None, |
| 67 | ), |
| 68 | TestCase( |
| 69 | model_name=MODEL_NAME, |
| 70 | base_url=[ROOT_PATH, "v1"], # http://localhost:8000/llm/v1 |
| 71 | api_key=API_KEY, |
| 72 | expected_error=None, |
| 73 | ), |
| 74 | ], |
| 75 | ) |
| 76 | async def test_chat_session_root_path_with_api_key( |
| 77 | server: RemoteOpenAIServer, test_case: TestCase |
| 78 | ): |
| 79 | saying: str = "Here is a common saying about apple. An apple a day, keeps" |
| 80 | ctx = contextlib.nullcontext() |
| 81 | if test_case.expected_error is not None: |
| 82 | ctx = pytest.raises(test_case.expected_error) |
| 83 | with ctx: |
| 84 | client = openai.AsyncOpenAI( |
| 85 | api_key=test_case.api_key, |
| 86 | base_url=server.url_for(*test_case.base_url), |
| 87 | max_retries=0, |
| 88 | ) |
| 89 | chat_completion = await client.chat.completions.create( |
| 90 | model=test_case.model_name, |
| 91 | messages=[ |
| 92 | {"role": "user", "content": "tell me a common saying"}, |
| 93 | {"role": "assistant", "content": saying}, |
| 94 | ], |
| 95 | extra_body={"continue_final_message": True, "add_generation_prompt": False}, |
| 96 | ) |
| 97 | |
| 98 | assert chat_completion.id is not None |
| 99 | assert len(chat_completion.choices) == 1 |
| 100 | choice = chat_completion.choices[0] |
| 101 | assert choice.finish_reason == "stop" |
| 102 | message = choice.message |
| 103 | assert len(message.content) > 0 |
| 104 | assert message.role == "assistant" |