@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d04b13a380
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bug]: Authorization ignored when root_path is set (#10606)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

Chauncey committed Nov 26, 2024 at 00:21 UTC d04b13a380da422afa1883efc81e0d4c4b18d091
2 files changed +107 -2
tests/entrypoints/openai/test_root_path.py new
+103
@@ -0,0 +1,103 @@
1 +import contextlib
2 +import os
3 +from typing import Any, List, NamedTuple
4 +
5 +import openai # use the official client for correctness check
6 +import pytest
7 +
8 +from ...utils import RemoteOpenAIServer
9 +
10 +# # any model with a chat template should work here
11 +MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
12 +DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
13 +API_KEY = "abc-123"
14 +ERROR_API_KEY = "abc"
15 +ROOT_PATH = "llm"
16 +
17 +
18 +@pytest.fixture(scope="module")
19 +def server():
20 + args = [
21 + # use half precision for speed and memory savings in CI environment
22 + "--dtype",
23 + "float16",
24 + "--enforce-eager",
25 + "--max-model-len",
26 + "4080",
27 + "--root-path", # use --root-path=/llm for testing
28 + "/" + ROOT_PATH,
29 + "--chat-template",
30 + DUMMY_CHAT_TEMPLATE,
31 + ]
32 + envs = os.environ.copy()
33 +
34 + envs["VLLM_API_KEY"] = API_KEY
35 + with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
36 + yield remote_server
37 +
38 +
39 +class TestCase(NamedTuple):
40 + model_name: str
41 + base_url: List[str]
42 + api_key: str
43 + expected_error: Any
44 +
45 +
46 +@pytest.mark.asyncio
47 +@pytest.mark.parametrize(
48 + "test_case",
49 + [
50 + TestCase(
51 + model_name=MODEL_NAME,
52 + base_url=["v1"], # http://localhost:8000/v1
53 + api_key=ERROR_API_KEY,
54 + expected_error=openai.AuthenticationError),
55 + TestCase(
56 + model_name=MODEL_NAME,
57 + base_url=[ROOT_PATH, "v1"], # http://localhost:8000/llm/v1
58 + api_key=ERROR_API_KEY,
59 + expected_error=openai.AuthenticationError),
60 + TestCase(
61 + model_name=MODEL_NAME,
62 + base_url=["v1"], # http://localhost:8000/v1
63 + api_key=API_KEY,
64 + expected_error=None),
65 + TestCase(
66 + model_name=MODEL_NAME,
67 + base_url=[ROOT_PATH, "v1"], # http://localhost:8000/llm/v1
68 + api_key=API_KEY,
69 + expected_error=None),
70 + ],
71 +)
72 +async def test_chat_session_root_path_with_api_key(server: RemoteOpenAIServer,
73 + test_case: TestCase):
74 + saying: str = "Here is a common saying about apple. An apple a day, keeps"
75 + ctx = contextlib.nullcontext()
76 + if test_case.expected_error is not None:
77 + ctx = pytest.raises(test_case.expected_error)
78 + with ctx:
79 + client = openai.AsyncOpenAI(
80 + api_key=test_case.api_key,
81 + base_url=server.url_for(*test_case.base_url),
82 + max_retries=0)
83 + chat_completion = await client.chat.completions.create(
84 + model=test_case.model_name,
85 + messages=[{
86 + "role": "user",
87 + "content": "tell me a common saying"
88 + }, {
89 + "role": "assistant",
90 + "content": saying
91 + }],
92 + extra_body={
93 + "continue_final_message": True,
94 + "add_generation_prompt": False
95 + })
96 +
97 + assert chat_completion.id is not None
98 + assert len(chat_completion.choices) == 1
99 + choice = chat_completion.choices[0]
100 + assert choice.finish_reason == "stop"
101 + message = choice.message
102 + assert len(message.content) > 0
103 + assert message.role == "assistant"
vllm/entrypoints/openai/api_server.py
+4 -2
@@ -499,10 +499,12 @@ def build_app(args: Namespace) -> FastAPI:
499
500 @app.middleware("http")
501 async def authentication(request: Request, call_next):
502 - root_path = "" if args.root_path is None else args.root_path
502 if request.method == "OPTIONS":
503 return await call_next(request)
505 - if not request.url.path.startswith(f"{root_path}/v1"):
504 + url_path = request.url.path
505 + if app.root_path and url_path.startswith(app.root_path):
506 + url_path = url_path[len(app.root_path):]
507 + if not url_path.startswith("/v1"):
508 return await call_next(request)
509 if request.headers.get("Authorization") != "Bearer " + token:
510 return JSONResponse(content={"error": "Unauthorized"},