A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bug]: Authorization ignored when root_path is set (#10606)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Nov 26, 2024 at 00:21 UTC
d04b13a380da422afa1883efc81e0d4c4b18d091
2 files changed
+107
-2
tests/entrypoints/openai/test_root_path.py
new
+103
@@ -0,0 +1,103 @@
1
+import contextlib
2
+import os
3
+from typing import Any, List, NamedTuple
4
+
5
+import openai # use the official client for correctness check
6
+import pytest
7
+
8
+from ...utils import RemoteOpenAIServer
9
+
10
+# # any model with a chat template should work here
11
+MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
12
+DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
13
+API_KEY = "abc-123"
14
+ERROR_API_KEY = "abc"
15
+ROOT_PATH = "llm"
16
+
17
+
18
+@pytest.fixture(scope="module")
19
+def server():
20
+ args = [
21
+ # use half precision for speed and memory savings in CI environment
22
+ "--dtype",
23
+ "float16",
24
+ "--enforce-eager",
25
+ "--max-model-len",
26
+ "4080",
27
+ "--root-path", # use --root-path=/llm for testing
28
+ "/" + ROOT_PATH,
29
+ "--chat-template",
30
+ DUMMY_CHAT_TEMPLATE,
31
+ ]
32
+ envs = os.environ.copy()
33
+
34
+ envs["VLLM_API_KEY"] = API_KEY
35
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
36
+ yield remote_server
37
+
38
+
39
+class TestCase(NamedTuple):
40
+ model_name: str
41
+ base_url: List[str]
42
+ api_key: str
43
+ expected_error: Any
44
+
45
+
46
+@pytest.mark.asyncio
47
+@pytest.mark.parametrize(
48
+ "test_case",
49
+ [
50
+ TestCase(
51
+ model_name=MODEL_NAME,
52
+ base_url=["v1"], # http://localhost:8000/v1
53
+ api_key=ERROR_API_KEY,
54
+ expected_error=openai.AuthenticationError),
55
+ TestCase(
56
+ model_name=MODEL_NAME,
57
+ base_url=[ROOT_PATH, "v1"], # http://localhost:8000/llm/v1
58
+ api_key=ERROR_API_KEY,
59
+ expected_error=openai.AuthenticationError),
60
+ TestCase(
61
+ model_name=MODEL_NAME,
62
+ base_url=["v1"], # http://localhost:8000/v1
63
+ api_key=API_KEY,
64
+ expected_error=None),
65
+ TestCase(
66
+ model_name=MODEL_NAME,
67
+ base_url=[ROOT_PATH, "v1"], # http://localhost:8000/llm/v1
68
+ api_key=API_KEY,
69
+ expected_error=None),
70
+ ],
71
+)
72
+async def test_chat_session_root_path_with_api_key(server: RemoteOpenAIServer,
73
+ test_case: TestCase):
74
+ saying: str = "Here is a common saying about apple. An apple a day, keeps"
75
+ ctx = contextlib.nullcontext()
76
+ if test_case.expected_error is not None:
77
+ ctx = pytest.raises(test_case.expected_error)
78
+ with ctx:
79
+ client = openai.AsyncOpenAI(
80
+ api_key=test_case.api_key,
81
+ base_url=server.url_for(*test_case.base_url),
82
+ max_retries=0)
83
+ chat_completion = await client.chat.completions.create(
84
+ model=test_case.model_name,
85
+ messages=[{
86
+ "role": "user",
87
+ "content": "tell me a common saying"
88
+ }, {
89
+ "role": "assistant",
90
+ "content": saying
91
+ }],
92
+ extra_body={
93
+ "continue_final_message": True,
94
+ "add_generation_prompt": False
95
+ })
96
+
97
+ assert chat_completion.id is not None
98
+ assert len(chat_completion.choices) == 1
99
+ choice = chat_completion.choices[0]
100
+ assert choice.finish_reason == "stop"
101
+ message = choice.message
102
+ assert len(message.content) > 0
103
+ assert message.role == "assistant"
vllm/entrypoints/openai/api_server.py
+4
-2
@@ -499,10 +499,12 @@ def build_app(args: Namespace) -> FastAPI:
499
500
@app.middleware("http")
501
async def authentication(request: Request, call_next):
502
- root_path = "" if args.root_path is None else args.root_path
502
if request.method == "OPTIONS":
503
return await call_next(request)
505
- if not request.url.path.startswith(f"{root_path}/v1"):
504
+ url_path = request.url.path
505
+ if app.root_path and url_path.startswith(app.root_path):
506
+ url_path = url_path[len(app.root_path):]
507
+ if not url_path.startswith("/v1"):
508
return await call_next(request)
509
if request.headers.get("Authorization") != "Bearer " + token:
510
return JSONResponse(content={"error": "Unauthorized"},