A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Frontend] add `add_request_id` middleware (#9594)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
cjackal committed
Nov 9, 2024 at 19:18 UTC
d88bff1b96c6f4c8abbd3d5ab4758bdc040f7b62
2 files changed
+34
docs/source/serving/openai_compatible_server.md
+26
@@ -62,6 +62,32 @@ completion = client.chat.completions.create(
62
)
63
```
64
65
+### Extra HTTP Headers
66
+
67
+Only `X-Request-Id` HTTP request header is supported for now.
68
+
69
+```python
70
+completion = client.chat.completions.create(
71
+ model="NousResearch/Meta-Llama-3-8B-Instruct",
72
+ messages=[
73
+ {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
74
+ ],
75
+ extra_headers={
76
+ "x-request-id": "sentiment-classification-00001",
77
+ }
78
+)
79
+print(completion._request_id)
80
+
81
+completion = client.completions.create(
82
+ model="NousResearch/Meta-Llama-3-8B-Instruct",
83
+ prompt="A robot may not injure a human being",
84
+ extra_headers={
85
+ "x-request-id": "completion-test",
86
+ }
87
+)
88
+print(completion._request_id)
89
+```
90
+
91
### Extra Parameters for Completions API
92
93
The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
vllm/entrypoints/openai/api_server.py
+8
@@ -7,6 +7,7 @@ import re
7
import signal
8
import socket
9
import tempfile
10
+import uuid
11
from argparse import Namespace
12
from contextlib import asynccontextmanager
13
from functools import partial
@@ -475,6 +476,13 @@ def build_app(args: Namespace) -> FastAPI:
476
status_code=401)
477
return await call_next(request)
478
479
+ @app.middleware("http")
480
+ async def add_request_id(request: Request, call_next):
481
+ request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex
482
+ response = await call_next(request)
483
+ response.headers["X-Request-Id"] = request_id
484
+ return response
485
+
486
for middleware in args.middleware:
487
module_path, object_name = middleware.rsplit(".", 1)
488
imported = getattr(importlib.import_module(module_path), object_name)