@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d88bff1b96
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Frontend] add `add_request_id` middleware (#9594)

Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>

cjackal committed Nov 9, 2024 at 19:18 UTC d88bff1b96c6f4c8abbd3d5ab4758bdc040f7b62
2 files changed +34
docs/source/serving/openai_compatible_server.md
+26
@@ -62,6 +62,32 @@ completion = client.chat.completions.create(
62 )
63 ```
64
65 +### Extra HTTP Headers
66 +
67 +Only `X-Request-Id` HTTP request header is supported for now.
68 +
69 +```python
70 +completion = client.chat.completions.create(
71 + model="NousResearch/Meta-Llama-3-8B-Instruct",
72 + messages=[
73 + {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
74 + ],
75 + extra_headers={
76 + "x-request-id": "sentiment-classification-00001",
77 + }
78 +)
79 +print(completion._request_id)
80 +
81 +completion = client.completions.create(
82 + model="NousResearch/Meta-Llama-3-8B-Instruct",
83 + prompt="A robot may not injure a human being",
84 + extra_headers={
85 + "x-request-id": "completion-test",
86 + }
87 +)
88 +print(completion._request_id)
89 +```
90 +
91 ### Extra Parameters for Completions API
92
93 The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
vllm/entrypoints/openai/api_server.py
+8
@@ -7,6 +7,7 @@ import re
7 import signal
8 import socket
9 import tempfile
10 +import uuid
11 from argparse import Namespace
12 from contextlib import asynccontextmanager
13 from functools import partial
@@ -475,6 +476,13 @@ def build_app(args: Namespace) -> FastAPI:
476 status_code=401)
477 return await call_next(request)
478
479 + @app.middleware("http")
480 + async def add_request_id(request: Request, call_next):
481 + request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex
482 + response = await call_next(request)
483 + response.headers["X-Request-Id"] = request_id
484 + return response
485 +
486 for middleware in args.middleware:
487 module_path, object_name = middleware.rsplit(".", 1)
488 imported = getattr(importlib.import_module(module_path), object_name)