22170354d8 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/serving/online_serving/openai_compatible_server.md

+15 -2

@@ -83,8 +83,8 @@ completion = client.chat.completions.create(

83

84 ## Extra HTTP Headers

85

86 -Only `X-Request-Id` HTTP request header is supported for now. It can be enabled

87 -with `--enable-request-id-headers`.

86 +The `X-Request-Id` HTTP request header can be enabled with

87 +`--enable-request-id-headers`.

88

89 ??? code

90

@@ -110,6 +110,19 @@ with `--enable-request-id-headers`.

110 print(completion._request_id)

111 ```

112

113 +The Completions, Chat Completions, and Responses APIs also support the

114 +`X-Vllm-Priority` request header. Its value must be an integer and overrides

115 +the `priority` value in the JSON request body. Non-zero priorities require the

116 +server to use priority scheduling.

117 +

118 +```python

119 +completion = client.chat.completions.create(

120 + model="NousResearch/Meta-Llama-3-8B-Instruct",

121 + messages=[{"role": "user", "content": "Hello!"}],

122 + extra_headers={"X-Vllm-Priority": "-10"},

123 +)

124 +```

125 +

126 ## API Reference

127

128 ### Completions API

rust/src/server/src/routes/openai/chat_completions/convert.rs

+18 -1

@@ -178,7 +178,7 @@ pub(super) fn prepare_chat_request(

178 min_tokens: request.min_tokens.unwrap_or(0),

179 },

180 intermediate: request.stream,

181 - priority: request.priority.unwrap_or(0),

181 + priority: ctx.priority.or(request.priority).unwrap_or(0),

182 documents: request.documents,

183 cache_salt: request.cache_salt,

184 add_special_tokens: request.add_special_tokens,

@@ -1264,6 +1264,23 @@ mod tests {

1264 );

1265 }

1266

1267 + #[test]

1268 + fn prepare_chat_request_header_priority_overrides_body() {

1269 + let mut headers = HeaderMap::new();

1270 + headers.insert("X-Vllm-Priority", "-5".parse().unwrap());

1271 + let request = ChatCompletionRequest {

1272 + priority: Some(10),

1273 + ..base_request()

1274 + };

1275 + let prepared = prepare_chat_request(

1276 + request,

1277 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

1278 + request_context(&headers, None),

1279 + )

1280 + .expect("request is valid");

1281 + assert_eq!(prepared.chat_request.priority, -5);

1282 + }

1283 +

1284 #[test]

1285 fn prepare_chat_request_ignores_correlation_header() {

1286 let mut headers = HeaderMap::new();

rust/src/server/src/routes/openai/completions/convert.rs

+22 -1

@@ -149,7 +149,7 @@ pub(super) fn prepare_completion_request(

149 min_tokens: request.min_tokens.unwrap_or(0),

150 },

151 intermediate: request.stream,

152 - priority: request.priority.unwrap_or(0),

152 + priority: ctx.priority.or(request.priority).unwrap_or(0),

153 cache_salt: request.cache_salt,

154 add_special_tokens: request.add_special_tokens,

155 data_parallel_rank: ctx.data_parallel_rank,

@@ -619,6 +619,27 @@ mod tests {

619 );

620 }

621

622 + #[test]

623 + fn prepare_completion_request_header_priority_overrides_body() {

624 + let request: CompletionRequest = serde_json::from_value(json!({

625 + "model": "Qwen/Qwen1.5-0.5B-Chat",

626 + "prompt": "hello",

627 + "priority": 10,

628 + }))

629 + .expect("parse request");

630 +

631 + let mut headers = HeaderMap::new();

632 + headers.insert("X-Vllm-Priority", "-5".parse().unwrap());

633 + let prepared = prepare_completion_request(

634 + request,

635 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

636 + request_context(&headers, None),

637 + &test_tokenizer(),

638 + )

639 + .expect("prepare");

640 + assert_eq!(prepared.text_request.priority, -5);

641 + }

642 +

643 #[test]

644 fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() {

645 let request: CompletionRequest = serde_json::from_value(json!({

rust/src/server/src/utils.rs

+8 -1

@@ -15,6 +15,7 @@ use crate::error::ApiError;

15 pub struct ResolvedRequestContext {

16 pub request_id: String,

17 pub data_parallel_rank: Option<u32>,

18 + pub priority: Option<i32>,

19 pub session_id: Option<String>,

20 }

21

@@ -111,7 +112,8 @@ pub fn convert_logit_bias(

112 }

113

114 /// Extract common request metadata from HTTP headers: the external request ID,

114 -/// session ID, and the optional data-parallel rank used for engine routing.

115 +/// session ID, priority, and the optional data-parallel rank used for engine

116 +/// routing.

117 pub fn resolve_request_context(

118 headers: &HeaderMap,

119 request_id: Option<&str>,

@@ -121,6 +123,10 @@ pub fn resolve_request_context(

123 .get("X-data-parallel-rank")

124 .and_then(|v| v.to_str().ok())

125 .and_then(|s| s.trim().parse().ok());

126 + let priority = headers

127 + .get("X-Vllm-Priority")

128 + .and_then(|value| value.to_str().ok())

129 + .and_then(|value| value.trim().parse().ok());

130

131 // Extract request id from header.

132 let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());

@@ -134,6 +140,7 @@ pub fn resolve_request_context(

140 ResolvedRequestContext {

141 request_id,

142 data_parallel_rank,

143 + priority,

144 session_id,

145 }

146 }

vllm/entrypoints/generate/base/serving.py

+15

@@ -42,6 +42,7 @@ logger = init_logger(__name__)

42 RequestT = TypeVar("RequestT", bound=AnyRequest)

43 _T = TypeVar("_T")

44 SESSION_ID_HEADER = "X-Session-ID"

45 +PRIORITY_HEADER = "X-Vllm-Priority"

46

47

48 def build_per_request_timing_metrics(

@@ -239,6 +240,20 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):

240 return session_id

241 return None

242

243 + @staticmethod

244 + def _get_priority(

245 + request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,

246 + raw_request: Request | None,

247 + ) -> int:

248 + if raw_request is not None:

249 + priority = raw_request.headers.get(PRIORITY_HEADER)

250 + if priority is not None:

251 + try:

252 + return int(priority)

253 + except ValueError:

254 + pass

255 + return request.priority

256 +

257 async def _with_kv_transfer_rejection_cleanup(

258 self,

259 awaitable: Awaitable[_T],

vllm/entrypoints/openai/chat_completion/serving.py

+1 -1

@@ -348,7 +348,7 @@ class OpenAIServingChat(GenerateBaseServing):

348 sub_request_id,

349 lora_request=lora_request,

350 trace_headers=trace_headers,

351 - priority=request.priority,

351 + priority=self._get_priority(request, raw_request),

352 data_parallel_rank=data_parallel_rank,

353 session_id=session_id,

354 reasoning_ended=reasoning_ended,

vllm/entrypoints/openai/completion/serving.py

+1 -1

@@ -212,7 +212,7 @@ class OpenAIServingCompletion(GenerateBaseServing):

212 request_id_item,

213 lora_request=lora_request,

214 trace_headers=trace_headers,

215 - priority=request.priority,

215 + priority=self._get_priority(request, raw_request),

216 data_parallel_rank=data_parallel_rank,

217 session_id=session_id,

218 )

vllm/entrypoints/openai/responses/serving.py

+1 -1

@@ -515,7 +515,7 @@ class OpenAIServingResponses(GenerateBaseServing):

515 sampling_params=sampling_params,

516 context=context,

517 lora_request=lora_request,

518 - priority=request.priority,

518 + priority=self._get_priority(request, raw_request),

519 trace_headers=trace_headers,

520 session_id=session_id,

521 reasoning_parser_kwargs=reasoning_parser_kwargs

[Feature] Parse request priority from HTTP header (#51089)