8df14cfc8c · tduncan/Nvidia-Optimization-Stack-VLLM

.buildkite/test_areas/misc.yaml

+2

@@ -89,6 +89,7 @@ steps:

89 - tests/v1/simple_kv_offload

90 - tests/v1/worker

91 - tests/v1/kv_connector/unit

92 + - tests/v1/ec_connector/unit

93 - tests/v1/metrics

94 - tests/entrypoints/openai/correctness/test_lmeval.py

95 commands:

@@ -101,6 +102,7 @@ steps:

102 - pytest -v -s v1/simple_kv_offload

103 - pytest -v -s v1/worker

104 - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit

105 + - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit

106 - pytest -v -s -m 'not cpu_test' v1/metrics

107 # Integration test for streaming correctness (requires special branch).

108 - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api

rust/proto/vllm_grpc.proto

+4

@@ -107,6 +107,9 @@ message KVCacheParameters {

107

108 // KV Connector transfer parameters

109 google.protobuf.Struct kv_transfer_params = 3;

110 +

111 + // Encoder cache connector transfer parameters

112 + google.protobuf.Struct ec_transfer_params = 4;

113 }

114

115 // Controls which extra candidate tokens at each position should be returned

@@ -173,6 +176,7 @@ message FinishInfo {

176

177 google.protobuf.Struct kv_transfer_params = 6;

178 //uint64 seed = 7;

179 + google.protobuf.Struct ec_transfer_params = 8;

180 }

181

182 // Info for candidate tokens other than the input/sampled

rust/src/chat/src/event.rs

+3

@@ -202,5 +202,8 @@ pub enum ChatEvent {

202 finish_reason: FinishReason,

203 /// Connector-specific KV transfer parameters for disaggregated serving.

204 kv_transfer_params: Option<serde_json::Value>,

205 + /// Connector-specific encoder cache transfer parameters for

206 + /// disaggregated serving.

207 + ec_transfer_params: Option<serde_json::Value>,

208 },

209 }

rust/src/chat/src/output/default/unified.rs

+4

@@ -257,6 +257,7 @@ pub(crate) async fn unified_event_stream(

257 usage: finished.usage,

258 finish_reason: finished.finish_reason,

259 kv_transfer_params: finished.kv_transfer_params,

260 + ec_transfer_params: finished.ec_transfer_params,

261 })

262 .await;

263 }

@@ -387,6 +388,7 @@ mod tests {

388 usage: vllm_llm::TokenUsage::default(),

389 finish_reason: crate::FinishReason::Stop(None),

390 kv_transfer_params: None,

391 + ec_transfer_params: None,

392 }),

393 }

394 }

@@ -628,6 +630,7 @@ mod tests {

630 usage: vllm_llm::TokenUsage::default(),

631 finish_reason: crate::FinishReason::Stop(None),

632 kv_transfer_params: None,

633 + ec_transfer_params: None,

634 },

635 ]

636 );

@@ -671,6 +674,7 @@ mod tests {

674 usage: vllm_llm::TokenUsage::default(),

675 finish_reason: crate::FinishReason::Stop(None),

676 kv_transfer_params: None,

677 + ec_transfer_params: None,

678 },

679 ]

680 );

rust/src/chat/src/output/harmony/mod.rs

+1

@@ -370,6 +370,7 @@ async fn harmony_assistant_event_stream(

370 usage: finished.usage,

371 finish_reason: finished.finish_reason,

372 kv_transfer_params: finished.kv_transfer_params,

373 + ec_transfer_params: finished.ec_transfer_params,

374 })

375 .await;

376 }

rust/src/chat/src/output/harmony/tests.rs

+3

@@ -52,6 +52,7 @@ fn finished() -> Finished {

52 },

53 finish_reason: FinishReason::stop_eos(),

54 kv_transfer_params: None,

55 + ec_transfer_params: None,

56 }

57 }

58

@@ -115,6 +116,7 @@ fn interrupted_final_message_is_preserved() {

116 },

117 finish_reason: FinishReason::stop_eos(),

118 kv_transfer_params: None,

119 + ec_transfer_params: None,

120 })

121 );

122 }

@@ -175,6 +177,7 @@ fn interrupted_analysis_message_is_preserved() {

177 },

178 finish_reason: FinishReason::stop_eos(),

179 kv_transfer_params: None,

180 + ec_transfer_params: None,

181 })

182 );

183 }

rust/src/chat/src/output/mod.rs

+3

@@ -48,6 +48,9 @@ pub(crate) enum AssistantEvent {

48 finish_reason: FinishReason,

49 /// Connector-specific KV transfer parameters for disaggregated serving.

50 kv_transfer_params: Option<serde_json::Value>,

51 + /// Connector-specific encoder cache transfer parameters for

52 + /// disaggregated serving.

53 + ec_transfer_params: Option<serde_json::Value>,

54 },

55 }

56

rust/src/chat/src/output/structured.rs

+11 -1

@@ -146,6 +146,7 @@ impl StructuredEventState {

146 usage: vllm_llm::TokenUsage,

147 finish_reason: FinishReason,

148 kv_transfer_params: Option<serde_json::Value>,

149 + ec_transfer_params: Option<serde_json::Value>,

150 ) -> Result<Vec<ChatEvent>> {

151 let mut events = Vec::new();

152 self.close_open_text_block(&mut events);

@@ -155,6 +156,7 @@ impl StructuredEventState {

156 usage,

157 finish_reason,

158 kv_transfer_params,

159 + ec_transfer_params,

160 });

161 Ok(events)

162 }

@@ -296,8 +298,11 @@ pub(crate) async fn structured_chat_event_stream(

298 usage,

299 finish_reason,

300 kv_transfer_params,

301 + ec_transfer_params,

302 } => {

300 - for next in state.finish(usage, finish_reason, kv_transfer_params)? {

303 + for next in

304 + state.finish(usage, finish_reason, kv_transfer_params, ec_transfer_params)?

305 + {

306 y.yield_ok(next).await;

307 }

308 }

@@ -334,6 +339,7 @@ mod tests {

339 },

340 finish_reason: FinishReason::stop_eos(),

341 kv_transfer_params: None,

342 + ec_transfer_params: None,

343 }),

344 ]);

345

@@ -388,6 +394,7 @@ mod tests {

394 },

395 finish_reason: FinishReason::stop_eos(),

396 kv_transfer_params: None,

397 + ec_transfer_params: None,

398 }),

399 ]);

400

@@ -439,6 +446,7 @@ mod tests {

446 },

447 finish_reason: FinishReason::stop_eos(),

448 kv_transfer_params: None,

449 + ec_transfer_params: None,

450 }),

451 ]);

452

@@ -490,6 +498,7 @@ mod tests {

498 },

499 finish_reason: FinishReason::stop_eos(),

500 kv_transfer_params: None,

501 + ec_transfer_params: None,

502 }),

503 ]);

504

@@ -557,6 +566,7 @@ mod tests {

566 },

567 finish_reason: FinishReason::stop_eos(),

568 kv_transfer_params: None,

569 + ec_transfer_params: None,

570 }),

571 ]);

572

rust/src/chat/src/stream.rs

+7

@@ -22,6 +22,9 @@ pub struct CollectedAssistantMessage {

22 pub finish_reason: FinishReason,

23 /// Connector-specific KV transfer parameters for disaggregated serving.

24 pub kv_transfer_params: Option<serde_json::Value>,

25 + /// Connector-specific encoder cache transfer parameters for disaggregated

26 + /// serving.

27 + pub ec_transfer_params: Option<serde_json::Value>,

28 }

29

30 /// Per-request stream of chat events.

@@ -77,6 +80,7 @@ impl ChatEventStream {

80 usage,

81 finish_reason,

82 kv_transfer_params,

83 + ec_transfer_params,

84 } => {

85 return Ok(CollectedAssistantMessage {

86 message: done,

@@ -89,6 +93,7 @@ impl ChatEventStream {

93 usage,

94 finish_reason,

95 kv_transfer_params,

96 + ec_transfer_params,

97 });

98 }

99 ChatEvent::ToolCallEnd { call, .. } => {

@@ -194,6 +199,7 @@ mod tests {

199 },

200 finish_reason: FinishReason::stop_eos(),

201 kv_transfer_params: None,

202 + ec_transfer_params: None,

203 }),

204 ]),

205 );

@@ -234,6 +240,7 @@ mod tests {

240 },

241 finish_reason: FinishReason::stop_eos(),

242 kv_transfer_params: None,

243 + ec_transfer_params: None,

244 }

245 );

246 }

rust/src/chat/tests/chat.rs

+2

@@ -50,6 +50,7 @@ fn request_output(

50 stop_reason,

51 events: None,

52 kv_transfer_params: None,

53 + ec_transfer_params: None,

54 trace_headers: None,

55 prefill_stats: None,

56 routed_experts: None,

@@ -75,6 +76,7 @@ fn request_output_with_logprobs(

76 stop_reason,

77 events: None,

78 kv_transfer_params: None,

79 + ec_transfer_params: None,

80 trace_headers: None,

81 prefill_stats: None,

82 routed_experts: None,

rust/src/chat/tests/roundtrip.rs

+2

@@ -676,6 +676,7 @@ fn decoded_completion_stream(

676 usage: Default::default(),

677 finish_reason: FinishReason::stop_eos(),

678 kv_transfer_params: None,

679 + ec_transfer_params: None,

680 }),

681 }

682 });

@@ -686,6 +687,7 @@ fn decoded_completion_stream(

687 usage: Default::default(),

688 finish_reason: FinishReason::stop_eos(),

689 kv_transfer_params: None,

690 + ec_transfer_params: None,

691 });

692 events.push(DecodedTextEvent::TextDelta {

693 delta: chunk.delta,

rust/src/engine-core-client/src/protocol/output.rs

+4

@@ -97,6 +97,8 @@ pub struct EngineCoreOutput {

97 #[serde(default)]

98 pub kv_transfer_params: Option<serde_json::Value>,

99 #[serde(default)]

100 + pub ec_transfer_params: Option<serde_json::Value>,

101 + #[serde(default)]

102 pub trace_headers: Option<OpaqueValue>,

103 /// Breakdown of the scheduled prefill computation, set on the first output

104 /// of a newly scheduled prefill and elided for subsequent decode outputs.

@@ -374,6 +376,7 @@ mod tests {

376 stop_reason: Some(StopReason::Text("stop".to_string())),

377 events: None,

378 kv_transfer_params: None,

379 + ec_transfer_params: None,

380 trace_headers: None,

381 prefill_stats: None,

382 routed_experts: None,

@@ -426,6 +429,7 @@ mod tests {

429 stop_reason: None,

430 events: None,

431 kv_transfer_params: None,

432 + ec_transfer_params: None,

433 trace_headers: None,

434 prefill_stats: None,

435 routed_experts: None,

rust/src/engine-core-client/src/tests/client.rs

+2

@@ -226,6 +226,7 @@ fn request_output(

226 stop_reason: None,

227 events: None,

228 kv_transfer_params: None,

229 + ec_transfer_params: None,

230 trace_headers: None,

231 prefill_stats: None,

232 routed_experts: None,

@@ -2517,6 +2518,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {

2518 stop_reason: None,

2519 events: None,

2520 kv_transfer_params: None,

2521 + ec_transfer_params: None,

2522 trace_headers: None,

2523 prefill_stats: None,

2524 routed_experts: None,

rust/src/engine-core-client/src/tests/python_compat.py

+1

@@ -91,6 +91,7 @@ class EngineCoreOutput(

91 stop_reason: int | str | None = None

92 events: object | None = None

93 kv_transfer_params: object | None = None

94 + ec_transfer_params: object | None = None

95 trace_headers: object | None = None

96 prefill_stats: object | None = None

97 routed_experts: object | None = None

rust/src/llm/src/output.rs

+10

@@ -38,6 +38,9 @@ pub struct CollectedGenerateOutput {

38 pub usage: TokenUsage,

39 /// Connector-specific KV transfer parameters for disaggregated serving.

40 pub kv_transfer_params: Option<serde_json::Value>,

41 + /// Connector-specific encoder cache transfer parameters for disaggregated

42 + /// serving.

43 + pub ec_transfer_params: Option<serde_json::Value>,

44 }

45

46 /// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for

@@ -146,6 +149,9 @@ pub struct GenerateOutput {

149 pub cached_token_count: usize,

150 /// Connector-specific KV transfer parameters for disaggregated serving.

151 pub kv_transfer_params: Option<serde_json::Value>,

152 + /// Connector-specific encoder cache transfer parameters for disaggregated

153 + /// serving.

154 + pub ec_transfer_params: Option<serde_json::Value>,

155 }

156

157 impl GenerateOutput {

@@ -192,6 +198,7 @@ impl GenerateOutput {

198 finish_reason,

199 cached_token_count: 0,

200 kv_transfer_params: None,

201 + ec_transfer_params: None,

202 }

203 }

204 }

@@ -280,6 +287,7 @@ impl Stream for GenerateOutputStream {

287 finish_reason,

288 cached_token_count,

289 kv_transfer_params: raw.kv_transfer_params,

290 + ec_transfer_params: raw.ec_transfer_params,

291 };

292

293 Poll::Ready(Some(Ok(output)))

@@ -362,6 +370,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {

370 cached_token_count,

371 },

372 kv_transfer_params: None,

373 + ec_transfer_params: None,

374 });

375 }

376

@@ -374,6 +383,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {

383 cached_token_count,

384 };

385 collected.kv_transfer_params = output.kv_transfer_params;

386 + collected.ec_transfer_params = output.ec_transfer_params;

387 return Ok(collected);

388 }

389 }

rust/src/llm/tests/generate.rs

+5

@@ -51,6 +51,7 @@ fn request_output_with_events(

51 stop_reason: None,

52 events,

53 kv_transfer_params: None,

54 + ec_transfer_params: None,

55 trace_headers: None,

56 prefill_stats: None,

57 routed_experts: None,

@@ -75,6 +76,7 @@ fn request_output_with_logprobs(

76 stop_reason: None,

77 events: None,

78 kv_transfer_params: None,

79 + ec_transfer_params: None,

80 trace_headers: None,

81 prefill_stats: None,

82 routed_experts: None,

@@ -89,6 +91,7 @@ fn request_output_with_logprobs_and_kv(

91 new_logprobs: Option<Logprobs>,

92 prompt_logprobs: Option<Logprobs>,

93 kv_transfer_params: Option<serde_json::Value>,

94 + ec_transfer_params: Option<serde_json::Value>,

95 ) -> EngineCoreOutput {

96 EngineCoreOutput {

97 request_id: request_id.to_string(),

@@ -100,6 +103,7 @@ fn request_output_with_logprobs_and_kv(

103 stop_reason: None,

104 events: None,

105 kv_transfer_params,

106 + ec_transfer_params,

107 trace_headers: None,

108 prefill_stats: None,

109 routed_experts: None,

@@ -356,6 +360,7 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() {

360 Some(logprobs_for_position(44, -0.3, 1, 88, -0.4)),

361 None,

362 Some(serde_json::json!({"connector": "x"})),

363 + None,

364 ),

365 ],

366 ..Default::default()

rust/src/server/src/grpc/convert.rs

+7

@@ -69,6 +69,11 @@ pub fn to_text_request(

69 let map = sampling_params.vllm_xargs.get_or_insert_with(Default::default);

70 map.insert("kv_transfer_params".to_string(), kv_json);

71 }

72 + if let Some(ec_struct) = kv.ec_transfer_params.as_ref() {

73 + let ec_json = proto_struct_to_json(ec_struct);

74 + let map = sampling_params.vllm_xargs.get_or_insert_with(Default::default);

75 + map.insert("ec_transfer_params".to_string(), ec_json);

76 + }

77 if kv.bypass_prefix_cache {

78 sampling_params.skip_reading_prefix_cache = Some(true);

79 }

@@ -343,6 +348,7 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {

348 finish_reason,

349 stop_reason,

350 kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),

351 + ec_transfer_params: finished.ec_transfer_params.as_ref().and_then(json_to_proto_struct),

352 }

353 }

354

@@ -586,6 +592,7 @@ mod tests {

592 },

593 finish_reason: reason,

594 kv_transfer_params: None,

595 + ec_transfer_params: None,

596 }

597 }

598

rust/src/server/src/grpc/mod.rs

+1

@@ -71,6 +71,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl {

71 usage: collected.usage,

72 finish_reason: collected.finish_reason,

73 kv_transfer_params: collected.kv_transfer_params,

74 + ec_transfer_params: collected.ec_transfer_params,

75 };

76

77 let outputs = convert::to_sequence_output(

rust/src/server/src/grpc/tests.rs

+1

@@ -104,6 +104,7 @@ fn request_output(

104 stop_reason: None,

105 events: None,

106 kv_transfer_params: None,

107 + ec_transfer_params: None,

108 trace_headers: None,

109 prefill_stats: None,

110 routed_experts: None,

rust/src/server/src/routes/http_client_tests.rs

+1

@@ -100,6 +100,7 @@ fn request_output(

100 stop_reason: None,

101 events: None,

102 kv_transfer_params: None,

103 + ec_transfer_params: None,

104 trace_headers: None,

105 prefill_stats: None,

106 routed_experts: None,

rust/src/server/src/routes/inference/generate.rs

+3

@@ -266,6 +266,7 @@ fn collect_generate(

266 }],

267 prompt_logprobs,

268 kv_transfer_params: collected.kv_transfer_params,

269 + ec_transfer_params: collected.ec_transfer_params,

270 })

271 }

272

@@ -404,6 +405,7 @@ mod tests {

405 finish_reason: None,

406 cached_token_count: 0,

407 kv_transfer_params: None,

408 + ec_transfer_params: None,

409 }),

410 Ok(GenerateOutput {

411 request_id: String::new(),

@@ -416,6 +418,7 @@ mod tests {

418 finish_reason: Some(FinishReason::stop_eos()),

419 cached_token_count: 2,

420 kv_transfer_params: None,

421 + ec_transfer_params: None,

422 }),

423 ]);

424

rust/src/server/src/routes/inference/generate/convert.rs

+5 -1

@@ -4,7 +4,7 @@ use super::types::GenerateRequest;

4 use super::validate;

5 use crate::error::ApiError;

6 use crate::lora::LoraModelResolution;

7 -use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};

7 +use crate::utils::{ResolvedRequestContext, merge_ec_transfer_params, merge_kv_transfer_params};

8

9 /// Lowered generate request plus the response request ID.

10 #[derive(Debug, Clone, PartialEq)]

@@ -56,6 +56,10 @@ pub(super) fn prepare_generate_request(

56 sampling_params.vllm_xargs,

57 request.kv_transfer_params.as_ref(),

58 );

59 + sampling_params.vllm_xargs = merge_ec_transfer_params(

60 + sampling_params.vllm_xargs,

61 + request.ec_transfer_params.as_ref(),

62 + );

63

64 let text_request = TextRequest {

65 request_id: ctx.request_id.clone(),

rust/src/server/src/routes/inference/generate/types.rs

+2

@@ -22,6 +22,7 @@ pub struct GenerateRequest {

22 #[serde(default)]

23 pub priority: i32,

24 pub kv_transfer_params: Option<HashMap<String, Value>>,

25 + pub ec_transfer_params: Option<HashMap<String, Value>>,

26 #[serde(flatten)]

27 pub other: Map<String, Value>,

28 }

@@ -66,6 +67,7 @@ pub(super) struct GenerateResponse {

67 pub choices: Vec<GenerateResponseChoice>,

68 pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,

69 pub kv_transfer_params: Option<Value>,

70 + pub ec_transfer_params: Option<Value>,

71 }

72

73 /// Mirrors the Python vLLM `Logprob` class used in prompt-logprobs payloads.

rust/src/server/src/routes/openai/chat_completions.rs

+8

@@ -144,6 +144,7 @@ async fn collect_chat_completion(

144 usage,

145 finish_reason,

146 kv_transfer_params,

147 + ec_transfer_params,

148 } = collected;

149 let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);

150 let saw_tool_calls = message.tool_calls().next().is_some();

@@ -224,6 +225,7 @@ async fn collect_chat_completion(

225 prompt_logprobs,

226 prompt_token_ids: return_token_ids.then(|| prompt_token_ids.to_vec()),

227 kv_transfer_params,

228 + ec_transfer_params,

229 })

230 }

231

@@ -951,6 +953,7 @@ mod tests {

953 },

954 finish_reason: FinishReason::stop_eos(),

955 kv_transfer_params: None,

956 + ec_transfer_params: None,

957 }),

958 ]);

959

@@ -1031,6 +1034,7 @@ mod tests {

1034 },

1035 finish_reason: FinishReason::stop_eos(),

1036 kv_transfer_params: None,

1037 + ec_transfer_params: None,

1038 }),

1039 ]);

1040

@@ -1086,6 +1090,7 @@ mod tests {

1090 },

1091 finish_reason: FinishReason::stop_eos(),

1092 kv_transfer_params: None,

1093 + ec_transfer_params: None,

1094 }),

1095 ]);

1096

@@ -1167,6 +1172,7 @@ mod tests {

1172 },

1173 finish_reason: FinishReason::stop_eos(),

1174 kv_transfer_params: None,

1175 + ec_transfer_params: None,

1176 }),

1177 ]);

1178

@@ -1300,6 +1306,7 @@ mod tests {

1306 },

1307 finish_reason: FinishReason::stop_eos(),

1308 kv_transfer_params: None,

1309 + ec_transfer_params: None,

1310 }),

1311 ]);

1312

@@ -1381,6 +1388,7 @@ mod tests {

1388 },

1389 finish_reason: FinishReason::stop_eos(),

1390 kv_transfer_params: None,

1391 + ec_transfer_params: None,

1392 }),

1393 ]);

1394

rust/src/server/src/routes/openai/chat_completions/convert.rs

+4 -2

@@ -13,7 +13,9 @@ use crate::routes::openai::utils::structured_outputs::convert_from_response_form

13 use crate::routes::openai::utils::types::{

14 ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,

15 };

16 -use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};

16 +use crate::utils::{

17 + ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,

18 +};

19

20 /// Lowered chat request plus the public response metadata carried by every SSE

21 /// chunk.

@@ -132,7 +134,7 @@ pub(super) fn prepare_chat_request(

134 structured_outputs,

135 skip_reading_prefix_cache: None,

136 vllm_xargs: merge_kv_transfer_params(

135 - request.vllm_xargs,

137 + merge_ec_transfer_params(request.vllm_xargs, request.ec_transfer_params.as_ref()),

138 request.kv_transfer_params.as_ref(),

139 ),

140 },

rust/src/server/src/routes/openai/chat_completions/types.rs

+5

@@ -232,6 +232,9 @@ pub struct ChatCompletionRequest {

232 /// KV transfer parameters for disaggregated serving

233 pub kv_transfer_params: Option<HashMap<String, Value>>,

234

235 + /// Encoder cache transfer parameters for disaggregated serving

236 + pub ec_transfer_params: Option<HashMap<String, Value>>,

237 +

238 /// Additional request parameters with string or numeric values for custom

239 /// extensions

240 pub vllm_xargs: Option<HashMap<String, Value>>,

@@ -299,6 +302,7 @@ impl Default for ChatCompletionRequest {

302 return_token_ids: None,

303 cache_salt: None,

304 kv_transfer_params: None,

305 + ec_transfer_params: None,

306 vllm_xargs: None,

307 repetition_detection: None,

308 }

@@ -346,6 +350,7 @@ pub(super) struct ChatCompletionResponse {

350 pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,

351 pub prompt_token_ids: Option<Vec<u32>>,

352 pub kv_transfer_params: Option<Value>,

353 + pub ec_transfer_params: Option<Value>,

354 }

355

356 /// Mirrors the Python vLLM `ChatCompletionResponseChoice` class.

rust/src/server/src/routes/openai/completions.rs

+7

@@ -212,6 +212,7 @@ async fn collect_completion(

212 usage: Some(usage),

213 system_fingerprint: None,

214 kv_transfer_params: collected.kv_transfer_params,

215 + ec_transfer_params: collected.ec_transfer_params,

216 })

217 }

218

@@ -682,6 +683,7 @@ mod tests {

683 "repetition_detected".to_string(),

684 ))),

685 kv_transfer_params: None,

686 + ec_transfer_params: None,

687 }),

688 }),

689 ]);

@@ -786,6 +788,7 @@ mod tests {

788 },

789 finish_reason: FinishReason::Length,

790 kv_transfer_params: None,

791 + ec_transfer_params: None,

792 }),

793 }),

794 ]);

@@ -837,6 +840,7 @@ mod tests {

840 },

841 finish_reason: FinishReason::Length,

842 kv_transfer_params: None,

843 + ec_transfer_params: None,

844 }),

845 }),

846 ]);

@@ -891,6 +895,7 @@ mod tests {

895 },

896 finish_reason: FinishReason::Length,

897 kv_transfer_params: None,

898 + ec_transfer_params: None,

899 }),

900 }),

901 ]);

@@ -962,6 +967,7 @@ mod tests {

967 },

968 finish_reason: FinishReason::Length,

969 kv_transfer_params: None,

970 + ec_transfer_params: None,

971 }),

972 }),

973 ]);

@@ -1035,6 +1041,7 @@ mod tests {

1041 },

1042 finish_reason: FinishReason::Length,

1043 kv_transfer_params: None,

1044 + ec_transfer_params: None,

1045 }),

1046 }),

1047 ]);

rust/src/server/src/routes/openai/completions/convert.rs

+4 -2

@@ -7,7 +7,9 @@ use crate::error::ApiError;

7 use crate::lora::LoraModelResolution;

8 use crate::routes::openai::completions::validate;

9 use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;

10 -use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};

10 +use crate::utils::{

11 + ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,

12 +};

13

14 /// Lowered completion request plus the public response metadata carried by

15 /// every SSE chunk.

@@ -127,7 +129,7 @@ pub(super) fn prepare_completion_request(

129 structured_outputs,

130 skip_reading_prefix_cache: None,

131 vllm_xargs: merge_kv_transfer_params(

130 - request.vllm_xargs,

132 + merge_ec_transfer_params(request.vllm_xargs, request.ec_transfer_params.as_ref()),

133 request.kv_transfer_params.as_ref(),

134 ),

135 },

rust/src/server/src/routes/openai/completions/types.rs

+4

@@ -174,6 +174,9 @@ pub struct CompletionRequest {

174 /// KV transfer parameters for disaggregated serving

175 pub kv_transfer_params: Option<HashMap<String, Value>>,

176

177 + /// Encoder cache transfer parameters for disaggregated serving

178 + pub ec_transfer_params: Option<HashMap<String, Value>>,

179 +

180 /// Additional request parameters with string or numeric values for custom

181 /// extensions

182 pub vllm_xargs: Option<HashMap<String, Value>>,

@@ -209,6 +212,7 @@ pub(super) struct CompletionResponse {

212 pub usage: Option<Usage>,

213 pub system_fingerprint: Option<String>,

214 pub kv_transfer_params: Option<Value>,

215 + pub ec_transfer_params: Option<Value>,

216 }

217

218 /// Mirrors the Python vLLM `CompletionResponseChoice` class.

rust/src/server/src/routes/tests.rs

+5

@@ -76,6 +76,7 @@ fn request_output_with_stop_reason(

76 stop_reason,

77 events: None,

78 kv_transfer_params: None,

79 + ec_transfer_params: None,

80 trace_headers: None,

81 prefill_stats: None,

82 routed_experts: None,

@@ -101,6 +102,7 @@ fn request_output_with_logprobs(

102 stop_reason,

103 events: None,

104 kv_transfer_params: None,

105 + ec_transfer_params: None,

106 trace_headers: None,

107 prefill_stats: None,

108 routed_experts: None,

@@ -116,6 +118,7 @@ fn request_output_with_logprobs_and_kv(

118 new_logprobs: Option<Logprobs>,

119 new_prompt_logprobs_tensors: Option<Logprobs>,

120 kv_transfer_params: Option<serde_json::Value>,

121 + ec_transfer_params: Option<serde_json::Value>,

122 ) -> EngineCoreOutput {

123 EngineCoreOutput {

124 request_id: request_id.to_string(),

@@ -127,6 +130,7 @@ fn request_output_with_logprobs_and_kv(

130 stop_reason,

131 events: None,

132 kv_transfer_params,

133 + ec_transfer_params,

134 trace_headers: None,

135 prefill_stats: None,

136 routed_experts: None,

@@ -3634,6 +3638,7 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {

3638 Some(sample_logprobs_for_token(44, 45)),

3639 None,

3640 Some(json!({"connector": "x"})),

3641 + None,

3642 ),

3643 ],

3644 ..Default::default()

rust/src/server/src/utils.rs

+18

@@ -45,6 +45,24 @@ pub fn merge_kv_transfer_params(

45 xargs

46 }

47

48 +/// Merge `ec_transfer_params` into the `vllm_xargs` map, mirroring the Python

49 +/// vLLM behavior where `ec_transfer_params` is injected into `extra_args` for

50 +/// engine-core consumption.

51 +pub fn merge_ec_transfer_params(

52 + mut xargs: Option<HashMap<String, Value>>,

53 + ec_transfer_params: Option<&HashMap<String, Value>>,

54 +) -> Option<HashMap<String, Value>> {

55 + if let Some(ec_params) = ec_transfer_params {

56 + let map = xargs.get_or_insert_with(HashMap::new);

57 + map.insert(

58 + "ec_transfer_params".to_string(),

59 + // This is safe because we know that `ec_params` is already valid JSON.

60 + serde_json::to_value(ec_params).unwrap(),

61 + );

62 + }

63 + xargs

64 +}

65 +

66 /// Convert OpenAI-style `logit_bias` with string token-ID keys into the

67 /// internal `HashMap<u32, f32>` representation, validating that every key

68 /// parses as a `u32`.

rust/src/text/src/output/decoded.rs

+5

@@ -44,6 +44,9 @@ pub struct Finished {

44 pub finish_reason: FinishReason,

45 /// Connector-specific KV transfer parameters for disaggregated serving.

46 pub kv_transfer_params: Option<serde_json::Value>,

47 + /// Connector-specific encoder cache transfer parameters for disaggregated

48 + /// serving.

49 + pub ec_transfer_params: Option<serde_json::Value>,

50 }

51

52 /// Internal decoded-text event emitted before higher-level assistant

@@ -151,6 +154,7 @@ pub async fn decoded_text_event_stream(

154 let decoder = decoder.as_mut().unwrap();

155

156 let kv_transfer_params = output.kv_transfer_params;

157 + let ec_transfer_params = output.ec_transfer_params;

158 let mut finish_reason = output.finish_reason;

159 let mut stop_str_matched = false;

160 let suppress_terminal_stop_token = finish_reason.as_ref().is_some_and(|r| r.is_stop())

@@ -275,6 +279,7 @@ pub async fn decoded_text_event_stream(

279 },

280 finish_reason: reason,

281 kv_transfer_params,

282 + ec_transfer_params,

283 }),

284 })

285 .await;

rust/src/text/src/output/mod.rs

+7

@@ -26,6 +26,9 @@ pub struct CollectedTextOutput {

26 pub usage: vllm_llm::TokenUsage,

27 /// Connector-specific KV transfer parameters for disaggregated serving.

28 pub kv_transfer_params: Option<serde_json::Value>,

29 + /// Connector-specific encoder cache transfer parameters for disaggregated

30 + /// serving.

31 + pub ec_transfer_params: Option<serde_json::Value>,

32 }

33

34 #[allow(clippy::manual_async_fn, reason = "specify `Send` bound")]

@@ -77,6 +80,7 @@ impl<T: TextOutputStream> T {

80 finish_reason: FinishReason::Error,

81 usage: vllm_llm::TokenUsage::default(),

82 kv_transfer_params: None,

83 + ec_transfer_params: None,

84 })

85 };

86

@@ -85,6 +89,7 @@ impl<T: TextOutputStream> T {

89 collected.finish_reason = finished.finish_reason;

90 collected.usage = finished.usage;

91 collected.kv_transfer_params = finished.kv_transfer_params;

92 + collected.ec_transfer_params = finished.ec_transfer_params;

93 return Ok(collected);

94 }

95 }

@@ -156,6 +161,7 @@ mod tests {

161 },

162 finish_reason: FinishReason::stop_eos(),

163 kv_transfer_params: None,

164 + ec_transfer_params: None,

165 }),

166 }),

167 ]);

@@ -273,6 +279,7 @@ mod tests {

279 },

280 finish_reason: FinishReason::stop_eos(),

281 kv_transfer_params: None,

282 + ec_transfer_params: None,

283 }),

284 }),

285 ]);

tests/v1/core/test_async_scheduler.py

+1 -1

@@ -294,7 +294,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():

294 def free_request(req, delay_free_blocks=False):

295 scheduler.finished_req_ids.add(req.request_id)

296 scheduler.requests.pop(req.request_id, None)

297 - return None

297 + return None, None

298

299 scheduler._free_request = Mock(side_effect=free_request)

300

tests/v1/core/test_scheduler.py

+1 -1

@@ -2968,7 +2968,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():

2968 def free_request(req: Request, delay_free_blocks: bool = False):

2969 scheduler.finished_req_ids.add(req.request_id)

2970 scheduler.requests.pop(req.request_id, None)

2971 - return None

2971 + return None, None

2972

2973 scheduler._free_request = Mock(side_effect=free_request)

2974

tests/v1/ec_connector/unit/test_ec_transfer_params.py new

+118

@@ -0,0 +1,118 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Sanity tests for ec_transfer_params protocol plumbing.

4 +

5 +No running engine required.

6 +"""

7 +

8 +from unittest.mock import MagicMock

9 +

10 +from tests.v1.core.utils import create_scheduler

11 +from vllm.entrypoints.openai.chat_completion.protocol import (

12 + ChatCompletionRequest,

13 +)

14 +from vllm.outputs import CompletionOutput, RequestOutput

15 +from vllm.sampling_params import SamplingParams

16 +from vllm.v1.request import Request, RequestStatus

17 +

18 +EC_PARAMS: dict = {"mm_hash_abc": {"peer_host": "10.0.0.1", "peer_port": 5501}}

19 +

20 +

21 +def test_ec_transfer_params_routed_to_sampling_params_extra_args():

22 + """ec_transfer_params on the request must land in SamplingParams.extra_args."""

23 + req = ChatCompletionRequest(

24 + model="test-model",

25 + messages=[{"role": "user", "content": "hi"}],

26 + max_tokens=5,

27 + ec_transfer_params=EC_PARAMS,

28 + )

29 + sp = req.to_sampling_params(max_tokens=5, default_sampling_params={})

30 + assert sp.extra_args is not None

31 + assert sp.extra_args.get("ec_transfer_params") == EC_PARAMS

32 +

33 +

34 +def test_request_output_add_propagates_ec_transfer_params():

35 + """RequestOutput.add() must carry ec_transfer_params forward to the caller."""

36 +

37 + def _out(ec_params):

38 + return RequestOutput(

39 + request_id="r1",

40 + prompt="p",

41 + prompt_token_ids=[1],

42 + prompt_logprobs=None,

43 + outputs=[

44 + CompletionOutput(

45 + index=0,

46 + text="",

47 + token_ids=[],

48 + cumulative_logprob=0.0,

49 + logprobs=None,

50 + finish_reason=None,

51 + )

52 + ],

53 + finished=False,

54 + ec_transfer_params=ec_params,

55 + )

56 +

57 + accumulated = _out(None)

58 + accumulated.add(_out(EC_PARAMS), aggregate=True)

59 + assert accumulated.ec_transfer_params == EC_PARAMS

60 +

61 +

62 +def test_request_reads_ec_transfer_params_from_extra_args():

63 + """v1 Request must pull ec_transfer_params out of SamplingParams.extra_args."""

64 + sp = SamplingParams(extra_args={"ec_transfer_params": EC_PARAMS})

65 + req = Request(

66 + request_id="r1",

67 + prompt_token_ids=[1, 2, 3],

68 + sampling_params=sp,

69 + pooling_params=None,

70 + )

71 + assert req.ec_transfer_params == EC_PARAMS

72 +

73 +

74 +def test_free_request_calls_ec_connector_and_surfaces_params():

75 + """_free_request must call ec_connector.request_finished() and return its params."""

76 + sp = SamplingParams(max_tokens=1)

77 + sp.update_from_generation_config({}, 50256)

78 + request = Request(

79 + request_id="test-req",

80 + prompt_token_ids=[1, 2, 3],

81 + sampling_params=sp,

82 + pooling_params=None,

83 + client_index=0,

84 + )

85 + scheduler = create_scheduler(use_ec_connector=True, ec_role="ec_producer")

86 + scheduler.add_request(request)

87 + request.status = RequestStatus.FINISHED_STOPPED

88 +

89 + mock_ec = MagicMock()

90 + mock_ec.request_finished.return_value = (False, EC_PARAMS)

91 + scheduler.ec_connector = mock_ec

92 +

93 + kv_params, ec_params = scheduler._free_request(request)

94 +

95 + mock_ec.request_finished.assert_called_once_with(request)

96 + assert ec_params == EC_PARAMS

97 + assert kv_params is None

98 +

99 +

100 +def test_free_request_without_ec_connector_returns_none():

101 + """When no EC connector is configured, ec_transfer_params must be None."""

102 + sp = SamplingParams(max_tokens=1)

103 + sp.update_from_generation_config({}, 50256)

104 + request = Request(

105 + request_id="test-req",

106 + prompt_token_ids=[1, 2, 3],

107 + sampling_params=sp,

108 + pooling_params=None,

109 + client_index=0,

110 + )

111 + scheduler = create_scheduler(use_ec_connector=True, ec_role="ec_producer")

112 + scheduler.add_request(request)

113 + request.status = RequestStatus.FINISHED_STOPPED

114 +

115 + kv_params, ec_params = scheduler._free_request(request)

116 +

117 + assert ec_params is None

118 + assert kv_params is None

vllm/entrypoints/anthropic/protocol.py

+9

@@ -137,6 +137,12 @@ class AnthropicMessagesRequest(BaseModel):

137 default=None,

138 description="KVTransfer parameters used for disaggregated serving.",

139 )

140 + ec_transfer_params: dict[str, Any] | None = Field(

141 + default=None,

142 + description=(

143 + "ECTransfer parameters used for encoder-cache disaggregated serving."

144 + ),

145 + )

146 chat_template_kwargs: dict[str, Any] | None = Field(

147 default=None,

148 description=(

@@ -218,6 +224,9 @@ class AnthropicMessagesResponse(BaseModel):

224 kv_transfer_params: dict[str, Any] | None = Field(

225 default=None, description="KVTransfer parameters."

226 )

227 + ec_transfer_params: dict[str, Any] | None = Field(

228 + default=None, description="ECTransfer parameters."

229 + )

230

231 def model_post_init(self, __context):

232 if not self.id:

vllm/entrypoints/anthropic/serving.py

+2

@@ -491,6 +491,7 @@ class AnthropicServingMessages(OpenAIServingChat):

491 top_p=anthropic_request.top_p,

492 top_k=anthropic_request.top_k,

493 kv_transfer_params=anthropic_request.kv_transfer_params,

494 + ec_transfer_params=anthropic_request.ec_transfer_params,

495 chat_template_kwargs=anthropic_request.chat_template_kwargs,

496 )

497

@@ -630,6 +631,7 @@ class AnthropicServingMessages(OpenAIServingChat):

631 generator.usage,

632 ),

633 kv_transfer_params=generator.kv_transfer_params,

634 + ec_transfer_params=generator.ec_transfer_params,

635 )

636 choice = generator.choices[0]

637 if choice.finish_reason == "stop":

vllm/entrypoints/openai/chat_completion/protocol.py

+13

@@ -134,6 +134,9 @@ class ChatCompletionResponse(OpenAIBaseModel):

134 kv_transfer_params: dict[str, Any] | None = Field(

135 default=None, description="KVTransfer parameters."

136 )

137 + ec_transfer_params: dict[str, Any] | None = Field(

138 + default=None, description="ECTransfer parameters."

139 + )

140 metrics: PerRequestTimingMetrics | None = None

141

142

@@ -439,6 +442,13 @@ class ChatCompletionRequest(OpenAIBaseModel):

442 description="KVTransfer parameters used for disaggregated serving.",

443 )

444

445 + ec_transfer_params: dict[str, Any] | None = Field(

446 + default=None,

447 + description=(

448 + "ECTransfer parameters used for encoder-cache disaggregated serving."

449 + ),

450 + )

451 +

453 default=None,

454 description=(

@@ -665,6 +675,9 @@ class ChatCompletionRequest(OpenAIBaseModel):

675 if self.kv_transfer_params:

676 # Pass in kv_transfer_params via extra_args

677 extra_args["kv_transfer_params"] = self.kv_transfer_params

678 + if self.ec_transfer_params:

679 + # Pass in ec_transfer_params via extra_args

680 + extra_args["ec_transfer_params"] = self.ec_transfer_params

681 return SamplingParams.from_optional(

682 n=self.n,

683 presence_penalty=self.presence_penalty,

vllm/entrypoints/openai/chat_completion/serving.py

+1

@@ -1057,6 +1057,7 @@ class OpenAIServingChat(GenerateBaseServing):

1057 ),

1058 prompt_text=prompt_text,

1059 kv_transfer_params=final_res.kv_transfer_params,

1060 + ec_transfer_params=final_res.ec_transfer_params,

1061 metrics=per_request_metrics,

1062 )

1063

vllm/entrypoints/openai/completion/protocol.py

+13

@@ -189,6 +189,13 @@ class CompletionRequest(OpenAIBaseModel):

189 description="KVTransfer parameters used for disaggregated serving.",

190 )

191

192 + ec_transfer_params: dict[str, Any] | None = Field(

193 + default=None,

194 + description=(

195 + "ECTransfer parameters used for encoder-cache disaggregated serving."

196 + ),

197 + )

198 +

199 vllm_xargs: dict[str, str | int | float] | None = Field(

200 default=None,

201 description=(

@@ -346,6 +353,9 @@ class CompletionRequest(OpenAIBaseModel):

353 if self.kv_transfer_params:

354 # Pass in kv_transfer_params via extra_args

355 extra_args["kv_transfer_params"] = self.kv_transfer_params

356 + if self.ec_transfer_params:

357 + # Pass in ec_transfer_params via extra_args

358 + extra_args["ec_transfer_params"] = self.ec_transfer_params

359 return SamplingParams.from_optional(

360 n=self.n,

361 presence_penalty=self.presence_penalty,

@@ -595,6 +605,9 @@ class CompletionResponse(OpenAIBaseModel):

605 kv_transfer_params: dict[str, Any] | None = Field(

606 default=None, description="KVTransfer parameters."

607 )

608 + ec_transfer_params: dict[str, Any] | None = Field(

609 + default=None, description="ECTransfer parameters."

610 + )

611 metrics: PerRequestTimingMetrics | None = None

612

613

vllm/entrypoints/openai/completion/serving.py

+4

@@ -510,6 +510,7 @@ class OpenAIServingCompletion(GenerateBaseServing):

510 num_prompt_tokens = 0

511 num_generated_tokens = 0

512 kv_transfer_params = None

513 + ec_transfer_params = None

514 last_final_res = None

515 for final_res in final_res_batch:

516 last_final_res = final_res

@@ -632,6 +633,8 @@ class OpenAIServingCompletion(GenerateBaseServing):

633

634 if final_res_batch:

635 kv_transfer_params = final_res_batch[0].kv_transfer_params

636 + ec_transfer_params = final_res_batch[0].ec_transfer_params

637 +

638 return CompletionResponse(

639 id=request_id,

640 created=created_time,

@@ -640,6 +643,7 @@ class OpenAIServingCompletion(GenerateBaseServing):

643 usage=usage,

644 system_fingerprint=self.system_fingerprint,

645 kv_transfer_params=kv_transfer_params,

646 + ec_transfer_params=ec_transfer_params,

647 metrics=per_request_metrics,

648 )

649

vllm/entrypoints/openai/responses/context.py

+10

@@ -200,6 +200,7 @@ class SimpleContext(ConversationContext):

200

201 self.input_messages: list[ResponseRawMessageAndToken] = []

202 self.kv_transfer_params: dict[str, Any] | None = None

203 + self.ec_transfer_params: dict[str, Any] | None = None

204

205 def append_output(self, output) -> None:

206 self.last_output = output

@@ -210,6 +211,8 @@ class SimpleContext(ConversationContext):

211 self.num_output_tokens += len(output.outputs[0].token_ids or [])

212 if output.kv_transfer_params is not None:

213 self.kv_transfer_params = output.kv_transfer_params

214 + if output.ec_transfer_params is not None:

215 + self.ec_transfer_params = output.ec_transfer_params

216

217 # Accumulate text, token_ids, and logprobs for streaming mode

218 delta_output = output.outputs[0]

@@ -328,6 +331,7 @@ class ParsableContext(ConversationContext):

331 self.output_messages: list[ResponseRawMessageAndToken] = []

332 self._accumulated_token_ids: list[int] = []

333 self.kv_transfer_params: dict[str, Any] | None = None

334 + self.ec_transfer_params: dict[str, Any] | None = None

335

336 def append_output(self, output: RequestOutput) -> None:

337 self.num_prompt_tokens = len(output.prompt_token_ids or [])

@@ -336,6 +340,9 @@ class ParsableContext(ConversationContext):

340 if output.kv_transfer_params is not None:

341 self.kv_transfer_params = output.kv_transfer_params

342

343 + if output.ec_transfer_params is not None:

344 + self.ec_transfer_params = output.ec_transfer_params

345 +

346 completion = output.outputs[0]

347 self.finish_reason = completion.finish_reason

348

@@ -630,6 +637,7 @@ class HarmonyContext(ConversationContext):

637 self.is_first_turn = True

638 self.first_tok_of_message = True

639 self.kv_transfer_params: dict[str, Any] | None = None

640 + self.ec_transfer_params: dict[str, Any] | None = None

641

642 def append_output(self, output: RequestOutput) -> None:

643 if self.first_tok_of_message:

@@ -645,6 +653,8 @@ class HarmonyContext(ConversationContext):

653 self._update_decode_token_usage(output)

654 if output.kv_transfer_params is not None:

655 self.kv_transfer_params = output.kv_transfer_params

656 + if output.ec_transfer_params is not None:

657 + self.ec_transfer_params = output.ec_transfer_params

658

659 if output.finished:

660 self.finish_reason = output.outputs[0].finish_reason

vllm/entrypoints/openai/responses/protocol.py

+13

@@ -285,6 +285,12 @@ class ResponsesRequest(OpenAIBaseModel):

285 default=None,

286 description="KVTransfer parameters used for disaggregated serving.",

287 )

288 + ec_transfer_params: dict[str, Any] | None = Field(

289 + default=None,

290 + description=(

291 + "ECTransfer parameters used for encoder-cache disaggregated serving."

292 + ),

293 + )

294 chat_template_kwargs: dict[str, Any] | None = Field(

295 default=None,

296 description=(

@@ -409,6 +415,8 @@ class ResponsesRequest(OpenAIBaseModel):

415 extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}

416 if self.kv_transfer_params:

417 extra_args["kv_transfer_params"] = self.kv_transfer_params

418 + if self.ec_transfer_params:

419 + extra_args["ec_transfer_params"] = self.ec_transfer_params

420

421 return SamplingParams.from_optional(

422 temperature=temperature,

@@ -675,6 +683,9 @@ class ResponsesResponse(OpenAIBaseModel):

683 kv_transfer_params: dict[str, Any] | None = Field(

684 default=None, description="KVTransfer parameters."

685 )

686 + ec_transfer_params: dict[str, Any] | None = Field(

687 + default=None, description="ECTransfer parameters."

688 + )

689

690 # --8<-- [start:responses-response-extra-params]

691 # These are populated when enable_response_messages is set to True

@@ -720,6 +731,7 @@ class ResponsesResponse(OpenAIBaseModel):

731 input_messages: ResponseInputOutputMessage | None = None,

732 output_messages: ResponseInputOutputMessage | None = None,

733 kv_transfer_params: dict[str, Any] | None = None,

734 + ec_transfer_params: dict[str, Any] | None = None,

735 ) -> "ResponsesResponse":

736 incomplete_details: IncompleteDetails | None = None

737 if status == "incomplete":

@@ -758,6 +770,7 @@ class ResponsesResponse(OpenAIBaseModel):

770 user=request.user,

771 usage=usage,

772 kv_transfer_params=kv_transfer_params,

773 + ec_transfer_params=ec_transfer_params,

774 )

775

776

vllm/entrypoints/openai/responses/serving.py

+1

@@ -932,6 +932,7 @@ class OpenAIServingResponses(GenerateBaseServing):

932 status=status,

933 usage=usage,

934 kv_transfer_params=context.kv_transfer_params,

935 + ec_transfer_params=context.ec_transfer_params,

936 )

937

938 if request.store:

vllm/entrypoints/scale_out/token_in_token_out/protocol.py

+12

@@ -132,6 +132,12 @@ class GenerateRequest(BaseModel):

132 default=None,

133 description="KVTransfer parameters used for disaggregated serving.",

134 )

135 + ec_transfer_params: dict[str, Any] | None = Field(

136 + default=None,

137 + description=(

138 + "ECTransfer parameters used for encoder-cache disaggregated serving."

139 + ),

140 + )

141

142 # Tracks which keys the caller explicitly set inside ``sampling_params``

143 # when the request was parsed from a JSON body. Lets the server tell

@@ -238,6 +244,12 @@ class GenerateResponse(BaseModel):

244 default=None,

245 description="KVTransfer parameters used for disaggregated serving.",

246 )

247 + ec_transfer_params: dict[str, Any] | None = Field(

248 + default=None,

249 + description=(

250 + "ECTransfer parameters used for encoder-cache disaggregated serving."

251 + ),

252 + )

253

254

255 ####### Derender (postprocessing) #######

vllm/entrypoints/scale_out/token_in_token_out/serving.py

+1

@@ -330,6 +330,7 @@ class ServingTokens(GenerateBaseServing):

330 usage=usage,

331 prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),

332 kv_transfer_params=final_res.kv_transfer_params,

333 + ec_transfer_params=final_res.ec_transfer_params,

334 )

335

336 # Log complete response if output logging is enabled

vllm/envs.py

+12

@@ -207,6 +207,8 @@ if TYPE_CHECKING:

207 VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False

208 VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"

209 VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600

210 + VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost"

211 + VLLM_EC_SIDE_CHANNEL_PORT: int = 5601

212 VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998

213 VLLM_MOONCAKE_STORE_TIER_LOG: bool = False

214 VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1

@@ -1565,6 +1567,16 @@ environment_variables: dict[str, Callable[[], Any]] = {

1567 "VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(

1568 os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")

1569 ),

1570 + # IP address used for the EC connector's ZMQ side channel

1571 + # (producer ROUTER bind, consumer DEALER dial).

1572 + "VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv(

1573 + "VLLM_EC_SIDE_CHANNEL_HOST", "localhost"

1574 + ),

1575 + # Port for the EC connector's ZMQ side channel; advertised to peers

1576 + # via `ec_transfer_params.peer_port` on the producer's response.

1577 + "VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(

1578 + os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")

1579 + ),

1580 # Port used for Mooncake handshake between remote agents.

1581 "VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(

1582 os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")

vllm/outputs.py

+4

@@ -104,6 +104,7 @@ class RequestOutput:

104 None if decoder-only.

105 num_cached_tokens: The number of tokens with prefix cache hit.

106 kv_transfer_params: The params for remote K/V transfer.

107 + ec_transfer_params: The params for remote encoder-cache transfer.

108 """

109

110 def __init__(

@@ -121,6 +122,7 @@ class RequestOutput:

122 num_cached_tokens: int | None = None,

123 *,

124 kv_transfer_params: dict[str, Any] | None = None,

125 + ec_transfer_params: dict[str, Any] | None = None,

126 # Forward compatibility, code that uses args added in new release can

127 # still run with older versions of vLLM without breaking.

128 **kwargs: Any,

@@ -141,12 +143,14 @@ class RequestOutput:

143 self.encoder_prompt_token_ids = encoder_prompt_token_ids

144 self.num_cached_tokens = num_cached_tokens

145 self.kv_transfer_params = kv_transfer_params

146 + self.ec_transfer_params = ec_transfer_params

147

148 def add(self, next_output: "RequestOutput", aggregate: bool) -> None:

149 """Merge subsequent RequestOutput into this one"""

150

151 self.finished |= next_output.finished

152 self.kv_transfer_params = next_output.kv_transfer_params

153 + self.ec_transfer_params = next_output.ec_transfer_params

154

155 for next_completion in next_output.outputs:

156 for i, completion in enumerate(self.outputs):

vllm/v1/core/sched/interface.py

+4

@@ -9,6 +9,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

9

10 if TYPE_CHECKING:

11 from vllm.config import VllmConfig

12 + from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase

13 from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorBase_V1

14 from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput

15 from vllm.v1.engine import EngineCoreOutputs

@@ -248,3 +249,6 @@ class SchedulerInterface(ABC):

249

250 def get_kv_connector(self) -> "KVConnectorBase_V1 | None":

251 return None

252 +

253 + def get_ec_connector(self) -> "ECConnectorBase | None":

254 + return None

vllm/v1/core/sched/scheduler.py

+20 -3

@@ -10,6 +10,7 @@ from typing import Any

10 from vllm.compilation.cuda_graph import CUDAGraphStat

11 from vllm.config import VllmConfig

12 from vllm.distributed.ec_transfer.ec_connector.base import (

13 + ECConnectorBase,

14 ECConnectorMetadata,

15 ECConnectorRole,

16 )

@@ -1675,6 +1676,7 @@ class Scheduler(SchedulerInterface):

1676 new_token_ids = generated_token_ids

1677 pooler_output = pooler_outputs[req_index] if pooler_outputs else None

1678 kv_transfer_params = None

1679 + ec_transfer_params = None

1680 status_before_stop = request.status

1681 num_output_tokens_before = len(request._output_token_ids)

1682

@@ -1761,7 +1763,7 @@ class Scheduler(SchedulerInterface):

1763 finish_reason = request.get_finished_reason()

1764 finished = self._handle_stopped_request(request)

1765 if finished:

1764 - kv_transfer_params = self._free_request(request)

1766 + kv_transfer_params, ec_transfer_params = self._free_request(request)

1767

1768 if status_before_stop == RequestStatus.RUNNING:

1769 stopped_running_reqs.add(request)

@@ -1785,6 +1787,7 @@ class Scheduler(SchedulerInterface):

1787 new_token_ids

1788 or pooler_output is not None

1789 or kv_transfer_params

1790 + or ec_transfer_params

1791 or stopped

1792 ):

1793 # Add EngineCoreOutput for this Request.

@@ -1800,6 +1803,7 @@ class Scheduler(SchedulerInterface):

1803 events=request.take_events(),

1804 prefill_stats=request.take_prefill_stats(),

1805 kv_transfer_params=kv_transfer_params,

1806 + ec_transfer_params=ec_transfer_params,

1807 trace_headers=request.trace_headers,

1808 routed_experts=routed_experts,

1809 num_nans_in_logits=request.num_nans_in_logits,

@@ -2151,11 +2155,21 @@ class Scheduler(SchedulerInterface):

2155

2156 def _free_request(

2157 self, request: Request, delay_free_blocks: bool = False

2154 - ) -> dict[str, Any] | None:

2158 + ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:

2159 assert request.is_finished()

2160

2161 self._inflight_prefills.discard(request)

2162 connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)

2163 +

2164 + # EC Connector: mirror the KV hook. The contract requires firing

2165 + # before the encoder cache is freed so the connector can inspect

2166 + # per-request state (e.g. which mm_hashes it recorded during

2167 + # save_caches()) and emit ec_transfer_params for the response body.

2168 + ec_xfer_params: dict[str, Any] | None = None

2169 + if self.ec_connector is not None:

2170 + ec_delay_free, ec_xfer_params = self.ec_connector.request_finished(request)

2171 + connector_delay_free_blocks |= ec_delay_free

2172 +

2173 self.encoder_cache_manager.free(request)

2174 request_id = request.request_id

2175 self.finished_req_ids.add(request_id)

@@ -2166,7 +2180,7 @@ class Scheduler(SchedulerInterface):

2180 if not delay_free_blocks:

2181 self._free_blocks(request)

2182

2169 - return kv_xfer_params

2183 + return kv_xfer_params, ec_xfer_params

2184

2185 def _free_blocks(self, request: Request):

2186 assert request.is_finished()

@@ -2414,6 +2428,9 @@ class Scheduler(SchedulerInterface):

2428 def get_kv_connector(self) -> KVConnectorBase_V1 | None:

2429 return self.connector

2430

2431 + def get_ec_connector(self) -> ECConnectorBase | None:

2432 + return self.ec_connector

2433 +

2434 def _connector_finished(

2435 self, request: Request

2436 ) -> tuple[bool, dict[str, Any] | None]:

vllm/v1/engine/__init__.py

+1

@@ -190,6 +190,7 @@ class EngineCoreOutput(

190 stop_reason: int | str | None = None

191 events: list[EngineCoreEvent] | None = None

192 kv_transfer_params: dict[str, Any] | None = None

193 + ec_transfer_params: dict[str, Any] | None = None

194

195 trace_headers: Mapping[str, str] | None = None

196

vllm/v1/engine/core.py

+9

@@ -400,6 +400,15 @@ class EngineCore:

400 "Disabling KVTransfer for this request."

401 )

402

403 + if (

404 + request.ec_transfer_params is not None

405 + and self.scheduler.get_ec_connector() is None

406 + ):

407 + logger.warning(

408 + "Got ec_transfer_params, but no ECConnector found. "

409 + "Disabling ECTransfer for this request."

410 + )

411 +

412 self.scheduler.add_request(request)

413 if request.abort_immediately:

414 # Immediately abort so the connector's request_finished hook runs

vllm/v1/engine/output_processor.py

+11 -1

@@ -276,6 +276,7 @@ class RequestState:

276 finish_reason: FinishReason | None,

277 stop_reason: int | str | None,

278 kv_transfer_params: dict[str, Any] | None = None,

279 + ec_transfer_params: dict[str, Any] | None = None,

280 ) -> RequestOutput | PoolingRequestOutput | None:

281 finished = finish_reason is not None

282 final_only = self.output_kind == RequestOutputKind.FINAL_ONLY

@@ -327,7 +328,11 @@ class RequestState:

328 external_req_id = self.parent_req.external_req_id

329

330 return self._new_request_output(

330 - external_req_id, outputs, finished, kv_transfer_params

331 + external_req_id,

332 + outputs,

333 + finished,

334 + kv_transfer_params,

335 + ec_transfer_params,

336 )

337

338 def _new_request_output(

@@ -336,6 +341,7 @@ class RequestState:

341 outputs: list[CompletionOutput] | list[PoolingOutput],

342 finished: bool,

343 kv_transfer_params: dict[str, Any] | None = None,

344 + ec_transfer_params: dict[str, Any] | None = None,

345 ) -> RequestOutput | PoolingRequestOutput:

346 # If prompt embeds were used, put placeholder prompt token ids

347 prompt_token_ids = self.prompt_token_ids

@@ -369,6 +375,7 @@ class RequestState:

375 outputs=cast(list[CompletionOutput], outputs),

376 finished=finished,

377 kv_transfer_params=kv_transfer_params,

378 + ec_transfer_params=ec_transfer_params,

379 num_cached_tokens=self.num_cached_tokens,

380 metrics=self.stats,

381 )

@@ -497,6 +504,7 @@ class OutputProcessor:

504 finish_reason=FinishReason.ABORT,

505 stop_reason=None,

506 kv_transfer_params=None,

507 + ec_transfer_params=None,

508 )

509 ):

510 req_state.queue.put(request_output)

@@ -620,6 +628,7 @@ class OutputProcessor:

628 finish_reason = engine_core_output.finish_reason

629 stop_reason = engine_core_output.stop_reason

630 kv_transfer_params = engine_core_output.kv_transfer_params

631 + ec_transfer_params = engine_core_output.ec_transfer_params

632 if engine_core_output.routed_experts is not None:

633 req_state.routed_experts_chunks.append(

634 engine_core_output.routed_experts

@@ -654,6 +663,7 @@ class OutputProcessor:

663 finish_reason,

664 stop_reason,

665 kv_transfer_params,

666 + ec_transfer_params,

667 ):

668 if req_state.streaming_input:

669 request_output.finished = False

vllm/v1/request.py

+5

@@ -100,6 +100,8 @@ class Request:

100

101 # P/D: Connector-specific KV transfer parameters.

102 self.kv_transfer_params: dict[str, Any] | None = None

103 + # E/P/D: Connector-specific encoder-cache transfer parameters.

104 + self.ec_transfer_params: dict[str, Any] | None = None

105

106 if pooling_params is not None:

107 # Pooling models.

@@ -115,6 +117,9 @@ class Request:

117 self.kv_transfer_params = sampling_params.extra_args.get(

118 "kv_transfer_params"

119 )

120 + self.ec_transfer_params = sampling_params.extra_args.get(

121 + "ec_transfer_params"

122 + )

123 self.kv_cache_report_mode = sampling_params.extra_args.get(

124 "kv_cache_report_mode", "incremental"

125 )

[EC Connector] Add EC Transfer Params (#42433)