A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Misc] Fixed the abnormally high TTFT issue in the PD disaggregation example (#18644)
Signed-off-by: zhaohaidao <zhaohaidao2008@hotmail.com> Signed-off-by: zhaohaiyuan <zhaohaiyuan@xiaohongshu.com> Co-authored-by: zhaohaiyuan <zhaohaiyuan@xiaohongshu.com>
AlexZhao committed
May 26, 2025 at 13:51 UTC
8820821b59121192205077d0cdc56490e215b9fb
1 file changed
+2
-2
examples/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py
+2
-2
@@ -135,7 +135,7 @@ async def handle_completions(request: Request):
135
yield chunk
136
137
return StreamingResponse(generate_stream(),
138
- media_type="application/json")
138
+ media_type="text/event-stream")
139
140
except Exception as e:
141
import sys
@@ -172,7 +172,7 @@ async def handle_chat_completions(request: Request):
172
yield chunk
173
174
return StreamingResponse(generate_stream(),
175
- media_type="application/json")
175
+ media_type="text/event-stream")
176
177
except Exception as e:
178
import sys