A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Bugfix] Fix small typo in the example of Streaming delimiter (#14793)
WeiCheng committed
Mar 14, 2025 at 16:05 UTC
54cc46f3ebcde68f072c37f2142ff37d60077ff0
2 files changed
+2
-2
examples/online_serving/api_client.py
+1
-1
@@ -42,7 +42,7 @@ def post_http_request(prompt: str,
42
def get_streaming_response(response: requests.Response) -> Iterable[list[str]]:
43
for chunk in response.iter_lines(chunk_size=8192,
44
decode_unicode=False,
45
- delimiter=b"\0"):
45
+ delimiter=b"\n"):
46
if chunk:
47
data = json.loads(chunk.decode("utf-8"))
48
output = data["text"]
examples/online_serving/gradio_webserver.py
+1
-1
@@ -21,7 +21,7 @@ def http_bot(prompt):
21
22
for chunk in response.iter_lines(chunk_size=8192,
23
decode_unicode=False,
24
- delimiter=b"\0"):
24
+ delimiter=b"\n"):
25
if chunk:
26
data = json.loads(chunk.decode("utf-8"))
27
output = data["text"][0]