A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
[Bugfix] fix offline chat output prompt (#32076)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Jan 10, 2026 at 15:50 UTC
14fc7a68c7d671d7bcea951a7edc6ac718ced964
1 file changed
+13
-7
examples/offline_inference/basic/chat.py
+13
-7
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from vllm import LLM, EngineArgs
5
+from vllm.outputs import RequestOutput
6
from vllm.utils.argparse_utils import FlexibleArgumentParser
7
8
@@ -44,12 +45,12 @@ def main(args: dict):
45
if top_k is not None:
46
sampling_params.top_k = top_k
47
47
- def print_outputs(outputs):
48
+ def print_outputs(outputs: list[RequestOutput], prompts: list):
49
+ assert len(outputs) == len(prompts)
50
print("\nGenerated Outputs:\n" + "-" * 80)
49
- for output in outputs:
50
- prompt = output.prompt
51
+ for i, output in enumerate(outputs):
52
generated_text = output.outputs[0].text
52
- print(f"Prompt: {prompt!r}\n")
53
+ print(f"Prompt: {prompts[i]!r}\n")
54
print(f"Generated text: {generated_text!r}")
55
print("-" * 80)
56
@@ -66,14 +67,19 @@ def main(args: dict):
67
},
68
]
69
outputs = llm.chat(conversation, sampling_params, use_tqdm=False)
69
- print_outputs(outputs)
70
+ print_outputs(
71
+ outputs,
72
+ [
73
+ conversation,
74
+ ],
75
+ )
76
77
# You can run batch inference with llm.chat API
78
conversations = [conversation for _ in range(10)]
79
80
# We turn on tqdm progress bar to verify it's indeed running batch inference
81
outputs = llm.chat(conversations, sampling_params, use_tqdm=True)
76
- print_outputs(outputs)
82
+ print_outputs(outputs, conversations)
83
84
# A chat template can be optionally supplied.
85
# If not, the model will use its default chat template.
@@ -87,7 +93,7 @@ def main(args: dict):
93
use_tqdm=False,
94
chat_template=chat_template,
95
)
90
- print_outputs(outputs)
96
+ print_outputs(outputs, conversations)
97
98
99
if __name__ == "__main__":