A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 07:45 UTC
fix offline inference chat response prompt (#32088)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Jan 11, 2026 at 22:01 UTC
d74132ca3b8aea4840478bfbca56fd59273b325e
2 files changed
+14
-9
examples/offline_inference/context_extension.py
+9
-7
@@ -9,7 +9,7 @@ Usage:
9
python examples/offline_inference/context_extension.py
10
"""
11
12
-from vllm import LLM, SamplingParams
12
+from vllm import LLM, RequestOutput, SamplingParams
13
14
15
def create_llm():
@@ -45,13 +45,15 @@ def run_llm_chat(llm):
45
{"role": "assistant", "content": "Hello! How can I assist you today?"},
46
]
47
outputs = llm.chat(conversation, sampling_params, use_tqdm=False)
48
- return outputs
48
+ return outputs, [
49
+ conversation,
50
+ ]
51
52
51
-def print_outputs(outputs):
53
+def print_outputs(outputs: list[RequestOutput], conversations: list):
54
print("\nGenerated Outputs:\n" + "-" * 80)
53
- for output in outputs:
54
- prompt = output.prompt
55
+ for i, output in enumerate(outputs):
56
+ prompt = conversations[i]
57
generated_text = output.outputs[0].text
58
print(f"Prompt: {prompt!r}\n")
59
print(f"Generated text: {generated_text!r}")
@@ -60,8 +62,8 @@ def print_outputs(outputs):
62
63
def main():
64
llm = create_llm()
63
- outputs = run_llm_chat(llm)
64
- print_outputs(outputs)
65
+ outputs, conversations = run_llm_chat(llm)
66
+ print_outputs(outputs, conversations)
67
68
69
if __name__ == "__main__":
examples/offline_inference/spec_decode.py
+5
-2
@@ -152,9 +152,12 @@ def main(args):
152
153
# print the generated text
154
if args.print_output:
155
- for output in outputs:
155
+ for i, output in enumerate(outputs):
156
print("-" * 50)
157
- print(f"prompt: {output.prompt}")
157
+ if not args.custom_mm_prompts:
158
+ print(f"prompt: {prompts[i].prompt}")
159
+ else:
160
+ print(f"prompt: {prompts[i]}")
161
print(f"generated text: {output.outputs[0].text}")
162
print("-" * 50)
163