A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Misc] improve example script output (#15528)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Mar 26, 2025 at 18:12 UTC
4ec2cee000af209a9499e0696993834af4f45035
5 files changed
+16
-6
examples/offline_inference/basic/basic.py
+4
-1
@@ -18,7 +18,10 @@ llm = LLM(model="facebook/opt-125m")
18
# that contain the prompt, generated text, and other information.
19
outputs = llm.generate(prompts, sampling_params)
20
# Print the outputs.
21
+print("\nGenerated Outputs:\n" + "-" * 60)
22
for output in outputs:
23
prompt = output.prompt
24
generated_text = output.outputs[0].text
24
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
\ No newline at end of file
25
+ print(f"Prompt: {prompt!r}")
26
+ print(f"Output: {generated_text!r}")
27
+ print("-" * 60)
\ No newline at end of file
examples/offline_inference/basic/chat.py
+3
-2
@@ -27,12 +27,13 @@ def main(args: dict):
27
sampling_params.top_k = top_k
28
29
def print_outputs(outputs):
30
+ print("\nGenerated Outputs:\n" + "-" * 80)
31
for output in outputs:
32
prompt = output.prompt
33
generated_text = output.outputs[0].text
33
- print(f"Prompt: {prompt!r}")
34
+ print(f"Prompt: {prompt!r}\n")
35
print(f"Generated text: {generated_text!r}")
35
- print("-" * 80)
36
+ print("-" * 80)
37
38
print("=" * 80)
39
examples/offline_inference/basic/classify.py
+3
-1
@@ -23,12 +23,14 @@ def main(args: Namespace):
23
outputs = model.classify(prompts)
24
25
# Print the outputs.
26
+ print("\nGenerated Outputs:\n" + "-" * 60)
27
for prompt, output in zip(prompts, outputs):
28
probs = output.outputs.probs
29
probs_trimmed = ((str(probs[:16])[:-1] +
30
", ...]") if len(probs) > 16 else probs)
30
- print(f"Prompt: {prompt!r} | "
31
+ print(f"Prompt: {prompt!r} \n"
32
f"Class Probabilities: {probs_trimmed} (size={len(probs)})")
33
+ print("-" * 60)
34
35
36
if __name__ == "__main__":
examples/offline_inference/basic/embed.py
+3
-1
@@ -23,12 +23,14 @@ def main(args: Namespace):
23
outputs = model.embed(prompts)
24
25
# Print the outputs.
26
+ print("\nGenerated Outputs:\n" + "-" * 60)
27
for prompt, output in zip(prompts, outputs):
28
embeds = output.outputs.embedding
29
embeds_trimmed = ((str(embeds[:16])[:-1] +
30
", ...]") if len(embeds) > 16 else embeds)
30
- print(f"Prompt: {prompt!r} | "
31
+ print(f"Prompt: {prompt!r} \n"
32
f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
33
+ print("-" * 60)
34
35
36
if __name__ == "__main__":
examples/offline_inference/basic/score.py
+3
-1
@@ -22,9 +22,11 @@ def main(args: Namespace):
22
outputs = model.score(text_1, texts_2)
23
24
# Print the outputs.
25
+ print("\nGenerated Outputs:\n" + "-" * 60)
26
for text_2, output in zip(texts_2, outputs):
27
score = output.outputs.score
27
- print(f"Pair: {[text_1, text_2]!r} | Score: {score}")
28
+ print(f"Pair: {[text_1, text_2]!r} \nScore: {score}")
29
+ print("-" * 60)
30
31
32
if __name__ == "__main__":