A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[doc] add the print result (#17584)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
May 2, 2025 at 20:24 UTC
6d1479ca4b5a3904b6c5b4a1d741dda43efdc289
1 file changed
+3
docs/source/features/quantization/fp8.md
+3
@@ -30,6 +30,7 @@ from vllm import LLM
30
model = LLM("facebook/opt-125m", quantization="fp8")
31
# INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB
32
result = model.generate("Hello, my name is")
33
+print(result[0].outputs[0].text)
34
```
35
36
:::{warning}
@@ -106,6 +107,7 @@ Load and run the model in `vllm`:
107
from vllm import LLM
108
model = LLM("./Meta-Llama-3-8B-Instruct-FP8-Dynamic")
109
model.generate("Hello my name is")
110
+print(result[0].outputs[0].text)
111
```
112
113
Evaluate accuracy with `lm_eval` (for example on 250 samples of `gsm8k`):
@@ -188,4 +190,5 @@ from vllm import LLM
190
model = LLM(model="Meta-Llama-3-8B-Instruct-FP8/")
191
# INFO 06-10 21:15:41 model_runner.py:159] Loading model weights took 8.4596 GB
192
result = model.generate("Hello, my name is")
193
+print(result[0].outputs[0].text)
194
```