A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix sampling_params passed incorrectly in Phi3v example (#5684)
Isotr0py committed
Jun 19, 2024 at 17:58 UTC
7d46c8d37864993162bbeb61dc19b5ad6043646d
1 file changed
+6
-6
examples/phi3v_example.py
+6
-6
@@ -12,7 +12,6 @@ def run_phi3v():
12
llm = LLM(
13
model=model_path,
14
trust_remote_code=True,
15
- max_model_len=4096,
15
image_input_type="pixel_values",
16
image_token_id=32044,
17
image_input_shape="1,3,1008,1344",
@@ -28,11 +27,12 @@ def run_phi3v():
27
28
sampling_params = SamplingParams(temperature=0, max_tokens=64)
29
31
- outputs = llm.generate({
32
- "prompt": prompt,
33
- "sampling_params": sampling_params,
34
- "multi_modal_data": ImagePixelData(image),
35
- })
30
+ outputs = llm.generate(
31
+ {
32
+ "prompt": prompt,
33
+ "multi_modal_data": ImagePixelData(image),
34
+ },
35
+ sampling_params=sampling_params)
36
for o in outputs:
37
generated_text = o.outputs[0].text
38
print(generated_text)