@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7d46c8d378
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix sampling_params passed incorrectly in Phi3v example (#5684)

Isotr0py committed Jun 19, 2024 at 17:58 UTC 7d46c8d37864993162bbeb61dc19b5ad6043646d
1 file changed +6 -6
examples/phi3v_example.py
+6 -6
@@ -12,7 +12,6 @@ def run_phi3v():
12 llm = LLM(
13 model=model_path,
14 trust_remote_code=True,
15 - max_model_len=4096,
15 image_input_type="pixel_values",
16 image_token_id=32044,
17 image_input_shape="1,3,1008,1344",
@@ -28,11 +27,12 @@ def run_phi3v():
27
28 sampling_params = SamplingParams(temperature=0, max_tokens=64)
29
31 - outputs = llm.generate({
32 - "prompt": prompt,
33 - "sampling_params": sampling_params,
34 - "multi_modal_data": ImagePixelData(image),
35 - })
30 + outputs = llm.generate(
31 + {
32 + "prompt": prompt,
33 + "multi_modal_data": ImagePixelData(image),
34 + },
35 + sampling_params=sampling_params)
36 for o in outputs:
37 generated_text = o.outputs[0].text
38 print(generated_text)