A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[BugFix] Fix test_phi3v.py (#5725)
Chang Su committed
Jun 20, 2024 at 21:45 UTC
c35e4a3dd74fa5952b04354a3c7cfd0ed09e2eb0
2 files changed
+9
-5
tests/conftest.py
+3
-1
@@ -233,11 +233,13 @@ class HfRunner:
233
prompts: List[str],
234
max_tokens: int,
235
images: Optional[List[Image.Image]] = None,
236
+ **kwargs,
237
) -> List[Tuple[List[int], str]]:
238
outputs = self.generate(prompts,
239
do_sample=False,
240
max_new_tokens=max_tokens,
240
- images=images)
241
+ images=images,
242
+ **kwargs)
243
244
return [(output_ids[0], output_str[0])
245
for output_ids, output_str in outputs]
tests/models/test_phi3v.py
+6
-4
@@ -77,7 +77,7 @@ if is_cpu():
77
# numeric difference for longer context and test can't pass
78
@pytest.mark.parametrize("model_and_config", model_and_vl_config)
79
@pytest.mark.parametrize("dtype", [target_dtype])
80
-@pytest.mark.parametrize("max_tokens", [8])
80
+@pytest.mark.parametrize("max_tokens", [128])
81
def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
82
model_and_config, dtype: str, max_tokens: int) -> None:
83
"""Inference result should be the same between hf and vllm.
@@ -95,9 +95,11 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
95
hf_model_kwargs = {"_attn_implementation": "eager"}
96
with hf_runner(model_id, dtype=dtype,
97
model_kwargs=hf_model_kwargs) as hf_model:
98
- hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
99
- max_tokens,
100
- images=hf_images)
98
+ hf_outputs = hf_model.generate_greedy(
99
+ HF_IMAGE_PROMPTS,
100
+ max_tokens,
101
+ images=hf_images,
102
+ eos_token_id=hf_model.processor.tokenizer.eos_token_id)
103
104
vllm_image_prompts = [
105
p.replace("<|image_1|>",