@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c35e4a3dd7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[BugFix] Fix test_phi3v.py (#5725)

Chang Su committed Jun 20, 2024 at 21:45 UTC c35e4a3dd74fa5952b04354a3c7cfd0ed09e2eb0
2 files changed +9 -5
tests/conftest.py
+3 -1
@@ -233,11 +233,13 @@ class HfRunner:
233 prompts: List[str],
234 max_tokens: int,
235 images: Optional[List[Image.Image]] = None,
236 + **kwargs,
237 ) -> List[Tuple[List[int], str]]:
238 outputs = self.generate(prompts,
239 do_sample=False,
240 max_new_tokens=max_tokens,
240 - images=images)
241 + images=images,
242 + **kwargs)
243
244 return [(output_ids[0], output_str[0])
245 for output_ids, output_str in outputs]
tests/models/test_phi3v.py
+6 -4
@@ -77,7 +77,7 @@ if is_cpu():
77 # numeric difference for longer context and test can't pass
78 @pytest.mark.parametrize("model_and_config", model_and_vl_config)
79 @pytest.mark.parametrize("dtype", [target_dtype])
80 -@pytest.mark.parametrize("max_tokens", [8])
80 +@pytest.mark.parametrize("max_tokens", [128])
81 def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
82 model_and_config, dtype: str, max_tokens: int) -> None:
83 """Inference result should be the same between hf and vllm.
@@ -95,9 +95,11 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
95 hf_model_kwargs = {"_attn_implementation": "eager"}
96 with hf_runner(model_id, dtype=dtype,
97 model_kwargs=hf_model_kwargs) as hf_model:
98 - hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
99 - max_tokens,
100 - images=hf_images)
98 + hf_outputs = hf_model.generate_greedy(
99 + HF_IMAGE_PROMPTS,
100 + max_tokens,
101 + images=hf_images,
102 + eos_token_id=hf_model.processor.tokenizer.eos_token_id)
103
104 vllm_image_prompts = [
105 p.replace("<|image_1|>",