@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 241ad7b301
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[ci] Fix sampler tests (#11922)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 10, 2025 at 20:45 UTC 241ad7b301facac0728e2b3312d71fe47acc8c9e
2 files changed +10 -2
.buildkite/test-pipeline.yaml
+1
@@ -214,6 +214,7 @@ steps:
214 - vllm/model_executor/layers
215 - vllm/sampling_metadata.py
216 - tests/samplers
217 + - tests/conftest.py
218 commands:
219 - pytest -v -s samplers
220 - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers
tests/conftest.py
+9 -2
@@ -28,12 +28,13 @@ from vllm.distributed import (cleanup_dist_env_and_memory,
28 init_distributed_environment,
29 initialize_model_parallel)
30 from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
31 - to_enc_dec_tuple_list, zip_enc_dec_prompts)
31 + TokensPrompt, to_enc_dec_tuple_list,
32 + zip_enc_dec_prompts)
33 from vllm.logger import init_logger
34 from vllm.outputs import RequestOutput
35 from vllm.sampling_params import BeamSearchParams
36 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
36 - identity)
37 + identity, is_list_of)
38
39 logger = init_logger(__name__)
40
@@ -886,6 +887,12 @@ class VllmRunner:
887 beam_width: int,
888 max_tokens: int,
889 ) -> List[Tuple[List[List[int]], List[str]]]:
890 + if is_list_of(prompts, str, check="all"):
891 + prompts = [TextPrompt(prompt=prompt) for prompt in prompts]
892 + else:
893 + prompts = [
894 + TokensPrompt(prompt_token_ids=tokens) for tokens in prompts
895 + ]
896 outputs = self.model.beam_search(
897 prompts,
898 BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens))