A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[ci] Fix sampler tests (#11922)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 10, 2025 at 20:45 UTC
241ad7b301facac0728e2b3312d71fe47acc8c9e
2 files changed
+10
-2
.buildkite/test-pipeline.yaml
+1
@@ -214,6 +214,7 @@ steps:
214
- vllm/model_executor/layers
215
- vllm/sampling_metadata.py
216
- tests/samplers
217
+ - tests/conftest.py
218
commands:
219
- pytest -v -s samplers
220
- VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers
tests/conftest.py
+9
-2
@@ -28,12 +28,13 @@ from vllm.distributed import (cleanup_dist_env_and_memory,
28
init_distributed_environment,
29
initialize_model_parallel)
30
from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
31
- to_enc_dec_tuple_list, zip_enc_dec_prompts)
31
+ TokensPrompt, to_enc_dec_tuple_list,
32
+ zip_enc_dec_prompts)
33
from vllm.logger import init_logger
34
from vllm.outputs import RequestOutput
35
from vllm.sampling_params import BeamSearchParams
36
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
36
- identity)
37
+ identity, is_list_of)
38
39
logger = init_logger(__name__)
40
@@ -886,6 +887,12 @@ class VllmRunner:
887
beam_width: int,
888
max_tokens: int,
889
) -> List[Tuple[List[List[int]], List[str]]]:
890
+ if is_list_of(prompts, str, check="all"):
891
+ prompts = [TextPrompt(prompt=prompt) for prompt in prompts]
892
+ else:
893
+ prompts = [
894
+ TokensPrompt(prompt_token_ids=tokens) for tokens in prompts
895
+ ]
896
outputs = self.model.beam_search(
897
prompts,
898
BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens))