A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Disable enforce_eager for V1 TPU sampler and structured output tests (#17016)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Apr 24, 2025 at 03:50 UTC
14288d1332602846d3f36c4a0d65b13d335cf557
3 files changed
+7
-2
.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh
+1
@@ -19,6 +19,7 @@ docker run --privileged --net host --shm-size=16G -it \
19
vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git \
20
&& python3 -m pip install pytest pytest-asyncio tpu-info \
21
&& python3 -m pip install lm_eval[api]==0.4.4 \
22
+ && export VLLM_XLA_CACHE_PATH= \
23
&& export VLLM_USE_V1=1 \
24
&& export VLLM_XLA_CHECK_RECOMPILATION=1 \
25
&& echo HARDWARE \
tests/v1/entrypoints/llm/test_struct_output_generate.py
+5
-1
@@ -13,6 +13,7 @@ from pydantic import BaseModel
13
14
from vllm.entrypoints.llm import LLM
15
from vllm.outputs import RequestOutput
16
+from vllm.platforms import current_platform
17
from vllm.sampling_params import GuidedDecodingParams, SamplingParams
18
19
PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [
@@ -63,10 +64,13 @@ def test_structured_output(
64
):
65
monkeypatch.setenv("VLLM_USE_V1", "1")
66
67
+ # Don't use eager execution on TPUs because we want to test for no
68
+ # recompilation at runtime
69
+ enforce_eager = bool(not current_platform.is_tpu())
70
# Use a single LLM instance for several scenarios to
71
# speed up the test suite.
72
llm = LLM(model=model_name,
69
- enforce_eager=True,
73
+ enforce_eager=enforce_eager,
74
max_model_len=1024,
75
guided_decoding_backend=guided_decoding_backend,
76
tokenizer_mode=tokenizer_mode)
tests/v1/tpu/test_sampler.py
+1
-1
@@ -23,7 +23,7 @@ def test_sampler_different(model_name: str):
23
different results.
24
"""
25
llm = LLM(model_name,
26
- enforce_eager=True,
26
+ enforce_eager=False,
27
max_num_seqs=1,
28
max_model_len=512,
29
max_num_batched_tokens=512)