A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI] Basic Integration Test For TPU (#9968)
Signed-off-by: Robert Shaw <rshaw@neuralmagic.com>
Robert Shaw committed
Nov 4, 2024 at 14:34 UTC
1c45f4c38576db6a27a52f36af9b693807d862b7
2 files changed
+16
-3
.buildkite/run-tpu-test.sh
+1
-1
@@ -12,4 +12,4 @@ remove_docker_container
12
# For HF_TOKEN.
13
source /etc/environment
14
# Run a simple end-to-end example.
15
-docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
15
+docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
tests/entrypoints/openai/test_accuracy.py
+15
-2
@@ -10,6 +10,8 @@ AsyncLLMEngine are working correctly.
10
import lm_eval
11
import pytest
12
13
+from vllm.platforms import current_platform
14
+
15
from ...utils import RemoteOpenAIServer
16
17
MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
@@ -18,12 +20,21 @@ TASK = "gsm8k"
20
FILTER = "exact_match,strict-match"
21
RTOL = 0.03
22
EXPECTED_VALUE = 0.58
21
-DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
23
+DEFAULT_ARGS = ["--max-model-len", "2048", "--disable-log-requests"]
24
MORE_ARGS_LIST = [
25
+ [], # Default
26
["--enable-chunked-prefill"], # Chunked
27
["--num-scheduler-steps", "8"], # MS
28
["--num-scheduler-steps", "8", "--multi-step-stream-outputs"] # MS+Stream
29
]
30
+MAX_WAIT_SECONDS = None
31
+
32
+if current_platform.is_tpu():
33
+ MORE_ARGS_LIST = [
34
+ [], # Default
35
+ # ["--num-scheduler-steps", "8"], # Multi-step << currently fails
36
+ ]
37
+ MAX_WAIT_SECONDS = 600
38
39
40
@pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
@@ -33,7 +44,9 @@ def test_lm_eval_accuracy(more_args):
44
45
print(f"Running with: {args}")
46
36
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
47
+ with RemoteOpenAIServer(
48
+ MODEL_NAME, args,
49
+ max_wait_seconds=MAX_WAIT_SECONDS) as remote_server:
50
url = f"{remote_server.url_for('v1')}/completions"
51
52
model_args = (