@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1c45f4c385
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI] Basic Integration Test For TPU (#9968)

Signed-off-by: Robert Shaw <rshaw@neuralmagic.com>

Robert Shaw committed Nov 4, 2024 at 14:34 UTC 1c45f4c38576db6a27a52f36af9b693807d862b7
2 files changed +16 -3
.buildkite/run-tpu-test.sh
+1 -1
@@ -12,4 +12,4 @@ remove_docker_container
12 # For HF_TOKEN.
13 source /etc/environment
14 # Run a simple end-to-end example.
15 -docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
15 +docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
tests/entrypoints/openai/test_accuracy.py
+15 -2
@@ -10,6 +10,8 @@ AsyncLLMEngine are working correctly.
10 import lm_eval
11 import pytest
12
13 +from vllm.platforms import current_platform
14 +
15 from ...utils import RemoteOpenAIServer
16
17 MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
@@ -18,12 +20,21 @@ TASK = "gsm8k"
20 FILTER = "exact_match,strict-match"
21 RTOL = 0.03
22 EXPECTED_VALUE = 0.58
21 -DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
23 +DEFAULT_ARGS = ["--max-model-len", "2048", "--disable-log-requests"]
24 MORE_ARGS_LIST = [
25 + [], # Default
26 ["--enable-chunked-prefill"], # Chunked
27 ["--num-scheduler-steps", "8"], # MS
28 ["--num-scheduler-steps", "8", "--multi-step-stream-outputs"] # MS+Stream
29 ]
30 +MAX_WAIT_SECONDS = None
31 +
32 +if current_platform.is_tpu():
33 + MORE_ARGS_LIST = [
34 + [], # Default
35 + # ["--num-scheduler-steps", "8"], # Multi-step << currently fails
36 + ]
37 + MAX_WAIT_SECONDS = 600
38
39
40 @pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
@@ -33,7 +44,9 @@ def test_lm_eval_accuracy(more_args):
44
45 print(f"Running with: {args}")
46
36 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
47 + with RemoteOpenAIServer(
48 + MODEL_NAME, args,
49 + max_wait_seconds=MAX_WAIT_SECONDS) as remote_server:
50 url = f"{remote_server.url_for('v1')}/completions"
51
52 model_args = (