A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[CI] Add Buildkite (#2355)
Simon Mo committed
Jan 14, 2024 at 12:37 UTC
6e01e8c1c8ea323d30e3f57050469b2df66b56c6
13 files changed
+192
-37
.buildkite/run-benchmarks.sh
new
+24
@@ -0,0 +1,24 @@
1
+# This script is run by buildkite to run the benchmarks and upload the results to buildkite
2
+
3
+set -ex
4
+
5
+# cd into parent directory of this file
6
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
7
+
8
+# run benchmarks and upload the result to buildkite
9
+python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
10
+
11
+python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
12
+
13
+# write the results into a markdown file
14
+echo "### Latency Benchmarks" >> benchmark_results.md
15
+sed -n '1p' benchmark_latency.txt >> benchmark_results.md
16
+echo "" >> benchmark_results.md
17
+sed -n '$p' benchmark_latency.txt >> benchmark_results.md
18
+echo "### Throughput Benchmarks" >> benchmark_results.md
19
+sed -n '1p' benchmark_throughput.txt >> benchmark_results.md
20
+echo "" >> benchmark_results.md
21
+sed -n '$p' benchmark_throughput.txt >> benchmark_results.md
22
+
23
+# upload the results to buildkite
24
+/workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
.buildkite/test-pipeline.yaml
new
+41
@@ -0,0 +1,41 @@
1
+# In this file, you can add more tests to run either by adding a new step or
2
+# adding a new command to an existing step. See different options here for examples.
3
+# This script will be feed into Jinja template in `test-template.j2` to generate
4
+# the final pipeline yaml file.
5
+
6
+steps:
7
+- label: Regression Test
8
+ command: pytest -v -s test_regression.py
9
+ working_dir: "/vllm-workspace/tests" # optional
10
+
11
+- label: AsyncEngine Test
12
+ command: pytest -v -s async_engine
13
+
14
+- label: Distributed Test
15
+ command: pytest -v -s test_comm_ops.py
16
+ working_dir: "/vllm-workspace/tests/distributed"
17
+ num_gpus: 2 # only support 1 or 2 for now.
18
+
19
+- label: Engine Test
20
+ command: pytest -v -s engine
21
+
22
+- label: Kernels Test
23
+ command: pytest -v -s kernels
24
+ soft_fail: true
25
+
26
+- label: Models Test
27
+ commands:
28
+ - pytest -v -s models --forked
29
+ soft_fail: true
30
+
31
+- label: Samplers Test
32
+ command: pytest -v -s samplers --forked
33
+
34
+- label: Worker Test
35
+ command: pytest -v -s worker
36
+
37
+- label: Benchmarks
38
+ working_dir: "/vllm-workspace/.buildkite"
39
+ commands:
40
+ - pip install aiohttp
41
+ - bash run-benchmarks.sh
.buildkite/test-template.j2
new
+46
@@ -0,0 +1,46 @@
1
+{% set docker_image = "us-central1-docker.pkg.dev/vllm-405802/vllm-ci-test-repo/vllm-test:$BUILDKITE_COMMIT" %}
2
+{% set default_num_gpu = 1 %}
3
+{% set default_working_dir = "/vllm-workspace/tests" %}
4
+
5
+steps:
6
+ - label: ":docker: build image"
7
+ commands:
8
+ - "docker build --tag {{ docker_image }} --target test --progress plain ."
9
+ - "docker push {{ docker_image }}"
10
+ env:
11
+ DOCKER_BUILDKIT: "1"
12
+ - wait
13
+
14
+ {% for step in steps %}
15
+ - label: "{{ step.label }}"
16
+ agents:
17
+ queue: kubernetes
18
+ soft_fail: {{ step.soft_fail or false }}
19
+ plugins:
20
+ - kubernetes:
21
+ podSpec:
22
+ volumes:
23
+ - name: dshm
24
+ emptyDir:
25
+ medium: Memory
26
+ containers:
27
+ - image: "{{ docker_image }}"
28
+ command: ["bash"]
29
+ args:
30
+ - "-c"
31
+ - "'cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}'"
32
+ resources:
33
+ requests:
34
+ nvidia.com/gpu: "{{ step.num_gpus or default_num_gpu }}"
35
+ limits:
36
+ nvidia.com/gpu: "{{ step.num_gpus or default_num_gpu }}"
37
+ env:
38
+ - name: HF_TOKEN
39
+ valueFrom:
40
+ secretKeyRef:
41
+ name: hf-token-secret
42
+ key: token
43
+ volumeMounts:
44
+ - mountPath: /dev/shm
45
+ name: dshm
46
+ {% endfor %}
Dockerfile
+24
-12
@@ -1,7 +1,11 @@
1
+# The vLLM Dockerfile is used to construct vLLM image that can be directly used
2
+# to run the OpenAI compatible server.
3
+
4
+#################### BASE BUILD IMAGE ####################
5
FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
6
7
RUN apt-get update -y \
4
- && apt-get install -y python3-pip
8
+ && apt-get install -y python3-pip git
9
10
WORKDIR /workspace
11
@@ -14,8 +18,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
18
COPY requirements-dev.txt requirements-dev.txt
19
RUN --mount=type=cache,target=/root/.cache/pip \
20
pip install -r requirements-dev.txt
21
+#################### BASE BUILD IMAGE ####################
22
+
23
18
-# image to build pytorch extensions
24
+#################### EXTENSION BUILD IMAGE ####################
25
FROM dev AS build
26
27
# install build dependencies
@@ -30,6 +36,7 @@ COPY requirements.txt requirements.txt
36
COPY pyproject.toml pyproject.toml
37
COPY vllm/__init__.py vllm/__init__.py
38
39
+# cuda arch list used by torch
40
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
41
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
42
# max jobs used by Ninja to build extensions
@@ -40,18 +47,26 @@ ARG nvcc_threads=8
47
ENV NVCC_THREADS=$nvcc_threads
48
49
RUN python3 setup.py build_ext --inplace
50
+#################### EXTENSION Build IMAGE ####################
51
52
+
53
+#################### TEST IMAGE ####################
54
# image to run unit testing suite
55
FROM dev AS test
56
57
# copy pytorch extensions separately to avoid having to rebuild
58
# when python code changes
49
-COPY --from=build /workspace/vllm/*.so /workspace/vllm/
50
-COPY tests tests
51
-COPY vllm vllm
59
+WORKDIR /vllm-workspace
60
+# ADD is used to preserve directory structure
61
+ADD . /vllm-workspace/
62
+COPY --from=build /workspace/vllm/*.so /vllm-workspace/vllm/
63
+# ignore build dependencies installation because we are using pre-complied extensions
64
+RUN rm pyproject.toml
65
+RUN --mount=type=cache,target=/root/.cache/pip VLLM_USE_PRECOMPILED=1 pip install . --verbose
66
+#################### TEST IMAGE ####################
67
53
-ENTRYPOINT ["python3", "-m", "pytest", "tests"]
68
69
+#################### RUNTIME BASE IMAGE ####################
70
# use CUDA base as CUDA runtime dependencies are already installed via pip
71
FROM nvidia/cuda:12.1.0-base-ubuntu22.04 AS vllm-base
72
@@ -63,14 +78,10 @@ WORKDIR /workspace
78
COPY requirements.txt requirements.txt
79
RUN --mount=type=cache,target=/root/.cache/pip \
80
pip install -r requirements.txt
81
+#################### RUNTIME BASE IMAGE ####################
82
67
-FROM vllm-base AS vllm
68
-COPY --from=build /workspace/vllm/*.so /workspace/vllm/
69
-COPY vllm vllm
70
-
71
-EXPOSE 8000
72
-ENTRYPOINT ["python3", "-m", "vllm.entrypoints.api_server"]
83
84
+#################### OPENAI API SERVER ####################
85
# openai api server alternative
86
FROM vllm-base AS vllm-openai
87
# install additional dependencies for openai api server
@@ -81,3 +92,4 @@ COPY --from=build /workspace/vllm/*.so /workspace/vllm/
92
COPY vllm vllm
93
94
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
95
+#################### OPENAI API SERVER ####################
requirements-dev.txt
+3
-1
@@ -13,4 +13,6 @@ types-setuptools
13
pytest
14
pytest-forked
15
pytest-asyncio
16
-
16
+httpx
17
+einops # required for MPT
18
+flash_attn # required for HuggingFace's llama implementation
setup.py
+6
-1
@@ -293,6 +293,11 @@ def get_requirements() -> List[str]:
293
return requirements
294
295
296
+package_data = {"vllm": ["py.typed"]}
297
+if os.environ.get("VLLM_USE_PRECOMPILED"):
298
+ ext_modules = []
299
+ package_data["vllm"].append("*.so")
300
+
301
setuptools.setup(
302
name="vllm",
303
version=get_vllm_version(),
@@ -321,5 +326,5 @@ setuptools.setup(
326
install_requires=get_requirements(),
327
ext_modules=ext_modules,
328
cmdclass={"build_ext": BuildExtension},
324
- package_data={"vllm": ["py.typed"]},
329
+ package_data=package_data,
330
)
tests/async_engine/test_api_server.py
+10
-2
@@ -29,8 +29,13 @@ def api_server():
29
script_path = Path(__file__).parent.joinpath(
30
"api_server_async_engine.py").absolute()
31
uvicorn_process = subprocess.Popen([
32
- sys.executable, "-u",
33
- str(script_path), "--model", "facebook/opt-125m"
32
+ sys.executable,
33
+ "-u",
34
+ str(script_path),
35
+ "--model",
36
+ "facebook/opt-125m",
37
+ "--host",
38
+ "127.0.0.1",
39
])
40
yield
41
uvicorn_process.terminate()
@@ -81,6 +86,9 @@ def test_api_server(api_server):
86
pool.join()
87
88
# check cancellation stats
89
+ # give it some times to update the stats
90
+ time.sleep(1)
91
+
92
num_aborted_requests = requests.get(
93
"http://localhost:8000/stats").json()["num_aborted_requests"]
94
assert num_aborted_requests > 0
tests/async_engine/test_openai_server.py
+9
-6
@@ -1,19 +1,24 @@
1
from argparse import Namespace
2
from dataclasses import dataclass
3
+import os
4
+import pathlib
5
6
import pytest
7
from fastapi.testclient import TestClient
8
9
from vllm.entrypoints.openai.api_server import *
10
11
+chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
12
+ __file__))).parent.parent / "examples/template_chatml.jinja"
13
+assert chatml_jinja_path.exists()
14
+
15
# Define models, templates, and their corresponding expected outputs
16
MODEL_TEMPLATE_GENERATON_OUTPUT = [
17
("facebook/opt-125m", None, True,
18
"Hello</s>Hi there!</s>What is the capital of</s>"),
19
("facebook/opt-125m", None, False,
20
"Hello</s>Hi there!</s>What is the capital of</s>"),
15
- ("facebook/opt-125m", "../../examples/template_chatml.jinja", True,
16
- """<|im_start|>user
21
+ ("facebook/opt-125m", chatml_jinja_path, True, """<|im_start|>user
22
Hello<|im_end|>
23
<|im_start|>assistant
24
Hi there!<|im_end|>
@@ -21,8 +26,7 @@ Hi there!<|im_end|>
26
What is the capital of<|im_end|>
27
<|im_start|>assistant
28
"""),
24
- ("facebook/opt-125m", "../../examples/template_chatml.jinja", False,
25
- """<|im_start|>user
29
+ ("facebook/opt-125m", chatml_jinja_path, False, """<|im_start|>user
30
Hello<|im_end|>
31
<|im_start|>assistant
32
Hi there!<|im_end|>
@@ -54,8 +58,7 @@ class MockTokenizer:
58
59
def test_load_chat_template():
60
# Testing chatml template
57
- template = "../../examples/template_chatml.jinja"
58
- mock_args = Namespace(chat_template=template)
61
+ mock_args = Namespace(chat_template=chatml_jinja_path)
62
tokenizer = MockTokenizer()
63
64
# Call the function with the mocked args
tests/distributed/test_comm_ops.py
+14
-12
@@ -2,10 +2,9 @@
2
3
Run `pytest tests/distributed/test_comm_ops.py --forked`.
4
"""
5
-from multiprocessing import Process, set_start_method
6
-
5
import pytest
6
import torch
7
+import ray
8
9
from vllm.config import ParallelConfig
10
from vllm.utils import get_open_port
@@ -23,11 +22,11 @@ def init_test_distributed_environment(pipeline_parallel_size: int,
22
tensor_parallel_size,
23
worker_use_ray=True)
24
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
26
- torch.cuda.set_device(rank)
25
_init_distributed_environment(parallel_config, rank,
26
distributed_init_method)
27
28
29
+@ray.remote(num_gpus=1, max_calls=1)
30
def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
31
distributed_init_port: str):
32
init_test_distributed_environment(1, tensor_parallel_size, rank,
@@ -43,6 +42,7 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
42
assert torch.allclose(t, expected)
43
44
45
+@ray.remote(num_gpus=1, max_calls=1)
46
def all_gather_test_worker(tensor_parallel_size: int, rank: int,
47
distributed_init_port: str):
48
init_test_distributed_environment(1, tensor_parallel_size, rank,
@@ -70,14 +70,16 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
70
@pytest.mark.parametrize("test_target",
71
[all_reduce_test_worker, all_gather_test_worker])
72
def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
73
- set_start_method("spawn", force=True)
73
+ # Using ray helps debugging the error when it failed
74
+ # as compared to multiprocessing.
75
+ ray.init()
76
+
77
distributed_init_port = get_open_port()
75
- processes = []
78
+ refs = []
79
for rank in range(tensor_parallel_size):
77
- p = Process(target=test_target,
78
- args=(tensor_parallel_size, rank, distributed_init_port))
79
- p.start()
80
- processes.append(p)
81
- for p in processes:
82
- p.join()
83
- assert all(p.exitcode == 0 for p in processes)
80
+ refs.append(
81
+ test_target.remote(tensor_parallel_size, rank,
82
+ distributed_init_port))
83
+ ray.get(refs)
84
+
85
+ ray.shutdown()
tests/kernels/test_attention.py
+1
-1
@@ -13,7 +13,7 @@ FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
13
# This will change depending on the compute capability.
14
# - 512 as a buffer
15
MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
16
-NUM_BLOCKS = 40000 # Arbitrary values for testing
16
+NUM_BLOCKS = 12000 # Arbitrary values for testing
17
PARTITION_SIZE = 512
18
19
DTYPES = [torch.half, torch.bfloat16, torch.float]
tests/kernels/test_cache.py
+2
-2
@@ -6,12 +6,12 @@ import torch
6
from vllm._C import cache_ops
7
8
DTYPES = [torch.half, torch.bfloat16, torch.float]
9
-NUM_TOKENS = [83] # Arbitrary values for testing
9
+NUM_TOKENS = [42] # Arbitrary values for testing
10
NUM_LAYERS = [1] # Arbitrary values for testing
11
NUM_HEADS = [8] # Arbitrary values for testing
12
HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13
BLOCK_SIZES = [8, 16, 32]
14
-NUM_BLOCKS = [1024, 36000] # Arbitrary values for testing
14
+NUM_BLOCKS = [1024, 3600] # Arbitrary values for testing
15
NUM_MAPPINGS = [256] # Arbitrary values for testing
16
SEEDS = [0]
17
DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
tests/samplers/test_logprobs.py
+1
@@ -30,6 +30,7 @@ def test_get_prompt_logprobs(
30
temperature=0.0)
31
vllm_results = vllm_model.model.generate(
32
example_prompts, sampling_params=vllm_sampling_params)
33
+ del vllm_model
34
35
# Test whether logprobs are included in the results.
36
for result in vllm_results:
tests/samplers/test_sampler.py
+11
@@ -75,6 +75,8 @@ def test_sampler_all_greedy(seed: int):
75
for nth_output in sequence_output.samples:
76
assert nth_output.output_token == expected[i].item()
77
78
+ del model_runner
79
+
80
81
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
82
def test_sampler_all_random(seed: int):
@@ -111,6 +113,8 @@ def test_sampler_all_random(seed: int):
113
for nth_output in sequence_output.samples:
114
assert nth_output.output_token == i
115
116
+ del model_runner
117
+
118
119
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
120
def test_sampler_all_beam(seed: int):
@@ -144,6 +148,7 @@ def test_sampler_all_beam(seed: int):
148
# the outputs are expected - in other words, this just tests
149
# whether there are no exceptions in the sampler
150
# when handling an all-beam search case.
151
+ del model_runner
152
153
154
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
@@ -198,6 +203,8 @@ def test_sampler_mixed(seed: int):
203
for nth_output in sequence_output.samples:
204
assert nth_output.output_token in expected_tokens
205
206
+ del model_runner
207
+
208
209
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
210
def test_sampler_logits_processors(seed: int):
@@ -235,6 +242,8 @@ def test_sampler_logits_processors(seed: int):
242
for idx, nth_output in enumerate(sequence_output.samples):
243
assert nth_output.output_token == idx
244
245
+ del model_runner
246
+
247
248
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
249
def test_sampler_top_k_top_p(seed: int):
@@ -296,3 +305,5 @@ def test_sampler_top_k_top_p(seed: int):
305
hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
306
assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
307
assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
308
+
309
+ del model_runner