@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6e01e8c1c8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[CI] Add Buildkite (#2355)

Simon Mo committed Jan 14, 2024 at 12:37 UTC 6e01e8c1c8ea323d30e3f57050469b2df66b56c6
13 files changed +192 -37
.buildkite/run-benchmarks.sh new
+24
@@ -0,0 +1,24 @@
1 +# This script is run by buildkite to run the benchmarks and upload the results to buildkite
2 +
3 +set -ex
4 +
5 +# cd into parent directory of this file
6 +cd "$(dirname "${BASH_SOURCE[0]}")/.."
7 +
8 +# run benchmarks and upload the result to buildkite
9 +python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
10 +
11 +python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
12 +
13 +# write the results into a markdown file
14 +echo "### Latency Benchmarks" >> benchmark_results.md
15 +sed -n '1p' benchmark_latency.txt >> benchmark_results.md
16 +echo "" >> benchmark_results.md
17 +sed -n '$p' benchmark_latency.txt >> benchmark_results.md
18 +echo "### Throughput Benchmarks" >> benchmark_results.md
19 +sed -n '1p' benchmark_throughput.txt >> benchmark_results.md
20 +echo "" >> benchmark_results.md
21 +sed -n '$p' benchmark_throughput.txt >> benchmark_results.md
22 +
23 +# upload the results to buildkite
24 +/workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
.buildkite/test-pipeline.yaml new
+41
@@ -0,0 +1,41 @@
1 +# In this file, you can add more tests to run either by adding a new step or
2 +# adding a new command to an existing step. See different options here for examples.
3 +# This script will be feed into Jinja template in `test-template.j2` to generate
4 +# the final pipeline yaml file.
5 +
6 +steps:
7 +- label: Regression Test
8 + command: pytest -v -s test_regression.py
9 + working_dir: "/vllm-workspace/tests" # optional
10 +
11 +- label: AsyncEngine Test
12 + command: pytest -v -s async_engine
13 +
14 +- label: Distributed Test
15 + command: pytest -v -s test_comm_ops.py
16 + working_dir: "/vllm-workspace/tests/distributed"
17 + num_gpus: 2 # only support 1 or 2 for now.
18 +
19 +- label: Engine Test
20 + command: pytest -v -s engine
21 +
22 +- label: Kernels Test
23 + command: pytest -v -s kernels
24 + soft_fail: true
25 +
26 +- label: Models Test
27 + commands:
28 + - pytest -v -s models --forked
29 + soft_fail: true
30 +
31 +- label: Samplers Test
32 + command: pytest -v -s samplers --forked
33 +
34 +- label: Worker Test
35 + command: pytest -v -s worker
36 +
37 +- label: Benchmarks
38 + working_dir: "/vllm-workspace/.buildkite"
39 + commands:
40 + - pip install aiohttp
41 + - bash run-benchmarks.sh
.buildkite/test-template.j2 new
+46
@@ -0,0 +1,46 @@
1 +{% set docker_image = "us-central1-docker.pkg.dev/vllm-405802/vllm-ci-test-repo/vllm-test:$BUILDKITE_COMMIT" %}
2 +{% set default_num_gpu = 1 %}
3 +{% set default_working_dir = "/vllm-workspace/tests" %}
4 +
5 +steps:
6 + - label: ":docker: build image"
7 + commands:
8 + - "docker build --tag {{ docker_image }} --target test --progress plain ."
9 + - "docker push {{ docker_image }}"
10 + env:
11 + DOCKER_BUILDKIT: "1"
12 + - wait
13 +
14 + {% for step in steps %}
15 + - label: "{{ step.label }}"
16 + agents:
17 + queue: kubernetes
18 + soft_fail: {{ step.soft_fail or false }}
19 + plugins:
20 + - kubernetes:
21 + podSpec:
22 + volumes:
23 + - name: dshm
24 + emptyDir:
25 + medium: Memory
26 + containers:
27 + - image: "{{ docker_image }}"
28 + command: ["bash"]
29 + args:
30 + - "-c"
31 + - "'cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}'"
32 + resources:
33 + requests:
34 + nvidia.com/gpu: "{{ step.num_gpus or default_num_gpu }}"
35 + limits:
36 + nvidia.com/gpu: "{{ step.num_gpus or default_num_gpu }}"
37 + env:
38 + - name: HF_TOKEN
39 + valueFrom:
40 + secretKeyRef:
41 + name: hf-token-secret
42 + key: token
43 + volumeMounts:
44 + - mountPath: /dev/shm
45 + name: dshm
46 + {% endfor %}
Dockerfile
+24 -12
@@ -1,7 +1,11 @@
1 +# The vLLM Dockerfile is used to construct vLLM image that can be directly used
2 +# to run the OpenAI compatible server.
3 +
4 +#################### BASE BUILD IMAGE ####################
5 FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
6
7 RUN apt-get update -y \
4 - && apt-get install -y python3-pip
8 + && apt-get install -y python3-pip git
9
10 WORKDIR /workspace
11
@@ -14,8 +18,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
18 COPY requirements-dev.txt requirements-dev.txt
19 RUN --mount=type=cache,target=/root/.cache/pip \
20 pip install -r requirements-dev.txt
21 +#################### BASE BUILD IMAGE ####################
22 +
23
18 -# image to build pytorch extensions
24 +#################### EXTENSION BUILD IMAGE ####################
25 FROM dev AS build
26
27 # install build dependencies
@@ -30,6 +36,7 @@ COPY requirements.txt requirements.txt
36 COPY pyproject.toml pyproject.toml
37 COPY vllm/__init__.py vllm/__init__.py
38
39 +# cuda arch list used by torch
40 ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
41 ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
42 # max jobs used by Ninja to build extensions
@@ -40,18 +47,26 @@ ARG nvcc_threads=8
47 ENV NVCC_THREADS=$nvcc_threads
48
49 RUN python3 setup.py build_ext --inplace
50 +#################### EXTENSION Build IMAGE ####################
51
52 +
53 +#################### TEST IMAGE ####################
54 # image to run unit testing suite
55 FROM dev AS test
56
57 # copy pytorch extensions separately to avoid having to rebuild
58 # when python code changes
49 -COPY --from=build /workspace/vllm/*.so /workspace/vllm/
50 -COPY tests tests
51 -COPY vllm vllm
59 +WORKDIR /vllm-workspace
60 +# ADD is used to preserve directory structure
61 +ADD . /vllm-workspace/
62 +COPY --from=build /workspace/vllm/*.so /vllm-workspace/vllm/
63 +# ignore build dependencies installation because we are using pre-complied extensions
64 +RUN rm pyproject.toml
65 +RUN --mount=type=cache,target=/root/.cache/pip VLLM_USE_PRECOMPILED=1 pip install . --verbose
66 +#################### TEST IMAGE ####################
67
53 -ENTRYPOINT ["python3", "-m", "pytest", "tests"]
68
69 +#################### RUNTIME BASE IMAGE ####################
70 # use CUDA base as CUDA runtime dependencies are already installed via pip
71 FROM nvidia/cuda:12.1.0-base-ubuntu22.04 AS vllm-base
72
@@ -63,14 +78,10 @@ WORKDIR /workspace
78 COPY requirements.txt requirements.txt
79 RUN --mount=type=cache,target=/root/.cache/pip \
80 pip install -r requirements.txt
81 +#################### RUNTIME BASE IMAGE ####################
82
67 -FROM vllm-base AS vllm
68 -COPY --from=build /workspace/vllm/*.so /workspace/vllm/
69 -COPY vllm vllm
70 -
71 -EXPOSE 8000
72 -ENTRYPOINT ["python3", "-m", "vllm.entrypoints.api_server"]
83
84 +#################### OPENAI API SERVER ####################
85 # openai api server alternative
86 FROM vllm-base AS vllm-openai
87 # install additional dependencies for openai api server
@@ -81,3 +92,4 @@ COPY --from=build /workspace/vllm/*.so /workspace/vllm/
92 COPY vllm vllm
93
94 ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
95 +#################### OPENAI API SERVER ####################
requirements-dev.txt
+3 -1
@@ -13,4 +13,6 @@ types-setuptools
13 pytest
14 pytest-forked
15 pytest-asyncio
16 -
16 +httpx
17 +einops # required for MPT
18 +flash_attn # required for HuggingFace's llama implementation
setup.py
+6 -1
@@ -293,6 +293,11 @@ def get_requirements() -> List[str]:
293 return requirements
294
295
296 +package_data = {"vllm": ["py.typed"]}
297 +if os.environ.get("VLLM_USE_PRECOMPILED"):
298 + ext_modules = []
299 + package_data["vllm"].append("*.so")
300 +
301 setuptools.setup(
302 name="vllm",
303 version=get_vllm_version(),
@@ -321,5 +326,5 @@ setuptools.setup(
326 install_requires=get_requirements(),
327 ext_modules=ext_modules,
328 cmdclass={"build_ext": BuildExtension},
324 - package_data={"vllm": ["py.typed"]},
329 + package_data=package_data,
330 )
tests/async_engine/test_api_server.py
+10 -2
@@ -29,8 +29,13 @@ def api_server():
29 script_path = Path(__file__).parent.joinpath(
30 "api_server_async_engine.py").absolute()
31 uvicorn_process = subprocess.Popen([
32 - sys.executable, "-u",
33 - str(script_path), "--model", "facebook/opt-125m"
32 + sys.executable,
33 + "-u",
34 + str(script_path),
35 + "--model",
36 + "facebook/opt-125m",
37 + "--host",
38 + "127.0.0.1",
39 ])
40 yield
41 uvicorn_process.terminate()
@@ -81,6 +86,9 @@ def test_api_server(api_server):
86 pool.join()
87
88 # check cancellation stats
89 + # give it some times to update the stats
90 + time.sleep(1)
91 +
92 num_aborted_requests = requests.get(
93 "http://localhost:8000/stats").json()["num_aborted_requests"]
94 assert num_aborted_requests > 0
tests/async_engine/test_openai_server.py
+9 -6
@@ -1,19 +1,24 @@
1 from argparse import Namespace
2 from dataclasses import dataclass
3 +import os
4 +import pathlib
5
6 import pytest
7 from fastapi.testclient import TestClient
8
9 from vllm.entrypoints.openai.api_server import *
10
11 +chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
12 + __file__))).parent.parent / "examples/template_chatml.jinja"
13 +assert chatml_jinja_path.exists()
14 +
15 # Define models, templates, and their corresponding expected outputs
16 MODEL_TEMPLATE_GENERATON_OUTPUT = [
17 ("facebook/opt-125m", None, True,
18 "Hello</s>Hi there!</s>What is the capital of</s>"),
19 ("facebook/opt-125m", None, False,
20 "Hello</s>Hi there!</s>What is the capital of</s>"),
15 - ("facebook/opt-125m", "../../examples/template_chatml.jinja", True,
16 - """<|im_start|>user
21 + ("facebook/opt-125m", chatml_jinja_path, True, """<|im_start|>user
22 Hello<|im_end|>
23 <|im_start|>assistant
24 Hi there!<|im_end|>
@@ -21,8 +26,7 @@ Hi there!<|im_end|>
26 What is the capital of<|im_end|>
27 <|im_start|>assistant
28 """),
24 - ("facebook/opt-125m", "../../examples/template_chatml.jinja", False,
25 - """<|im_start|>user
29 + ("facebook/opt-125m", chatml_jinja_path, False, """<|im_start|>user
30 Hello<|im_end|>
31 <|im_start|>assistant
32 Hi there!<|im_end|>
@@ -54,8 +58,7 @@ class MockTokenizer:
58
59 def test_load_chat_template():
60 # Testing chatml template
57 - template = "../../examples/template_chatml.jinja"
58 - mock_args = Namespace(chat_template=template)
61 + mock_args = Namespace(chat_template=chatml_jinja_path)
62 tokenizer = MockTokenizer()
63
64 # Call the function with the mocked args
tests/distributed/test_comm_ops.py
+14 -12
@@ -2,10 +2,9 @@
2
3 Run `pytest tests/distributed/test_comm_ops.py --forked`.
4 """
5 -from multiprocessing import Process, set_start_method
6 -
5 import pytest
6 import torch
7 +import ray
8
9 from vllm.config import ParallelConfig
10 from vllm.utils import get_open_port
@@ -23,11 +22,11 @@ def init_test_distributed_environment(pipeline_parallel_size: int,
22 tensor_parallel_size,
23 worker_use_ray=True)
24 distributed_init_method = f"tcp://localhost:{distributed_init_port}"
26 - torch.cuda.set_device(rank)
25 _init_distributed_environment(parallel_config, rank,
26 distributed_init_method)
27
28
29 +@ray.remote(num_gpus=1, max_calls=1)
30 def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
31 distributed_init_port: str):
32 init_test_distributed_environment(1, tensor_parallel_size, rank,
@@ -43,6 +42,7 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
42 assert torch.allclose(t, expected)
43
44
45 +@ray.remote(num_gpus=1, max_calls=1)
46 def all_gather_test_worker(tensor_parallel_size: int, rank: int,
47 distributed_init_port: str):
48 init_test_distributed_environment(1, tensor_parallel_size, rank,
@@ -70,14 +70,16 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
70 @pytest.mark.parametrize("test_target",
71 [all_reduce_test_worker, all_gather_test_worker])
72 def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
73 - set_start_method("spawn", force=True)
73 + # Using ray helps debugging the error when it failed
74 + # as compared to multiprocessing.
75 + ray.init()
76 +
77 distributed_init_port = get_open_port()
75 - processes = []
78 + refs = []
79 for rank in range(tensor_parallel_size):
77 - p = Process(target=test_target,
78 - args=(tensor_parallel_size, rank, distributed_init_port))
79 - p.start()
80 - processes.append(p)
81 - for p in processes:
82 - p.join()
83 - assert all(p.exitcode == 0 for p in processes)
80 + refs.append(
81 + test_target.remote(tensor_parallel_size, rank,
82 + distributed_init_port))
83 + ray.get(refs)
84 +
85 + ray.shutdown()
tests/kernels/test_attention.py
+1 -1
@@ -13,7 +13,7 @@ FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
13 # This will change depending on the compute capability.
14 # - 512 as a buffer
15 MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
16 -NUM_BLOCKS = 40000 # Arbitrary values for testing
16 +NUM_BLOCKS = 12000 # Arbitrary values for testing
17 PARTITION_SIZE = 512
18
19 DTYPES = [torch.half, torch.bfloat16, torch.float]
tests/kernels/test_cache.py
+2 -2
@@ -6,12 +6,12 @@ import torch
6 from vllm._C import cache_ops
7
8 DTYPES = [torch.half, torch.bfloat16, torch.float]
9 -NUM_TOKENS = [83] # Arbitrary values for testing
9 +NUM_TOKENS = [42] # Arbitrary values for testing
10 NUM_LAYERS = [1] # Arbitrary values for testing
11 NUM_HEADS = [8] # Arbitrary values for testing
12 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13 BLOCK_SIZES = [8, 16, 32]
14 -NUM_BLOCKS = [1024, 36000] # Arbitrary values for testing
14 +NUM_BLOCKS = [1024, 3600] # Arbitrary values for testing
15 NUM_MAPPINGS = [256] # Arbitrary values for testing
16 SEEDS = [0]
17 DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
tests/samplers/test_logprobs.py
+1
@@ -30,6 +30,7 @@ def test_get_prompt_logprobs(
30 temperature=0.0)
31 vllm_results = vllm_model.model.generate(
32 example_prompts, sampling_params=vllm_sampling_params)
33 + del vllm_model
34
35 # Test whether logprobs are included in the results.
36 for result in vllm_results:
tests/samplers/test_sampler.py
+11
@@ -75,6 +75,8 @@ def test_sampler_all_greedy(seed: int):
75 for nth_output in sequence_output.samples:
76 assert nth_output.output_token == expected[i].item()
77
78 + del model_runner
79 +
80
81 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
82 def test_sampler_all_random(seed: int):
@@ -111,6 +113,8 @@ def test_sampler_all_random(seed: int):
113 for nth_output in sequence_output.samples:
114 assert nth_output.output_token == i
115
116 + del model_runner
117 +
118
119 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
120 def test_sampler_all_beam(seed: int):
@@ -144,6 +148,7 @@ def test_sampler_all_beam(seed: int):
148 # the outputs are expected - in other words, this just tests
149 # whether there are no exceptions in the sampler
150 # when handling an all-beam search case.
151 + del model_runner
152
153
154 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
@@ -198,6 +203,8 @@ def test_sampler_mixed(seed: int):
203 for nth_output in sequence_output.samples:
204 assert nth_output.output_token in expected_tokens
205
206 + del model_runner
207 +
208
209 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
210 def test_sampler_logits_processors(seed: int):
@@ -235,6 +242,8 @@ def test_sampler_logits_processors(seed: int):
242 for idx, nth_output in enumerate(sequence_output.samples):
243 assert nth_output.output_token == idx
244
245 + del model_runner
246 +
247
248 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
249 def test_sampler_top_k_top_p(seed: int):
@@ -296,3 +305,5 @@ def test_sampler_top_k_top_p(seed: int):
305 hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
306 assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
307 assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
308 +
309 + del model_runner