@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd2f63fb36
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[CI/CD] add neuron docker and ci test scripts (#3571)

Liangfu Chen committed Apr 18, 2024 at 15:26 UTC cd2f63fb362b2c53b993e7edf6565ab6a5f9f260
6 files changed +103 -4
.buildkite/run-neuron-test.sh new
+37
@@ -0,0 +1,37 @@
1 +# This script build the Neuron docker image and run the API server inside the container.
2 +# It serves a sanity check for compilation and basic model usage.
3 +set -e
4 +
5 +# Try building the docker image
6 +aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com
7 +docker build -t neuron -f Dockerfile.neuron .
8 +
9 +# Setup cleanup
10 +remove_docker_container() { docker rm -f neuron || true; }
11 +trap remove_docker_container EXIT
12 +remove_docker_container
13 +
14 +# Run the image
15 +docker run --device=/dev/neuron0 --device=/dev/neuron1 --network host --name neuron neuron python3 -m vllm.entrypoints.api_server \
16 + --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --max-num-seqs 8 --max-model-len 128 --block-size 128 --device neuron --tensor-parallel-size 2 &
17 +
18 +# Wait for the server to start
19 +wait_for_server_to_start() {
20 + timeout=300
21 + counter=0
22 +
23 + while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/health)" != "200" ]; do
24 + sleep 1
25 + counter=$((counter + 1))
26 + if [ $counter -ge $timeout ]; then
27 + echo "Timeout after $timeout seconds"
28 + break
29 + fi
30 + done
31 +}
32 +wait_for_server_to_start
33 +
34 +# Test a simple prompt
35 +curl -X POST -H "Content-Type: application/json" \
36 + localhost:8000/generate \
37 + -d '{"prompt": "San Francisco is a"}'
.buildkite/test-template.j2
+5
@@ -21,6 +21,11 @@ steps:
21 queue: amd
22 command: bash .buildkite/run-amd-test.sh
23
24 + - label: "Neuron Test"
25 + agents:
26 + queue: neuron
27 + command: bash .buildkite/run-neuron-test.sh
28 +
29 - label: "CPU Test"
30 command: bash .buildkite/run-cpu-test.sh
31
Dockerfile.neuron new
+36
@@ -0,0 +1,36 @@
1 +# default base image
2 +ARG BASE_IMAGE="763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference-neuronx:2.1.1-neuronx-py310-sdk2.17.0-ubuntu20.04"
3 +
4 +FROM $BASE_IMAGE
5 +
6 +RUN echo "Base image is $BASE_IMAGE"
7 +
8 +# Install some basic utilities
9 +RUN apt-get update && apt-get install python3 python3-pip -y
10 +
11 +### Mount Point ###
12 +# When launching the container, mount the code directory to /app
13 +ARG APP_MOUNT=/app
14 +VOLUME [ ${APP_MOUNT} ]
15 +WORKDIR ${APP_MOUNT}
16 +
17 +RUN python3 -m pip install --upgrade pip
18 +RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
19 +RUN python3 -m pip install sentencepiece transformers==4.36.2 -U
20 +RUN python3 -m pip install transformers-neuronx --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
21 +RUN python3 -m pip install --pre neuronx-cc==2.12.* --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
22 +
23 +COPY ./vllm /app/vllm/vllm
24 +COPY ./setup.py /app/vllm/setup.py
25 +COPY ./requirements-common.txt /app/vllm/requirements-common.txt
26 +COPY ./requirements-neuron.txt /app/vllm/requirements-neuron.txt
27 +
28 +RUN cd /app/vllm \
29 + && python3 -m pip install -U -r requirements-neuron.txt
30 +
31 +ENV VLLM_BUILD_WITH_NEURON 1
32 +RUN cd /app/vllm \
33 + && pip install -e . \
34 + && cd ..
35 +
36 +CMD ["/bin/bash"]
setup.py
+2 -1
@@ -204,7 +204,8 @@ def _is_neuron() -> bool:
204 subprocess.run(["neuron-ls"], capture_output=True, check=True)
205 except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
206 torch_neuronx_installed = False
207 - return torch_neuronx_installed
207 + return torch_neuronx_installed or os.environ.get("VLLM_BUILD_WITH_NEURON",
208 + False)
209
210
211 def _is_cpu() -> bool:
vllm/engine/async_llm_engine.py
+2 -2
@@ -335,8 +335,8 @@ class AsyncLLMEngine:
335 engine_config = engine_args.create_engine_config()
336
337 if engine_config.device_config.device_type == "neuron":
338 - raise NotImplementedError("Neuron is not supported for "
339 - "async engine yet.")
338 + from vllm.executor.neuron_executor import NeuronExecutorAsync
339 + executor_class = NeuronExecutorAsync
340 elif engine_config.parallel_config.worker_use_ray:
341 initialize_ray_cluster(engine_config.parallel_config)
342 from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
vllm/executor/neuron_executor.py
+21 -1
@@ -1,9 +1,10 @@
1 from typing import Dict, List, Set, Tuple
2
3 -from vllm.executor.executor_base import ExecutorBase
3 +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
6 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 +from vllm.utils import make_async
8
9 logger = init_logger(__name__)
10
@@ -73,3 +74,22 @@ class NeuronExecutor(ExecutorBase):
74 # NeuronExecutor will always be healthy as long as
75 # it's running.
76 return
77 +
78 +
79 +class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
80 +
81 + async def execute_model_async(
82 + self,
83 + seq_group_metadata_list: List[SequenceGroupMetadata],
84 + blocks_to_swap_in: Dict[int, int],
85 + blocks_to_swap_out: Dict[int, int],
86 + blocks_to_copy: Dict[int, List[int]],
87 + ) -> SamplerOutput:
88 + output = await make_async(self.driver_worker.execute_model)(
89 + seq_group_metadata_list=seq_group_metadata_list, )
90 + return output
91 +
92 + async def check_health_async(self) -> None:
93 + # NeuronExecutor will always be healthy as long as
94 + # it's running.
95 + return