@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 898cdf033e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[CI] Fix neuron CI and run offline tests (#11779)

Signed-off-by: Liangfu Chen <liangfc@amazon.com>

Liangfu Chen committed Jan 6, 2025 at 21:36 UTC 898cdf033e31dc28042f7181b1565c78d905196e
3 files changed +35 -37
.buildkite/run-neuron-test.sh
+27 -26
@@ -3,6 +3,18 @@
3 # This script build the Neuron docker image and run the API server inside the container.
4 # It serves a sanity check for compilation and basic model usage.
5 set -e
6 +set -v
7 +
8 +image_name="neuron/vllm-ci"
9 +container_name="neuron_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
10 +
11 +HF_CACHE="$(realpath ~)/huggingface"
12 +mkdir -p "${HF_CACHE}"
13 +HF_MOUNT="/root/.cache/huggingface"
14 +
15 +NEURON_COMPILE_CACHE_URL="$(realpath ~)/neuron_compile_cache"
16 +mkdir -p "${NEURON_COMPILE_CACHE_URL}"
17 +NEURON_COMPILE_CACHE_MOUNT="/root/.cache/neuron_compile_cache"
18
19 # Try building the docker image
20 aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com
@@ -13,41 +25,30 @@ if [ -f /tmp/neuron-docker-build-timestamp ]; then
25 last_build=$(cat /tmp/neuron-docker-build-timestamp)
26 current_time=$(date +%s)
27 if [ $((current_time - last_build)) -gt 86400 ]; then
28 + docker image prune -f
29 docker system prune -f
30 + rm -rf "${HF_MOUNT:?}/*"
31 + rm -rf "${NEURON_COMPILE_CACHE_MOUNT:?}/*"
32 echo "$current_time" > /tmp/neuron-docker-build-timestamp
33 fi
34 else
35 date "+%s" > /tmp/neuron-docker-build-timestamp
36 fi
37
23 -docker build -t neuron -f Dockerfile.neuron .
38 +docker build -t "${image_name}" -f Dockerfile.neuron .
39
40 # Setup cleanup
26 -remove_docker_container() { docker rm -f neuron || true; }
41 +remove_docker_container() {
42 + docker image rm -f "${image_name}" || true;
43 +}
44 trap remove_docker_container EXIT
28 -remove_docker_container
45
46 # Run the image
31 -docker run --device=/dev/neuron0 --device=/dev/neuron1 --network host --name neuron neuron python3 -m vllm.entrypoints.api_server \
32 - --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --max-num-seqs 8 --max-model-len 128 --block-size 128 --device neuron --tensor-parallel-size 2 &
33 -
34 -# Wait for the server to start
35 -wait_for_server_to_start() {
36 - timeout=300
37 - counter=0
38 -
39 - while [ "$(curl -s -o /dev/null -w '%{http_code}' localhost:8000/health)" != "200" ]; do
40 - sleep 1
41 - counter=$((counter + 1))
42 - if [ $counter -ge $timeout ]; then
43 - echo "Timeout after $timeout seconds"
44 - break
45 - fi
46 - done
47 -}
48 -wait_for_server_to_start
49 -
50 -# Test a simple prompt
51 -curl -X POST -H "Content-Type: application/json" \
52 - localhost:8000/generate \
53 - -d '{"prompt": "San Francisco is a"}'
47 +docker run --rm -it --device=/dev/neuron0 --device=/dev/neuron1 --network host \
48 + -v "${HF_CACHE}:${HF_MOUNT}" \
49 + -e "HF_HOME=${HF_MOUNT}" \
50 + -v "${NEURON_COMPILE_CACHE_URL}:${NEURON_COMPILE_CACHE_MOUNT}" \
51 + -e "NEURON_COMPILE_CACHE_URL=${NEURON_COMPILE_CACHE_MOUNT}" \
52 + --name "${container_name}" \
53 + ${image_name} \
54 + /bin/bash -c "python3 /workspace/vllm/examples/offline_inference_neuron.py"
Dockerfile.neuron
+6 -2
@@ -15,8 +15,8 @@ RUN apt-get update && \
15 ffmpeg libsm6 libxext6 libgl1
16
17 ### Mount Point ###
18 -# When launching the container, mount the code directory to /app
19 -ARG APP_MOUNT=/app
18 +# When launching the container, mount the code directory to /workspace
19 +ARG APP_MOUNT=/workspace
20 VOLUME [ ${APP_MOUNT} ]
21 WORKDIR ${APP_MOUNT}/vllm
22
@@ -25,6 +25,7 @@ RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
25 RUN python3 -m pip install sentencepiece transformers==4.45.2 -U
26 RUN python3 -m pip install transformers-neuronx --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
27 RUN python3 -m pip install neuronx-cc==2.16.345.0 --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
28 +RUN python3 -m pip install pytest
29
30 COPY . .
31 ARG GIT_REPO_CHECK=0
@@ -42,4 +43,7 @@ RUN --mount=type=bind,source=.git,target=.git \
43 # install development dependencies (for testing)
44 RUN python3 -m pip install -e tests/vllm_test_utils
45
46 +# overwrite entrypoint to run bash script
47 +RUN echo "import subprocess; import sys; subprocess.check_call(sys.argv[1:])" > /usr/local/bin/dockerd-entrypoint.py
48 +
49 CMD ["/bin/bash"]
examples/offline_inference_neuron.py
+2 -9
@@ -1,12 +1,5 @@
1 -import os
2 -
1 from vllm import LLM, SamplingParams
2
5 -# creates XLA hlo graphs for all the context length buckets.
6 -os.environ['NEURON_CONTEXT_LENGTH_BUCKETS'] = "128,512,1024,2048"
7 -# creates XLA hlo graphs for all the token gen buckets.
8 -os.environ['NEURON_TOKEN_GEN_BUCKETS'] = "128,512,1024,2048"
9 -
3 # Sample prompts.
4 prompts = [
5 "Hello, my name is",
@@ -26,8 +19,8 @@ llm = LLM(
19 # Currently, this is a known limitation in continuous batching support
20 # in transformers-neuronx.
21 # TODO(liangfu): Support paged-attention in transformers-neuronx.
29 - max_model_len=2048,
30 - block_size=2048,
22 + max_model_len=1024,
23 + block_size=1024,
24 # The device can be automatically detected when AWS Neuron SDK is installed.
25 # The device argument can be either unspecified for automated detection,
26 # or explicitly assigned.