A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[CI/Build][TPU] Add TPU CI test (#6277)
Co-authored-by: kevin <kevin@anyscale.com>
Woosuk Kwon committed
Jul 15, 2024 at 14:31 UTC
4552e37b55e3b8d7ba54d05945ac06b680964db6
3 files changed
+44
-5
.buildkite/run-tpu-test.sh
new
+16
@@ -0,0 +1,16 @@
1
+set -e
2
+
3
+# Build the docker image.
4
+docker build -f Dockerfile.tpu -t vllm-tpu .
5
+
6
+# Set up cleanup.
7
+remove_docker_container() { docker rm -f tpu-test || true; }
8
+trap remove_docker_container EXIT
9
+# Remove the container that might not be cleaned up in the previous run.
10
+remove_docker_container
11
+
12
+# For HF_TOKEN.
13
+source /etc/environment
14
+# Run a simple end-to-end example.
15
+docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu \
16
+ python3 /workspace/vllm/examples/offline_inference_tpu.py
Dockerfile.tpu
-5
@@ -15,9 +15,4 @@ COPY . /workspace/vllm
15
ENV VLLM_TARGET_DEVICE="tpu"
16
RUN cd /workspace/vllm && python setup.py develop
17
18
-# Re-install outlines to avoid dependency errors.
19
-# The outlines version must follow requirements-common.txt.
20
-RUN pip uninstall outlines -y
21
-RUN pip install "outlines>=0.0.43"
22
-
18
CMD ["/bin/bash"]
examples/offline_inference_tpu.py
new
+28
@@ -0,0 +1,28 @@
1
+from vllm import LLM, SamplingParams
2
+
3
+prompts = [
4
+ "A robot may not injure a human being",
5
+ "It is only with the heart that one can see rightly;",
6
+ "The greatest glory in living lies not in never falling,",
7
+]
8
+answers = [
9
+ " or, through inaction, allow a human being to come to harm.",
10
+ " what is essential is invisible to the eye.",
11
+ " but in rising every time we fall.",
12
+]
13
+N = 1
14
+# Currently, top-p sampling is disabled. `top_p` should be 1.0.
15
+sampling_params = SamplingParams(temperature=0.7,
16
+ top_p=1.0,
17
+ n=N,
18
+ max_tokens=16)
19
+
20
+# Set `enforce_eager=True` to avoid ahead-of-time compilation.
21
+# In real workloads, `enforace_eager` should be `False`.
22
+llm = LLM(model="google/gemma-2b", enforce_eager=True)
23
+outputs = llm.generate(prompts, sampling_params)
24
+for output, answer in zip(outputs, answers):
25
+ prompt = output.prompt
26
+ generated_text = output.outputs[0].text
27
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
28
+ assert generated_text.startswith(answer)