A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[TPU] Support tensor parallelism in async llm engine (#6891)
Earthwalker committed
Jul 30, 2024 at 03:42 UTC
7f8d612d24c66e9b5f8c0aa6cb562e853e9523a0
2 files changed
+11
-2
Dockerfile.tpu
+3
@@ -12,6 +12,9 @@ RUN pip install "numpy<2"
12
RUN pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
13
RUN pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
14
15
+# Fix FastAPI dependence
16
+RUN pip install "starlette<0.38.0"
17
+
18
# Build vLLM.
19
COPY . /workspace/vllm
20
ENV VLLM_TARGET_DEVICE="tpu"
vllm/engine/async_llm_engine.py
+8
-2
@@ -407,8 +407,14 @@ class AsyncLLMEngine:
407
from vllm.executor.neuron_executor import NeuronExecutorAsync
408
executor_class = NeuronExecutorAsync
409
elif engine_config.device_config.device_type == "tpu":
410
- from vllm.executor.tpu_executor import TPUExecutorAsync
411
- executor_class = TPUExecutorAsync
410
+ if distributed_executor_backend == "ray":
411
+ initialize_ray_cluster(engine_config.parallel_config)
412
+ from vllm.executor.ray_tpu_executor import RayTPUExecutorAsync
413
+ executor_class = RayTPUExecutorAsync
414
+ else:
415
+ assert distributed_executor_backend is None
416
+ from vllm.executor.tpu_executor import TPUExecutorAsync
417
+ executor_class = TPUExecutorAsync
418
elif engine_config.device_config.device_type == "cpu":
419
from vllm.executor.cpu_executor import CPUExecutorAsync
420
executor_class = CPUExecutorAsync