@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7f8d612d24
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[TPU] Support tensor parallelism in async llm engine (#6891)

Earthwalker committed Jul 30, 2024 at 03:42 UTC 7f8d612d24c66e9b5f8c0aa6cb562e853e9523a0
2 files changed +11 -2
Dockerfile.tpu
+3
@@ -12,6 +12,9 @@ RUN pip install "numpy<2"
12 RUN pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
13 RUN pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
14
15 +# Fix FastAPI dependence
16 +RUN pip install "starlette<0.38.0"
17 +
18 # Build vLLM.
19 COPY . /workspace/vllm
20 ENV VLLM_TARGET_DEVICE="tpu"
vllm/engine/async_llm_engine.py
+8 -2
@@ -407,8 +407,14 @@ class AsyncLLMEngine:
407 from vllm.executor.neuron_executor import NeuronExecutorAsync
408 executor_class = NeuronExecutorAsync
409 elif engine_config.device_config.device_type == "tpu":
410 - from vllm.executor.tpu_executor import TPUExecutorAsync
411 - executor_class = TPUExecutorAsync
410 + if distributed_executor_backend == "ray":
411 + initialize_ray_cluster(engine_config.parallel_config)
412 + from vllm.executor.ray_tpu_executor import RayTPUExecutorAsync
413 + executor_class = RayTPUExecutorAsync
414 + else:
415 + assert distributed_executor_backend is None
416 + from vllm.executor.tpu_executor import TPUExecutorAsync
417 + executor_class = TPUExecutorAsync
418 elif engine_config.device_config.device_type == "cpu":
419 from vllm.executor.cpu_executor import CPUExecutorAsync
420 executor_class = CPUExecutorAsync