@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf0d97d786
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

Update requirements-tpu.txt to support python 3.9 and 3.11 (#11695)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Jan 3, 2025 at 17:36 UTC bf0d97d78619b290ed273199ad3800b57b638603
2 files changed +7 -3
requirements-tpu.txt
+3 -1
@@ -18,6 +18,8 @@ ray[default]
18 --find-links https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
19 torch==2.6.0.dev20241126+cpu
20 torchvision==0.20.0.dev20241126+cpu
21 -torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241126-cp310-cp310-linux_x86_64.whl
21 +torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241126-cp39-cp39-linux_x86_64.whl ; python_version == "3.9"
22 +torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241126-cp310-cp310-linux_x86_64.whl ; python_version == "3.10"
23 +torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241126-cp311-cp311-linux_x86_64.whl ; python_version == "3.11"
24 jaxlib==0.4.36.dev20241122
25 jax==0.4.36.dev20241122
vllm/worker/tpu_model_runner.py
+4 -2
@@ -126,8 +126,10 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
126 logger.warning(
127 "The max_model_len (%d) is too large. This may degrade the "
128 "performance due to the insufficient smem size. Consider "
129 - "setting --max-model-len to a smaller value.",
130 - self.model_config.max_model_len)
129 + "setting --max-model-len to a smaller value, like %d.",
130 + self.model_config.max_model_len,
131 + self.model_config.max_model_len /
132 + (block_table_size / smem_size))
133
134 def load_model(self) -> None:
135 self.device = self.device_config.device