@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 42de2cefcb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Add a wrapper for torch.inference_mode (#6618)

Woosuk Kwon committed Jul 21, 2024 at 18:43 UTC 42de2cefcbfc1e640f3337b36d5dddfd3133ef0e
5 files changed +49 -4
vllm/platforms/__init__.py
+7 -2
@@ -2,7 +2,9 @@ from typing import Optional
2
3 import torch
4
5 -from .interface import Platform, PlatformEnum
5 +from vllm.utils import is_tpu
6 +
7 +from .interface import Platform, PlatformEnum, UnspecifiedPlatform
8
9 current_platform: Optional[Platform]
10
@@ -12,7 +14,10 @@ if torch.version.cuda is not None:
14 elif torch.version.hip is not None:
15 from .rocm import RocmPlatform
16 current_platform = RocmPlatform()
17 +elif is_tpu():
18 + from .tpu import TpuPlatform
19 + current_platform = TpuPlatform()
20 else:
16 - current_platform = None
21 + current_platform = UnspecifiedPlatform()
22
23 __all__ = ['Platform', 'PlatformEnum', 'current_platform']
vllm/platforms/interface.py
+21
@@ -1,10 +1,14 @@
1 import enum
2 from typing import Tuple
3
4 +import torch
5 +
6
7 class PlatformEnum(enum.Enum):
8 CUDA = enum.auto()
9 ROCM = enum.auto()
10 + TPU = enum.auto()
11 + UNSPECIFIED = enum.auto()
12
13
14 class Platform:
@@ -16,6 +20,23 @@ class Platform:
20 def is_rocm(self) -> bool:
21 return self._enum == PlatformEnum.ROCM
22
23 + def is_tpu(self) -> bool:
24 + return self._enum == PlatformEnum.TPU
25 +
26 @staticmethod
27 def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
28 raise NotImplementedError
29 +
30 + @staticmethod
31 + def inference_mode():
32 + """A device-specific wrapper of `torch.inference_mode`.
33 +
34 + This wrapper is recommended because some hardware backends such as TPU
35 + do not support `torch.inference_mode`. In such a case, they will fall
36 + back to `torch.no_grad` by overriding this method.
37 + """
38 + return torch.inference_mode(mode=True)
39 +
40 +
41 +class UnspecifiedPlatform(Platform):
42 + _enum = PlatformEnum.UNSPECIFIED
vllm/platforms/tpu.py new
+17
@@ -0,0 +1,17 @@
1 +from typing import Tuple
2 +
3 +import torch
4 +
5 +from .interface import Platform, PlatformEnum
6 +
7 +
8 +class TpuPlatform(Platform):
9 + _enum = PlatformEnum.TPU
10 +
11 + @staticmethod
12 + def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
13 + raise RuntimeError("TPU does not have device capability.")
14 +
15 + @staticmethod
16 + def inference_mode():
17 + return torch.no_grad()
vllm/worker/model_runner_base.py
+2 -1
@@ -5,6 +5,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
5
6 import torch
7
8 +from vllm.platforms import current_platform
9 from vllm.sequence import (IntermediateTensors, SamplerOutput,
10 SequenceGroupMetadata)
11
@@ -163,7 +164,7 @@ class ModelRunnerBase(ABC, Generic[T]):
164 """
165 raise NotImplementedError
166
166 - @torch.inference_mode()
167 + @current_platform.inference_mode()
168 def execute_model(
169 self,
170 model_input: T,
vllm/worker/worker_base.py
+2 -1
@@ -9,6 +9,7 @@ import torch
9 from vllm.distributed import broadcast_tensor_dict, get_pp_group
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 +from vllm.platforms import current_platform
13 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
14 SamplerOutput)
15 from vllm.utils import (enable_trace_function_call_for_thread,
@@ -53,7 +54,7 @@ class WorkerBase(ABC):
54 """
55 raise NotImplementedError
56
56 - @torch.inference_mode()
57 + @current_platform.inference_mode()
58 def start_worker_execution_loop(self) -> None:
59 """Execute model loop in parallel worker.
60