A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc] Add a wrapper for torch.inference_mode (#6618)
Woosuk Kwon committed
Jul 21, 2024 at 18:43 UTC
42de2cefcbfc1e640f3337b36d5dddfd3133ef0e
5 files changed
+49
-4
vllm/platforms/__init__.py
+7
-2
@@ -2,7 +2,9 @@ from typing import Optional
2
3
import torch
4
5
-from .interface import Platform, PlatformEnum
5
+from vllm.utils import is_tpu
6
+
7
+from .interface import Platform, PlatformEnum, UnspecifiedPlatform
8
9
current_platform: Optional[Platform]
10
@@ -12,7 +14,10 @@ if torch.version.cuda is not None:
14
elif torch.version.hip is not None:
15
from .rocm import RocmPlatform
16
current_platform = RocmPlatform()
17
+elif is_tpu():
18
+ from .tpu import TpuPlatform
19
+ current_platform = TpuPlatform()
20
else:
16
- current_platform = None
21
+ current_platform = UnspecifiedPlatform()
22
23
__all__ = ['Platform', 'PlatformEnum', 'current_platform']
vllm/platforms/interface.py
+21
@@ -1,10 +1,14 @@
1
import enum
2
from typing import Tuple
3
4
+import torch
5
+
6
7
class PlatformEnum(enum.Enum):
8
CUDA = enum.auto()
9
ROCM = enum.auto()
10
+ TPU = enum.auto()
11
+ UNSPECIFIED = enum.auto()
12
13
14
class Platform:
@@ -16,6 +20,23 @@ class Platform:
20
def is_rocm(self) -> bool:
21
return self._enum == PlatformEnum.ROCM
22
23
+ def is_tpu(self) -> bool:
24
+ return self._enum == PlatformEnum.TPU
25
+
26
@staticmethod
27
def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
28
raise NotImplementedError
29
+
30
+ @staticmethod
31
+ def inference_mode():
32
+ """A device-specific wrapper of `torch.inference_mode`.
33
+
34
+ This wrapper is recommended because some hardware backends such as TPU
35
+ do not support `torch.inference_mode`. In such a case, they will fall
36
+ back to `torch.no_grad` by overriding this method.
37
+ """
38
+ return torch.inference_mode(mode=True)
39
+
40
+
41
+class UnspecifiedPlatform(Platform):
42
+ _enum = PlatformEnum.UNSPECIFIED
vllm/platforms/tpu.py
new
+17
@@ -0,0 +1,17 @@
1
+from typing import Tuple
2
+
3
+import torch
4
+
5
+from .interface import Platform, PlatformEnum
6
+
7
+
8
+class TpuPlatform(Platform):
9
+ _enum = PlatformEnum.TPU
10
+
11
+ @staticmethod
12
+ def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
13
+ raise RuntimeError("TPU does not have device capability.")
14
+
15
+ @staticmethod
16
+ def inference_mode():
17
+ return torch.no_grad()
vllm/worker/model_runner_base.py
+2
-1
@@ -5,6 +5,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
5
6
import torch
7
8
+from vllm.platforms import current_platform
9
from vllm.sequence import (IntermediateTensors, SamplerOutput,
10
SequenceGroupMetadata)
11
@@ -163,7 +164,7 @@ class ModelRunnerBase(ABC, Generic[T]):
164
"""
165
raise NotImplementedError
166
166
- @torch.inference_mode()
167
+ @current_platform.inference_mode()
168
def execute_model(
169
self,
170
model_input: T,
vllm/worker/worker_base.py
+2
-1
@@ -9,6 +9,7 @@ import torch
9
from vllm.distributed import broadcast_tensor_dict, get_pp_group
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
+from vllm.platforms import current_platform
13
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
14
SamplerOutput)
15
from vllm.utils import (enable_trace_function_call_for_thread,
@@ -53,7 +54,7 @@ class WorkerBase(ABC):
54
"""
55
raise NotImplementedError
56
56
- @torch.inference_mode()
57
+ @current_platform.inference_mode()
58
def start_worker_execution_loop(self) -> None:
59
"""Execute model loop in parallel worker.
60