A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Do not pin memory when in WSL (#312)
Zhuohan Li committed
Jun 29, 2023 at 15:00 UTC
85de0934727dc2c7b740b1d4a90d1a2e3c2d0585
2 files changed
+17
-2
vllm/utils.py
+5
@@ -1,4 +1,5 @@
1
import enum
2
+from platform import uname
3
import uuid
4
5
import psutil
@@ -36,3 +37,7 @@ def get_cpu_memory() -> int:
37
38
def random_uuid() -> str:
39
return str(uuid.uuid4().hex)
40
+
41
+def in_wsl() -> bool:
42
+ # Reference: https://github.com/microsoft/WSL/issues/4071
43
+ return "microsoft" in " ".join(uname()).lower()
vllm/worker/cache_engine.py
+12
-2
@@ -5,6 +5,10 @@ import torch
5
6
from vllm import cache_ops
7
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8
+from vllm.logger import init_logger
9
+from vllm.utils import in_wsl
10
+
11
+logger = init_logger(__name__)
12
13
KVCache = Tuple[torch.Tensor, torch.Tensor]
14
@@ -85,16 +89,22 @@ class CacheEngine:
89
cpu_cache: List[KVCache] = []
90
key_block_shape = self.get_key_block_shape()
91
value_block_shape = self.get_value_block_shape()
92
+ pin_memory = not in_wsl()
93
+ if not pin_memory:
94
+ # Pinning memory in WSL is not supported.
95
+ # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
96
+ logger.warn("Using 'pin_memory=False' as WSL is detected. "
97
+ "This may slow down the performance.")
98
for _ in range(self.num_layers):
99
key_blocks = torch.empty(
100
size=(self.num_cpu_blocks, *key_block_shape),
101
dtype=self.dtype,
92
- pin_memory=True,
102
+ pin_memory=pin_memory,
103
)
104
value_blocks = torch.empty(
105
size=(self.num_cpu_blocks, *value_block_shape),
106
dtype=self.dtype,
97
- pin_memory=True,
107
+ pin_memory=pin_memory,
108
)
109
cpu_cache.append((key_blocks, value_blocks))
110
return cpu_cache