@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 85de093472
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Do not pin memory when in WSL (#312)

Zhuohan Li committed Jun 29, 2023 at 15:00 UTC 85de0934727dc2c7b740b1d4a90d1a2e3c2d0585
2 files changed +17 -2
vllm/utils.py
+5
@@ -1,4 +1,5 @@
1 import enum
2 +from platform import uname
3 import uuid
4
5 import psutil
@@ -36,3 +37,7 @@ def get_cpu_memory() -> int:
37
38 def random_uuid() -> str:
39 return str(uuid.uuid4().hex)
40 +
41 +def in_wsl() -> bool:
42 + # Reference: https://github.com/microsoft/WSL/issues/4071
43 + return "microsoft" in " ".join(uname()).lower()
vllm/worker/cache_engine.py
+12 -2
@@ -5,6 +5,10 @@ import torch
5
6 from vllm import cache_ops
7 from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8 +from vllm.logger import init_logger
9 +from vllm.utils import in_wsl
10 +
11 +logger = init_logger(__name__)
12
13 KVCache = Tuple[torch.Tensor, torch.Tensor]
14
@@ -85,16 +89,22 @@ class CacheEngine:
89 cpu_cache: List[KVCache] = []
90 key_block_shape = self.get_key_block_shape()
91 value_block_shape = self.get_value_block_shape()
92 + pin_memory = not in_wsl()
93 + if not pin_memory:
94 + # Pinning memory in WSL is not supported.
95 + # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
96 + logger.warn("Using 'pin_memory=False' as WSL is detected. "
97 + "This may slow down the performance.")
98 for _ in range(self.num_layers):
99 key_blocks = torch.empty(
100 size=(self.num_cpu_blocks, *key_block_shape),
101 dtype=self.dtype,
92 - pin_memory=True,
102 + pin_memory=pin_memory,
103 )
104 value_blocks = torch.empty(
105 size=(self.num_cpu_blocks, *value_block_shape),
106 dtype=self.dtype,
97 - pin_memory=True,
107 + pin_memory=pin_memory,
108 )
109 cpu_cache.append((key_blocks, value_blocks))
110 return cpu_cache