A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Minor] More fix of test_cache.py CI test failure (#2750)
Lily Liu committed
Feb 6, 2024 at 11:38 UTC
fe6d09ae61f2281417e35f53a948b6fa898a4eba
2 files changed
+16
-11
tests/kernels/test_cache.py
+4
-5
@@ -181,16 +181,15 @@ def test_swap_blocks(
181
num_blocks: int,
182
dtype: torch.dtype,
183
seed: int,
184
- device: int,
184
+ device: str,
185
) -> None:
186
random.seed(seed)
187
torch.random.manual_seed(seed)
188
if torch.cuda.is_available():
189
torch.cuda.manual_seed(seed)
190
- src_device = f"{direction[0]}:{device}" if direction[
191
- 0] == "cuda" else direction[0]
192
- dst_device = f"{direction[1]}:{device}" if direction[
193
- 1] == "cuda" else direction[1]
190
+
191
+ src_device = device if direction[0] == "cuda" else 'cpu'
192
+ dst_device = device if direction[1] == "cuda" else 'cpu'
193
194
src_blocks = random.sample(range(num_blocks), num_mappings)
195
# For the same device, mapping must not overlap
vllm/utils.py
+12
-6
@@ -258,10 +258,13 @@ def create_kv_caches_with_random(
258
key_cache = torch.empty(size=key_cache_shape,
259
dtype=torch_dtype,
260
device=device)
261
- if cache_dtype in ["auto", "half", "bfloat16", "float"]:
262
- key_cache.uniform_(-scale, scale)
263
- elif cache_dtype == 'fp8_e5m2':
261
+ if cache_dtype == 'fp8_e5m2':
262
_generate_random_fp8_e5m2(key_cache, -scale, scale)
263
+ elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
264
+ key_cache.uniform_(-scale, scale)
265
+ else:
266
+ raise ValueError(
267
+ f"Does not support key cache of type {cache_dtype}")
268
key_caches.append(key_cache)
269
270
value_cache_shape = (num_blocks, num_heads, head_size, block_size)
@@ -270,9 +273,12 @@ def create_kv_caches_with_random(
273
value_cache = torch.empty(size=value_cache_shape,
274
dtype=torch_dtype,
275
device=device)
273
- if cache_dtype in ["auto", "half", "bfloat16", "float"]:
274
- value_cache.uniform_(-scale, scale)
275
- elif cache_dtype == 'fp8_e5m2':
276
+ if cache_dtype == 'fp8_e5m2':
277
_generate_random_fp8_e5m2(value_cache, -scale, scale)
278
+ elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
279
+ value_cache.uniform_(-scale, scale)
280
+ else:
281
+ raise ValueError(
282
+ f"Does not support value cache of type {cache_dtype}")
283
value_caches.append(value_cache)
284
return key_caches, value_caches