@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fe6d09ae61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Minor] More fix of test_cache.py CI test failure (#2750)

Lily Liu committed Feb 6, 2024 at 11:38 UTC fe6d09ae61f2281417e35f53a948b6fa898a4eba
2 files changed +16 -11
tests/kernels/test_cache.py
+4 -5
@@ -181,16 +181,15 @@ def test_swap_blocks(
181 num_blocks: int,
182 dtype: torch.dtype,
183 seed: int,
184 - device: int,
184 + device: str,
185 ) -> None:
186 random.seed(seed)
187 torch.random.manual_seed(seed)
188 if torch.cuda.is_available():
189 torch.cuda.manual_seed(seed)
190 - src_device = f"{direction[0]}:{device}" if direction[
191 - 0] == "cuda" else direction[0]
192 - dst_device = f"{direction[1]}:{device}" if direction[
193 - 1] == "cuda" else direction[1]
190 +
191 + src_device = device if direction[0] == "cuda" else 'cpu'
192 + dst_device = device if direction[1] == "cuda" else 'cpu'
193
194 src_blocks = random.sample(range(num_blocks), num_mappings)
195 # For the same device, mapping must not overlap
vllm/utils.py
+12 -6
@@ -258,10 +258,13 @@ def create_kv_caches_with_random(
258 key_cache = torch.empty(size=key_cache_shape,
259 dtype=torch_dtype,
260 device=device)
261 - if cache_dtype in ["auto", "half", "bfloat16", "float"]:
262 - key_cache.uniform_(-scale, scale)
263 - elif cache_dtype == 'fp8_e5m2':
261 + if cache_dtype == 'fp8_e5m2':
262 _generate_random_fp8_e5m2(key_cache, -scale, scale)
263 + elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
264 + key_cache.uniform_(-scale, scale)
265 + else:
266 + raise ValueError(
267 + f"Does not support key cache of type {cache_dtype}")
268 key_caches.append(key_cache)
269
270 value_cache_shape = (num_blocks, num_heads, head_size, block_size)
@@ -270,9 +273,12 @@ def create_kv_caches_with_random(
273 value_cache = torch.empty(size=value_cache_shape,
274 dtype=torch_dtype,
275 device=device)
273 - if cache_dtype in ["auto", "half", "bfloat16", "float"]:
274 - value_cache.uniform_(-scale, scale)
275 - elif cache_dtype == 'fp8_e5m2':
276 + if cache_dtype == 'fp8_e5m2':
277 _generate_random_fp8_e5m2(value_cache, -scale, scale)
278 + elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
279 + value_cache.uniform_(-scale, scale)
280 + else:
281 + raise ValueError(
282 + f"Does not support value cache of type {cache_dtype}")
283 value_caches.append(value_cache)
284 return key_caches, value_caches