A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core]fix type annotation for `swap_blocks` (#4726)
Kunshang Ji committed
May 10, 2024 at 20:52 UTC
64b77dfd7e1378853ec7b189f3d7d0e51ce18855
1 file changed
+2
-2
vllm/_custom_ops.py
+2
-2
@@ -1,4 +1,4 @@
1
-from typing import Dict, Optional, Tuple
1
+from typing import Optional, Tuple
2
3
import torch
4
@@ -266,7 +266,7 @@ def copy_blocks(key_caches: torch.Tensor, value_caches: torch.Tensor,
266
267
268
def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
269
- block_mapping: Dict[int, int]) -> None:
269
+ block_mapping: torch.Tensor) -> None:
270
vllm_cache_ops.swap_blocks(src, dst, block_mapping)
271
272