A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] fix type hint for py 3.8 (#4036)
youkaichao committed
Apr 12, 2024 at 12:35 UTC
96b6a6d790115d04bb87d410f3bdd5d7d85b43f1
1 file changed
+2
-2
vllm/executor/executor_base.py
+2
-2
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Dict, List, Optional
2
+from typing import Dict, List, Optional, Tuple
3
4
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5
ParallelConfig, SchedulerConfig, SpeculativeConfig,
@@ -31,7 +31,7 @@ class ExecutorBase(ABC):
31
raise NotImplementedError
32
33
@abstractmethod
34
- def determine_num_available_blocks(self) -> tuple[int, int]:
34
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
35
"""Determine the number of available blocks for the GPU KV cache and
36
swappable CPU KV cache.
37