@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96b6a6d790
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] fix type hint for py 3.8 (#4036)

youkaichao committed Apr 12, 2024 at 12:35 UTC 96b6a6d790115d04bb87d410f3bdd5d7d85b43f1
1 file changed +2 -2
vllm/executor/executor_base.py
+2 -2
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Dict, List, Optional
2 +from typing import Dict, List, Optional, Tuple
3
4 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5 ParallelConfig, SchedulerConfig, SpeculativeConfig,
@@ -31,7 +31,7 @@ class ExecutorBase(ABC):
31 raise NotImplementedError
32
33 @abstractmethod
34 - def determine_num_available_blocks(self) -> tuple[int, int]:
34 + def determine_num_available_blocks(self) -> Tuple[int, int]:
35 """Determine the number of available blocks for the GPU KV cache and
36 swappable CPU KV cache.
37