A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] More type hint fixes for py 3.8 (#4039)
Dylan Hawk committed
Apr 12, 2024 at 21:07 UTC
5c2e66e4871917c5d59cc4a8b89ef53e690e9bd9
4 files changed
+8
-8
vllm/executor/executor_base.py
+1
-1
@@ -39,7 +39,7 @@ class ExecutorBase(ABC):
39
ExecutorBase may require modification of the result, e.g. to ensure the
40
selected cache sizes are compatible with all workers.
41
42
- Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
42
+ Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
43
are blocks that are "active" on the device and can be appended to.
44
num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
45
appended to.
vllm/worker/cpu_worker.py
+2
-2
@@ -1,5 +1,5 @@
1
"""A CPU worker class."""
2
-from typing import Dict, List, Optional
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
import torch.distributed
@@ -157,7 +157,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
157
def load_model(self):
158
self.model_runner.load_model()
159
160
- def determine_num_available_blocks(self) -> tuple[int, int]:
160
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
161
"""Determine the number of blocks available for the KV cache.
162
163
This determines how many KV blocks can fit into the configured CPU
vllm/worker/neuron_worker.py
+2
-2
@@ -1,5 +1,5 @@
1
"""A Neuron worker class."""
2
-from typing import List, Optional
2
+from typing import List, Optional, Tuple
3
4
import torch
5
import torch.distributed
@@ -40,7 +40,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
40
def load_model(self):
41
self.model_runner.load_model()
42
43
- def determine_num_available_blocks(self) -> tuple[int, int]:
43
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
44
"""Determine the number of available KV blocks.
45
46
Swapping is not yet supported, so always return num_cpu_blocks=0.
vllm/worker/worker_base.py
+3
-3
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Dict, List
2
+from typing import Dict, List, Tuple
3
4
from vllm.lora.request import LoRARequest
5
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -18,14 +18,14 @@ class WorkerBase(ABC):
18
raise NotImplementedError
19
20
@abstractmethod
21
- def determine_num_available_blocks(self) -> tuple[int, int]:
21
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
22
"""Determine the number of available blocks for the GPU KV cache and
23
swappable CPU KV cache.
24
25
The implementation may run profiling or other heuristics to determine
26
the size of caches.
27
28
- Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
28
+ Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
29
are blocks that are "active" on the device and can be appended to.
30
num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
31
appended to.