@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5c2e66e487
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] More type hint fixes for py 3.8 (#4039)

Dylan Hawk committed Apr 12, 2024 at 21:07 UTC 5c2e66e4871917c5d59cc4a8b89ef53e690e9bd9
4 files changed +8 -8
vllm/executor/executor_base.py
+1 -1
@@ -39,7 +39,7 @@ class ExecutorBase(ABC):
39 ExecutorBase may require modification of the result, e.g. to ensure the
40 selected cache sizes are compatible with all workers.
41
42 - Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
42 + Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
43 are blocks that are "active" on the device and can be appended to.
44 num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
45 appended to.
vllm/worker/cpu_worker.py
+2 -2
@@ -1,5 +1,5 @@
1 """A CPU worker class."""
2 -from typing import Dict, List, Optional
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5 import torch.distributed
@@ -157,7 +157,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
157 def load_model(self):
158 self.model_runner.load_model()
159
160 - def determine_num_available_blocks(self) -> tuple[int, int]:
160 + def determine_num_available_blocks(self) -> Tuple[int, int]:
161 """Determine the number of blocks available for the KV cache.
162
163 This determines how many KV blocks can fit into the configured CPU
vllm/worker/neuron_worker.py
+2 -2
@@ -1,5 +1,5 @@
1 """A Neuron worker class."""
2 -from typing import List, Optional
2 +from typing import List, Optional, Tuple
3
4 import torch
5 import torch.distributed
@@ -40,7 +40,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
40 def load_model(self):
41 self.model_runner.load_model()
42
43 - def determine_num_available_blocks(self) -> tuple[int, int]:
43 + def determine_num_available_blocks(self) -> Tuple[int, int]:
44 """Determine the number of available KV blocks.
45
46 Swapping is not yet supported, so always return num_cpu_blocks=0.
vllm/worker/worker_base.py
+3 -3
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Dict, List
2 +from typing import Dict, List, Tuple
3
4 from vllm.lora.request import LoRARequest
5 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -18,14 +18,14 @@ class WorkerBase(ABC):
18 raise NotImplementedError
19
20 @abstractmethod
21 - def determine_num_available_blocks(self) -> tuple[int, int]:
21 + def determine_num_available_blocks(self) -> Tuple[int, int]:
22 """Determine the number of available blocks for the GPU KV cache and
23 swappable CPU KV cache.
24
25 The implementation may run profiling or other heuristics to determine
26 the size of caches.
27
28 - Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
28 + Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
29 are blocks that are "active" on the device and can be appended to.
30 num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
31 appended to.