A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Minor
Woosuk Kwon committed
Mar 26, 2023 at 08:00 UTC
d359cda5fae1c9a6fe54ba12b940572edfbf87ac
3 files changed
+5
-4
cacheflow/master/block_manager.py
+3
-3
@@ -7,7 +7,7 @@ from cacheflow.sequence import SequenceStatus
7
from cacheflow.utils import Device
8
9
10
-class BlockManager:
10
+class BlockAllocator:
11
12
def __init__(
13
self,
@@ -65,8 +65,8 @@ class BlockSpaceManager:
65
self.num_total_gpu_blocks = num_gpu_blocks
66
self.num_total_cpu_blocks = num_cpu_blocks
67
68
- self.gpu_allocator = BlockManager(Device.GPU, block_size, num_gpu_blocks)
69
- self.cpu_allocator = BlockManager(Device.CPU, block_size, num_cpu_blocks)
68
+ self.gpu_allocator = BlockAllocator(Device.GPU, block_size, num_gpu_blocks)
69
+ self.cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
70
71
# Mapping: seq_id -> BlockTable.
72
self.block_tables: Dict[int, BlockTable] = {}
cacheflow/models/sample.py
+1
@@ -8,6 +8,7 @@ from cacheflow.sampling_params import SamplingParams
8
from cacheflow.sequence import SequenceOutputs
9
from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_parallel_region
10
11
+
12
class Sampler(nn.Module):
13
14
def __init__(self) -> None:
cacheflow/sequence.py
+1
-1
@@ -30,7 +30,7 @@ class Sequence:
30
31
self.status = SequenceStatus.PENDING
32
self.output_logprobs: List[Dict[int, float]] = []
33
- self.cumulative_logprobs = 1.0
33
+ self.cumulative_logprobs = 0.0
34
35
def add_block(self) -> None:
36
block = LogicalTokenBlock(