@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d359cda5fa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Minor

Woosuk Kwon committed Mar 26, 2023 at 08:00 UTC d359cda5fae1c9a6fe54ba12b940572edfbf87ac
3 files changed +5 -4
cacheflow/master/block_manager.py
+3 -3
@@ -7,7 +7,7 @@ from cacheflow.sequence import SequenceStatus
7 from cacheflow.utils import Device
8
9
10 -class BlockManager:
10 +class BlockAllocator:
11
12 def __init__(
13 self,
@@ -65,8 +65,8 @@ class BlockSpaceManager:
65 self.num_total_gpu_blocks = num_gpu_blocks
66 self.num_total_cpu_blocks = num_cpu_blocks
67
68 - self.gpu_allocator = BlockManager(Device.GPU, block_size, num_gpu_blocks)
69 - self.cpu_allocator = BlockManager(Device.CPU, block_size, num_cpu_blocks)
68 + self.gpu_allocator = BlockAllocator(Device.GPU, block_size, num_gpu_blocks)
69 + self.cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
70
71 # Mapping: seq_id -> BlockTable.
72 self.block_tables: Dict[int, BlockTable] = {}
cacheflow/models/sample.py
+1
@@ -8,6 +8,7 @@ from cacheflow.sampling_params import SamplingParams
8 from cacheflow.sequence import SequenceOutputs
9 from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_parallel_region
10
11 +
12 class Sampler(nn.Module):
13
14 def __init__(self) -> None:
cacheflow/sequence.py
+1 -1
@@ -30,7 +30,7 @@ class Sequence:
30
31 self.status = SequenceStatus.PENDING
32 self.output_logprobs: List[Dict[int, float]] = []
33 - self.cumulative_logprobs = 1.0
33 + self.cumulative_logprobs = 0.0
34
35 def add_block(self) -> None:
36 block = LogicalTokenBlock(