A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add docstrings to some modules and classes (#100)
Woosuk Kwon committed
May 14, 2023 at 22:32 UTC
b322fd1607597930993dfd005343c4f04c6545d4
17 files changed
+166
-31
cacheflow/block.py
+9
-2
@@ -1,11 +1,17 @@
1
+"""Token blocks."""
2
from typing import List
3
4
from cacheflow.utils import Device
5
5
-BLANK_TOKEN_ID = -1
6
+_BLANK_TOKEN_ID = -1
7
8
9
class LogicalTokenBlock:
10
+ """A block that stores a contiguous chunk of tokens from left to right.
11
+
12
+ Logical blocks are used to represent the states of the corresponding
13
+ physical blocks in the KV cache.
14
+ """
15
16
def __init__(
17
self,
@@ -15,7 +21,7 @@ class LogicalTokenBlock:
21
self.block_number = block_number
22
self.block_size = block_size
23
18
- self.token_ids = [BLANK_TOKEN_ID] * block_size
24
+ self.token_ids = [_BLANK_TOKEN_ID] * block_size
25
self.num_tokens = 0
26
27
def is_empty(self) -> bool:
@@ -41,6 +47,7 @@ class LogicalTokenBlock:
47
48
49
class PhysicalTokenBlock:
50
+ """Represents the state of a block in the KV cache."""
51
52
def __init__(
53
self,
cacheflow/core/block_manager.py
+20
-15
@@ -1,13 +1,18 @@
1
+"""A block manager that manages token blocks."""
2
from typing import Dict, List, Optional, Set, Tuple
3
4
from cacheflow.block import PhysicalTokenBlock
4
-from cacheflow.sequence import Sequence
5
-from cacheflow.sequence import SequenceGroup
6
-from cacheflow.sequence import SequenceStatus
5
+from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
6
from cacheflow.utils import Device
7
8
9
class BlockAllocator:
10
+ """Manages free physical token blocks for a device.
11
+
12
+ The allocator maintains a list of free blocks and allocates a block when
13
+ requested. When a block is freed, its reference count is decremented. If
14
+ the reference count becomes zero, the block is added back to the free list.
15
+ """
16
17
def __init__(
18
self,
@@ -20,24 +25,22 @@ class BlockAllocator:
25
self.num_blocks = num_blocks
26
27
# Initialize the free blocks.
23
- # TODO(woosuk): Make this a priority queue.
24
- self.free_blocks = [
25
- PhysicalTokenBlock(device=device, block_number=i, block_size=block_size)
26
- for i in range(num_blocks)
27
- ]
28
+ self.free_blocks: List[PhysicalTokenBlock] = []
29
+ for i in range(num_blocks):
30
+ block = PhysicalTokenBlock(
31
+ device=device, block_number=i, block_size=block_size)
32
+ self.free_blocks.append(block)
33
34
def allocate(self) -> PhysicalTokenBlock:
35
if not self.free_blocks:
31
- raise ValueError('Out of memory! '
32
- f'No more free blocks are available.')
36
+ raise ValueError("Out of memory! No free blocks are available.")
37
block = self.free_blocks.pop()
38
block.ref_count = 1
39
return block
40
41
def free(self, block: PhysicalTokenBlock) -> None:
42
if block.ref_count == 0:
39
- raise ValueError('Double free! '
40
- f'The block {block} is already freed.')
43
+ raise ValueError(f"Double free! {block} is already freed.")
44
block.ref_count -= 1
45
if block.ref_count == 0:
46
self.free_blocks.append(block)
@@ -51,6 +54,7 @@ BlockTable = List[PhysicalTokenBlock]
54
55
56
class BlockSpaceManager:
57
+ """Manages the mapping between logical and physical token blocks."""
58
59
def __init__(
60
self,
@@ -66,9 +70,10 @@ class BlockSpaceManager:
70
assert watermark >= 0.0
71
72
self.watermark_blocks = int(watermark * num_gpu_blocks)
69
- self.gpu_allocator = BlockAllocator(Device.GPU, block_size, num_gpu_blocks)
70
- self.cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
71
-
73
+ self.gpu_allocator = BlockAllocator(Device.GPU, block_size,
74
+ num_gpu_blocks)
75
+ self.cpu_allocator = BlockAllocator(Device.CPU, block_size,
76
+ num_cpu_blocks)
77
# Mapping: seq_id -> BlockTable.
78
self.block_tables: Dict[int, BlockTable] = {}
79
cacheflow/core/server.py
+2
-2
@@ -1,12 +1,12 @@
1
import argparse
2
-from typing import List, Tuple, Optional
2
import random
3
+from typing import List, Optional, Tuple
4
5
-import torch
5
try:
6
import ray
7
except ImportError:
8
ray = None
9
+import torch
10
11
from cacheflow.core.scheduler import Scheduler
12
from cacheflow.frontend.simple_frontend import SimpleFrontend
cacheflow/model_executor/layers/activation.py
+5
@@ -1,3 +1,4 @@
1
+"""Custom activation functions."""
2
import torch
3
import torch.nn as nn
4
@@ -5,6 +6,10 @@ from cacheflow import activation_ops
6
7
8
class SiluAndMul(nn.Module):
9
+ """An activation function for SwiGLU.
10
+
11
+ The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[1] // 2.
12
+ """
13
14
def __init__(self):
15
super().__init__()
cacheflow/model_executor/layers/attention.py
+28
-1
@@ -1,3 +1,4 @@
1
+"""Multi-head attention."""
2
from typing import Optional
3
4
import torch
@@ -11,6 +12,32 @@ from cacheflow.model_executor.input_metadata import InputMetadata
12
13
14
class GPTCacheFlowAttention(nn.Module):
15
+ """GPT-style multi-head attention.
16
+
17
+ This class takes flattened 1D query, key, and value tensors as input. The
18
+ input 1D tensors can be split into three parts: the prompt tokens, the
19
+ generation tokens, and the paddings.
20
+
21
+ |<------------------------------------- num_valid_tokens ------------------------------------->|
22
+ |<--------------- num_prompt_tokens -------------->|<------- num_generation_tokens (M) ------->|
23
+ |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|<--generation_0-->|...|<--generation_M-1-->|<--padding-->|
24
+
25
+ The prompts might have different lengths, while the generation tokens always
26
+ have length 1. The paddings are appended to make the input length a multiple
27
+ of 8, which is desirable for Tensor Cores.
28
+
29
+ The class does the following:
30
+ 1. Perform multi_query_kv_attention for the prompts. This operation does
31
+ not use the KV cache.
32
+ 2. Wait for the cache operations (e.g., swap, copy) to finish. The cache
33
+ operations are issued by the cache engine before executing the forward
34
+ pass of the model, and they are executed asynchronously.
35
+ 3. Reshape and store the input key and value tensors in the KV cache.
36
+ 4. Perform single_query_cached_kv_attention for the generation tokens.
37
+ This operation reads the previous key and value tensors from the KV
38
+ cache.
39
+ 5. Output a flattened 1D tensor.
40
+ """
41
42
def __init__(self, scale: float) -> None:
43
super().__init__()
@@ -157,7 +184,7 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
184
torch_dtype = torch.get_default_dtype()
185
cache = cache.to(torch_dtype)
186
# Embedding size: [max_position, rotary_dim]
160
- self.register_buffer('cos_sin_cache', cache, persistent=False)
187
+ self.register_buffer("cos_sin_cache", cache, persistent=False)
188
189
def forward(
190
self,
cacheflow/model_executor/layers/layernorm.py
+6
@@ -1,3 +1,4 @@
1
+"""Custom normalization layers."""
2
import torch
3
import torch.nn as nn
4
@@ -5,6 +6,11 @@ from cacheflow import layernorm_ops
6
7
8
class RMSNorm(nn.Module):
9
+ """Root mean square normalization.
10
+
11
+ Computes x -> w * x / sqrt(E[x^2] + eps) where w is the learned weight.
12
+ Refer to https://arxiv.org/abs/1910.07467
13
+ """
14
15
def __init__(
16
self,
cacheflow/model_executor/layers/sampler.py
+14
@@ -1,3 +1,4 @@
1
+"""A layer that samples the next tokens from the model's outputs."""
2
from typing import Dict, List, Tuple
3
4
import numpy as np
@@ -12,6 +13,19 @@ from cacheflow.sequence import SequenceOutputs
13
14
15
class Sampler(nn.Module):
16
+ """Samples the next tokens from the model's outputs.
17
+
18
+ This layer does the following:
19
+ 1. Discard the hidden states that are not used for sampling (i.e., all
20
+ tokens except the final one in each prompt).
21
+ 2. Compute the logits for the next tokens.
22
+ 3. Apply presence and frequency penalties.
23
+ 4. Apply temperature scaling.
24
+ 5. Apply top-p and top-k truncation.
25
+ 6. Sample the next tokens.
26
+ Here, each sequence group within the batch can have different sampling
27
+ parameters (e.g., sampling method, temperature, top-p, top-k, etc.).
28
+ """
29
30
def __init__(self, vocab_size: int) -> None:
31
super().__init__()
cacheflow/model_executor/model_loader.py
+2
-2
@@ -1,9 +1,9 @@
1
+"""Utilities for selecting and loading models."""
2
from typing import Optional
3
4
import torch
5
import torch.nn as nn
5
-from transformers import AutoConfig
6
-from transformers import PretrainedConfig
6
+from transformers import AutoConfig, PretrainedConfig
7
8
from cacheflow.model_executor.memory_analyzer import (
9
CacheFlowMemoryAnalyzer, GPT2MemoryAnalyzer, GPTNeoXMemoryAnalyzer,
cacheflow/model_executor/models/gpt2.py
+5
-1
@@ -15,7 +15,11 @@
15
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
# See the License for the specific language governing permissions and
17
# limitations under the License.
18
-"""1D GPT-2 model compatible with HuggingFace weights."""
18
+"""Inference-only GPT-2 model compatible with HuggingFace weights.
19
+
20
+The input of the model is flattened to a 1D tensor of tokens. The model uses
21
+InputMetadata to extract the original 2D shape of the input.
22
+"""
23
from typing import Dict, List, Optional, Tuple
24
25
import torch
cacheflow/model_executor/models/gpt_neox.py
+6
-1
@@ -14,7 +14,11 @@
14
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
17
-"""1D GPT-NeoX model compatible with HuggingFace weights."""
17
+"""Inference-only GPT-NeoX model compatible with HuggingFace weights.
18
+
19
+The input of the model is flattened to a 1D tensor of tokens. The model uses
20
+InputMetadata to extract the original 2D shape of the input.
21
+"""
22
from typing import Dict, List, Optional, Tuple
23
24
import torch
@@ -79,6 +83,7 @@ class GPTNeoXAttention(nn.Module):
83
84
85
class GPTNeoXMLP(nn.Module):
86
+
87
def __init__(self, config: GPTNeoXConfig):
88
super().__init__()
89
self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
cacheflow/model_executor/models/llama.py
+5
-1
@@ -19,7 +19,11 @@
19
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
-"""1D LLaMA model compatible with HuggingFace weights."""
22
+"""Inference-only LLaMA model compatible with HuggingFace weights.
23
+
24
+The input of the model is flattened to a 1D tensor of tokens. The model uses
25
+InputMetadata to extract the original 2D shape of the input.
26
+"""
27
from typing import Dict, List, Optional, Tuple
28
29
import torch
cacheflow/model_executor/models/opt.py
+5
-1
@@ -14,7 +14,11 @@
14
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
17
-"""1D OPT model compatible with HuggingFace weights."""
17
+"""Inference-only OPT model compatible with HuggingFace weights.
18
+
19
+The input of the model is flattened to a 1D tensor of tokens. The model uses
20
+InputMetadata to extract the original 2D shape of the input.
21
+"""
22
from typing import Dict, List, Optional, Tuple
23
24
import torch
cacheflow/model_executor/utils.py
+6
-5
@@ -1,3 +1,4 @@
1
+"""Utils for model executor."""
2
import random
3
from typing import Union
4
@@ -9,11 +10,11 @@ from cacheflow.model_executor.parallel_utils.tensor_parallel import model_parall
10
11
12
_STR_DTYPE_TO_TORCH_DTYPE = {
12
- 'half': torch.half,
13
- 'float': torch.float,
14
- 'float16': torch.float16,
15
- 'float32': torch.float32,
16
- 'bfloat16': torch.bfloat16,
13
+ "half": torch.half,
14
+ "float": torch.float,
15
+ "float16": torch.float16,
16
+ "float32": torch.float32,
17
+ "bfloat16": torch.bfloat16,
18
}
19
20
cacheflow/model_executor/weight_utils.py
+8
@@ -1,3 +1,4 @@
1
+"""Utilities for downloading and initializing model weights."""
2
import filelock
3
import glob
4
import json
@@ -106,5 +107,12 @@ def initialize_dummy_weights(
107
low: float = -1e-3,
108
high: float = 1e-3,
109
) -> None:
110
+ """Initialize model weights with random values.
111
+
112
+ The model weights must be randomly initialized for accurate performance
113
+ measurements. Additionally, the model weights should not cause NaNs in the
114
+ forward pass. We empirically found that initializing the weights with
115
+ values between -1e-3 and 1e-3 works well for most models.
116
+ """
117
for param in model.state_dict().values():
118
param.data.uniform_(low, high)
cacheflow/sampling_params.py
+30
@@ -1,7 +1,37 @@
1
+"""Sampling parameters for text generation."""
2
from typing import Set
3
4
5
class SamplingParams:
6
+ """Sampling parameters for text generation.
7
+
8
+ Overall, we follow the sampling parameters from the OpenAI text completion
9
+ API (https://platform.openai.com/docs/api-reference/completions/create).
10
+ In addition, we support beam search, which is not supported by OpenAI.
11
+
12
+ Args:
13
+ n: Number of output sequences to generate from the given prompt. This is
14
+ regarded as the beam width when using beam search.
15
+ presence_penalty: Float that penalizes new tokens based on whether they
16
+ appear in the generated text so far. Values > 0 encourage the model
17
+ to use new tokens, while values < 0 encourage the model to repeat
18
+ tokens.
19
+ frequency_penalty: Float that penalizes new tokens based on their
20
+ frequency in the generated text so far. Values > 0 encourage the
21
+ model to use new tokens, while values < 0 encourage the model to
22
+ repeat tokens.
23
+ temperature: Float that controls the randomness of the sampling. Lower
24
+ values make the model more deterministic, while higher values make
25
+ the model more random. Zero means greedy sampling.
26
+ top_p: Float that controls the cumulative probability of the top tokens
27
+ to consider. Must be in (0, 1]. Set to 1 to consider all tokens.
28
+ top_k: Integer that controls the number of top tokens to consider. Set
29
+ to -1 to consider all tokens.
30
+ use_beam_search: Whether to use beam search instead of sampling.
31
+ stop_token_ids: Set of token IDs that indicate the end of a sequence.
32
+ max_tokens: Maximum number of tokens to generate per output sequence.
33
+ logprobs: Number of log probabilities to return per output token.
34
+ """
35
36
def __init__(
37
self,
cacheflow/worker/cache_engine.py
+8
@@ -1,12 +1,20 @@
1
+"""CacheEngine class for managing the KV cache."""
2
from typing import Dict, List, Tuple
3
4
import torch
5
+
6
from cacheflow import cache_ops
7
8
KVCache = Tuple[torch.Tensor, torch.Tensor]
9
10
11
class CacheEngine:
12
+ """Manages the KV cache.
13
+
14
+ This class is responsible for initializing and managing the GPU and CPU KV
15
+ caches. It also provides methods for performing KV cache operations, such
16
+ as swapping and copying.
17
+ """
18
19
def __init__(
20
self,
cacheflow/worker/worker.py
+7
@@ -1,3 +1,4 @@
1
+"""A GPU worker class."""
2
from typing import Dict, List, Optional, Tuple
3
4
import torch
@@ -14,6 +15,12 @@ from cacheflow.worker.cache_engine import CacheEngine
15
16
17
class Worker:
18
+ """A worker class that executes (a partition of) the model on a GPU.
19
+
20
+ Each worker is associated with a single GPU. The worker is responsible for
21
+ maintaining the KV cache and executing the model on the GPU. In case of
22
+ distributed inference, each worker is assigned a partition of the model.
23
+ """
24
25
def __init__(
26
self,