@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b322fd1607
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add docstrings to some modules and classes (#100)

Woosuk Kwon committed May 14, 2023 at 22:32 UTC b322fd1607597930993dfd005343c4f04c6545d4
17 files changed +166 -31
cacheflow/block.py
+9 -2
@@ -1,11 +1,17 @@
1 +"""Token blocks."""
2 from typing import List
3
4 from cacheflow.utils import Device
5
5 -BLANK_TOKEN_ID = -1
6 +_BLANK_TOKEN_ID = -1
7
8
9 class LogicalTokenBlock:
10 + """A block that stores a contiguous chunk of tokens from left to right.
11 +
12 + Logical blocks are used to represent the states of the corresponding
13 + physical blocks in the KV cache.
14 + """
15
16 def __init__(
17 self,
@@ -15,7 +21,7 @@ class LogicalTokenBlock:
21 self.block_number = block_number
22 self.block_size = block_size
23
18 - self.token_ids = [BLANK_TOKEN_ID] * block_size
24 + self.token_ids = [_BLANK_TOKEN_ID] * block_size
25 self.num_tokens = 0
26
27 def is_empty(self) -> bool:
@@ -41,6 +47,7 @@ class LogicalTokenBlock:
47
48
49 class PhysicalTokenBlock:
50 + """Represents the state of a block in the KV cache."""
51
52 def __init__(
53 self,
cacheflow/core/block_manager.py
+20 -15
@@ -1,13 +1,18 @@
1 +"""A block manager that manages token blocks."""
2 from typing import Dict, List, Optional, Set, Tuple
3
4 from cacheflow.block import PhysicalTokenBlock
4 -from cacheflow.sequence import Sequence
5 -from cacheflow.sequence import SequenceGroup
6 -from cacheflow.sequence import SequenceStatus
5 +from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
6 from cacheflow.utils import Device
7
8
9 class BlockAllocator:
10 + """Manages free physical token blocks for a device.
11 +
12 + The allocator maintains a list of free blocks and allocates a block when
13 + requested. When a block is freed, its reference count is decremented. If
14 + the reference count becomes zero, the block is added back to the free list.
15 + """
16
17 def __init__(
18 self,
@@ -20,24 +25,22 @@ class BlockAllocator:
25 self.num_blocks = num_blocks
26
27 # Initialize the free blocks.
23 - # TODO(woosuk): Make this a priority queue.
24 - self.free_blocks = [
25 - PhysicalTokenBlock(device=device, block_number=i, block_size=block_size)
26 - for i in range(num_blocks)
27 - ]
28 + self.free_blocks: List[PhysicalTokenBlock] = []
29 + for i in range(num_blocks):
30 + block = PhysicalTokenBlock(
31 + device=device, block_number=i, block_size=block_size)
32 + self.free_blocks.append(block)
33
34 def allocate(self) -> PhysicalTokenBlock:
35 if not self.free_blocks:
31 - raise ValueError('Out of memory! '
32 - f'No more free blocks are available.')
36 + raise ValueError("Out of memory! No free blocks are available.")
37 block = self.free_blocks.pop()
38 block.ref_count = 1
39 return block
40
41 def free(self, block: PhysicalTokenBlock) -> None:
42 if block.ref_count == 0:
39 - raise ValueError('Double free! '
40 - f'The block {block} is already freed.')
43 + raise ValueError(f"Double free! {block} is already freed.")
44 block.ref_count -= 1
45 if block.ref_count == 0:
46 self.free_blocks.append(block)
@@ -51,6 +54,7 @@ BlockTable = List[PhysicalTokenBlock]
54
55
56 class BlockSpaceManager:
57 + """Manages the mapping between logical and physical token blocks."""
58
59 def __init__(
60 self,
@@ -66,9 +70,10 @@ class BlockSpaceManager:
70 assert watermark >= 0.0
71
72 self.watermark_blocks = int(watermark * num_gpu_blocks)
69 - self.gpu_allocator = BlockAllocator(Device.GPU, block_size, num_gpu_blocks)
70 - self.cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
71 -
73 + self.gpu_allocator = BlockAllocator(Device.GPU, block_size,
74 + num_gpu_blocks)
75 + self.cpu_allocator = BlockAllocator(Device.CPU, block_size,
76 + num_cpu_blocks)
77 # Mapping: seq_id -> BlockTable.
78 self.block_tables: Dict[int, BlockTable] = {}
79
cacheflow/core/server.py
+2 -2
@@ -1,12 +1,12 @@
1 import argparse
2 -from typing import List, Tuple, Optional
2 import random
3 +from typing import List, Optional, Tuple
4
5 -import torch
5 try:
6 import ray
7 except ImportError:
8 ray = None
9 +import torch
10
11 from cacheflow.core.scheduler import Scheduler
12 from cacheflow.frontend.simple_frontend import SimpleFrontend
cacheflow/model_executor/layers/activation.py
+5
@@ -1,3 +1,4 @@
1 +"""Custom activation functions."""
2 import torch
3 import torch.nn as nn
4
@@ -5,6 +6,10 @@ from cacheflow import activation_ops
6
7
8 class SiluAndMul(nn.Module):
9 + """An activation function for SwiGLU.
10 +
11 + The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[1] // 2.
12 + """
13
14 def __init__(self):
15 super().__init__()
cacheflow/model_executor/layers/attention.py
+28 -1
@@ -1,3 +1,4 @@
1 +"""Multi-head attention."""
2 from typing import Optional
3
4 import torch
@@ -11,6 +12,32 @@ from cacheflow.model_executor.input_metadata import InputMetadata
12
13
14 class GPTCacheFlowAttention(nn.Module):
15 + """GPT-style multi-head attention.
16 +
17 + This class takes flattened 1D query, key, and value tensors as input. The
18 + input 1D tensors can be split into three parts: the prompt tokens, the
19 + generation tokens, and the paddings.
20 +
21 + |<------------------------------------- num_valid_tokens ------------------------------------->|
22 + |<--------------- num_prompt_tokens -------------->|<------- num_generation_tokens (M) ------->|
23 + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|<--generation_0-->|...|<--generation_M-1-->|<--padding-->|
24 +
25 + The prompts might have different lengths, while the generation tokens always
26 + have length 1. The paddings are appended to make the input length a multiple
27 + of 8, which is desirable for Tensor Cores.
28 +
29 + The class does the following:
30 + 1. Perform multi_query_kv_attention for the prompts. This operation does
31 + not use the KV cache.
32 + 2. Wait for the cache operations (e.g., swap, copy) to finish. The cache
33 + operations are issued by the cache engine before executing the forward
34 + pass of the model, and they are executed asynchronously.
35 + 3. Reshape and store the input key and value tensors in the KV cache.
36 + 4. Perform single_query_cached_kv_attention for the generation tokens.
37 + This operation reads the previous key and value tensors from the KV
38 + cache.
39 + 5. Output a flattened 1D tensor.
40 + """
41
42 def __init__(self, scale: float) -> None:
43 super().__init__()
@@ -157,7 +184,7 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
184 torch_dtype = torch.get_default_dtype()
185 cache = cache.to(torch_dtype)
186 # Embedding size: [max_position, rotary_dim]
160 - self.register_buffer('cos_sin_cache', cache, persistent=False)
187 + self.register_buffer("cos_sin_cache", cache, persistent=False)
188
189 def forward(
190 self,
cacheflow/model_executor/layers/layernorm.py
+6
@@ -1,3 +1,4 @@
1 +"""Custom normalization layers."""
2 import torch
3 import torch.nn as nn
4
@@ -5,6 +6,11 @@ from cacheflow import layernorm_ops
6
7
8 class RMSNorm(nn.Module):
9 + """Root mean square normalization.
10 +
11 + Computes x -> w * x / sqrt(E[x^2] + eps) where w is the learned weight.
12 + Refer to https://arxiv.org/abs/1910.07467
13 + """
14
15 def __init__(
16 self,
cacheflow/model_executor/layers/sampler.py
+14
@@ -1,3 +1,4 @@
1 +"""A layer that samples the next tokens from the model's outputs."""
2 from typing import Dict, List, Tuple
3
4 import numpy as np
@@ -12,6 +13,19 @@ from cacheflow.sequence import SequenceOutputs
13
14
15 class Sampler(nn.Module):
16 + """Samples the next tokens from the model's outputs.
17 +
18 + This layer does the following:
19 + 1. Discard the hidden states that are not used for sampling (i.e., all
20 + tokens except the final one in each prompt).
21 + 2. Compute the logits for the next tokens.
22 + 3. Apply presence and frequency penalties.
23 + 4. Apply temperature scaling.
24 + 5. Apply top-p and top-k truncation.
25 + 6. Sample the next tokens.
26 + Here, each sequence group within the batch can have different sampling
27 + parameters (e.g., sampling method, temperature, top-p, top-k, etc.).
28 + """
29
30 def __init__(self, vocab_size: int) -> None:
31 super().__init__()
cacheflow/model_executor/model_loader.py
+2 -2
@@ -1,9 +1,9 @@
1 +"""Utilities for selecting and loading models."""
2 from typing import Optional
3
4 import torch
5 import torch.nn as nn
5 -from transformers import AutoConfig
6 -from transformers import PretrainedConfig
6 +from transformers import AutoConfig, PretrainedConfig
7
8 from cacheflow.model_executor.memory_analyzer import (
9 CacheFlowMemoryAnalyzer, GPT2MemoryAnalyzer, GPTNeoXMemoryAnalyzer,
cacheflow/model_executor/models/gpt2.py
+5 -1
@@ -15,7 +15,11 @@
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 -"""1D GPT-2 model compatible with HuggingFace weights."""
18 +"""Inference-only GPT-2 model compatible with HuggingFace weights.
19 +
20 +The input of the model is flattened to a 1D tensor of tokens. The model uses
21 +InputMetadata to extract the original 2D shape of the input.
22 +"""
23 from typing import Dict, List, Optional, Tuple
24
25 import torch
cacheflow/model_executor/models/gpt_neox.py
+6 -1
@@ -14,7 +14,11 @@
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 -"""1D GPT-NeoX model compatible with HuggingFace weights."""
17 +"""Inference-only GPT-NeoX model compatible with HuggingFace weights.
18 +
19 +The input of the model is flattened to a 1D tensor of tokens. The model uses
20 +InputMetadata to extract the original 2D shape of the input.
21 +"""
22 from typing import Dict, List, Optional, Tuple
23
24 import torch
@@ -79,6 +83,7 @@ class GPTNeoXAttention(nn.Module):
83
84
85 class GPTNeoXMLP(nn.Module):
86 +
87 def __init__(self, config: GPTNeoXConfig):
88 super().__init__()
89 self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
cacheflow/model_executor/models/llama.py
+5 -1
@@ -19,7 +19,11 @@
19 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 -"""1D LLaMA model compatible with HuggingFace weights."""
22 +"""Inference-only LLaMA model compatible with HuggingFace weights.
23 +
24 +The input of the model is flattened to a 1D tensor of tokens. The model uses
25 +InputMetadata to extract the original 2D shape of the input.
26 +"""
27 from typing import Dict, List, Optional, Tuple
28
29 import torch
cacheflow/model_executor/models/opt.py
+5 -1
@@ -14,7 +14,11 @@
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 -"""1D OPT model compatible with HuggingFace weights."""
17 +"""Inference-only OPT model compatible with HuggingFace weights.
18 +
19 +The input of the model is flattened to a 1D tensor of tokens. The model uses
20 +InputMetadata to extract the original 2D shape of the input.
21 +"""
22 from typing import Dict, List, Optional, Tuple
23
24 import torch
cacheflow/model_executor/utils.py
+6 -5
@@ -1,3 +1,4 @@
1 +"""Utils for model executor."""
2 import random
3 from typing import Union
4
@@ -9,11 +10,11 @@ from cacheflow.model_executor.parallel_utils.tensor_parallel import model_parall
10
11
12 _STR_DTYPE_TO_TORCH_DTYPE = {
12 - 'half': torch.half,
13 - 'float': torch.float,
14 - 'float16': torch.float16,
15 - 'float32': torch.float32,
16 - 'bfloat16': torch.bfloat16,
13 + "half": torch.half,
14 + "float": torch.float,
15 + "float16": torch.float16,
16 + "float32": torch.float32,
17 + "bfloat16": torch.bfloat16,
18 }
19
20
cacheflow/model_executor/weight_utils.py
+8
@@ -1,3 +1,4 @@
1 +"""Utilities for downloading and initializing model weights."""
2 import filelock
3 import glob
4 import json
@@ -106,5 +107,12 @@ def initialize_dummy_weights(
107 low: float = -1e-3,
108 high: float = 1e-3,
109 ) -> None:
110 + """Initialize model weights with random values.
111 +
112 + The model weights must be randomly initialized for accurate performance
113 + measurements. Additionally, the model weights should not cause NaNs in the
114 + forward pass. We empirically found that initializing the weights with
115 + values between -1e-3 and 1e-3 works well for most models.
116 + """
117 for param in model.state_dict().values():
118 param.data.uniform_(low, high)
cacheflow/sampling_params.py
+30
@@ -1,7 +1,37 @@
1 +"""Sampling parameters for text generation."""
2 from typing import Set
3
4
5 class SamplingParams:
6 + """Sampling parameters for text generation.
7 +
8 + Overall, we follow the sampling parameters from the OpenAI text completion
9 + API (https://platform.openai.com/docs/api-reference/completions/create).
10 + In addition, we support beam search, which is not supported by OpenAI.
11 +
12 + Args:
13 + n: Number of output sequences to generate from the given prompt. This is
14 + regarded as the beam width when using beam search.
15 + presence_penalty: Float that penalizes new tokens based on whether they
16 + appear in the generated text so far. Values > 0 encourage the model
17 + to use new tokens, while values < 0 encourage the model to repeat
18 + tokens.
19 + frequency_penalty: Float that penalizes new tokens based on their
20 + frequency in the generated text so far. Values > 0 encourage the
21 + model to use new tokens, while values < 0 encourage the model to
22 + repeat tokens.
23 + temperature: Float that controls the randomness of the sampling. Lower
24 + values make the model more deterministic, while higher values make
25 + the model more random. Zero means greedy sampling.
26 + top_p: Float that controls the cumulative probability of the top tokens
27 + to consider. Must be in (0, 1]. Set to 1 to consider all tokens.
28 + top_k: Integer that controls the number of top tokens to consider. Set
29 + to -1 to consider all tokens.
30 + use_beam_search: Whether to use beam search instead of sampling.
31 + stop_token_ids: Set of token IDs that indicate the end of a sequence.
32 + max_tokens: Maximum number of tokens to generate per output sequence.
33 + logprobs: Number of log probabilities to return per output token.
34 + """
35
36 def __init__(
37 self,
cacheflow/worker/cache_engine.py
+8
@@ -1,12 +1,20 @@
1 +"""CacheEngine class for managing the KV cache."""
2 from typing import Dict, List, Tuple
3
4 import torch
5 +
6 from cacheflow import cache_ops
7
8 KVCache = Tuple[torch.Tensor, torch.Tensor]
9
10
11 class CacheEngine:
12 + """Manages the KV cache.
13 +
14 + This class is responsible for initializing and managing the GPU and CPU KV
15 + caches. It also provides methods for performing KV cache operations, such
16 + as swapping and copying.
17 + """
18
19 def __init__(
20 self,
cacheflow/worker/worker.py
+7
@@ -1,3 +1,4 @@
1 +"""A GPU worker class."""
2 from typing import Dict, List, Optional, Tuple
3
4 import torch
@@ -14,6 +15,12 @@ from cacheflow.worker.cache_engine import CacheEngine
15
16
17 class Worker:
18 + """A worker class that executes (a partition of) the model on a GPU.
19 +
20 + Each worker is associated with a single GPU. The worker is responsible for
21 + maintaining the KV cache and executing the model on the GPU. In case of
22 + distributed inference, each worker is assigned a partition of the model.
23 + """
24
25 def __init__(
26 self,