A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add contributing guideline and mypy config (#122)
Woosuk Kwon committed
May 23, 2023 at 17:58 UTC
a283ec2eece57454ec9301e5542cffa1201e175f
16 files changed
+128
-44
CONTRIBUTING.md
new
+74
@@ -0,0 +1,74 @@
1
+# Contributing to CacheFlow
2
+
3
+Thank you for your interest in contributing to CacheFlow!
4
+Our community is open to everyone and welcomes all kinds of contributions, no matter how small or large.
5
+There are several ways you can contribute to the project:
6
+
7
+- Identify and report any issues or bugs.
8
+- Request or add a new model.
9
+- Suggest or implement new features.
10
+
11
+However, remember that contributions aren't just about code.
12
+We believe in the power of community support; thus, answering queries, assisting others, and enhancing the documentation are highly regarded and beneficial contributions.
13
+
14
+Finally, one of the most impactful ways to support us is by raising awareness about CacheFlow.
15
+Talk about it in your blog posts, highlighting how it's driving your incredible projects.
16
+Express your support on Twitter if CacheFlow aids you, or simply offer your appreciation by starring our repository.
17
+
18
+
19
+## Setup for development
20
+
21
+### Build from source
22
+
23
+```bash
24
+pip install -r requirements.txt
25
+pip install -e . # This may take several minutes.
26
+```
27
+
28
+### Testing
29
+
30
+```bash
31
+pip install -r requirements-dev.txt
32
+
33
+# Static type checking
34
+mypy
35
+# Unit tests
36
+pytest tests/
37
+```
38
+**Note:** Currently, the repository does not pass the mypy tests.
39
+
40
+
41
+## Contributing Guidelines
42
+
43
+### Issue Reporting
44
+
45
+If you encounter a bug or have a feature request, please check our issues page first to see if someone else has already reported it.
46
+If not, please file a new issue, providing as much relevant information as possible.
47
+
48
+### Coding Style Guide
49
+
50
+In general, we adhere to [Google Python style guide](https://google.github.io/styleguide/pyguide.html) and [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
51
+
52
+### Pull Requests
53
+
54
+When submitting a pull request:
55
+
56
+1. Make sure your code has been rebased on top of the latest commit on the main branch.
57
+2. Include a detailed description of the changes in the pull request.
58
+Explain why you made the changes you did.
59
+If your pull request fixes an open issue, please include a reference to it in the description.
60
+
61
+### Code Reviews
62
+
63
+All submissions, including submissions by project members, require a code review.
64
+To make the review process as smooth as possible, please:
65
+
66
+1. Keep your changes as concise as possible.
67
+If your pull request involves multiple unrelated changes, consider splitting it into separate pull requests.
68
+2. Respond to all comments within a reasonable time frame.
69
+If a comment isn't clear or you disagree with a suggestion, feel free to ask for clarification or discuss the suggestion.
70
+
71
+### Thank You
72
+
73
+Finally, thank you for taking the time to read these guidelines and for your interest in contributing to CacheFlow.
74
+Your contributions make CacheFlow a great tool for everyone!
cacheflow/core/scheduler.py
+1
-1
@@ -87,7 +87,7 @@ class Scheduler:
87
def has_unfinished_seqs(self) -> bool:
88
return self.waiting or self.running or self.swapped
89
90
- def _schedule(self) -> Tuple[SchedulerOutputs, List[int]]:
90
+ def _schedule(self) -> Tuple[SchedulerOutputs, List[str]]:
91
# Blocks that need to be swaped or copied before model execution.
92
blocks_to_swap_in: Dict[int, int] = {}
93
blocks_to_swap_out: Dict[int, int] = {}
cacheflow/model_executor/layers/attention.py
+2
-2
@@ -61,7 +61,7 @@ class GPTCacheFlowAttention(nn.Module):
61
key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
62
value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
63
attn_bias: xops.AttentionBias,
64
- ) -> None:
64
+ ) -> torch.Tensor:
65
# TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
66
out = xops.memory_efficient_attention_forward(
67
query.unsqueeze(0),
@@ -197,7 +197,7 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
197
198
def forward(
199
self,
200
- positions: torch.LongTensor, # [num_tokens]
200
+ positions: torch.Tensor, # [num_tokens]
201
query: torch.Tensor, # [num_tokens, num_heads * head_size]
202
key: torch.Tensor, # [num_tokens, num_heads * head_size]
203
value: torch.Tensor, # [num_tokens, num_heads * head_size]
cacheflow/model_executor/layers/sampler.py
+1
-1
@@ -347,7 +347,7 @@ def _sample_from_generation_tokens(
347
# Greedy sampling.
348
assert len(seq_ids) == 1
349
next_token_id = torch.argmax(probs, dim=-1)
350
- next_token_ids = [next_token_id.item()]
350
+ next_token_ids = [int(next_token_id.item())]
351
parent_seq_ids = seq_ids
352
else:
353
# Random sampling.
cacheflow/model_executor/model_loader.py
+3
-1
@@ -1,4 +1,6 @@
1
"""Utilities for selecting and loading models."""
2
+from typing import Type
3
+
4
import torch
5
import torch.nn as nn
6
from transformers import PretrainedConfig
@@ -17,7 +19,7 @@ _MODEL_REGISTRY = {
19
}
20
21
20
-def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
22
+def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
23
architectures = getattr(config, "architectures", [])
24
for arch in architectures:
25
if arch in _MODEL_REGISTRY:
cacheflow/model_executor/models/gpt2.py
+4
-4
@@ -168,8 +168,8 @@ class GPT2Model(nn.Module):
168
169
def forward(
170
self,
171
- input_ids: torch.LongTensor,
172
- position_ids: torch.LongTensor,
171
+ input_ids: torch.Tensor,
172
+ position_ids: torch.Tensor,
173
kv_caches: List[KVCache],
174
input_metadata: InputMetadata,
175
cache_events: Optional[List[torch.cuda.Event]],
@@ -204,8 +204,8 @@ class GPT2LMHeadModel(nn.Module):
204
205
def forward(
206
self,
207
- input_ids: torch.LongTensor,
208
- positions: torch.LongTensor,
207
+ input_ids: torch.Tensor,
208
+ positions: torch.Tensor,
209
kv_caches: List[KVCache],
210
input_metadata: InputMetadata,
211
cache_events: Optional[List[torch.cuda.Event]],
cacheflow/model_executor/models/gpt_neox.py
+6
-6
@@ -67,7 +67,7 @@ class GPTNeoXAttention(nn.Module):
67
68
def forward(
69
self,
70
- position_ids: torch.LongTensor,
70
+ position_ids: torch.Tensor,
71
hidden_states: torch.Tensor,
72
kv_cache: KVCache,
73
input_metadata: InputMetadata,
@@ -118,7 +118,7 @@ class GPTNeoXLayer(nn.Module):
118
119
def forward(
120
self,
121
- position_ids: torch.LongTensor,
121
+ position_ids: torch.Tensor,
122
hidden_states: torch.Tensor,
123
kv_cache: KVCache,
124
input_metadata: InputMetadata,
@@ -162,8 +162,8 @@ class GPTNeoXModel(nn.Module):
162
163
def forward(
164
self,
165
- input_ids: torch.LongTensor,
166
- position_ids: torch.LongTensor,
165
+ input_ids: torch.Tensor,
166
+ position_ids: torch.Tensor,
167
kv_caches: List[KVCache],
168
input_metadata: InputMetadata,
169
cache_events: Optional[List[torch.cuda.Event]],
@@ -199,8 +199,8 @@ class GPTNeoXForCausalLM(nn.Module):
199
200
def forward(
201
self,
202
- input_ids: torch.LongTensor,
203
- positions: torch.LongTensor,
202
+ input_ids: torch.Tensor,
203
+ positions: torch.Tensor,
204
kv_caches: List[KVCache],
205
input_metadata: InputMetadata,
206
cache_events: Optional[List[torch.cuda.Event]],
cacheflow/model_executor/models/llama.py
+6
-6
@@ -109,7 +109,7 @@ class LlamaAttention(nn.Module):
109
110
def forward(
111
self,
112
- positions: torch.LongTensor,
112
+ positions: torch.Tensor,
113
hidden_states: torch.Tensor,
114
kv_cache: KVCache,
115
input_metadata: InputMetadata,
@@ -143,7 +143,7 @@ class LlamaDecoderLayer(nn.Module):
143
144
def forward(
145
self,
146
- positions: torch.LongTensor,
146
+ positions: torch.Tensor,
147
hidden_states: torch.Tensor,
148
kv_cache: KVCache,
149
input_metadata: InputMetadata,
@@ -184,8 +184,8 @@ class LlamaModel(nn.Module):
184
185
def forward(
186
self,
187
- input_ids: torch.LongTensor,
188
- positions: torch.LongTensor,
187
+ input_ids: torch.Tensor,
188
+ positions: torch.Tensor,
189
kv_caches: List[KVCache],
190
input_metadata: InputMetadata,
191
cache_events: Optional[List[torch.cuda.Event]],
@@ -222,8 +222,8 @@ class LlamaForCausalLM(nn.Module):
222
223
def forward(
224
self,
225
- input_ids: torch.LongTensor,
226
- positions: torch.LongTensor,
225
+ input_ids: torch.Tensor,
226
+ positions: torch.Tensor,
227
kv_caches: List[KVCache],
228
input_metadata: InputMetadata,
229
cache_events: Optional[List[torch.cuda.Event]],
cacheflow/model_executor/models/opt.py
+7
-7
@@ -47,7 +47,7 @@ class OPTLearnedPositionalEmbedding(nn.Embedding):
47
self.offset = 2
48
super().__init__(num_embeddings + self.offset, embedding_dim)
49
50
- def forward(self, positions: torch.LongTensor):
50
+ def forward(self, positions: torch.Tensor):
51
return super().forward(positions + self.offset)
52
53
@@ -199,8 +199,8 @@ class OPTDecoder(nn.Module):
199
200
def forward(
201
self,
202
- input_ids: torch.LongTensor,
203
- positions: torch.LongTensor,
202
+ input_ids: torch.Tensor,
203
+ positions: torch.Tensor,
204
kv_caches: List[KVCache],
205
input_metadata: InputMetadata,
206
cache_events: Optional[List[torch.cuda.Event]],
@@ -235,8 +235,8 @@ class OPTModel(nn.Module):
235
236
def forward(
237
self,
238
- input_ids: torch.LongTensor,
239
- positions: torch.LongTensor,
238
+ input_ids: torch.Tensor,
239
+ positions: torch.Tensor,
240
kv_caches: List[KVCache],
241
input_metadata: InputMetadata,
242
cache_events: Optional[List[torch.cuda.Event]],
@@ -258,8 +258,8 @@ class OPTForCausalLM(nn.Module):
258
259
def forward(
260
self,
261
- input_ids: torch.LongTensor,
262
- positions: torch.LongTensor,
261
+ input_ids: torch.Tensor,
262
+ positions: torch.Tensor,
263
kv_caches: List[KVCache],
264
input_metadata: InputMetadata,
265
cache_events: Optional[List[torch.cuda.Event]],
cacheflow/outputs.py
+1
-1
@@ -31,7 +31,7 @@ class RequestOutput:
31
32
def __init__(
33
self,
34
- request_id: int,
34
+ request_id: str,
35
prompt: str,
36
prompt_token_ids: List[int],
37
outputs: List[CompletionOutput],
cacheflow/sequence.py
+5
-2
@@ -116,10 +116,11 @@ class Sequence:
116
def get_cumulative_logprob(self) -> float:
117
return self.data.cumulative_logprob
118
119
- def fork(self, child_seq: 'Sequence') -> 'Sequence':
119
+ def fork(self, child_seq: 'Sequence') -> None:
120
child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
121
child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
122
child_seq.data = copy.deepcopy(self.data)
123
+ return None
124
125
def __repr__(self) -> str:
126
return (f'Sequence(seq_id={self.seq_id}, '
@@ -205,7 +206,9 @@ class SequenceOutputs:
206
f'output_token={self.output_token}), '
207
f'logprobs={self.logprobs}')
208
208
- def __eq__(self, other: 'SequenceOutputs') -> bool:
209
+ def __eq__(self, other: object) -> bool:
210
+ if not isinstance(other, SequenceOutputs):
211
+ return NotImplemented
212
return (self.seq_id == other.seq_id and
213
self.parent_seq_id == other.parent_seq_id and
214
self.output_token == other.output_token and
cacheflow/server/ray_utils.py
+1
-1
@@ -8,7 +8,7 @@ except ImportError:
8
9
from cacheflow.config import ParallelConfig
10
11
-DeviceID = Tuple[int, str, int] # rank, node resource (node IP), device id
11
+DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), device id
12
13
14
def initialize_cluster(
cacheflow/worker/worker.py
+6
-11
@@ -132,7 +132,7 @@ class Worker:
132
def _prepare_inputs(
133
self,
134
seq_group_metadata_list: List[SequenceGroupMetadata],
135
- ) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
135
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
136
seq_groups: List[Tuple[List[int], SamplingParams]] = []
137
input_tokens: List[int] = []
138
input_positions: List[int] = []
@@ -216,19 +216,14 @@ class Worker:
216
input_positions = _pad_to_alignment(input_positions, multiple_of=8)
217
218
# Convert to tensors.
219
- tokens_tensor = torch.tensor(
220
- input_tokens, dtype=torch.long, device='cuda')
221
- positions_tensor = torch.tensor(
222
- input_positions, dtype=torch.long, device='cuda')
223
- slot_mapping_tensor = torch.tensor(
224
- slot_mapping, dtype=torch.int, device='cuda')
225
- context_lens_tensor = torch.tensor(
226
- context_lens, dtype=torch.int, device='cuda')
219
+ tokens_tensor = torch.cuda.LongTensor(input_tokens)
220
+ positions_tensor = torch.cuda.LongTensor(input_positions)
221
+ slot_mapping_tensor = torch.cuda.IntTensor(slot_mapping)
222
+ context_lens_tensor = torch.cuda.IntTensor(context_lens)
223
padded_block_tables = [
224
_pad_to_max(block_table, max_num_blocks_per_seq)
225
for block_table in generation_block_tables]
230
- block_tables_tensor = torch.tensor(
231
- padded_block_tables, dtype=torch.int, device='cuda')
226
+ block_tables_tensor = torch.cuda.IntTensor(padded_block_tables)
227
228
seq_data: Dict[int, SequenceData] = {}
229
for seq_group_metadata in seq_group_metadata_list:
mypy.ini
new
+8
@@ -0,0 +1,8 @@
1
+[mypy]
2
+python_version = 3.8
3
+
4
+ignore_missing_imports = True
5
+
6
+files = cacheflow
7
+# TODO(woosuk): Include the code from Megatron and HuggingFace.
8
+exclude = cacheflow/model_executor/parallel_utils/|cacheflow/model_executor/models/
requirements-dev.txt
new
+2
@@ -0,0 +1,2 @@
1
+mypy
2
+pytest
tests/kernels/test_pos_encoding.py
+1
-1
@@ -49,7 +49,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
49
50
def forward(
51
self,
52
- positions: torch.LongTensor, # [num_tokens]
52
+ positions: torch.Tensor, # [num_tokens]
53
query: torch.Tensor, # [num_tokens, num_heads, head_size]
54
key: torch.Tensor, # [num_tokens, num_heads, head_size]
55
) -> Tuple[torch.Tensor, torch.Tensor]: