@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a283ec2eec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add contributing guideline and mypy config (#122)

Woosuk Kwon committed May 23, 2023 at 17:58 UTC a283ec2eece57454ec9301e5542cffa1201e175f
16 files changed +128 -44
CONTRIBUTING.md new
+74
@@ -0,0 +1,74 @@
1 +# Contributing to CacheFlow
2 +
3 +Thank you for your interest in contributing to CacheFlow!
4 +Our community is open to everyone and welcomes all kinds of contributions, no matter how small or large.
5 +There are several ways you can contribute to the project:
6 +
7 +- Identify and report any issues or bugs.
8 +- Request or add a new model.
9 +- Suggest or implement new features.
10 +
11 +However, remember that contributions aren't just about code.
12 +We believe in the power of community support; thus, answering queries, assisting others, and enhancing the documentation are highly regarded and beneficial contributions.
13 +
14 +Finally, one of the most impactful ways to support us is by raising awareness about CacheFlow.
15 +Talk about it in your blog posts, highlighting how it's driving your incredible projects.
16 +Express your support on Twitter if CacheFlow aids you, or simply offer your appreciation by starring our repository.
17 +
18 +
19 +## Setup for development
20 +
21 +### Build from source
22 +
23 +```bash
24 +pip install -r requirements.txt
25 +pip install -e . # This may take several minutes.
26 +```
27 +
28 +### Testing
29 +
30 +```bash
31 +pip install -r requirements-dev.txt
32 +
33 +# Static type checking
34 +mypy
35 +# Unit tests
36 +pytest tests/
37 +```
38 +**Note:** Currently, the repository does not pass the mypy tests.
39 +
40 +
41 +## Contributing Guidelines
42 +
43 +### Issue Reporting
44 +
45 +If you encounter a bug or have a feature request, please check our issues page first to see if someone else has already reported it.
46 +If not, please file a new issue, providing as much relevant information as possible.
47 +
48 +### Coding Style Guide
49 +
50 +In general, we adhere to [Google Python style guide](https://google.github.io/styleguide/pyguide.html) and [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
51 +
52 +### Pull Requests
53 +
54 +When submitting a pull request:
55 +
56 +1. Make sure your code has been rebased on top of the latest commit on the main branch.
57 +2. Include a detailed description of the changes in the pull request.
58 +Explain why you made the changes you did.
59 +If your pull request fixes an open issue, please include a reference to it in the description.
60 +
61 +### Code Reviews
62 +
63 +All submissions, including submissions by project members, require a code review.
64 +To make the review process as smooth as possible, please:
65 +
66 +1. Keep your changes as concise as possible.
67 +If your pull request involves multiple unrelated changes, consider splitting it into separate pull requests.
68 +2. Respond to all comments within a reasonable time frame.
69 +If a comment isn't clear or you disagree with a suggestion, feel free to ask for clarification or discuss the suggestion.
70 +
71 +### Thank You
72 +
73 +Finally, thank you for taking the time to read these guidelines and for your interest in contributing to CacheFlow.
74 +Your contributions make CacheFlow a great tool for everyone!
cacheflow/core/scheduler.py
+1 -1
@@ -87,7 +87,7 @@ class Scheduler:
87 def has_unfinished_seqs(self) -> bool:
88 return self.waiting or self.running or self.swapped
89
90 - def _schedule(self) -> Tuple[SchedulerOutputs, List[int]]:
90 + def _schedule(self) -> Tuple[SchedulerOutputs, List[str]]:
91 # Blocks that need to be swaped or copied before model execution.
92 blocks_to_swap_in: Dict[int, int] = {}
93 blocks_to_swap_out: Dict[int, int] = {}
cacheflow/model_executor/layers/attention.py
+2 -2
@@ -61,7 +61,7 @@ class GPTCacheFlowAttention(nn.Module):
61 key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
62 value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
63 attn_bias: xops.AttentionBias,
64 - ) -> None:
64 + ) -> torch.Tensor:
65 # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
66 out = xops.memory_efficient_attention_forward(
67 query.unsqueeze(0),
@@ -197,7 +197,7 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
197
198 def forward(
199 self,
200 - positions: torch.LongTensor, # [num_tokens]
200 + positions: torch.Tensor, # [num_tokens]
201 query: torch.Tensor, # [num_tokens, num_heads * head_size]
202 key: torch.Tensor, # [num_tokens, num_heads * head_size]
203 value: torch.Tensor, # [num_tokens, num_heads * head_size]
cacheflow/model_executor/layers/sampler.py
+1 -1
@@ -347,7 +347,7 @@ def _sample_from_generation_tokens(
347 # Greedy sampling.
348 assert len(seq_ids) == 1
349 next_token_id = torch.argmax(probs, dim=-1)
350 - next_token_ids = [next_token_id.item()]
350 + next_token_ids = [int(next_token_id.item())]
351 parent_seq_ids = seq_ids
352 else:
353 # Random sampling.
cacheflow/model_executor/model_loader.py
+3 -1
@@ -1,4 +1,6 @@
1 """Utilities for selecting and loading models."""
2 +from typing import Type
3 +
4 import torch
5 import torch.nn as nn
6 from transformers import PretrainedConfig
@@ -17,7 +19,7 @@ _MODEL_REGISTRY = {
19 }
20
21
20 -def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
22 +def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
23 architectures = getattr(config, "architectures", [])
24 for arch in architectures:
25 if arch in _MODEL_REGISTRY:
cacheflow/model_executor/models/gpt2.py
+4 -4
@@ -168,8 +168,8 @@ class GPT2Model(nn.Module):
168
169 def forward(
170 self,
171 - input_ids: torch.LongTensor,
172 - position_ids: torch.LongTensor,
171 + input_ids: torch.Tensor,
172 + position_ids: torch.Tensor,
173 kv_caches: List[KVCache],
174 input_metadata: InputMetadata,
175 cache_events: Optional[List[torch.cuda.Event]],
@@ -204,8 +204,8 @@ class GPT2LMHeadModel(nn.Module):
204
205 def forward(
206 self,
207 - input_ids: torch.LongTensor,
208 - positions: torch.LongTensor,
207 + input_ids: torch.Tensor,
208 + positions: torch.Tensor,
209 kv_caches: List[KVCache],
210 input_metadata: InputMetadata,
211 cache_events: Optional[List[torch.cuda.Event]],
cacheflow/model_executor/models/gpt_neox.py
+6 -6
@@ -67,7 +67,7 @@ class GPTNeoXAttention(nn.Module):
67
68 def forward(
69 self,
70 - position_ids: torch.LongTensor,
70 + position_ids: torch.Tensor,
71 hidden_states: torch.Tensor,
72 kv_cache: KVCache,
73 input_metadata: InputMetadata,
@@ -118,7 +118,7 @@ class GPTNeoXLayer(nn.Module):
118
119 def forward(
120 self,
121 - position_ids: torch.LongTensor,
121 + position_ids: torch.Tensor,
122 hidden_states: torch.Tensor,
123 kv_cache: KVCache,
124 input_metadata: InputMetadata,
@@ -162,8 +162,8 @@ class GPTNeoXModel(nn.Module):
162
163 def forward(
164 self,
165 - input_ids: torch.LongTensor,
166 - position_ids: torch.LongTensor,
165 + input_ids: torch.Tensor,
166 + position_ids: torch.Tensor,
167 kv_caches: List[KVCache],
168 input_metadata: InputMetadata,
169 cache_events: Optional[List[torch.cuda.Event]],
@@ -199,8 +199,8 @@ class GPTNeoXForCausalLM(nn.Module):
199
200 def forward(
201 self,
202 - input_ids: torch.LongTensor,
203 - positions: torch.LongTensor,
202 + input_ids: torch.Tensor,
203 + positions: torch.Tensor,
204 kv_caches: List[KVCache],
205 input_metadata: InputMetadata,
206 cache_events: Optional[List[torch.cuda.Event]],
cacheflow/model_executor/models/llama.py
+6 -6
@@ -109,7 +109,7 @@ class LlamaAttention(nn.Module):
109
110 def forward(
111 self,
112 - positions: torch.LongTensor,
112 + positions: torch.Tensor,
113 hidden_states: torch.Tensor,
114 kv_cache: KVCache,
115 input_metadata: InputMetadata,
@@ -143,7 +143,7 @@ class LlamaDecoderLayer(nn.Module):
143
144 def forward(
145 self,
146 - positions: torch.LongTensor,
146 + positions: torch.Tensor,
147 hidden_states: torch.Tensor,
148 kv_cache: KVCache,
149 input_metadata: InputMetadata,
@@ -184,8 +184,8 @@ class LlamaModel(nn.Module):
184
185 def forward(
186 self,
187 - input_ids: torch.LongTensor,
188 - positions: torch.LongTensor,
187 + input_ids: torch.Tensor,
188 + positions: torch.Tensor,
189 kv_caches: List[KVCache],
190 input_metadata: InputMetadata,
191 cache_events: Optional[List[torch.cuda.Event]],
@@ -222,8 +222,8 @@ class LlamaForCausalLM(nn.Module):
222
223 def forward(
224 self,
225 - input_ids: torch.LongTensor,
226 - positions: torch.LongTensor,
225 + input_ids: torch.Tensor,
226 + positions: torch.Tensor,
227 kv_caches: List[KVCache],
228 input_metadata: InputMetadata,
229 cache_events: Optional[List[torch.cuda.Event]],
cacheflow/model_executor/models/opt.py
+7 -7
@@ -47,7 +47,7 @@ class OPTLearnedPositionalEmbedding(nn.Embedding):
47 self.offset = 2
48 super().__init__(num_embeddings + self.offset, embedding_dim)
49
50 - def forward(self, positions: torch.LongTensor):
50 + def forward(self, positions: torch.Tensor):
51 return super().forward(positions + self.offset)
52
53
@@ -199,8 +199,8 @@ class OPTDecoder(nn.Module):
199
200 def forward(
201 self,
202 - input_ids: torch.LongTensor,
203 - positions: torch.LongTensor,
202 + input_ids: torch.Tensor,
203 + positions: torch.Tensor,
204 kv_caches: List[KVCache],
205 input_metadata: InputMetadata,
206 cache_events: Optional[List[torch.cuda.Event]],
@@ -235,8 +235,8 @@ class OPTModel(nn.Module):
235
236 def forward(
237 self,
238 - input_ids: torch.LongTensor,
239 - positions: torch.LongTensor,
238 + input_ids: torch.Tensor,
239 + positions: torch.Tensor,
240 kv_caches: List[KVCache],
241 input_metadata: InputMetadata,
242 cache_events: Optional[List[torch.cuda.Event]],
@@ -258,8 +258,8 @@ class OPTForCausalLM(nn.Module):
258
259 def forward(
260 self,
261 - input_ids: torch.LongTensor,
262 - positions: torch.LongTensor,
261 + input_ids: torch.Tensor,
262 + positions: torch.Tensor,
263 kv_caches: List[KVCache],
264 input_metadata: InputMetadata,
265 cache_events: Optional[List[torch.cuda.Event]],
cacheflow/outputs.py
+1 -1
@@ -31,7 +31,7 @@ class RequestOutput:
31
32 def __init__(
33 self,
34 - request_id: int,
34 + request_id: str,
35 prompt: str,
36 prompt_token_ids: List[int],
37 outputs: List[CompletionOutput],
cacheflow/sequence.py
+5 -2
@@ -116,10 +116,11 @@ class Sequence:
116 def get_cumulative_logprob(self) -> float:
117 return self.data.cumulative_logprob
118
119 - def fork(self, child_seq: 'Sequence') -> 'Sequence':
119 + def fork(self, child_seq: 'Sequence') -> None:
120 child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
121 child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
122 child_seq.data = copy.deepcopy(self.data)
123 + return None
124
125 def __repr__(self) -> str:
126 return (f'Sequence(seq_id={self.seq_id}, '
@@ -205,7 +206,9 @@ class SequenceOutputs:
206 f'output_token={self.output_token}), '
207 f'logprobs={self.logprobs}')
208
208 - def __eq__(self, other: 'SequenceOutputs') -> bool:
209 + def __eq__(self, other: object) -> bool:
210 + if not isinstance(other, SequenceOutputs):
211 + return NotImplemented
212 return (self.seq_id == other.seq_id and
213 self.parent_seq_id == other.parent_seq_id and
214 self.output_token == other.output_token and
cacheflow/server/ray_utils.py
+1 -1
@@ -8,7 +8,7 @@ except ImportError:
8
9 from cacheflow.config import ParallelConfig
10
11 -DeviceID = Tuple[int, str, int] # rank, node resource (node IP), device id
11 +DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), device id
12
13
14 def initialize_cluster(
cacheflow/worker/worker.py
+6 -11
@@ -132,7 +132,7 @@ class Worker:
132 def _prepare_inputs(
133 self,
134 seq_group_metadata_list: List[SequenceGroupMetadata],
135 - ) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
135 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
136 seq_groups: List[Tuple[List[int], SamplingParams]] = []
137 input_tokens: List[int] = []
138 input_positions: List[int] = []
@@ -216,19 +216,14 @@ class Worker:
216 input_positions = _pad_to_alignment(input_positions, multiple_of=8)
217
218 # Convert to tensors.
219 - tokens_tensor = torch.tensor(
220 - input_tokens, dtype=torch.long, device='cuda')
221 - positions_tensor = torch.tensor(
222 - input_positions, dtype=torch.long, device='cuda')
223 - slot_mapping_tensor = torch.tensor(
224 - slot_mapping, dtype=torch.int, device='cuda')
225 - context_lens_tensor = torch.tensor(
226 - context_lens, dtype=torch.int, device='cuda')
219 + tokens_tensor = torch.cuda.LongTensor(input_tokens)
220 + positions_tensor = torch.cuda.LongTensor(input_positions)
221 + slot_mapping_tensor = torch.cuda.IntTensor(slot_mapping)
222 + context_lens_tensor = torch.cuda.IntTensor(context_lens)
223 padded_block_tables = [
224 _pad_to_max(block_table, max_num_blocks_per_seq)
225 for block_table in generation_block_tables]
230 - block_tables_tensor = torch.tensor(
231 - padded_block_tables, dtype=torch.int, device='cuda')
226 + block_tables_tensor = torch.cuda.IntTensor(padded_block_tables)
227
228 seq_data: Dict[int, SequenceData] = {}
229 for seq_group_metadata in seq_group_metadata_list:
mypy.ini new
+8
@@ -0,0 +1,8 @@
1 +[mypy]
2 +python_version = 3.8
3 +
4 +ignore_missing_imports = True
5 +
6 +files = cacheflow
7 +# TODO(woosuk): Include the code from Megatron and HuggingFace.
8 +exclude = cacheflow/model_executor/parallel_utils/|cacheflow/model_executor/models/
requirements-dev.txt new
+2
@@ -0,0 +1,2 @@
1 +mypy
2 +pytest
tests/kernels/test_pos_encoding.py
+1 -1
@@ -49,7 +49,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
49
50 def forward(
51 self,
52 - positions: torch.LongTensor, # [num_tokens]
52 + positions: torch.Tensor, # [num_tokens]
53 query: torch.Tensor, # [num_tokens, num_heads, head_size]
54 key: torch.Tensor, # [num_tokens, num_heads, head_size]
55 ) -> Tuple[torch.Tensor, torch.Tensor]: