@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 37ca558103
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Optimize model execution with CUDA graph (#1926)

Co-authored-by: Chen Shen <scv119@gmail.com> Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Woosuk Kwon committed Dec 16, 2023 at 21:12 UTC 37ca5581039271d4bf69e5cb1f1ec8e04775777c
34 files changed +557 -254
benchmarks/benchmark_latency.py
+4
@@ -23,6 +23,7 @@ def main(args: argparse.Namespace):
23 tensor_parallel_size=args.tensor_parallel_size,
24 trust_remote_code=args.trust_remote_code,
25 dtype=args.dtype,
26 + enforce_eager=args.enforce_eager,
27 )
28
29 sampling_params = SamplingParams(
@@ -111,6 +112,9 @@ if __name__ == '__main__':
112 'The "auto" option will use FP16 precision '
113 'for FP32 and FP16 models, and BF16 precision '
114 'for BF16 models.')
115 + parser.add_argument('--enforce-eager',
116 + action='store_true',
117 + help='enforce eager mode and disable CUDA graph')
118 parser.add_argument(
119 '--profile',
120 action='store_true',
benchmarks/benchmark_throughput.py
+7 -2
@@ -69,7 +69,8 @@ def run_vllm(
69 use_beam_search: bool,
70 trust_remote_code: bool,
71 dtype: str,
72 - max_model_len: Optional[int] = None,
72 + max_model_len: Optional[int],
73 + enforce_eager: bool,
74 ) -> float:
75 from vllm import LLM, SamplingParams
76 llm = LLM(
@@ -81,6 +82,7 @@ def run_vllm(
82 trust_remote_code=trust_remote_code,
83 dtype=dtype,
84 max_model_len=max_model_len,
85 + enforce_eager=enforce_eager,
86 )
87
88 # Add the requests to the engine.
@@ -204,7 +206,7 @@ def main(args: argparse.Namespace):
206 args.quantization, args.tensor_parallel_size,
207 args.seed, args.n, args.use_beam_search,
208 args.trust_remote_code, args.dtype,
207 - args.max_model_len)
209 + args.max_model_len, args.enforce_eager)
210 elif args.backend == "hf":
211 assert args.tensor_parallel_size == 1
212 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -279,6 +281,9 @@ if __name__ == "__main__":
281 'The "auto" option will use FP16 precision '
282 'for FP32 and FP16 models, and BF16 precision '
283 'for BF16 models.')
284 + parser.add_argument("--enforce-eager",
285 + action="store_true",
286 + help="enforce eager execution")
287 args = parser.parse_args()
288 if args.tokenizer is None:
289 args.tokenizer = args.model
requirements.txt
+1
@@ -12,3 +12,4 @@ fastapi
12 uvicorn[standard]
13 pydantic == 1.10.13 # Required for OpenAI server.
14 aioprometheus[starlette]
15 +cupy-cuda12x # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead. # FIXME: Fix this in setup.py.
vllm/config.py
+17
@@ -49,6 +49,12 @@ class ModelConfig:
49 output). If None, will be derived from the model.
50 quantization: Quantization method that was used to quantize the model
51 weights. If None, we assume the model weights are not quantized.
52 + enforce_eager: Whether to enforce eager execution. If True, we will
53 + disable CUDA graph and always execute the model in eager mode.
54 + If False, we will use CUDA graph and eager execution in hybrid.
55 + max_context_len_to_capture: Maximum context len covered by CUDA graphs.
56 + When a sequence has context length larger than this, we fall back
57 + to eager mode.
58 """
59
60 def __init__(
@@ -65,6 +71,8 @@ class ModelConfig:
71 tokenizer_revision: Optional[str] = None,
72 max_model_len: Optional[int] = None,
73 quantization: Optional[str] = None,
74 + enforce_eager: bool = False,
75 + max_context_len_to_capture: Optional[int] = None,
76 ) -> None:
77 self.model = model
78 self.tokenizer = tokenizer
@@ -76,6 +84,8 @@ class ModelConfig:
84 self.revision = revision
85 self.tokenizer_revision = tokenizer_revision
86 self.quantization = quantization
87 + self.enforce_eager = enforce_eager
88 + self.max_context_len_to_capture = max_context_len_to_capture
89
90 if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
91 # download model from ModelScope hub,
@@ -95,6 +105,7 @@ class ModelConfig:
105 self._verify_load_format()
106 self._verify_tokenizer_mode()
107 self._verify_quantization()
108 + self._verify_cuda_graph()
109
110 def _verify_load_format(self) -> None:
111 load_format = self.load_format.lower()
@@ -169,6 +180,12 @@ class ModelConfig:
180 "optimized yet. The speed can be slower than "
181 "non-quantized models.")
182
183 + def _verify_cuda_graph(self) -> None:
184 + if self.max_context_len_to_capture is None:
185 + self.max_context_len_to_capture = self.max_model_len
186 + self.max_context_len_to_capture = min(self.max_context_len_to_capture,
187 + self.max_model_len)
188 +
189 def verify_with_parallel_config(
190 self,
191 parallel_config: "ParallelConfig",
vllm/engine/arg_utils.py
+15 -1
@@ -33,6 +33,8 @@ class EngineArgs:
33 revision: Optional[str] = None
34 tokenizer_revision: Optional[str] = None
35 quantization: Optional[str] = None
36 + enforce_eager: bool = False
37 + max_context_len_to_capture: int = 8192
38
39 def __post_init__(self):
40 if self.tokenizer is None:
@@ -182,6 +184,17 @@ class EngineArgs:
184 choices=['awq', 'gptq', 'squeezellm', None],
185 default=None,
186 help='Method used to quantize the weights')
187 + parser.add_argument('--enforce-eager',
188 + action='store_true',
189 + help='Always use eager-mode PyTorch. If False, '
190 + 'will use eager mode and CUDA graph in hybrid '
191 + 'for maximal performance and flexibility.')
192 + parser.add_argument('--max-context-len-to-capture',
193 + type=int,
194 + default=EngineArgs.max_context_len_to_capture,
195 + help='maximum context length covered by CUDA '
196 + 'graphs. When a sequence has context length '
197 + 'larger than this, we fall back to eager mode.')
198 return parser
199
200 @classmethod
@@ -200,7 +213,8 @@ class EngineArgs:
213 self.download_dir, self.load_format,
214 self.dtype, self.seed, self.revision,
215 self.tokenizer_revision, self.max_model_len,
203 - self.quantization)
216 + self.quantization, self.enforce_eager,
217 + self.max_context_len_to_capture)
218 cache_config = CacheConfig(self.block_size,
219 self.gpu_memory_utilization,
220 self.swap_space,
vllm/engine/llm_engine.py
+6 -1
@@ -17,7 +17,7 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
17 SequenceOutput, SequenceStatus)
18 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
19 get_tokenizer)
20 -from vllm.utils import Counter
20 +from vllm.utils import Counter, get_open_port
21
22 if ray:
23 from ray.air.util.torch_dist import init_torch_dist_process_group
@@ -84,6 +84,7 @@ class LLMEngine:
84 f"load_format={model_config.load_format}, "
85 f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
86 f"quantization={model_config.quantization}, "
87 + f"enforce_eager={model_config.enforce_eager}, "
88 f"seed={model_config.seed})")
89 # TODO(woosuk): Print more configs in debug mode.
90
@@ -189,6 +190,7 @@ class LLMEngine:
190 ))
191 self._run_workers(
192 "init_model",
193 + cupy_port=get_open_port(),
194 get_all_outputs=True,
195 )
196 self._run_workers(
@@ -232,6 +234,9 @@ class LLMEngine:
234
235 # Initialize the cache.
236 self._run_workers("init_cache_engine", cache_config=self.cache_config)
237 + # Warm up the model. This includes capturing the model into CUDA graph
238 + # if enforce_eager is False.
239 + self._run_workers("warm_up_model")
240
241 @classmethod
242 def from_engine_args(cls, engine_args: EngineArgs) -> "LLMEngine":
vllm/engine/ray_utils.py
+1 -8
@@ -1,9 +1,8 @@
1 -import socket
1 from typing import Optional, Tuple, TYPE_CHECKING
2
3 from vllm.config import ParallelConfig
4 from vllm.logger import init_logger
6 -from vllm.utils import is_hip
5 +from vllm.utils import get_open_port, is_hip
6
7 logger = init_logger(__name__)
8
@@ -43,12 +42,6 @@ if TYPE_CHECKING:
42 from ray.util.placement_group import PlacementGroup
43
44
46 -def get_open_port():
47 - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
48 - s.bind(("", 0))
49 - return s.getsockname()[1]
50 -
51 -
45 def initialize_cluster(
46 parallel_config: ParallelConfig,
47 engine_use_ray: bool = False,
vllm/entrypoints/llm.py
+10
@@ -56,6 +56,12 @@ class LLM:
56 when their `best_of` sampling parameters are larger than 1. If all
57 requests will have `best_of=1`, you can safely set this to 0.
58 Otherwise, too small values may cause out-of-memory (OOM) errors.
59 + enforce_eager: Whether to enforce eager execution. If True, we will
60 + disable CUDA graph and always execute the model in eager mode.
61 + If False, we will use CUDA graph and eager execution in hybrid.
62 + max_context_len_to_capture: Maximum context len covered by CUDA graphs.
63 + When a sequence has context length larger than this, we fall back
64 + to eager mode.
65 """
66
67 def __init__(
@@ -72,6 +78,8 @@ class LLM:
78 seed: int = 0,
79 gpu_memory_utilization: float = 0.9,
80 swap_space: int = 4,
81 + enforce_eager: bool = False,
82 + max_context_len_to_capture: int = 8192,
83 **kwargs,
84 ) -> None:
85 if "disable_log_stats" not in kwargs:
@@ -89,6 +97,8 @@ class LLM:
97 seed=seed,
98 gpu_memory_utilization=gpu_memory_utilization,
99 swap_space=swap_space,
100 + enforce_eager=enforce_eager,
101 + max_context_len_to_capture=max_context_len_to_capture,
102 **kwargs,
103 )
104 self.llm_engine = LLMEngine.from_engine_args(engine_args)
vllm/model_executor/input_metadata.py
+4 -1
@@ -21,12 +21,14 @@ class InputMetadata:
21 max_context_len: Optional[int],
22 context_lens: Optional[torch.Tensor],
23 block_tables: Optional[torch.Tensor],
24 + use_cuda_graph: bool,
25 ) -> None:
26 self.prompt_lens = prompt_lens
27 self.max_context_len = max_context_len
28 self.slot_mapping = slot_mapping
29 self.context_lens = context_lens
30 self.block_tables = block_tables
31 + self.use_cuda_graph = use_cuda_graph
32
33 self.is_prompt = len(prompt_lens) > 0
34 # Set during the execution of the first attention op.
@@ -39,4 +41,5 @@ class InputMetadata:
41 f"max_context_len={self.max_context_len}, "
42 f"slot_mapping={self.slot_mapping}, "
43 f"context_lens={self.context_lens}, "
42 - f"block_tables={self.block_tables})")
44 + f"block_tables={self.block_tables}, "
45 + f"use_cuda_graph={self.use_cuda_graph})")
vllm/model_executor/layers/attention.py
+18 -22
@@ -24,13 +24,10 @@ class PagedAttention(nn.Module):
24 can either contain prompt tokens or generation tokens.
25 The class does the following:
26
27 - 1. Wait for the cache operations (e.g., swap, copy) to finish. The cache
28 - operations are issued by the cache engine before executing the forward
29 - pass of the model, and they are executed asynchronously.
30 - 2. Reshape and store the input key and value tensors in the KV cache.
31 - 3. Perform (multi-head/multi-query/grouped-query) attention using either
27 + 1. Reshape and store the input key and value tensors in the KV cache.
28 + 2. Perform (multi-head/multi-query/grouped-query) attention using either
29 xformers or the PagedAttention custom op.
33 - 4. Return the output tensor.
30 + 3. Return the output tensor.
31 """
32
33 def __init__(
@@ -67,7 +64,6 @@ class PagedAttention(nn.Module):
64 key_cache: Optional[torch.Tensor],
65 value_cache: Optional[torch.Tensor],
66 input_metadata: InputMetadata,
70 - cache_event: Optional[torch.cuda.Event],
67 ) -> torch.Tensor:
68 """PagedAttention forward pass.
69
@@ -80,7 +76,6 @@ class PagedAttention(nn.Module):
76 value_cache: shape = [num_blocks, num_kv_heads, head_size,
77 block_size]
78 input_metadata: metadata for the inputs.
83 - cache_event: event to wait for the cache operations to finish.
79 Returns:
80 shape = [batch_size, seq_len, num_heads * head_size]
81 """
@@ -89,10 +84,6 @@ class PagedAttention(nn.Module):
84 query = query.view(-1, self.num_heads, self.head_size)
85 key = key.view(-1, self.num_kv_heads, self.head_size)
86 value = value.view(-1, self.num_kv_heads, self.head_size)
92 - slot_mapping = input_metadata.slot_mapping.flatten()
93 -
94 - if cache_event is not None:
95 - cache_event.wait()
87
88 # Reshape the keys and values and store them in the cache.
89 # If key_cache and value_cache are not provided, the new key and value
@@ -104,7 +95,7 @@ class PagedAttention(nn.Module):
95 value,
96 key_cache,
97 value_cache,
107 - slot_mapping,
98 + input_metadata.slot_mapping.flatten(),
99 )
100
101 if input_metadata.is_prompt:
@@ -165,15 +156,20 @@ class PagedAttention(nn.Module):
156 output = out.view_as(query)
157 else:
158 # Decoding run.
168 - output = _paged_attention(
169 - query,
170 - key_cache,
171 - value_cache,
172 - input_metadata,
173 - self.num_kv_heads,
174 - self.scale,
175 - self.alibi_slopes,
176 - )
159 + if key_cache is not None and value_cache is not None:
160 + output = _paged_attention(
161 + query,
162 + key_cache,
163 + value_cache,
164 + input_metadata,
165 + self.num_kv_heads,
166 + self.scale,
167 + self.alibi_slopes,
168 + )
169 + else:
170 + # This happens during the initial memory profiling run for
171 + # CUDA graphs.
172 + output = torch.zeros_like(query)
173
174 # Reshape the output tensor.
175 return output.view(batch_size, seq_len, hidden_size)
vllm/model_executor/models/aquila.py
+2 -10
@@ -158,14 +158,12 @@ class AquilaAttention(nn.Module):
158 hidden_states: torch.Tensor,
159 kv_cache: KVCache,
160 input_metadata: InputMetadata,
161 - cache_event: Optional[torch.cuda.Event],
161 ) -> torch.Tensor:
162 qkv, _ = self.qkv_proj(hidden_states)
163 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164 q, k = self.rotary_emb(positions, q, k)
165 k_cache, v_cache = kv_cache
167 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
168 - cache_event)
166 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
167 output, _ = self.o_proj(attn_output)
168 return output
169
@@ -209,7 +207,6 @@ class AquilaDecoderLayer(nn.Module):
207 hidden_states: torch.Tensor,
208 kv_cache: KVCache,
209 input_metadata: InputMetadata,
212 - cache_event: Optional[torch.cuda.Event],
210 ) -> torch.Tensor:
211 # Self Attention
212 residual = hidden_states
@@ -219,7 +216,6 @@ class AquilaDecoderLayer(nn.Module):
216 hidden_states=hidden_states,
217 kv_cache=kv_cache,
218 input_metadata=input_metadata,
222 - cache_event=cache_event,
219 )
220 hidden_states = residual + hidden_states
221
@@ -258,18 +254,15 @@ class AquilaModel(nn.Module):
254 positions: torch.Tensor,
255 kv_caches: List[KVCache],
256 input_metadata: InputMetadata,
261 - cache_events: Optional[List[torch.cuda.Event]],
257 ) -> torch.Tensor:
258 hidden_states = self.embed_tokens(input_ids)
259 for i in range(len(self.layers)):
265 - cache_event = None if cache_events is None else cache_events[i]
260 layer = self.layers[i]
261 hidden_states = layer(
262 positions,
263 hidden_states,
264 kv_caches[i],
265 input_metadata,
272 - cache_event,
266 )
267 hidden_states = self.norm(hidden_states)
268
@@ -296,10 +289,9 @@ class AquilaForCausalLM(nn.Module):
289 positions: torch.Tensor,
290 kv_caches: List[KVCache],
291 input_metadata: InputMetadata,
299 - cache_events: Optional[List[torch.cuda.Event]],
292 ) -> torch.Tensor:
293 hidden_states = self.model(input_ids, positions, kv_caches,
302 - input_metadata, cache_events)
294 + input_metadata)
295 return hidden_states
296
297 def sample(
vllm/model_executor/models/baichuan.py
+2 -10
@@ -172,15 +172,13 @@ class BaiChuanAttention(nn.Module):
172 hidden_states: torch.Tensor,
173 kv_cache: KVCache,
174 input_metadata: InputMetadata,
175 - cache_event: Optional[torch.cuda.Event],
175 ) -> torch.Tensor:
176 qkv, _ = self.W_pack(hidden_states)
177 q, k, v = qkv.chunk(chunks=3, dim=-1)
178 if self.postion_embedding != "ALIBI":
179 q, k = self.rotary_emb(positions, q, k)
180 k_cache, v_cache = kv_cache
182 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
183 - cache_event)
181 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
182 output, _ = self.o_proj(attn_output)
183 return output
184
@@ -221,7 +219,6 @@ class BaiChuanDecoderLayer(nn.Module):
219 hidden_states: torch.Tensor,
220 kv_cache: KVCache,
221 input_metadata: InputMetadata,
224 - cache_event: Optional[torch.cuda.Event],
222 residual: Optional[torch.Tensor],
223 ) -> Tuple[torch.Tensor, torch.Tensor]:
224 # Self Attention
@@ -236,7 +233,6 @@ class BaiChuanDecoderLayer(nn.Module):
233 hidden_states=hidden_states,
234 kv_cache=kv_cache,
235 input_metadata=input_metadata,
239 - cache_event=cache_event,
236 )
237
238 # Fully Connected
@@ -273,19 +269,16 @@ class BaiChuanModel(nn.Module):
269 positions: torch.Tensor,
270 kv_caches: List[KVCache],
271 input_metadata: InputMetadata,
276 - cache_events: Optional[List[torch.cuda.Event]],
272 ) -> torch.Tensor:
273 hidden_states = self.embed_tokens(input_ids)
274 residual = None
275 for i in range(len(self.layers)):
281 - cache_event = None if cache_events is None else cache_events[i]
276 layer = self.layers[i]
277 hidden_states, residual = layer(
278 positions,
279 hidden_states,
280 kv_caches[i],
281 input_metadata,
288 - cache_event,
282 residual,
283 )
284 hidden_states, _ = self.norm(hidden_states, residual)
@@ -311,10 +304,9 @@ class BaiChuanBaseForCausalLM(nn.Module):
304 positions: torch.Tensor,
305 kv_caches: List[KVCache],
306 input_metadata: InputMetadata,
314 - cache_events: Optional[List[torch.cuda.Event]],
307 ) -> torch.Tensor:
308 hidden_states = self.model(input_ids, positions, kv_caches,
317 - input_metadata, cache_events)
309 + input_metadata)
310 return hidden_states
311
312 def sample(
vllm/model_executor/models/bloom.py
+2 -10
@@ -118,14 +118,12 @@ class BloomAttention(nn.Module):
118 hidden_states: torch.Tensor,
119 kv_cache: KVCache,
120 input_metadata: InputMetadata,
121 - cache_event: Optional[torch.cuda.Event],
121 ) -> torch.Tensor:
122 del position_ids # Unused.
123 qkv, _ = self.query_key_value(hidden_states)
124 q, k, v = qkv.chunk(chunks=3, dim=-1)
125 k_cache, v_cache = kv_cache
127 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
128 - cache_event)
126 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
127 output, _ = self.dense(attn_output)
128 return output
129
@@ -184,7 +182,6 @@ class BloomBlock(nn.Module):
182 hidden_states: torch.Tensor,
183 kv_cache: KVCache,
184 input_metadata: InputMetadata,
187 - cache_event: Optional[torch.cuda.Event],
185 ) -> torch.Tensor:
186 # Layer norm at the beginning of the transformer layer.
187 layernorm_output = self.input_layernorm(hidden_states)
@@ -201,7 +198,6 @@ class BloomBlock(nn.Module):
198 hidden_states=layernorm_output,
199 kv_cache=kv_cache,
200 input_metadata=input_metadata,
204 - cache_event=cache_event,
201 )
202 attention_output = attention_output + residual
203 layernorm_output = self.post_attention_layernorm(attention_output)
@@ -250,19 +246,16 @@ class BloomModel(nn.Module):
246 position_ids: torch.Tensor,
247 kv_caches: List[KVCache],
248 input_metadata: InputMetadata,
253 - cache_events: Optional[List[torch.cuda.Event]],
249 ) -> torch.Tensor:
250 hidden_states = self.word_embeddings(input_ids)
251 hidden_states = self.word_embeddings_layernorm(hidden_states)
252 for i in range(len(self.h)):
258 - cache_event = None if cache_events is None else cache_events[i]
253 layer = self.h[i]
254 hidden_states = layer(
255 position_ids,
256 hidden_states,
257 kv_caches[i],
258 input_metadata,
265 - cache_event,
259 )
260 hidden_states = self.ln_f(hidden_states)
261 return hidden_states
@@ -288,10 +281,9 @@ class BloomForCausalLM(nn.Module):
281 positions: torch.Tensor,
282 kv_caches: List[KVCache],
283 input_metadata: InputMetadata,
291 - cache_events: Optional[List[torch.cuda.Event]],
284 ) -> torch.Tensor:
285 hidden_states = self.transformer(input_ids, positions, kv_caches,
294 - input_metadata, cache_events)
286 + input_metadata)
287 return hidden_states
288
289 def sample(
vllm/model_executor/models/chatglm.py
+2 -13
@@ -100,7 +100,6 @@ class GLMAttention(nn.Module):
100 position_ids: torch.Tensor,
101 kv_cache: KVCache,
102 input_metadata: InputMetadata,
103 - cache_event: Optional[torch.cuda.Event],
103 ) -> torch.Tensor:
104 qkv, _ = self.query_key_value(hidden_states)
105 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
@@ -113,7 +112,6 @@ class GLMAttention(nn.Module):
112 key_cache,
113 value_cache,
114 input_metadata,
116 - cache_event,
115 )
116 attn_output, _ = self.dense(context_layer)
117 return attn_output
@@ -203,7 +201,6 @@ class GLMBlock(nn.Module):
201 position_ids: torch.Tensor,
202 kv_cache: KVCache,
203 input_metadata: InputMetadata,
206 - cache_event: Optional[torch.cuda.Event],
204 ) -> torch.Tensor:
205 # hidden_states: [num_tokens, h]
206 # Layer norm at the beginning of the transformer layer.
@@ -214,7 +211,6 @@ class GLMBlock(nn.Module):
211 position_ids=position_ids,
212 kv_cache=kv_cache,
213 input_metadata=input_metadata,
217 - cache_event=cache_event,
214 )
215
216 # Residual connection.
@@ -269,17 +265,14 @@ class GLMTransformer(nn.Module):
265 position_ids: torch.Tensor,
266 kv_caches: List[KVCache],
267 input_metadata: InputMetadata,
272 - cache_events: Optional[List[torch.cuda.Event]],
268 ) -> torch.Tensor:
269 for i in range(self.num_layers):
275 - cache_event = None if cache_events is None else cache_events[i]
270 layer = self.layers[i]
271 hidden_states = layer(
272 hidden_states=hidden_states,
273 position_ids=position_ids,
274 kv_cache=kv_caches[i],
275 input_metadata=input_metadata,
282 - cache_event=cache_event,
276 )
277 # Final layer norm.
278 if self.post_layer_norm:
@@ -314,8 +307,7 @@ class ChatGLMModel(nn.Module):
307 position_ids: torch.Tensor,
308 kv_caches: List[KVCache],
309 input_metadata: InputMetadata,
317 - cache_events: Optional[List[torch.cuda.Event]],
318 - ):
310 + ) -> torch.Tensor:
311 inputs_embeds = self.embedding(input_ids)
312
313 # Run encoder.
@@ -324,9 +316,7 @@ class ChatGLMModel(nn.Module):
316 position_ids=position_ids,
317 kv_caches=kv_caches,
318 input_metadata=input_metadata,
327 - cache_events=cache_events,
319 )
329 -
320 return hidden_states
321
322
@@ -350,10 +340,9 @@ class ChatGLMForCausalLM(nn.Module):
340 positions: torch.Tensor,
341 kv_caches: List[KVCache],
342 input_metadata: InputMetadata,
353 - cache_events: Optional[List[torch.cuda.Event]],
343 ) -> torch.Tensor:
344 hidden_states = self.transformer(input_ids, positions, kv_caches,
356 - input_metadata, cache_events)
345 + input_metadata)
346 return hidden_states
347
348 def sample(
vllm/model_executor/models/falcon.py
+2 -12
@@ -178,7 +178,6 @@ class FalconAttention(nn.Module):
178 hidden_states: torch.Tensor,
179 kv_cache: KVCache,
180 input_metadata: InputMetadata,
181 - cache_event: Optional[torch.cuda.Event],
181 ) -> torch.Tensor:
182 qkv, bias = self.query_key_value(hidden_states)
183 if bias is not None:
@@ -187,8 +186,7 @@ class FalconAttention(nn.Module):
186 if self.use_rotary:
187 q, k = self.rotary_emb(positions, q, k)
188 k_cache, v_cache = kv_cache
190 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
191 - cache_event)
189 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
190 attn_output, bias = self.dense(attn_output)
191 return attn_output, bias
192
@@ -266,8 +264,7 @@ class FalconDecoderLayer(nn.Module):
264 hidden_states: torch.Tensor,
265 kv_cache: KVCache,
266 input_metadata: InputMetadata,
269 - cache_event: Optional[torch.cuda.Event],
270 - ):
267 + ) -> torch.Tensor:
268 residual = hidden_states
269
270 if self.config.new_decoder_architecture:
@@ -282,7 +279,6 @@ class FalconDecoderLayer(nn.Module):
279 hidden_states=attention_layernorm_out,
280 kv_cache=kv_cache,
281 input_metadata=input_metadata,
285 - cache_event=cache_event,
282 )
283 if self.reduce_row_parallel_results and attention_bias is not None:
284 attention_output += attention_bias
@@ -311,7 +307,6 @@ class FalconDecoderLayer(nn.Module):
307 mlp_output += mlp_bias
308
309 output = mlp_output + residual
314 -
310 return output
311
312
@@ -349,18 +344,15 @@ class FalconModel(nn.Module):
344 positions: torch.Tensor,
345 kv_caches: List[KVCache],
346 input_metadata: InputMetadata,
352 - cache_events: Optional[List[torch.cuda.Event]],
347 ) -> torch.Tensor:
348 hidden_states = self.word_embeddings(input_ids)
349 for i in range(len(self.h)):
356 - cache_event = None if cache_events is None else cache_events[i]
350 layer = self.h[i]
351 hidden_states = layer(
352 positions,
353 hidden_states,
354 kv_caches[i],
355 input_metadata,
363 - cache_event,
356 )
357 hidden_states = self.ln_f(hidden_states)
358 return hidden_states
@@ -389,14 +381,12 @@ class FalconForCausalLM(nn.Module):
381 positions: torch.Tensor,
382 kv_caches: List[KVCache],
383 input_metadata: InputMetadata,
392 - cache_events: Optional[List[torch.cuda.Event]],
384 ) -> torch.Tensor:
385 hidden_states = self.transformer(
386 input_ids,
387 positions,
388 kv_caches,
389 input_metadata,
399 - cache_events,
390 )
391 return hidden_states
392
vllm/model_executor/models/gpt2.py
+3 -10
@@ -82,13 +82,12 @@ class GPT2Attention(nn.Module):
82 hidden_states: torch.Tensor,
83 kv_cache: KVCache,
84 input_metadata: InputMetadata,
85 - cache_event: Optional[torch.cuda.Event],
85 ) -> torch.Tensor:
86 qkv, _ = self.c_attn(hidden_states)
87 q, k, v = qkv.chunk(chunks=3, dim=-1)
88 key_cache, value_cache = kv_cache
89 attn_output = self.attn(q, k, v, key_cache, value_cache,
91 - input_metadata, cache_event)
90 + input_metadata)
91 attn_output, _ = self.c_proj(attn_output)
92 return attn_output
93
@@ -148,7 +147,6 @@ class GPT2Block(nn.Module):
147 hidden_states: torch.Tensor,
148 kv_cache: KVCache,
149 input_metadata: InputMetadata,
151 - cache_event: Optional[torch.cuda.Event],
150 ) -> torch.Tensor:
151 residual = hidden_states
152 hidden_states = self.ln_1(hidden_states)
@@ -156,7 +154,6 @@ class GPT2Block(nn.Module):
154 hidden_states=hidden_states,
155 kv_cache=kv_cache,
156 input_metadata=input_metadata,
159 - cache_event=cache_event,
157 )
158 # residual connection
159 hidden_states = attn_output + residual
@@ -196,17 +193,14 @@ class GPT2Model(nn.Module):
193 position_ids: torch.Tensor,
194 kv_caches: List[KVCache],
195 input_metadata: InputMetadata,
199 - cache_events: Optional[List[torch.cuda.Event]],
196 ) -> torch.Tensor:
197 inputs_embeds = self.wte(input_ids)
198 position_embeds = self.wpe(position_ids)
199 hidden_states = inputs_embeds + position_embeds
200
201 for i in range(len(self.h)):
206 - cache_event = None if cache_events is None else cache_events[i]
202 layer = self.h[i]
208 - hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
209 - cache_event)
203 + hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
204
205 hidden_states = self.ln_f(hidden_states)
206 return hidden_states
@@ -232,10 +226,9 @@ class GPT2LMHeadModel(nn.Module):
226 positions: torch.Tensor,
227 kv_caches: List[KVCache],
228 input_metadata: InputMetadata,
235 - cache_events: Optional[List[torch.cuda.Event]],
229 ) -> torch.Tensor:
230 hidden_states = self.transformer(input_ids, positions, kv_caches,
238 - input_metadata, cache_events)
231 + input_metadata)
232 return hidden_states
233
234 def sample(
vllm/model_executor/models/gpt_bigcode.py
+3 -10
@@ -95,7 +95,6 @@ class GPTBigCodeAttention(nn.Module):
95 hidden_states: torch.Tensor,
96 kv_cache: KVCache,
97 input_metadata: InputMetadata,
98 - cache_event: Optional[torch.cuda.Event],
98 ) -> torch.Tensor:
99 qkv, _ = self.c_attn(hidden_states)
100 q, k, v = qkv.split(
@@ -107,7 +106,7 @@ class GPTBigCodeAttention(nn.Module):
106 )
107 key_cache, value_cache = kv_cache
108 attn_output = self.attn(q, k, v, key_cache, value_cache,
110 - input_metadata, cache_event)
109 + input_metadata)
110 attn_output, _ = self.c_proj(attn_output)
111 return attn_output
112
@@ -167,7 +166,6 @@ class GPTBigCodeBlock(nn.Module):
166 hidden_states: torch.Tensor,
167 kv_cache: KVCache,
168 input_metadata: InputMetadata,
170 - cache_event: Optional[torch.cuda.Event],
169 ) -> torch.Tensor:
170 residual = hidden_states
171 hidden_states = self.ln_1(hidden_states)
@@ -175,7 +173,6 @@ class GPTBigCodeBlock(nn.Module):
173 hidden_states=hidden_states,
174 kv_cache=kv_cache,
175 input_metadata=input_metadata,
178 - cache_event=cache_event,
176 )
177 # residual connection
178 hidden_states = attn_output + residual
@@ -215,17 +212,14 @@ class GPTBigCodeModel(nn.Module):
212 position_ids: torch.Tensor,
213 kv_caches: List[KVCache],
214 input_metadata: InputMetadata,
218 - cache_events: Optional[List[torch.cuda.Event]],
215 ) -> torch.Tensor:
216 inputs_embeds = self.wte(input_ids)
217 position_embeds = self.wpe(position_ids)
218 hidden_states = inputs_embeds + position_embeds
219
220 for i in range(len(self.h)):
225 - cache_event = None if cache_events is None else cache_events[i]
221 layer = self.h[i]
227 - hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
228 - cache_event)
222 + hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
223
224 hidden_states = self.ln_f(hidden_states)
225 return hidden_states
@@ -251,10 +245,9 @@ class GPTBigCodeForCausalLM(nn.Module):
245 positions: torch.Tensor,
246 kv_caches: List[KVCache],
247 input_metadata: InputMetadata,
254 - cache_events: Optional[List[torch.cuda.Event]],
248 ) -> torch.Tensor:
249 hidden_states = self.transformer(input_ids, positions, kv_caches,
257 - input_metadata, cache_events)
250 + input_metadata)
251 return hidden_states
252
253 def sample(
vllm/model_executor/models/gpt_j.py
+2 -10
@@ -94,14 +94,12 @@ class GPTJAttention(nn.Module):
94 hidden_states: torch.Tensor,
95 kv_cache: KVCache,
96 input_metadata: InputMetadata,
97 - cache_event: Optional[torch.cuda.Event],
97 ) -> torch.Tensor:
98 qkv, _ = self.qkv_proj(hidden_states)
99 q, k, v = qkv.chunk(chunks=3, dim=-1)
100 q, k = self.rotary_emb(position_ids, q, k)
101 k_cache, v_cache = kv_cache
103 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
104 - cache_event)
102 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
103 attn_output, _ = self.out_proj(attn_output)
104 return attn_output
105
@@ -156,7 +154,6 @@ class GPTJBlock(nn.Module):
154 hidden_states: torch.Tensor,
155 kv_cache: KVCache,
156 input_metadata: InputMetadata,
159 - cache_event: Optional[torch.cuda.Event],
157 ) -> torch.Tensor:
158 residual = hidden_states
159 hidden_states = self.ln_1(hidden_states)
@@ -165,7 +162,6 @@ class GPTJBlock(nn.Module):
162 hidden_states=hidden_states,
163 kv_cache=kv_cache,
164 input_metadata=input_metadata,
168 - cache_event=cache_event,
165 )
166 mlp_output = self.mlp(hidden_states)
167 hidden_states = attn_output + mlp_output + residual
@@ -196,18 +192,15 @@ class GPTJModel(nn.Module):
192 position_ids: torch.Tensor,
193 kv_caches: List[KVCache],
194 input_metadata: InputMetadata,
199 - cache_events: Optional[List[torch.cuda.Event]],
195 ) -> torch.Tensor:
196 hidden_states = self.wte(input_ids)
197 for i in range(len(self.h)):
203 - cache_event = None if cache_events is None else cache_events[i]
198 layer = self.h[i]
199 hidden_states = layer(
200 position_ids,
201 hidden_states,
202 kv_caches[i],
203 input_metadata,
210 - cache_event,
204 )
205 hidden_states = self.ln_f(hidden_states)
206 return hidden_states
@@ -238,10 +231,9 @@ class GPTJForCausalLM(nn.Module):
231 positions: torch.Tensor,
232 kv_caches: List[KVCache],
233 input_metadata: InputMetadata,
241 - cache_events: Optional[List[torch.cuda.Event]],
234 ) -> torch.Tensor:
235 hidden_states = self.transformer(input_ids, positions, kv_caches,
244 - input_metadata, cache_events)
236 + input_metadata)
237 return hidden_states
238
239 def sample(
vllm/model_executor/models/gpt_neox.py
+2 -10
@@ -92,14 +92,12 @@ class GPTNeoXAttention(nn.Module):
92 hidden_states: torch.Tensor,
93 kv_cache: KVCache,
94 input_metadata: InputMetadata,
95 - cache_event: Optional[torch.cuda.Event],
95 ) -> torch.Tensor:
96 qkv, _ = self.query_key_value(hidden_states)
97 q, k, v = qkv.chunk(chunks=3, dim=-1)
98 q, k = self.rotary_emb(position_ids, q, k)
99 k_cache, v_cache = kv_cache
101 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
102 - cache_event)
100 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
101 output, _ = self.dense(attn_output)
102 return output
103
@@ -155,7 +153,6 @@ class GPTNeoXLayer(nn.Module):
153 hidden_states: torch.Tensor,
154 kv_cache: KVCache,
155 input_metadata: InputMetadata,
158 - cache_event: Optional[torch.cuda.Event],
156 ) -> torch.Tensor:
157 attn_input = self.input_layernorm(hidden_states)
158 attn_output = self.attention(
@@ -163,7 +160,6 @@ class GPTNeoXLayer(nn.Module):
160 hidden_states=attn_input,
161 kv_cache=kv_cache,
162 input_metadata=input_metadata,
166 - cache_event=cache_event,
163 )
164
165 if self.use_parallel_residual:
@@ -210,18 +206,15 @@ class GPTNeoXModel(nn.Module):
206 position_ids: torch.Tensor,
207 kv_caches: List[KVCache],
208 input_metadata: InputMetadata,
213 - cache_events: Optional[List[torch.cuda.Event]],
209 ) -> torch.Tensor:
210 hidden_states = self.embed_in(input_ids)
211 for i in range(len(self.layers)):
217 - cache_event = None if cache_events is None else cache_events[i]
212 layer = self.layers[i]
213 hidden_states = layer(
214 position_ids,
215 hidden_states,
216 kv_caches[i],
217 input_metadata,
224 - cache_event,
218 )
219 hidden_states = self.final_layer_norm(hidden_states)
220 return hidden_states
@@ -250,10 +243,9 @@ class GPTNeoXForCausalLM(nn.Module):
243 positions: torch.Tensor,
244 kv_caches: List[KVCache],
245 input_metadata: InputMetadata,
253 - cache_events: Optional[List[torch.cuda.Event]],
246 ) -> torch.Tensor:
247 hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
256 - input_metadata, cache_events)
248 + input_metadata)
249 return hidden_states
250
251 def sample(
vllm/model_executor/models/internlm.py
+2 -10
@@ -110,14 +110,12 @@ class InternLMAttention(nn.Module):
110 hidden_states: torch.Tensor,
111 kv_cache: KVCache,
112 input_metadata: InputMetadata,
113 - cache_event: Optional[torch.cuda.Event],
113 ) -> torch.Tensor:
114 qkv, _ = self.qkv_proj(hidden_states)
115 q, k, v = qkv.chunk(chunks=3, dim=-1)
116 q, k = self.rotary_emb(positions, q, k)
117 k_cache, v_cache = kv_cache
119 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
120 - cache_event)
118 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
119 output, _ = self.o_proj(attn_output)
120 return output
121
@@ -160,7 +158,6 @@ class InternLMDecoderLayer(nn.Module):
158 hidden_states: torch.Tensor,
159 kv_cache: KVCache,
160 input_metadata: InputMetadata,
163 - cache_event: Optional[torch.cuda.Event],
161 residual: Optional[torch.Tensor],
162 ) -> Tuple[torch.Tensor, torch.Tensor]:
163 # Self Attention
@@ -175,7 +172,6 @@ class InternLMDecoderLayer(nn.Module):
172 hidden_states=hidden_states,
173 kv_cache=kv_cache,
174 input_metadata=input_metadata,
178 - cache_event=cache_event,
175 )
176
177 # Fully Connected
@@ -214,19 +210,16 @@ class InternLMModel(nn.Module):
210 positions: torch.Tensor,
211 kv_caches: List[KVCache],
212 input_metadata: InputMetadata,
217 - cache_events: Optional[List[torch.cuda.Event]],
213 ) -> torch.Tensor:
214 hidden_states = self.embed_tokens(input_ids)
215 residual = None
216 for i in range(len(self.layers)):
222 - cache_event = None if cache_events is None else cache_events[i]
217 layer = self.layers[i]
218 hidden_states, residual = layer(
219 positions,
220 hidden_states,
221 kv_caches[i],
222 input_metadata,
229 - cache_event,
223 residual,
224 )
225 hidden_states, _ = self.norm(hidden_states, residual)
@@ -253,10 +246,9 @@ class InternLMForCausalLM(nn.Module):
246 positions: torch.Tensor,
247 kv_caches: List[KVCache],
248 input_metadata: InputMetadata,
256 - cache_events: Optional[List[torch.cuda.Event]],
249 ) -> torch.Tensor:
250 hidden_states = self.model(input_ids, positions, kv_caches,
259 - input_metadata, cache_events)
251 + input_metadata)
252 return hidden_states
253
254 def sample(
vllm/model_executor/models/llama.py
+2 -10
@@ -147,14 +147,12 @@ class LlamaAttention(nn.Module):
147 hidden_states: torch.Tensor,
148 kv_cache: KVCache,
149 input_metadata: InputMetadata,
150 - cache_event: Optional[torch.cuda.Event],
150 ) -> torch.Tensor:
151 qkv, _ = self.qkv_proj(hidden_states)
152 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
153 q, k = self.rotary_emb(positions, q, k)
154 k_cache, v_cache = kv_cache
156 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
157 - cache_event)
155 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
156 output, _ = self.o_proj(attn_output)
157 return output
158
@@ -198,7 +196,6 @@ class LlamaDecoderLayer(nn.Module):
196 hidden_states: torch.Tensor,
197 kv_cache: KVCache,
198 input_metadata: InputMetadata,
201 - cache_event: Optional[torch.cuda.Event],
199 residual: Optional[torch.Tensor],
200 ) -> Tuple[torch.Tensor, torch.Tensor]:
201 # Self Attention
@@ -213,7 +210,6 @@ class LlamaDecoderLayer(nn.Module):
210 hidden_states=hidden_states,
211 kv_cache=kv_cache,
212 input_metadata=input_metadata,
216 - cache_event=cache_event,
213 )
214
215 # Fully Connected
@@ -250,19 +246,16 @@ class LlamaModel(nn.Module):
246 positions: torch.Tensor,
247 kv_caches: List[KVCache],
248 input_metadata: InputMetadata,
253 - cache_events: Optional[List[torch.cuda.Event]],
249 ) -> torch.Tensor:
250 hidden_states = self.embed_tokens(input_ids)
251 residual = None
252 for i in range(len(self.layers)):
258 - cache_event = None if cache_events is None else cache_events[i]
253 layer = self.layers[i]
254 hidden_states, residual = layer(
255 positions,
256 hidden_states,
257 kv_caches[i],
258 input_metadata,
265 - cache_event,
259 residual,
260 )
261 hidden_states, _ = self.norm(hidden_states, residual)
@@ -289,10 +282,9 @@ class LlamaForCausalLM(nn.Module):
282 positions: torch.Tensor,
283 kv_caches: List[KVCache],
284 input_metadata: InputMetadata,
292 - cache_events: Optional[List[torch.cuda.Event]],
285 ) -> torch.Tensor:
286 hidden_states = self.model(input_ids, positions, kv_caches,
295 - input_metadata, cache_events)
287 + input_metadata)
288 return hidden_states
289
290 def sample(
vllm/model_executor/models/mistral.py
+2 -10
@@ -145,14 +145,12 @@ class MistralAttention(nn.Module):
145 hidden_states: torch.Tensor,
146 kv_cache: KVCache,
147 input_metadata: InputMetadata,
148 - cache_event: Optional[torch.cuda.Event],
148 ) -> torch.Tensor:
149 qkv, _ = self.qkv_proj(hidden_states)
150 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
151 q, k = self.rotary_emb(positions, q, k)
152 k_cache, v_cache = kv_cache
154 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
155 - cache_event)
153 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
154 output, _ = self.o_proj(attn_output)
155 return output
156
@@ -193,7 +191,6 @@ class MistralDecoderLayer(nn.Module):
191 hidden_states: torch.Tensor,
192 kv_cache: KVCache,
193 input_metadata: InputMetadata,
196 - cache_event: Optional[torch.cuda.Event],
194 residual: Optional[torch.Tensor],
195 ) -> Tuple[torch.Tensor, torch.Tensor]:
196 # Self Attention
@@ -208,7 +205,6 @@ class MistralDecoderLayer(nn.Module):
205 hidden_states=hidden_states,
206 kv_cache=kv_cache,
207 input_metadata=input_metadata,
211 - cache_event=cache_event,
208 )
209
210 # Fully Connected
@@ -246,19 +242,16 @@ class MistralModel(nn.Module):
242 positions: torch.Tensor,
243 kv_caches: List[KVCache],
244 input_metadata: InputMetadata,
249 - cache_events: Optional[List[torch.cuda.Event]],
245 ) -> torch.Tensor:
246 hidden_states = self.embed_tokens(input_ids)
247 residual = None
248 for i in range(len(self.layers)):
254 - cache_event = None if cache_events is None else cache_events[i]
249 layer = self.layers[i]
250 hidden_states, residual = layer(
251 positions,
252 hidden_states,
253 kv_caches[i],
254 input_metadata,
261 - cache_event,
255 residual,
256 )
257 hidden_states, _ = self.norm(hidden_states, residual)
@@ -285,10 +278,9 @@ class MistralForCausalLM(nn.Module):
278 positions: torch.Tensor,
279 kv_caches: List[KVCache],
280 input_metadata: InputMetadata,
288 - cache_events: Optional[List[torch.cuda.Event]],
281 ) -> torch.Tensor:
282 hidden_states = self.model(input_ids, positions, kv_caches,
291 - input_metadata, cache_events)
283 + input_metadata)
284 return hidden_states
285
286 def sample(
vllm/model_executor/models/mixtral.py
+3 -10
@@ -253,14 +253,12 @@ class MixtralAttention(nn.Module):
253 hidden_states: torch.Tensor,
254 kv_cache: KVCache,
255 input_metadata: InputMetadata,
256 - cache_event: Optional[torch.cuda.Event],
256 ) -> torch.Tensor:
257 qkv, _ = self.qkv_proj(hidden_states)
258 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
259 q, k = self.rotary_emb(positions, q, k)
260 k_cache, v_cache = kv_cache
262 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
263 - cache_event)
261 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
262 output, _ = self.o_proj(attn_output)
263 return output
264
@@ -297,7 +295,6 @@ class MixtralDecoderLayer(nn.Module):
295 hidden_states: torch.Tensor,
296 kv_cache: KVCache,
297 input_metadata: InputMetadata,
300 - cache_event: Optional[torch.cuda.Event],
298 residual: Optional[torch.Tensor],
299 ) -> torch.Tensor:
300 # Self Attention
@@ -312,7 +309,6 @@ class MixtralDecoderLayer(nn.Module):
309 hidden_states=hidden_states,
310 kv_cache=kv_cache,
311 input_metadata=input_metadata,
315 - cache_event=cache_event,
312 )
313
314 # Fully Connected
@@ -349,16 +345,14 @@ class MixtralModel(nn.Module):
345 positions: torch.Tensor,
346 kv_caches: List[KVCache],
347 input_metadata: InputMetadata,
352 - cache_events: Optional[List[torch.cuda.Event]],
348 ) -> SamplerOutput:
349 hidden_states = self.embed_tokens(input_ids)
350 residual = None
351 for i in range(len(self.layers)):
357 - cache_event = None if cache_events is None else cache_events[i]
352 layer = self.layers[i]
353 hidden_states, residual = layer(positions, hidden_states,
354 kv_caches[i], input_metadata,
361 - cache_event, residual)
355 + residual)
356 hidden_states, _ = self.norm(hidden_states, residual)
357 return hidden_states
358
@@ -383,10 +377,9 @@ class MixtralForCausalLM(nn.Module):
377 positions: torch.Tensor,
378 kv_caches: List[KVCache],
379 input_metadata: InputMetadata,
386 - cache_events: Optional[List[torch.cuda.Event]],
380 ) -> torch.Tensor:
381 hidden_states = self.model(input_ids, positions, kv_caches,
389 - input_metadata, cache_events)
382 + input_metadata)
383 return hidden_states
384
385 def sample(
vllm/model_executor/models/mpt.py
+2 -10
@@ -117,7 +117,6 @@ class MPTAttention(nn.Module):
117 hidden_states: torch.Tensor,
118 kv_cache: KVCache,
119 input_metadata: InputMetadata,
120 - cache_event: Optional[torch.cuda.Event],
120 ) -> torch.Tensor:
121 del position_ids # unused.
122 qkv, _ = self.Wqkv(hidden_states)
@@ -128,8 +127,7 @@ class MPTAttention(nn.Module):
127 q = self.q_ln(q)
128 k = self.k_ln(k)
129 k_cache, v_cache = kv_cache
131 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
132 - cache_event)
130 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
131 output, _ = self.out_proj(attn_output)
132 return output
133
@@ -187,7 +185,6 @@ class MPTBlock(nn.Module):
185 hidden_states: torch.Tensor,
186 kv_cache: KVCache,
187 input_metadata: InputMetadata,
190 - cache_event: Optional[torch.cuda.Event],
188 ) -> torch.Tensor:
189 x = self.norm_1(hidden_states)
190 x = self.attn(
@@ -195,7 +192,6 @@ class MPTBlock(nn.Module):
192 hidden_states=x,
193 kv_cache=kv_cache,
194 input_metadata=input_metadata,
198 - cache_event=cache_event,
195 )
196 hidden_states = hidden_states + x
197 x = self.norm_2(hidden_states)
@@ -235,18 +231,15 @@ class MPTModel(nn.Module):
231 position_ids: torch.Tensor,
232 kv_caches: List[KVCache],
233 input_metadata: InputMetadata,
238 - cache_events: Optional[List[torch.cuda.Event]],
234 ) -> torch.Tensor:
235 hidden_states = self.wte(input_ids)
236 for i in range(len(self.blocks)):
242 - cache_event = None if cache_events is None else cache_events[i]
237 block = self.blocks[i]
238 hidden_states = block(
239 position_ids,
240 hidden_states,
241 kv_caches[i],
242 input_metadata,
249 - cache_event,
243 )
244 hidden_states = self.norm_f(hidden_states)
245 return hidden_states
@@ -274,10 +267,9 @@ class MPTForCausalLM(nn.Module):
267 positions: torch.Tensor,
268 kv_caches: List[KVCache],
269 input_metadata: InputMetadata,
277 - cache_events: Optional[List[torch.cuda.Event]],
270 ) -> torch.Tensor:
271 hidden_states = self.transformer(input_ids, positions, kv_caches,
280 - input_metadata, cache_events)
272 + input_metadata)
273 return hidden_states
274
275 def sample(
vllm/model_executor/models/opt.py
+5 -14
@@ -98,13 +98,12 @@ class OPTAttention(nn.Module):
98 hidden_states: torch.Tensor,
99 kv_cache: KVCache,
100 input_metadata: InputMetadata,
101 - cache_event: Optional[torch.cuda.Event],
101 ) -> torch.Tensor:
102 qkv, _ = self.qkv_proj(hidden_states)
103 q, k, v = qkv.chunk(chunks=3, dim=-1)
104 key_cache, value_cache = kv_cache
105 attn_output = self.attn(q, k, v, key_cache, value_cache,
107 - input_metadata, cache_event)
106 + input_metadata)
107 output, _ = self.out_proj(attn_output)
108 return output
109
@@ -154,7 +153,6 @@ class OPTDecoderLayer(nn.Module):
153 hidden_states: torch.Tensor,
154 kv_cache: KVCache,
155 input_metadata: InputMetadata,
157 - cache_event: Optional[torch.cuda.Event],
156 ) -> torch.Tensor:
157 # Self Attention
158 residual = hidden_states
@@ -163,8 +161,7 @@ class OPTDecoderLayer(nn.Module):
161 hidden_states = self.self_attn_layer_norm(hidden_states)
162 hidden_states = self.self_attn(hidden_states=hidden_states,
163 kv_cache=kv_cache,
166 - input_metadata=input_metadata,
167 - cache_event=cache_event)
164 + input_metadata=input_metadata)
165 hidden_states = residual + hidden_states
166 # 350m applies layer norm AFTER attention
167 if not self.do_layer_norm_before:
@@ -245,7 +242,6 @@ class OPTDecoder(nn.Module):
242 positions: torch.Tensor,
243 kv_caches: List[KVCache],
244 input_metadata: InputMetadata,
248 - cache_events: Optional[List[torch.cuda.Event]],
245 ) -> torch.Tensor:
246 inputs_embeds = self.embed_tokens(input_ids)
247 pos_embeds = self.embed_positions(positions)
@@ -254,10 +250,8 @@ class OPTDecoder(nn.Module):
250 hidden_states = inputs_embeds + pos_embeds
251
252 for i in range(len(self.layers)):
257 - cache_event = None if cache_events is None else cache_events[i]
253 layer = self.layers[i]
259 - hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
260 - cache_event)
254 + hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
255
256 if self.final_layer_norm is not None:
257 hidden_states = self.final_layer_norm(hidden_states)
@@ -282,10 +276,8 @@ class OPTModel(nn.Module):
276 positions: torch.Tensor,
277 kv_caches: List[KVCache],
278 input_metadata: InputMetadata,
285 - cache_events: Optional[List[torch.cuda.Event]],
279 ) -> torch.Tensor:
287 - return self.decoder(input_ids, positions, kv_caches, input_metadata,
288 - cache_events)
280 + return self.decoder(input_ids, positions, kv_caches, input_metadata)
281
282
283 class OPTForCausalLM(nn.Module):
@@ -308,10 +300,9 @@ class OPTForCausalLM(nn.Module):
300 positions: torch.Tensor,
301 kv_caches: List[KVCache],
302 input_metadata: InputMetadata,
311 - cache_events: Optional[List[torch.cuda.Event]],
303 ) -> torch.Tensor:
304 hidden_states = self.model(input_ids, positions, kv_caches,
314 - input_metadata, cache_events)
305 + input_metadata)
306 return hidden_states
307
308 def sample(
vllm/model_executor/models/phi_1_5.py
+2 -10
@@ -135,14 +135,12 @@ class PhiAttention(nn.Module):
135 hidden_states: torch.Tensor,
136 kv_cache: KVCache,
137 input_metadata: InputMetadata,
138 - cache_event: Optional[torch.cuda.Event],
138 ) -> torch.Tensor:
139 qkv, _ = self.Wqkv(hidden_states)
140 q, k, v = qkv.chunk(chunks=3, dim=-1)
141 q, k = self.rotary_emb(position_ids, q, k)
142 k_cache, v_cache = kv_cache
144 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
145 - cache_event)
143 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
144 output, _ = self.out_proj(attn_output)
145 return output
146
@@ -195,7 +193,6 @@ class PhiLayer(nn.Module):
193 hidden_states: torch.Tensor,
194 kv_cache: KVCache,
195 input_metadata: InputMetadata,
198 - cache_event: Optional[torch.cuda.Event],
196 ) -> torch.Tensor:
197 residual = hidden_states
198 hidden_states = self.ln(hidden_states)
@@ -204,7 +201,6 @@ class PhiLayer(nn.Module):
201 hidden_states=hidden_states,
202 kv_cache=kv_cache,
203 input_metadata=input_metadata,
207 - cache_event=cache_event,
204 )
205 feed_forward_hidden_states = self.mlp(hidden_states)
206 hidden_states = attn_outputs + feed_forward_hidden_states + residual
@@ -231,18 +227,15 @@ class PhiModel(nn.Module):
227 positions: torch.Tensor,
228 kv_caches: List[KVCache],
229 input_metadata: InputMetadata,
234 - cache_events: Optional[List[torch.cuda.Event]],
230 ) -> torch.Tensor:
231 hidden_states = self.embd(input_ids)
232 for i in range(self.config.num_hidden_layers):
238 - cache_event = None if cache_events is None else cache_events[i]
233 layer = self.h[i]
234 hidden_states = layer(
235 positions,
236 hidden_states,
237 kv_caches[i],
238 input_metadata,
245 - cache_event,
239 )
240 return hidden_states
241
@@ -277,10 +270,9 @@ class PhiForCausalLM(nn.Module):
270 positions: torch.Tensor,
271 kv_caches: List[KVCache],
272 input_metadata: InputMetadata,
280 - cache_events: Optional[List[torch.cuda.Event]],
273 ) -> torch.Tensor:
274 hidden_states = self.transformer(input_ids, positions, kv_caches,
283 - input_metadata, cache_events)
275 + input_metadata)
276 hidden_states = self.lm_head.ln(hidden_states)
277 return hidden_states
278
vllm/model_executor/models/qwen.py
+2 -10
@@ -112,14 +112,12 @@ class QWenAttention(nn.Module):
112 hidden_states: torch.Tensor,
113 kv_cache: KVCache,
114 input_metadata: InputMetadata,
115 - cache_event: Optional[torch.cuda.Event],
115 ) -> torch.Tensor:
116 qkv, _ = self.c_attn(hidden_states)
117 q, k, v = qkv.chunk(chunks=3, dim=-1)
118 q, k = self.rotary_emb(positions, q, k)
119 k_cache, v_cache = kv_cache
121 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
122 - cache_event)
120 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
121
122 output, _ = self.c_proj(attn_output)
123 return output
@@ -156,7 +154,6 @@ class QWenBlock(nn.Module):
154 hidden_states: torch.Tensor,
155 kv_cache: KVCache,
156 input_metadata: InputMetadata,
159 - cache_event: Optional[torch.cuda.Event],
157 residual: Optional[torch.Tensor],
158 ) -> Tuple[torch.Tensor, torch.Tensor]:
159 # Self Attention
@@ -170,7 +167,6 @@ class QWenBlock(nn.Module):
167 hidden_states=hidden_states,
168 kv_cache=kv_cache,
169 input_metadata=input_metadata,
173 - cache_event=cache_event,
170 )
171
172 # Fully Connected
@@ -206,19 +202,16 @@ class QWenModel(nn.Module):
202 positions: torch.Tensor,
203 kv_caches: List[KVCache],
204 input_metadata: InputMetadata,
209 - cache_events: Optional[List[torch.cuda.Event]],
205 ) -> torch.Tensor:
206 hidden_states = self.wte(input_ids)
207 residual = None
208 for i in range(len(self.h)):
214 - cache_event = None if cache_events is None else cache_events[i]
209 layer = self.h[i]
210 hidden_states, residual = layer(
211 positions,
212 hidden_states,
213 kv_caches[i],
214 input_metadata,
221 - cache_event,
215 residual,
216 )
217 hidden_states, _ = self.ln_f(hidden_states, residual)
@@ -245,10 +238,9 @@ class QWenLMHeadModel(nn.Module):
238 positions: torch.Tensor,
239 kv_caches: List[KVCache],
240 input_metadata: InputMetadata,
248 - cache_events: Optional[List[torch.cuda.Event]],
241 ) -> torch.Tensor:
242 hidden_states = self.transformer(input_ids, positions, kv_caches,
251 - input_metadata, cache_events)
243 + input_metadata)
244 return hidden_states
245
246 def sample(
vllm/model_executor/models/yi.py
+2 -10
@@ -146,14 +146,12 @@ class YiAttention(nn.Module):
146 hidden_states: torch.Tensor,
147 kv_cache: KVCache,
148 input_metadata: InputMetadata,
149 - cache_event: Optional[torch.cuda.Event],
149 ) -> torch.Tensor:
150 qkv, _ = self.qkv_proj(hidden_states)
151 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152 q, k = self.rotary_emb(positions, q, k)
153 k_cache, v_cache = kv_cache
155 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
156 - cache_event)
154 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155 output, _ = self.o_proj(attn_output)
156 return output
157
@@ -195,7 +193,6 @@ class YiDecoderLayer(nn.Module):
193 hidden_states: torch.Tensor,
194 kv_cache: KVCache,
195 input_metadata: InputMetadata,
198 - cache_event: Optional[torch.cuda.Event],
196 residual: Optional[torch.Tensor],
197 ) -> Tuple[torch.Tensor, torch.Tensor]:
198 # Self Attention
@@ -209,7 +206,6 @@ class YiDecoderLayer(nn.Module):
206 hidden_states=hidden_states,
207 kv_cache=kv_cache,
208 input_metadata=input_metadata,
212 - cache_event=cache_event,
209 )
210
211 # Fully Connected
@@ -245,19 +241,16 @@ class YiModel(nn.Module):
241 positions: torch.Tensor,
242 kv_caches: List[KVCache],
243 input_metadata: InputMetadata,
248 - cache_events: Optional[List[torch.cuda.Event]],
244 ) -> torch.Tensor:
245 hidden_states = self.embed_tokens(input_ids)
246 residual = None
247 for i in range(len(self.layers)):
253 - cache_event = None if cache_events is None else cache_events[i]
248 layer = self.layers[i]
249 hidden_states, residual = layer(
250 positions,
251 hidden_states,
252 kv_caches[i],
253 input_metadata,
260 - cache_event,
254 residual,
255 )
256 hidden_states, _ = self.norm(hidden_states, residual)
@@ -284,10 +277,9 @@ class YiForCausalLM(nn.Module):
277 positions: torch.Tensor,
278 kv_caches: List[KVCache],
279 input_metadata: InputMetadata,
287 - cache_events: Optional[List[torch.cuda.Event]],
280 ) -> torch.Tensor:
281 hidden_states = self.model(input_ids, positions, kv_caches,
290 - input_metadata, cache_events)
282 + input_metadata)
283 return hidden_states
284
285 def sample(
vllm/model_executor/parallel_utils/communication_op.py
+8 -2
@@ -1,8 +1,10 @@
1 import torch
2
3 +from vllm.model_executor.parallel_utils import cupy_utils
4 from vllm.model_executor.parallel_utils.parallel_state import (
5 get_tensor_model_parallel_world_size,
6 get_tensor_model_parallel_group,
7 + is_custom_nccl_enabled_for_all_reduce,
8 )
9
10
@@ -15,8 +17,12 @@ def tensor_model_parallel_all_reduce(input_):
17 if get_tensor_model_parallel_world_size() == 1:
18 return input_
19 # All-reduce.
18 - torch.distributed.all_reduce(input_,
19 - group=get_tensor_model_parallel_group())
20 + if is_custom_nccl_enabled_for_all_reduce():
21 + # TODO: support multiple parallel groups.
22 + cupy_utils.all_reduce(input_)
23 + else:
24 + torch.distributed.all_reduce(input_,
25 + group=get_tensor_model_parallel_group())
26 return input_
27
28
vllm/model_executor/parallel_utils/cupy_utils.py new
+115
@@ -0,0 +1,115 @@
1 +"""CuPy utilities for all-reduce.
2 +
3 +We use CuPy all-reduce instead of torch.distributed.all_reduce when capturing
4 +CUDA graphs, because torch.distributed.all_reduce causes errors when capturing
5 +CUDA graphs.
6 +
7 +TODO: Remove this file when torch.distributed.all_reduce is fixed.
8 +"""
9 +import contextlib
10 +
11 +import torch
12 +from torch.distributed import ReduceOp
13 +
14 +try:
15 + import cupy
16 + from cupyx.distributed import NCCLBackend
17 + from cupy.cuda import nccl
18 +except ImportError as e:
19 + cupy = e
20 + nccl = None
21 +
22 + class NCCLBackend:
23 + ...
24 +
25 +
26 +_OP_MAPPING = {
27 + ReduceOp.SUM: "sum",
28 + ReduceOp.PRODUCT: "prod",
29 + ReduceOp.MIN: "min",
30 + ReduceOp.MAX: "max",
31 +}
32 +
33 +
34 +class NCCLBackendWithBFloat16(NCCLBackend):
35 + # This is enough to add bfloat16 support for most operations,
36 + # but broadcast will fail (will require changes in compiled
37 + # cupy code).
38 + def _get_nccl_dtype_and_count(self, array, count=None):
39 + nccl_dtype, count = super()._get_nccl_dtype_and_count(array, count)
40 + torch_dtype = getattr(array, "_torch_dtype", None)
41 + if torch_dtype is torch.bfloat16:
42 + nccl_dtype = nccl.NCCL_BFLOAT16
43 + return nccl_dtype, count
44 +
45 +
46 +_NCCL_BACKEND = None
47 +_WORLD_SIZE = 0
48 +
49 +
50 +def is_initialized() -> bool:
51 + """Returns whether the NCCL backend is initialized."""
52 + return _NCCL_BACKEND is not None
53 +
54 +
55 +@contextlib.contextmanager
56 +def set_cupy_stream(stream: torch.cuda.Stream) -> None:
57 + """Set the cuda stream for communication"""
58 + cupy_stream = cupy.cuda.ExternalStream(stream.cuda_stream,
59 + stream.device_index)
60 + with cupy_stream:
61 + yield
62 +
63 +
64 +def init_process_group(world_size: int, rank: int, host: str,
65 + port: int) -> None:
66 + """Initializes the CuPy NCCL backend.
67 +
68 + # TODO: handle NCCL timeouts.
69 + """
70 + assert not is_initialized()
71 +
72 + if isinstance(cupy, Exception):
73 + raise ImportError(
74 + "NCCLBackend is not available. Please install cupy.") from cupy
75 +
76 + # TODO(woosuk): Create TP and PP process groups for CuPy.
77 + global _NCCL_BACKEND
78 + global _WORLD_SIZE
79 + assert world_size > 0, f"{world_size=} should be a positive integer"
80 + assert 0 <= rank < world_size, (
81 + f"{rank=} should be a integer between [0, {world_size})")
82 +
83 + cupy.cuda.runtime.setDevice(torch.cuda.current_device())
84 + _NCCL_BACKEND = NCCLBackendWithBFloat16(world_size, rank, host, port)
85 + _WORLD_SIZE = world_size
86 +
87 +
88 +def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
89 + """All-reduces the input tensor across the process group."""
90 + assert input_.is_cuda, f"{input_} should be a cuda tensor"
91 + # Hack to support bfloat16
92 + torch_dtype = input_.dtype
93 + if torch_dtype is torch.bfloat16:
94 + # We need to view as float16, otherwise
95 + # cupy will fail. This will not change
96 + # the underlying data.
97 + input_ = input_.view(torch.float16)
98 + cupy_input = cupy.asarray(input_)
99 + cupy_input._torch_dtype = torch_dtype # pylint: disable=protected-access
100 + _NCCL_BACKEND.all_reduce(in_array=cupy_input,
101 + out_array=cupy_input,
102 + op=_OP_MAPPING[op])
103 +
104 +
105 +def destroy_process_group() -> None:
106 + """Destroys the NCCL backend."""
107 + global _NCCL_BACKEND
108 + global _WORLD_SIZE
109 + _NCCL_BACKEND = None
110 + _WORLD_SIZE = 0
111 +
112 +
113 +def get_world_size() -> int:
114 + """Returns the world size."""
115 + return _WORLD_SIZE
vllm/model_executor/parallel_utils/parallel_state.py
+37
@@ -3,9 +3,12 @@
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 """Tensor and pipeline parallel groups."""
6 +import contextlib
7
8 import torch
9
10 +from vllm.model_executor.parallel_utils import cupy_utils
11 +
12 # Tensor model parallel group that the current rank belongs to.
13 _TENSOR_MODEL_PARALLEL_GROUP = None
14 # Pipeline model parallel group that the current rank belongs to.
@@ -177,3 +180,37 @@ def destroy_model_parallel():
180 _PIPELINE_MODEL_PARALLEL_GROUP = None
181 global _PIPELINE_GLOBAL_RANKS
182 _PIPELINE_GLOBAL_RANKS = None
183 +
184 + # Destroy the cupy states if any.
185 + cupy_utils.destroy_process_group()
186 +
187 +
188 +# Whether to use cupy for nccl all reduce.
189 +# We use cupy for all reduce when using CUDA graph, because torch.distributed
190 +# is not well supported by CUDA graph.
191 +_ENABLE_CUPY_FOR_ALL_REDUCE = False
192 +
193 +
194 +@contextlib.contextmanager
195 +def with_custom_nccl_for_all_reduce():
196 + """use custom nccl instead of torch.distributed for all reduce"""
197 + tp_size = get_tensor_model_parallel_world_size()
198 + if tp_size == 1:
199 + # No-op.
200 + # NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
201 + yield
202 + else:
203 + global _ENABLE_CUPY_FOR_ALL_REDUCE
204 + old = _ENABLE_CUPY_FOR_ALL_REDUCE
205 + _ENABLE_CUPY_FOR_ALL_REDUCE = True
206 +
207 + stream = torch.cuda.current_stream()
208 + with cupy_utils.set_cupy_stream(stream):
209 + yield
210 + _ENABLE_CUPY_FOR_ALL_REDUCE = old
211 +
212 +
213 +def is_custom_nccl_enabled_for_all_reduce():
214 + """check if custom nccl is enabled for all reduce"""
215 + global _ENABLE_CUPY_FOR_ALL_REDUCE
216 + return _ENABLE_CUPY_FOR_ALL_REDUCE
vllm/utils.py
+7
@@ -1,4 +1,5 @@
1 import enum
2 +import socket
3 import uuid
4 from platform import uname
5
@@ -52,3 +53,9 @@ def random_uuid() -> str:
53 def in_wsl() -> bool:
54 # Reference: https://github.com/microsoft/WSL/issues/4071
55 return "microsoft" in " ".join(uname()).lower()
56 +
57 +
58 +def get_open_port():
59 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
60 + s.bind(("", 0))
61 + return s.getsockname()[1]
vllm/worker/model_runner.py
+225 -16
@@ -1,16 +1,25 @@
1 -from typing import Dict, List, Optional, Tuple
1 +import time
2 +from typing import Dict, List, Tuple, Union
3
4 +import numpy as np
5 import torch
6 +import torch.nn as nn
7
8 from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
9 from vllm.logger import init_logger
10 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11 +from vllm.model_executor.parallel_utils.parallel_state import (
12 + with_custom_nccl_for_all_reduce)
13 from vllm.sampling_params import SamplingParams, SamplingType
14 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
16 logger = init_logger(__name__)
17
18 +KVCache = Tuple[torch.Tensor, torch.Tensor]
19 _PAD_SLOT_ID = -1
20 +# Capture graphs for batch size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
21 +# NOTE: _get_graph_batch_size needs to be updated if this list is changed.
22 +_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [8 * i for i in range(1, 33)]
23
24
25 class ModelRunner:
@@ -32,12 +41,31 @@ class ModelRunner:
41 self.model = None
42 self.block_size = None # Set after initial profiling.
43
44 + self.graph_runners: Dict[int, CUDAGraphRunner] = {}
45 + self.graph_memory_pool = None # Set during graph capture.
46 +
47 + self.max_context_len_to_capture = (
48 + self.model_config.max_context_len_to_capture
49 + if self.model_config is not None else 0)
50 + # When using CUDA graph, the input block tables must be padded to
51 + # max_context_len_to_capture. However, creating the block table in
52 + # Python can be expensive. To optimize this, we cache the block table
53 + # in numpy and only copy the actual input content at every iteration.
54 + # The shape of the cached block table will be
55 + # (max batch size to capture, max context len to capture / block size).
56 + self.graph_block_tables = None # Set after initial profiling.
57 +
58 def load_model(self) -> None:
59 self.model = get_model(self.model_config)
60
61 def set_block_size(self, block_size: int) -> None:
62 self.block_size = block_size
63
64 + max_num_blocks = (self.max_context_len_to_capture + block_size -
65 + 1) // block_size
66 + self.graph_block_tables = np.zeros(
67 + (max(_BATCH_SIZES_TO_CAPTURE), max_num_blocks), dtype=np.int32)
68 +
69 def _prepare_prompt(
70 self,
71 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -111,6 +139,7 @@ class ModelRunner:
139 max_context_len=None,
140 context_lens=None,
141 block_tables=None,
142 + use_cuda_graph=False,
143 )
144 return input_tokens, input_positions, input_metadata
145
@@ -154,27 +183,62 @@ class ModelRunner:
183 block_table = block_table[-sliding_window_blocks:]
184 block_tables.append(block_table)
185
186 + batch_size = len(input_tokens)
187 + max_context_len = max(context_lens)
188 + use_captured_graph = (
189 + not self.model_config.enforce_eager
190 + and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
191 + and max_context_len <= self.max_context_len_to_capture)
192 + if use_captured_graph:
193 + # Pad the input tokens, positions, and slot mapping to match the
194 + # batch size of the captured graph.
195 + graph_batch_size = _get_graph_batch_size(batch_size)
196 + assert graph_batch_size >= batch_size
197 + for _ in range(graph_batch_size - batch_size):
198 + input_tokens.append([])
199 + input_positions.append([])
200 + slot_mapping.append([])
201 + context_lens.append(1)
202 + block_tables.append([])
203 + batch_size = graph_batch_size
204 +
205 + # When using CUDA graph, we don't need to make the tensors on the GPU
206 + # because they will be eventually copied to the designated GPU buffer.
207 + device = "cpu" if use_captured_graph else "cuda"
208 input_tokens = _make_tensor_with_pad(input_tokens,
209 max_len=1,
210 pad=0,
160 - dtype=torch.long)
211 + dtype=torch.long,
212 + device=device)
213 input_positions = _make_tensor_with_pad(input_positions,
214 max_len=1,
215 pad=0,
164 - dtype=torch.long)
216 + dtype=torch.long,
217 + device=device)
218 slot_mapping = _make_tensor_with_pad(slot_mapping,
219 max_len=1,
220 pad=_PAD_SLOT_ID,
168 - dtype=torch.long)
169 - max_context_len = max(context_lens)
221 + dtype=torch.long,
222 + device=device)
223 context_lens = torch.tensor(context_lens,
224 dtype=torch.int,
172 - device="cuda")
173 - max_block_table_len = max([len(t) for t in block_tables])
174 - block_tables = _make_tensor_with_pad(block_tables,
175 - max_len=max_block_table_len,
176 - pad=0,
177 - dtype=torch.int)
225 + device=device)
226 +
227 + if use_captured_graph:
228 + # The shape of graph_block_tables is
229 + # [max batch size, max context len // block size].
230 + input_block_tables = self.graph_block_tables[:batch_size]
231 + for i, block_table in enumerate(block_tables):
232 + if block_table:
233 + input_block_tables[i, :len(block_table)] = block_table
234 + block_tables = torch.from_numpy(input_block_tables).to(device)
235 + else:
236 + block_tables = _make_tensor_with_pad(
237 + block_tables,
238 + max_len=max_context_len,
239 + pad=0,
240 + dtype=torch.int,
241 + )
242
243 input_metadata = InputMetadata(
244 prompt_lens=[],
@@ -182,6 +246,7 @@ class ModelRunner:
246 max_context_len=max_context_len,
247 context_lens=context_lens,
248 block_tables=block_tables,
249 + use_cuda_graph=use_captured_graph,
250 )
251 return input_tokens, input_positions, input_metadata
252
@@ -260,12 +325,11 @@ class ModelRunner:
325 self,
326 seq_group_metadata_list: List[SequenceGroupMetadata],
327 kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
263 - cache_events: Optional[List[torch.cuda.Event]] = None,
328 ) -> SamplerOutput:
329 # NOTE: We assume that all sequences in the group are all prompts or
330 # all decodes.
267 - # Prepare input tensors.
331 is_prompt = seq_group_metadata_list[0].is_prompt
332 + # Prepare input tensors.
333 if is_prompt:
334 inputs = self._prepare_prompt(seq_group_metadata_list)
335 input_tokens, input_positions, input_metadata = inputs
@@ -276,12 +340,16 @@ class ModelRunner:
340 input_metadata.prompt_lens)
341
342 # Execute the model.
279 - hidden_states = self.model(
343 + if input_metadata.use_cuda_graph:
344 + graph_batch_size = input_tokens.shape[0]
345 + model_executable = self.graph_runners[graph_batch_size]
346 + else:
347 + model_executable = self.model
348 + hidden_states = model_executable(
349 input_ids=input_tokens,
350 positions=input_positions,
351 kv_caches=kv_caches,
352 input_metadata=input_metadata,
284 - cache_events=cache_events,
353 )
354
355 # Sample the next token.
@@ -319,8 +387,139 @@ class ModelRunner:
387 num_layers = self.model_config.get_num_layers(self.parallel_config)
388 kv_caches = [(None, None)] * num_layers
389 self.execute_model(seqs, kv_caches)
390 + torch.cuda.synchronize()
391 return
392
393 + @torch.inference_mode()
394 + def capture_model(self, kv_caches: List[KVCache]) -> None:
395 + assert not self.model_config.enforce_eager
396 + logger.info("Capturing the model for CUDA graphs. This may lead to "
397 + "unexpected consequences if the model is not static. To "
398 + "run the model in eager mode, set 'enforce_eager=True' or "
399 + "use '--enforce-eager' in the CLI.")
400 + start_time = time.perf_counter()
401 +
402 + # Prepare dummy inputs. These will be reused for all batch sizes.
403 + max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
404 + input_tokens = torch.zeros(max_batch_size, 1, dtype=torch.long).cuda()
405 + input_positions = torch.zeros(max_batch_size, 1,
406 + dtype=torch.long).cuda()
407 + slot_mapping = torch.empty(max_batch_size, 1, dtype=torch.long).cuda()
408 + slot_mapping.fill_(_PAD_SLOT_ID)
409 + context_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
410 + block_tables = torch.from_numpy(self.graph_block_tables).cuda()
411 +
412 + # NOTE: Capturing the largest batch size first may help reduce the
413 + # memory usage of CUDA graph.
414 + for batch_size in reversed(_BATCH_SIZES_TO_CAPTURE):
415 + # Create dummy input_metadata.
416 + input_metadata = InputMetadata(
417 + prompt_lens=[],
418 + slot_mapping=slot_mapping[:batch_size],
419 + max_context_len=self.max_context_len_to_capture,
420 + context_lens=context_lens[:batch_size],
421 + block_tables=block_tables[:batch_size],
422 + use_cuda_graph=True,
423 + )
424 +
425 + graph_runner = CUDAGraphRunner(self.model)
426 + graph_runner.capture(
427 + input_tokens[:batch_size],
428 + input_positions[:batch_size],
429 + kv_caches,
430 + input_metadata,
431 + memory_pool=self.graph_memory_pool,
432 + )
433 + self.graph_memory_pool = graph_runner.graph.pool()
434 + self.graph_runners[batch_size] = graph_runner
435 +
436 + end_time = time.perf_counter()
437 + elapsed_time = end_time - start_time
438 + # This usually takes < 10 seconds.
439 + logger.info(f"Graph capturing finished in {elapsed_time:.0f} secs.")
440 +
441 +
442 +class CUDAGraphRunner:
443 +
444 + def __init__(self, model: nn.Module):
445 + self.model = model
446 + self.graph = None
447 + self.input_buffers: Dict[str, torch.Tensor] = {}
448 + self.output_buffers: Dict[str, torch.Tensor] = {}
449 +
450 + def capture(
451 + self,
452 + input_ids: torch.Tensor,
453 + positions: torch.Tensor,
454 + kv_caches: List[KVCache],
455 + input_metadata: InputMetadata,
456 + memory_pool,
457 + ) -> None:
458 + assert self.graph is None
459 + # Run the model once without capturing the graph.
460 + # This is to make sure that the captured graph does not include the
461 + # kernel launches for initial benchmarking (e.g., Triton autotune).
462 + with with_custom_nccl_for_all_reduce():
463 + self.model(
464 + input_ids,
465 + positions,
466 + kv_caches,
467 + input_metadata,
468 + )
469 + torch.cuda.synchronize()
470 +
471 + # Capture the graph.
472 + # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
473 + # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
474 + self.graph = torch.cuda.CUDAGraph()
475 + with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
476 + with with_custom_nccl_for_all_reduce():
477 + hidden_states = self.model(
478 + input_ids,
479 + positions,
480 + kv_caches,
481 + input_metadata,
482 + )
483 + torch.cuda.synchronize()
484 +
485 + # Save the input and output buffers.
486 + self.input_buffers = {
487 + "input_ids": input_ids,
488 + "positions": positions,
489 + "kv_caches": kv_caches,
490 + "slot_mapping": input_metadata.slot_mapping,
491 + "context_lens": input_metadata.context_lens,
492 + "block_tables": input_metadata.block_tables,
493 + }
494 + self.output_buffers = {"hidden_states": hidden_states}
495 + return
496 +
497 + def forward(
498 + self,
499 + input_ids: torch.Tensor,
500 + positions: torch.Tensor,
501 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
502 + input_metadata: InputMetadata,
503 + ) -> torch.Tensor:
504 + # KV caches are fixed tensors, so we don't need to copy them.
505 + del kv_caches
506 +
507 + # Copy the input tensors to the input buffers.
508 + self.input_buffers["input_ids"].copy_(input_ids)
509 + self.input_buffers["positions"].copy_(positions)
510 + self.input_buffers["slot_mapping"].copy_(input_metadata.slot_mapping)
511 + self.input_buffers["context_lens"].copy_(input_metadata.context_lens)
512 + self.input_buffers["block_tables"].copy_(input_metadata.block_tables)
513 +
514 + # Run the graph.
515 + self.graph.replay()
516 +
517 + # Return the output tensor.
518 + return self.output_buffers["hidden_states"]
519 +
520 + def __call__(self, *args, **kwargs):
521 + return self.forward(*args, **kwargs)
522 +
523
524 def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
525 assert len(x) <= max_len
@@ -332,6 +531,16 @@ def _make_tensor_with_pad(
531 max_len: int,
532 pad: int,
533 dtype: torch.dtype,
534 + device: Union[str, torch.device] = "cuda",
535 ) -> torch.Tensor:
536 padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
337 - return torch.tensor(padded_x, dtype=dtype, device="cuda")
537 + return torch.tensor(padded_x, dtype=dtype, device=device)
538 +
539 +
540 +def _get_graph_batch_size(batch_size: int) -> int:
541 + if batch_size <= 2:
542 + return batch_size
543 + elif batch_size <= 4:
544 + return 4
545 + else:
546 + return (batch_size + 7) // 8 * 8
vllm/worker/worker.py
+40 -12
@@ -8,6 +8,7 @@ import torch.distributed
8 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig)
10 from vllm.model_executor import set_random_seed
11 +from vllm.model_executor.parallel_utils import cupy_utils
12 from vllm.model_executor.parallel_utils.parallel_state import (
13 initialize_model_parallel)
14 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -46,7 +47,7 @@ class Worker:
47 self.cache_events = None
48 self.gpu_cache = None
49
49 - def init_model(self):
50 + def init_model(self, cupy_port: Optional[int] = None):
51 # This env var set by Ray causes exceptions with graph building.
52 os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
53 # Env vars will be set by Ray.
@@ -62,7 +63,7 @@ class Worker:
63
64 # Initialize the distributed environment.
65 _init_distributed_environment(self.parallel_config, self.rank,
65 - self.distributed_init_method)
66 + cupy_port, self.distributed_init_method)
67
68 # Initialize the model.
69 set_random_seed(self.model_config.seed)
@@ -100,10 +101,6 @@ class Worker:
101 num_gpu_blocks = max(num_gpu_blocks, 0)
102 num_cpu_blocks = max(num_cpu_blocks, 0)
103 torch.cuda.empty_cache()
103 -
104 - # Reset the seed to ensure that the random state is not affected by
105 - # the model initialization and profiling.
106 - set_random_seed(self.model_config.seed)
104 return num_gpu_blocks, num_cpu_blocks
105
106 def init_cache_engine(self, cache_config: CacheConfig) -> None:
@@ -114,6 +111,13 @@ class Worker:
111 self.gpu_cache = self.cache_engine.gpu_cache
112 self.model_runner.set_block_size(self.cache_engine.block_size)
113
114 + def warm_up_model(self) -> None:
115 + if not self.model_config.enforce_eager:
116 + self.model_runner.capture_model(self.gpu_cache)
117 + # Reset the seed to ensure that the random state is not affected by
118 + # the model initialization and profiling.
119 + set_random_seed(self.model_config.seed)
120 +
121 @torch.inference_mode()
122 def execute_model(
123 self,
@@ -136,21 +140,24 @@ class Worker:
140
141 cache_events = self.cache_events if issued_cache_op else None
142
143 + # Wait for cache operations to finish.
144 + # TODO(woosuk): Profile swapping overhead and optimize if needed.
145 + if cache_events is not None:
146 + for event in cache_events:
147 + event.wait()
148 # If there is no input, we don't need to execute the model.
149 if not seq_group_metadata_list:
141 - if cache_events is not None:
142 - for event in cache_events:
143 - event.wait()
150 return {}
151
152 output = self.model_runner.execute_model(seq_group_metadata_list,
147 - self.gpu_cache, cache_events)
153 + self.gpu_cache)
154 return output
155
156
157 def _init_distributed_environment(
158 parallel_config: ParallelConfig,
159 rank: int,
160 + cupy_port: Optional[int],
161 distributed_init_method: Optional[str] = None,
162 ) -> None:
163 """Initialize the distributed environment."""
@@ -173,8 +180,29 @@ def _init_distributed_environment(
180 init_method=distributed_init_method,
181 )
182
176 - # A small all_reduce for warmup.
177 - torch.distributed.all_reduce(torch.zeros(1).cuda())
183 + if cupy_utils.is_initialized():
184 + cupy_world_size = cupy_utils.get_world_size()
185 + if cupy_world_size != parallel_config.world_size:
186 + raise RuntimeError(
187 + "cupy.distributed is already initialized but the cupy world "
188 + "size does not match parallel_config.world_size "
189 + f"({cupy_world_size} vs. {parallel_config.world_size}).")
190 + elif parallel_config.world_size > 1:
191 + # NOTE(woosuk): We don't initialize CuPy process group when world size
192 + # is 1.
193 + # TODO(woosuk): Support multi-node connection.
194 + cupy_utils.init_process_group(
195 + world_size=parallel_config.world_size,
196 + rank=rank,
197 + host="localhost",
198 + port=cupy_port,
199 + )
200 +
201 + if parallel_config.world_size > 1:
202 + # A small all_reduce for warmup.
203 + torch.distributed.all_reduce(torch.zeros(1).cuda())
204 + cupy_utils.all_reduce(torch.zeros(1).cuda())
205 +
206 initialize_model_parallel(parallel_config.tensor_parallel_size,
207 parallel_config.pipeline_parallel_size)
208