A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Optimize model execution with CUDA graph (#1926)
Co-authored-by: Chen Shen <scv119@gmail.com> Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Woosuk Kwon committed
Dec 16, 2023 at 21:12 UTC
37ca5581039271d4bf69e5cb1f1ec8e04775777c
34 files changed
+557
-254
benchmarks/benchmark_latency.py
+4
@@ -23,6 +23,7 @@ def main(args: argparse.Namespace):
23
tensor_parallel_size=args.tensor_parallel_size,
24
trust_remote_code=args.trust_remote_code,
25
dtype=args.dtype,
26
+ enforce_eager=args.enforce_eager,
27
)
28
29
sampling_params = SamplingParams(
@@ -111,6 +112,9 @@ if __name__ == '__main__':
112
'The "auto" option will use FP16 precision '
113
'for FP32 and FP16 models, and BF16 precision '
114
'for BF16 models.')
115
+ parser.add_argument('--enforce-eager',
116
+ action='store_true',
117
+ help='enforce eager mode and disable CUDA graph')
118
parser.add_argument(
119
'--profile',
120
action='store_true',
benchmarks/benchmark_throughput.py
+7
-2
@@ -69,7 +69,8 @@ def run_vllm(
69
use_beam_search: bool,
70
trust_remote_code: bool,
71
dtype: str,
72
- max_model_len: Optional[int] = None,
72
+ max_model_len: Optional[int],
73
+ enforce_eager: bool,
74
) -> float:
75
from vllm import LLM, SamplingParams
76
llm = LLM(
@@ -81,6 +82,7 @@ def run_vllm(
82
trust_remote_code=trust_remote_code,
83
dtype=dtype,
84
max_model_len=max_model_len,
85
+ enforce_eager=enforce_eager,
86
)
87
88
# Add the requests to the engine.
@@ -204,7 +206,7 @@ def main(args: argparse.Namespace):
206
args.quantization, args.tensor_parallel_size,
207
args.seed, args.n, args.use_beam_search,
208
args.trust_remote_code, args.dtype,
207
- args.max_model_len)
209
+ args.max_model_len, args.enforce_eager)
210
elif args.backend == "hf":
211
assert args.tensor_parallel_size == 1
212
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -279,6 +281,9 @@ if __name__ == "__main__":
281
'The "auto" option will use FP16 precision '
282
'for FP32 and FP16 models, and BF16 precision '
283
'for BF16 models.')
284
+ parser.add_argument("--enforce-eager",
285
+ action="store_true",
286
+ help="enforce eager execution")
287
args = parser.parse_args()
288
if args.tokenizer is None:
289
args.tokenizer = args.model
requirements.txt
+1
@@ -12,3 +12,4 @@ fastapi
12
uvicorn[standard]
13
pydantic == 1.10.13 # Required for OpenAI server.
14
aioprometheus[starlette]
15
+cupy-cuda12x # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead. # FIXME: Fix this in setup.py.
vllm/config.py
+17
@@ -49,6 +49,12 @@ class ModelConfig:
49
output). If None, will be derived from the model.
50
quantization: Quantization method that was used to quantize the model
51
weights. If None, we assume the model weights are not quantized.
52
+ enforce_eager: Whether to enforce eager execution. If True, we will
53
+ disable CUDA graph and always execute the model in eager mode.
54
+ If False, we will use CUDA graph and eager execution in hybrid.
55
+ max_context_len_to_capture: Maximum context len covered by CUDA graphs.
56
+ When a sequence has context length larger than this, we fall back
57
+ to eager mode.
58
"""
59
60
def __init__(
@@ -65,6 +71,8 @@ class ModelConfig:
71
tokenizer_revision: Optional[str] = None,
72
max_model_len: Optional[int] = None,
73
quantization: Optional[str] = None,
74
+ enforce_eager: bool = False,
75
+ max_context_len_to_capture: Optional[int] = None,
76
) -> None:
77
self.model = model
78
self.tokenizer = tokenizer
@@ -76,6 +84,8 @@ class ModelConfig:
84
self.revision = revision
85
self.tokenizer_revision = tokenizer_revision
86
self.quantization = quantization
87
+ self.enforce_eager = enforce_eager
88
+ self.max_context_len_to_capture = max_context_len_to_capture
89
90
if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
91
# download model from ModelScope hub,
@@ -95,6 +105,7 @@ class ModelConfig:
105
self._verify_load_format()
106
self._verify_tokenizer_mode()
107
self._verify_quantization()
108
+ self._verify_cuda_graph()
109
110
def _verify_load_format(self) -> None:
111
load_format = self.load_format.lower()
@@ -169,6 +180,12 @@ class ModelConfig:
180
"optimized yet. The speed can be slower than "
181
"non-quantized models.")
182
183
+ def _verify_cuda_graph(self) -> None:
184
+ if self.max_context_len_to_capture is None:
185
+ self.max_context_len_to_capture = self.max_model_len
186
+ self.max_context_len_to_capture = min(self.max_context_len_to_capture,
187
+ self.max_model_len)
188
+
189
def verify_with_parallel_config(
190
self,
191
parallel_config: "ParallelConfig",
vllm/engine/arg_utils.py
+15
-1
@@ -33,6 +33,8 @@ class EngineArgs:
33
revision: Optional[str] = None
34
tokenizer_revision: Optional[str] = None
35
quantization: Optional[str] = None
36
+ enforce_eager: bool = False
37
+ max_context_len_to_capture: int = 8192
38
39
def __post_init__(self):
40
if self.tokenizer is None:
@@ -182,6 +184,17 @@ class EngineArgs:
184
choices=['awq', 'gptq', 'squeezellm', None],
185
default=None,
186
help='Method used to quantize the weights')
187
+ parser.add_argument('--enforce-eager',
188
+ action='store_true',
189
+ help='Always use eager-mode PyTorch. If False, '
190
+ 'will use eager mode and CUDA graph in hybrid '
191
+ 'for maximal performance and flexibility.')
192
+ parser.add_argument('--max-context-len-to-capture',
193
+ type=int,
194
+ default=EngineArgs.max_context_len_to_capture,
195
+ help='maximum context length covered by CUDA '
196
+ 'graphs. When a sequence has context length '
197
+ 'larger than this, we fall back to eager mode.')
198
return parser
199
200
@classmethod
@@ -200,7 +213,8 @@ class EngineArgs:
213
self.download_dir, self.load_format,
214
self.dtype, self.seed, self.revision,
215
self.tokenizer_revision, self.max_model_len,
203
- self.quantization)
216
+ self.quantization, self.enforce_eager,
217
+ self.max_context_len_to_capture)
218
cache_config = CacheConfig(self.block_size,
219
self.gpu_memory_utilization,
220
self.swap_space,
vllm/engine/llm_engine.py
+6
-1
@@ -17,7 +17,7 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
17
SequenceOutput, SequenceStatus)
18
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
19
get_tokenizer)
20
-from vllm.utils import Counter
20
+from vllm.utils import Counter, get_open_port
21
22
if ray:
23
from ray.air.util.torch_dist import init_torch_dist_process_group
@@ -84,6 +84,7 @@ class LLMEngine:
84
f"load_format={model_config.load_format}, "
85
f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
86
f"quantization={model_config.quantization}, "
87
+ f"enforce_eager={model_config.enforce_eager}, "
88
f"seed={model_config.seed})")
89
# TODO(woosuk): Print more configs in debug mode.
90
@@ -189,6 +190,7 @@ class LLMEngine:
190
))
191
self._run_workers(
192
"init_model",
193
+ cupy_port=get_open_port(),
194
get_all_outputs=True,
195
)
196
self._run_workers(
@@ -232,6 +234,9 @@ class LLMEngine:
234
235
# Initialize the cache.
236
self._run_workers("init_cache_engine", cache_config=self.cache_config)
237
+ # Warm up the model. This includes capturing the model into CUDA graph
238
+ # if enforce_eager is False.
239
+ self._run_workers("warm_up_model")
240
241
@classmethod
242
def from_engine_args(cls, engine_args: EngineArgs) -> "LLMEngine":
vllm/engine/ray_utils.py
+1
-8
@@ -1,9 +1,8 @@
1
-import socket
1
from typing import Optional, Tuple, TYPE_CHECKING
2
3
from vllm.config import ParallelConfig
4
from vllm.logger import init_logger
6
-from vllm.utils import is_hip
5
+from vllm.utils import get_open_port, is_hip
6
7
logger = init_logger(__name__)
8
@@ -43,12 +42,6 @@ if TYPE_CHECKING:
42
from ray.util.placement_group import PlacementGroup
43
44
46
-def get_open_port():
47
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
48
- s.bind(("", 0))
49
- return s.getsockname()[1]
50
-
51
-
45
def initialize_cluster(
46
parallel_config: ParallelConfig,
47
engine_use_ray: bool = False,
vllm/entrypoints/llm.py
+10
@@ -56,6 +56,12 @@ class LLM:
56
when their `best_of` sampling parameters are larger than 1. If all
57
requests will have `best_of=1`, you can safely set this to 0.
58
Otherwise, too small values may cause out-of-memory (OOM) errors.
59
+ enforce_eager: Whether to enforce eager execution. If True, we will
60
+ disable CUDA graph and always execute the model in eager mode.
61
+ If False, we will use CUDA graph and eager execution in hybrid.
62
+ max_context_len_to_capture: Maximum context len covered by CUDA graphs.
63
+ When a sequence has context length larger than this, we fall back
64
+ to eager mode.
65
"""
66
67
def __init__(
@@ -72,6 +78,8 @@ class LLM:
78
seed: int = 0,
79
gpu_memory_utilization: float = 0.9,
80
swap_space: int = 4,
81
+ enforce_eager: bool = False,
82
+ max_context_len_to_capture: int = 8192,
83
**kwargs,
84
) -> None:
85
if "disable_log_stats" not in kwargs:
@@ -89,6 +97,8 @@ class LLM:
97
seed=seed,
98
gpu_memory_utilization=gpu_memory_utilization,
99
swap_space=swap_space,
100
+ enforce_eager=enforce_eager,
101
+ max_context_len_to_capture=max_context_len_to_capture,
102
**kwargs,
103
)
104
self.llm_engine = LLMEngine.from_engine_args(engine_args)
vllm/model_executor/input_metadata.py
+4
-1
@@ -21,12 +21,14 @@ class InputMetadata:
21
max_context_len: Optional[int],
22
context_lens: Optional[torch.Tensor],
23
block_tables: Optional[torch.Tensor],
24
+ use_cuda_graph: bool,
25
) -> None:
26
self.prompt_lens = prompt_lens
27
self.max_context_len = max_context_len
28
self.slot_mapping = slot_mapping
29
self.context_lens = context_lens
30
self.block_tables = block_tables
31
+ self.use_cuda_graph = use_cuda_graph
32
33
self.is_prompt = len(prompt_lens) > 0
34
# Set during the execution of the first attention op.
@@ -39,4 +41,5 @@ class InputMetadata:
41
f"max_context_len={self.max_context_len}, "
42
f"slot_mapping={self.slot_mapping}, "
43
f"context_lens={self.context_lens}, "
42
- f"block_tables={self.block_tables})")
44
+ f"block_tables={self.block_tables}, "
45
+ f"use_cuda_graph={self.use_cuda_graph})")
vllm/model_executor/layers/attention.py
+18
-22
@@ -24,13 +24,10 @@ class PagedAttention(nn.Module):
24
can either contain prompt tokens or generation tokens.
25
The class does the following:
26
27
- 1. Wait for the cache operations (e.g., swap, copy) to finish. The cache
28
- operations are issued by the cache engine before executing the forward
29
- pass of the model, and they are executed asynchronously.
30
- 2. Reshape and store the input key and value tensors in the KV cache.
31
- 3. Perform (multi-head/multi-query/grouped-query) attention using either
27
+ 1. Reshape and store the input key and value tensors in the KV cache.
28
+ 2. Perform (multi-head/multi-query/grouped-query) attention using either
29
xformers or the PagedAttention custom op.
33
- 4. Return the output tensor.
30
+ 3. Return the output tensor.
31
"""
32
33
def __init__(
@@ -67,7 +64,6 @@ class PagedAttention(nn.Module):
64
key_cache: Optional[torch.Tensor],
65
value_cache: Optional[torch.Tensor],
66
input_metadata: InputMetadata,
70
- cache_event: Optional[torch.cuda.Event],
67
) -> torch.Tensor:
68
"""PagedAttention forward pass.
69
@@ -80,7 +76,6 @@ class PagedAttention(nn.Module):
76
value_cache: shape = [num_blocks, num_kv_heads, head_size,
77
block_size]
78
input_metadata: metadata for the inputs.
83
- cache_event: event to wait for the cache operations to finish.
79
Returns:
80
shape = [batch_size, seq_len, num_heads * head_size]
81
"""
@@ -89,10 +84,6 @@ class PagedAttention(nn.Module):
84
query = query.view(-1, self.num_heads, self.head_size)
85
key = key.view(-1, self.num_kv_heads, self.head_size)
86
value = value.view(-1, self.num_kv_heads, self.head_size)
92
- slot_mapping = input_metadata.slot_mapping.flatten()
93
-
94
- if cache_event is not None:
95
- cache_event.wait()
87
88
# Reshape the keys and values and store them in the cache.
89
# If key_cache and value_cache are not provided, the new key and value
@@ -104,7 +95,7 @@ class PagedAttention(nn.Module):
95
value,
96
key_cache,
97
value_cache,
107
- slot_mapping,
98
+ input_metadata.slot_mapping.flatten(),
99
)
100
101
if input_metadata.is_prompt:
@@ -165,15 +156,20 @@ class PagedAttention(nn.Module):
156
output = out.view_as(query)
157
else:
158
# Decoding run.
168
- output = _paged_attention(
169
- query,
170
- key_cache,
171
- value_cache,
172
- input_metadata,
173
- self.num_kv_heads,
174
- self.scale,
175
- self.alibi_slopes,
176
- )
159
+ if key_cache is not None and value_cache is not None:
160
+ output = _paged_attention(
161
+ query,
162
+ key_cache,
163
+ value_cache,
164
+ input_metadata,
165
+ self.num_kv_heads,
166
+ self.scale,
167
+ self.alibi_slopes,
168
+ )
169
+ else:
170
+ # This happens during the initial memory profiling run for
171
+ # CUDA graphs.
172
+ output = torch.zeros_like(query)
173
174
# Reshape the output tensor.
175
return output.view(batch_size, seq_len, hidden_size)
vllm/model_executor/models/aquila.py
+2
-10
@@ -158,14 +158,12 @@ class AquilaAttention(nn.Module):
158
hidden_states: torch.Tensor,
159
kv_cache: KVCache,
160
input_metadata: InputMetadata,
161
- cache_event: Optional[torch.cuda.Event],
161
) -> torch.Tensor:
162
qkv, _ = self.qkv_proj(hidden_states)
163
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164
q, k = self.rotary_emb(positions, q, k)
165
k_cache, v_cache = kv_cache
167
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
168
- cache_event)
166
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
167
output, _ = self.o_proj(attn_output)
168
return output
169
@@ -209,7 +207,6 @@ class AquilaDecoderLayer(nn.Module):
207
hidden_states: torch.Tensor,
208
kv_cache: KVCache,
209
input_metadata: InputMetadata,
212
- cache_event: Optional[torch.cuda.Event],
210
) -> torch.Tensor:
211
# Self Attention
212
residual = hidden_states
@@ -219,7 +216,6 @@ class AquilaDecoderLayer(nn.Module):
216
hidden_states=hidden_states,
217
kv_cache=kv_cache,
218
input_metadata=input_metadata,
222
- cache_event=cache_event,
219
)
220
hidden_states = residual + hidden_states
221
@@ -258,18 +254,15 @@ class AquilaModel(nn.Module):
254
positions: torch.Tensor,
255
kv_caches: List[KVCache],
256
input_metadata: InputMetadata,
261
- cache_events: Optional[List[torch.cuda.Event]],
257
) -> torch.Tensor:
258
hidden_states = self.embed_tokens(input_ids)
259
for i in range(len(self.layers)):
265
- cache_event = None if cache_events is None else cache_events[i]
260
layer = self.layers[i]
261
hidden_states = layer(
262
positions,
263
hidden_states,
264
kv_caches[i],
265
input_metadata,
272
- cache_event,
266
)
267
hidden_states = self.norm(hidden_states)
268
@@ -296,10 +289,9 @@ class AquilaForCausalLM(nn.Module):
289
positions: torch.Tensor,
290
kv_caches: List[KVCache],
291
input_metadata: InputMetadata,
299
- cache_events: Optional[List[torch.cuda.Event]],
292
) -> torch.Tensor:
293
hidden_states = self.model(input_ids, positions, kv_caches,
302
- input_metadata, cache_events)
294
+ input_metadata)
295
return hidden_states
296
297
def sample(
vllm/model_executor/models/baichuan.py
+2
-10
@@ -172,15 +172,13 @@ class BaiChuanAttention(nn.Module):
172
hidden_states: torch.Tensor,
173
kv_cache: KVCache,
174
input_metadata: InputMetadata,
175
- cache_event: Optional[torch.cuda.Event],
175
) -> torch.Tensor:
176
qkv, _ = self.W_pack(hidden_states)
177
q, k, v = qkv.chunk(chunks=3, dim=-1)
178
if self.postion_embedding != "ALIBI":
179
q, k = self.rotary_emb(positions, q, k)
180
k_cache, v_cache = kv_cache
182
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
183
- cache_event)
181
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
182
output, _ = self.o_proj(attn_output)
183
return output
184
@@ -221,7 +219,6 @@ class BaiChuanDecoderLayer(nn.Module):
219
hidden_states: torch.Tensor,
220
kv_cache: KVCache,
221
input_metadata: InputMetadata,
224
- cache_event: Optional[torch.cuda.Event],
222
residual: Optional[torch.Tensor],
223
) -> Tuple[torch.Tensor, torch.Tensor]:
224
# Self Attention
@@ -236,7 +233,6 @@ class BaiChuanDecoderLayer(nn.Module):
233
hidden_states=hidden_states,
234
kv_cache=kv_cache,
235
input_metadata=input_metadata,
239
- cache_event=cache_event,
236
)
237
238
# Fully Connected
@@ -273,19 +269,16 @@ class BaiChuanModel(nn.Module):
269
positions: torch.Tensor,
270
kv_caches: List[KVCache],
271
input_metadata: InputMetadata,
276
- cache_events: Optional[List[torch.cuda.Event]],
272
) -> torch.Tensor:
273
hidden_states = self.embed_tokens(input_ids)
274
residual = None
275
for i in range(len(self.layers)):
281
- cache_event = None if cache_events is None else cache_events[i]
276
layer = self.layers[i]
277
hidden_states, residual = layer(
278
positions,
279
hidden_states,
280
kv_caches[i],
281
input_metadata,
288
- cache_event,
282
residual,
283
)
284
hidden_states, _ = self.norm(hidden_states, residual)
@@ -311,10 +304,9 @@ class BaiChuanBaseForCausalLM(nn.Module):
304
positions: torch.Tensor,
305
kv_caches: List[KVCache],
306
input_metadata: InputMetadata,
314
- cache_events: Optional[List[torch.cuda.Event]],
307
) -> torch.Tensor:
308
hidden_states = self.model(input_ids, positions, kv_caches,
317
- input_metadata, cache_events)
309
+ input_metadata)
310
return hidden_states
311
312
def sample(
vllm/model_executor/models/bloom.py
+2
-10
@@ -118,14 +118,12 @@ class BloomAttention(nn.Module):
118
hidden_states: torch.Tensor,
119
kv_cache: KVCache,
120
input_metadata: InputMetadata,
121
- cache_event: Optional[torch.cuda.Event],
121
) -> torch.Tensor:
122
del position_ids # Unused.
123
qkv, _ = self.query_key_value(hidden_states)
124
q, k, v = qkv.chunk(chunks=3, dim=-1)
125
k_cache, v_cache = kv_cache
127
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
128
- cache_event)
126
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
127
output, _ = self.dense(attn_output)
128
return output
129
@@ -184,7 +182,6 @@ class BloomBlock(nn.Module):
182
hidden_states: torch.Tensor,
183
kv_cache: KVCache,
184
input_metadata: InputMetadata,
187
- cache_event: Optional[torch.cuda.Event],
185
) -> torch.Tensor:
186
# Layer norm at the beginning of the transformer layer.
187
layernorm_output = self.input_layernorm(hidden_states)
@@ -201,7 +198,6 @@ class BloomBlock(nn.Module):
198
hidden_states=layernorm_output,
199
kv_cache=kv_cache,
200
input_metadata=input_metadata,
204
- cache_event=cache_event,
201
)
202
attention_output = attention_output + residual
203
layernorm_output = self.post_attention_layernorm(attention_output)
@@ -250,19 +246,16 @@ class BloomModel(nn.Module):
246
position_ids: torch.Tensor,
247
kv_caches: List[KVCache],
248
input_metadata: InputMetadata,
253
- cache_events: Optional[List[torch.cuda.Event]],
249
) -> torch.Tensor:
250
hidden_states = self.word_embeddings(input_ids)
251
hidden_states = self.word_embeddings_layernorm(hidden_states)
252
for i in range(len(self.h)):
258
- cache_event = None if cache_events is None else cache_events[i]
253
layer = self.h[i]
254
hidden_states = layer(
255
position_ids,
256
hidden_states,
257
kv_caches[i],
258
input_metadata,
265
- cache_event,
259
)
260
hidden_states = self.ln_f(hidden_states)
261
return hidden_states
@@ -288,10 +281,9 @@ class BloomForCausalLM(nn.Module):
281
positions: torch.Tensor,
282
kv_caches: List[KVCache],
283
input_metadata: InputMetadata,
291
- cache_events: Optional[List[torch.cuda.Event]],
284
) -> torch.Tensor:
285
hidden_states = self.transformer(input_ids, positions, kv_caches,
294
- input_metadata, cache_events)
286
+ input_metadata)
287
return hidden_states
288
289
def sample(
vllm/model_executor/models/chatglm.py
+2
-13
@@ -100,7 +100,6 @@ class GLMAttention(nn.Module):
100
position_ids: torch.Tensor,
101
kv_cache: KVCache,
102
input_metadata: InputMetadata,
103
- cache_event: Optional[torch.cuda.Event],
103
) -> torch.Tensor:
104
qkv, _ = self.query_key_value(hidden_states)
105
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
@@ -113,7 +112,6 @@ class GLMAttention(nn.Module):
112
key_cache,
113
value_cache,
114
input_metadata,
116
- cache_event,
115
)
116
attn_output, _ = self.dense(context_layer)
117
return attn_output
@@ -203,7 +201,6 @@ class GLMBlock(nn.Module):
201
position_ids: torch.Tensor,
202
kv_cache: KVCache,
203
input_metadata: InputMetadata,
206
- cache_event: Optional[torch.cuda.Event],
204
) -> torch.Tensor:
205
# hidden_states: [num_tokens, h]
206
# Layer norm at the beginning of the transformer layer.
@@ -214,7 +211,6 @@ class GLMBlock(nn.Module):
211
position_ids=position_ids,
212
kv_cache=kv_cache,
213
input_metadata=input_metadata,
217
- cache_event=cache_event,
214
)
215
216
# Residual connection.
@@ -269,17 +265,14 @@ class GLMTransformer(nn.Module):
265
position_ids: torch.Tensor,
266
kv_caches: List[KVCache],
267
input_metadata: InputMetadata,
272
- cache_events: Optional[List[torch.cuda.Event]],
268
) -> torch.Tensor:
269
for i in range(self.num_layers):
275
- cache_event = None if cache_events is None else cache_events[i]
270
layer = self.layers[i]
271
hidden_states = layer(
272
hidden_states=hidden_states,
273
position_ids=position_ids,
274
kv_cache=kv_caches[i],
275
input_metadata=input_metadata,
282
- cache_event=cache_event,
276
)
277
# Final layer norm.
278
if self.post_layer_norm:
@@ -314,8 +307,7 @@ class ChatGLMModel(nn.Module):
307
position_ids: torch.Tensor,
308
kv_caches: List[KVCache],
309
input_metadata: InputMetadata,
317
- cache_events: Optional[List[torch.cuda.Event]],
318
- ):
310
+ ) -> torch.Tensor:
311
inputs_embeds = self.embedding(input_ids)
312
313
# Run encoder.
@@ -324,9 +316,7 @@ class ChatGLMModel(nn.Module):
316
position_ids=position_ids,
317
kv_caches=kv_caches,
318
input_metadata=input_metadata,
327
- cache_events=cache_events,
319
)
329
-
320
return hidden_states
321
322
@@ -350,10 +340,9 @@ class ChatGLMForCausalLM(nn.Module):
340
positions: torch.Tensor,
341
kv_caches: List[KVCache],
342
input_metadata: InputMetadata,
353
- cache_events: Optional[List[torch.cuda.Event]],
343
) -> torch.Tensor:
344
hidden_states = self.transformer(input_ids, positions, kv_caches,
356
- input_metadata, cache_events)
345
+ input_metadata)
346
return hidden_states
347
348
def sample(
vllm/model_executor/models/falcon.py
+2
-12
@@ -178,7 +178,6 @@ class FalconAttention(nn.Module):
178
hidden_states: torch.Tensor,
179
kv_cache: KVCache,
180
input_metadata: InputMetadata,
181
- cache_event: Optional[torch.cuda.Event],
181
) -> torch.Tensor:
182
qkv, bias = self.query_key_value(hidden_states)
183
if bias is not None:
@@ -187,8 +186,7 @@ class FalconAttention(nn.Module):
186
if self.use_rotary:
187
q, k = self.rotary_emb(positions, q, k)
188
k_cache, v_cache = kv_cache
190
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
191
- cache_event)
189
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
190
attn_output, bias = self.dense(attn_output)
191
return attn_output, bias
192
@@ -266,8 +264,7 @@ class FalconDecoderLayer(nn.Module):
264
hidden_states: torch.Tensor,
265
kv_cache: KVCache,
266
input_metadata: InputMetadata,
269
- cache_event: Optional[torch.cuda.Event],
270
- ):
267
+ ) -> torch.Tensor:
268
residual = hidden_states
269
270
if self.config.new_decoder_architecture:
@@ -282,7 +279,6 @@ class FalconDecoderLayer(nn.Module):
279
hidden_states=attention_layernorm_out,
280
kv_cache=kv_cache,
281
input_metadata=input_metadata,
285
- cache_event=cache_event,
282
)
283
if self.reduce_row_parallel_results and attention_bias is not None:
284
attention_output += attention_bias
@@ -311,7 +307,6 @@ class FalconDecoderLayer(nn.Module):
307
mlp_output += mlp_bias
308
309
output = mlp_output + residual
314
-
310
return output
311
312
@@ -349,18 +344,15 @@ class FalconModel(nn.Module):
344
positions: torch.Tensor,
345
kv_caches: List[KVCache],
346
input_metadata: InputMetadata,
352
- cache_events: Optional[List[torch.cuda.Event]],
347
) -> torch.Tensor:
348
hidden_states = self.word_embeddings(input_ids)
349
for i in range(len(self.h)):
356
- cache_event = None if cache_events is None else cache_events[i]
350
layer = self.h[i]
351
hidden_states = layer(
352
positions,
353
hidden_states,
354
kv_caches[i],
355
input_metadata,
363
- cache_event,
356
)
357
hidden_states = self.ln_f(hidden_states)
358
return hidden_states
@@ -389,14 +381,12 @@ class FalconForCausalLM(nn.Module):
381
positions: torch.Tensor,
382
kv_caches: List[KVCache],
383
input_metadata: InputMetadata,
392
- cache_events: Optional[List[torch.cuda.Event]],
384
) -> torch.Tensor:
385
hidden_states = self.transformer(
386
input_ids,
387
positions,
388
kv_caches,
389
input_metadata,
399
- cache_events,
390
)
391
return hidden_states
392
vllm/model_executor/models/gpt2.py
+3
-10
@@ -82,13 +82,12 @@ class GPT2Attention(nn.Module):
82
hidden_states: torch.Tensor,
83
kv_cache: KVCache,
84
input_metadata: InputMetadata,
85
- cache_event: Optional[torch.cuda.Event],
85
) -> torch.Tensor:
86
qkv, _ = self.c_attn(hidden_states)
87
q, k, v = qkv.chunk(chunks=3, dim=-1)
88
key_cache, value_cache = kv_cache
89
attn_output = self.attn(q, k, v, key_cache, value_cache,
91
- input_metadata, cache_event)
90
+ input_metadata)
91
attn_output, _ = self.c_proj(attn_output)
92
return attn_output
93
@@ -148,7 +147,6 @@ class GPT2Block(nn.Module):
147
hidden_states: torch.Tensor,
148
kv_cache: KVCache,
149
input_metadata: InputMetadata,
151
- cache_event: Optional[torch.cuda.Event],
150
) -> torch.Tensor:
151
residual = hidden_states
152
hidden_states = self.ln_1(hidden_states)
@@ -156,7 +154,6 @@ class GPT2Block(nn.Module):
154
hidden_states=hidden_states,
155
kv_cache=kv_cache,
156
input_metadata=input_metadata,
159
- cache_event=cache_event,
157
)
158
# residual connection
159
hidden_states = attn_output + residual
@@ -196,17 +193,14 @@ class GPT2Model(nn.Module):
193
position_ids: torch.Tensor,
194
kv_caches: List[KVCache],
195
input_metadata: InputMetadata,
199
- cache_events: Optional[List[torch.cuda.Event]],
196
) -> torch.Tensor:
197
inputs_embeds = self.wte(input_ids)
198
position_embeds = self.wpe(position_ids)
199
hidden_states = inputs_embeds + position_embeds
200
201
for i in range(len(self.h)):
206
- cache_event = None if cache_events is None else cache_events[i]
202
layer = self.h[i]
208
- hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
209
- cache_event)
203
+ hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
204
205
hidden_states = self.ln_f(hidden_states)
206
return hidden_states
@@ -232,10 +226,9 @@ class GPT2LMHeadModel(nn.Module):
226
positions: torch.Tensor,
227
kv_caches: List[KVCache],
228
input_metadata: InputMetadata,
235
- cache_events: Optional[List[torch.cuda.Event]],
229
) -> torch.Tensor:
230
hidden_states = self.transformer(input_ids, positions, kv_caches,
238
- input_metadata, cache_events)
231
+ input_metadata)
232
return hidden_states
233
234
def sample(
vllm/model_executor/models/gpt_bigcode.py
+3
-10
@@ -95,7 +95,6 @@ class GPTBigCodeAttention(nn.Module):
95
hidden_states: torch.Tensor,
96
kv_cache: KVCache,
97
input_metadata: InputMetadata,
98
- cache_event: Optional[torch.cuda.Event],
98
) -> torch.Tensor:
99
qkv, _ = self.c_attn(hidden_states)
100
q, k, v = qkv.split(
@@ -107,7 +106,7 @@ class GPTBigCodeAttention(nn.Module):
106
)
107
key_cache, value_cache = kv_cache
108
attn_output = self.attn(q, k, v, key_cache, value_cache,
110
- input_metadata, cache_event)
109
+ input_metadata)
110
attn_output, _ = self.c_proj(attn_output)
111
return attn_output
112
@@ -167,7 +166,6 @@ class GPTBigCodeBlock(nn.Module):
166
hidden_states: torch.Tensor,
167
kv_cache: KVCache,
168
input_metadata: InputMetadata,
170
- cache_event: Optional[torch.cuda.Event],
169
) -> torch.Tensor:
170
residual = hidden_states
171
hidden_states = self.ln_1(hidden_states)
@@ -175,7 +173,6 @@ class GPTBigCodeBlock(nn.Module):
173
hidden_states=hidden_states,
174
kv_cache=kv_cache,
175
input_metadata=input_metadata,
178
- cache_event=cache_event,
176
)
177
# residual connection
178
hidden_states = attn_output + residual
@@ -215,17 +212,14 @@ class GPTBigCodeModel(nn.Module):
212
position_ids: torch.Tensor,
213
kv_caches: List[KVCache],
214
input_metadata: InputMetadata,
218
- cache_events: Optional[List[torch.cuda.Event]],
215
) -> torch.Tensor:
216
inputs_embeds = self.wte(input_ids)
217
position_embeds = self.wpe(position_ids)
218
hidden_states = inputs_embeds + position_embeds
219
220
for i in range(len(self.h)):
225
- cache_event = None if cache_events is None else cache_events[i]
221
layer = self.h[i]
227
- hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
228
- cache_event)
222
+ hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
223
224
hidden_states = self.ln_f(hidden_states)
225
return hidden_states
@@ -251,10 +245,9 @@ class GPTBigCodeForCausalLM(nn.Module):
245
positions: torch.Tensor,
246
kv_caches: List[KVCache],
247
input_metadata: InputMetadata,
254
- cache_events: Optional[List[torch.cuda.Event]],
248
) -> torch.Tensor:
249
hidden_states = self.transformer(input_ids, positions, kv_caches,
257
- input_metadata, cache_events)
250
+ input_metadata)
251
return hidden_states
252
253
def sample(
vllm/model_executor/models/gpt_j.py
+2
-10
@@ -94,14 +94,12 @@ class GPTJAttention(nn.Module):
94
hidden_states: torch.Tensor,
95
kv_cache: KVCache,
96
input_metadata: InputMetadata,
97
- cache_event: Optional[torch.cuda.Event],
97
) -> torch.Tensor:
98
qkv, _ = self.qkv_proj(hidden_states)
99
q, k, v = qkv.chunk(chunks=3, dim=-1)
100
q, k = self.rotary_emb(position_ids, q, k)
101
k_cache, v_cache = kv_cache
103
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
104
- cache_event)
102
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
103
attn_output, _ = self.out_proj(attn_output)
104
return attn_output
105
@@ -156,7 +154,6 @@ class GPTJBlock(nn.Module):
154
hidden_states: torch.Tensor,
155
kv_cache: KVCache,
156
input_metadata: InputMetadata,
159
- cache_event: Optional[torch.cuda.Event],
157
) -> torch.Tensor:
158
residual = hidden_states
159
hidden_states = self.ln_1(hidden_states)
@@ -165,7 +162,6 @@ class GPTJBlock(nn.Module):
162
hidden_states=hidden_states,
163
kv_cache=kv_cache,
164
input_metadata=input_metadata,
168
- cache_event=cache_event,
165
)
166
mlp_output = self.mlp(hidden_states)
167
hidden_states = attn_output + mlp_output + residual
@@ -196,18 +192,15 @@ class GPTJModel(nn.Module):
192
position_ids: torch.Tensor,
193
kv_caches: List[KVCache],
194
input_metadata: InputMetadata,
199
- cache_events: Optional[List[torch.cuda.Event]],
195
) -> torch.Tensor:
196
hidden_states = self.wte(input_ids)
197
for i in range(len(self.h)):
203
- cache_event = None if cache_events is None else cache_events[i]
198
layer = self.h[i]
199
hidden_states = layer(
200
position_ids,
201
hidden_states,
202
kv_caches[i],
203
input_metadata,
210
- cache_event,
204
)
205
hidden_states = self.ln_f(hidden_states)
206
return hidden_states
@@ -238,10 +231,9 @@ class GPTJForCausalLM(nn.Module):
231
positions: torch.Tensor,
232
kv_caches: List[KVCache],
233
input_metadata: InputMetadata,
241
- cache_events: Optional[List[torch.cuda.Event]],
234
) -> torch.Tensor:
235
hidden_states = self.transformer(input_ids, positions, kv_caches,
244
- input_metadata, cache_events)
236
+ input_metadata)
237
return hidden_states
238
239
def sample(
vllm/model_executor/models/gpt_neox.py
+2
-10
@@ -92,14 +92,12 @@ class GPTNeoXAttention(nn.Module):
92
hidden_states: torch.Tensor,
93
kv_cache: KVCache,
94
input_metadata: InputMetadata,
95
- cache_event: Optional[torch.cuda.Event],
95
) -> torch.Tensor:
96
qkv, _ = self.query_key_value(hidden_states)
97
q, k, v = qkv.chunk(chunks=3, dim=-1)
98
q, k = self.rotary_emb(position_ids, q, k)
99
k_cache, v_cache = kv_cache
101
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
102
- cache_event)
100
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
101
output, _ = self.dense(attn_output)
102
return output
103
@@ -155,7 +153,6 @@ class GPTNeoXLayer(nn.Module):
153
hidden_states: torch.Tensor,
154
kv_cache: KVCache,
155
input_metadata: InputMetadata,
158
- cache_event: Optional[torch.cuda.Event],
156
) -> torch.Tensor:
157
attn_input = self.input_layernorm(hidden_states)
158
attn_output = self.attention(
@@ -163,7 +160,6 @@ class GPTNeoXLayer(nn.Module):
160
hidden_states=attn_input,
161
kv_cache=kv_cache,
162
input_metadata=input_metadata,
166
- cache_event=cache_event,
163
)
164
165
if self.use_parallel_residual:
@@ -210,18 +206,15 @@ class GPTNeoXModel(nn.Module):
206
position_ids: torch.Tensor,
207
kv_caches: List[KVCache],
208
input_metadata: InputMetadata,
213
- cache_events: Optional[List[torch.cuda.Event]],
209
) -> torch.Tensor:
210
hidden_states = self.embed_in(input_ids)
211
for i in range(len(self.layers)):
217
- cache_event = None if cache_events is None else cache_events[i]
212
layer = self.layers[i]
213
hidden_states = layer(
214
position_ids,
215
hidden_states,
216
kv_caches[i],
217
input_metadata,
224
- cache_event,
218
)
219
hidden_states = self.final_layer_norm(hidden_states)
220
return hidden_states
@@ -250,10 +243,9 @@ class GPTNeoXForCausalLM(nn.Module):
243
positions: torch.Tensor,
244
kv_caches: List[KVCache],
245
input_metadata: InputMetadata,
253
- cache_events: Optional[List[torch.cuda.Event]],
246
) -> torch.Tensor:
247
hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
256
- input_metadata, cache_events)
248
+ input_metadata)
249
return hidden_states
250
251
def sample(
vllm/model_executor/models/internlm.py
+2
-10
@@ -110,14 +110,12 @@ class InternLMAttention(nn.Module):
110
hidden_states: torch.Tensor,
111
kv_cache: KVCache,
112
input_metadata: InputMetadata,
113
- cache_event: Optional[torch.cuda.Event],
113
) -> torch.Tensor:
114
qkv, _ = self.qkv_proj(hidden_states)
115
q, k, v = qkv.chunk(chunks=3, dim=-1)
116
q, k = self.rotary_emb(positions, q, k)
117
k_cache, v_cache = kv_cache
119
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
120
- cache_event)
118
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
119
output, _ = self.o_proj(attn_output)
120
return output
121
@@ -160,7 +158,6 @@ class InternLMDecoderLayer(nn.Module):
158
hidden_states: torch.Tensor,
159
kv_cache: KVCache,
160
input_metadata: InputMetadata,
163
- cache_event: Optional[torch.cuda.Event],
161
residual: Optional[torch.Tensor],
162
) -> Tuple[torch.Tensor, torch.Tensor]:
163
# Self Attention
@@ -175,7 +172,6 @@ class InternLMDecoderLayer(nn.Module):
172
hidden_states=hidden_states,
173
kv_cache=kv_cache,
174
input_metadata=input_metadata,
178
- cache_event=cache_event,
175
)
176
177
# Fully Connected
@@ -214,19 +210,16 @@ class InternLMModel(nn.Module):
210
positions: torch.Tensor,
211
kv_caches: List[KVCache],
212
input_metadata: InputMetadata,
217
- cache_events: Optional[List[torch.cuda.Event]],
213
) -> torch.Tensor:
214
hidden_states = self.embed_tokens(input_ids)
215
residual = None
216
for i in range(len(self.layers)):
222
- cache_event = None if cache_events is None else cache_events[i]
217
layer = self.layers[i]
218
hidden_states, residual = layer(
219
positions,
220
hidden_states,
221
kv_caches[i],
222
input_metadata,
229
- cache_event,
223
residual,
224
)
225
hidden_states, _ = self.norm(hidden_states, residual)
@@ -253,10 +246,9 @@ class InternLMForCausalLM(nn.Module):
246
positions: torch.Tensor,
247
kv_caches: List[KVCache],
248
input_metadata: InputMetadata,
256
- cache_events: Optional[List[torch.cuda.Event]],
249
) -> torch.Tensor:
250
hidden_states = self.model(input_ids, positions, kv_caches,
259
- input_metadata, cache_events)
251
+ input_metadata)
252
return hidden_states
253
254
def sample(
vllm/model_executor/models/llama.py
+2
-10
@@ -147,14 +147,12 @@ class LlamaAttention(nn.Module):
147
hidden_states: torch.Tensor,
148
kv_cache: KVCache,
149
input_metadata: InputMetadata,
150
- cache_event: Optional[torch.cuda.Event],
150
) -> torch.Tensor:
151
qkv, _ = self.qkv_proj(hidden_states)
152
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
153
q, k = self.rotary_emb(positions, q, k)
154
k_cache, v_cache = kv_cache
156
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
157
- cache_event)
155
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
156
output, _ = self.o_proj(attn_output)
157
return output
158
@@ -198,7 +196,6 @@ class LlamaDecoderLayer(nn.Module):
196
hidden_states: torch.Tensor,
197
kv_cache: KVCache,
198
input_metadata: InputMetadata,
201
- cache_event: Optional[torch.cuda.Event],
199
residual: Optional[torch.Tensor],
200
) -> Tuple[torch.Tensor, torch.Tensor]:
201
# Self Attention
@@ -213,7 +210,6 @@ class LlamaDecoderLayer(nn.Module):
210
hidden_states=hidden_states,
211
kv_cache=kv_cache,
212
input_metadata=input_metadata,
216
- cache_event=cache_event,
213
)
214
215
# Fully Connected
@@ -250,19 +246,16 @@ class LlamaModel(nn.Module):
246
positions: torch.Tensor,
247
kv_caches: List[KVCache],
248
input_metadata: InputMetadata,
253
- cache_events: Optional[List[torch.cuda.Event]],
249
) -> torch.Tensor:
250
hidden_states = self.embed_tokens(input_ids)
251
residual = None
252
for i in range(len(self.layers)):
258
- cache_event = None if cache_events is None else cache_events[i]
253
layer = self.layers[i]
254
hidden_states, residual = layer(
255
positions,
256
hidden_states,
257
kv_caches[i],
258
input_metadata,
265
- cache_event,
259
residual,
260
)
261
hidden_states, _ = self.norm(hidden_states, residual)
@@ -289,10 +282,9 @@ class LlamaForCausalLM(nn.Module):
282
positions: torch.Tensor,
283
kv_caches: List[KVCache],
284
input_metadata: InputMetadata,
292
- cache_events: Optional[List[torch.cuda.Event]],
285
) -> torch.Tensor:
286
hidden_states = self.model(input_ids, positions, kv_caches,
295
- input_metadata, cache_events)
287
+ input_metadata)
288
return hidden_states
289
290
def sample(
vllm/model_executor/models/mistral.py
+2
-10
@@ -145,14 +145,12 @@ class MistralAttention(nn.Module):
145
hidden_states: torch.Tensor,
146
kv_cache: KVCache,
147
input_metadata: InputMetadata,
148
- cache_event: Optional[torch.cuda.Event],
148
) -> torch.Tensor:
149
qkv, _ = self.qkv_proj(hidden_states)
150
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
151
q, k = self.rotary_emb(positions, q, k)
152
k_cache, v_cache = kv_cache
154
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
155
- cache_event)
153
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
154
output, _ = self.o_proj(attn_output)
155
return output
156
@@ -193,7 +191,6 @@ class MistralDecoderLayer(nn.Module):
191
hidden_states: torch.Tensor,
192
kv_cache: KVCache,
193
input_metadata: InputMetadata,
196
- cache_event: Optional[torch.cuda.Event],
194
residual: Optional[torch.Tensor],
195
) -> Tuple[torch.Tensor, torch.Tensor]:
196
# Self Attention
@@ -208,7 +205,6 @@ class MistralDecoderLayer(nn.Module):
205
hidden_states=hidden_states,
206
kv_cache=kv_cache,
207
input_metadata=input_metadata,
211
- cache_event=cache_event,
208
)
209
210
# Fully Connected
@@ -246,19 +242,16 @@ class MistralModel(nn.Module):
242
positions: torch.Tensor,
243
kv_caches: List[KVCache],
244
input_metadata: InputMetadata,
249
- cache_events: Optional[List[torch.cuda.Event]],
245
) -> torch.Tensor:
246
hidden_states = self.embed_tokens(input_ids)
247
residual = None
248
for i in range(len(self.layers)):
254
- cache_event = None if cache_events is None else cache_events[i]
249
layer = self.layers[i]
250
hidden_states, residual = layer(
251
positions,
252
hidden_states,
253
kv_caches[i],
254
input_metadata,
261
- cache_event,
255
residual,
256
)
257
hidden_states, _ = self.norm(hidden_states, residual)
@@ -285,10 +278,9 @@ class MistralForCausalLM(nn.Module):
278
positions: torch.Tensor,
279
kv_caches: List[KVCache],
280
input_metadata: InputMetadata,
288
- cache_events: Optional[List[torch.cuda.Event]],
281
) -> torch.Tensor:
282
hidden_states = self.model(input_ids, positions, kv_caches,
291
- input_metadata, cache_events)
283
+ input_metadata)
284
return hidden_states
285
286
def sample(
vllm/model_executor/models/mixtral.py
+3
-10
@@ -253,14 +253,12 @@ class MixtralAttention(nn.Module):
253
hidden_states: torch.Tensor,
254
kv_cache: KVCache,
255
input_metadata: InputMetadata,
256
- cache_event: Optional[torch.cuda.Event],
256
) -> torch.Tensor:
257
qkv, _ = self.qkv_proj(hidden_states)
258
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
259
q, k = self.rotary_emb(positions, q, k)
260
k_cache, v_cache = kv_cache
262
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
263
- cache_event)
261
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
262
output, _ = self.o_proj(attn_output)
263
return output
264
@@ -297,7 +295,6 @@ class MixtralDecoderLayer(nn.Module):
295
hidden_states: torch.Tensor,
296
kv_cache: KVCache,
297
input_metadata: InputMetadata,
300
- cache_event: Optional[torch.cuda.Event],
298
residual: Optional[torch.Tensor],
299
) -> torch.Tensor:
300
# Self Attention
@@ -312,7 +309,6 @@ class MixtralDecoderLayer(nn.Module):
309
hidden_states=hidden_states,
310
kv_cache=kv_cache,
311
input_metadata=input_metadata,
315
- cache_event=cache_event,
312
)
313
314
# Fully Connected
@@ -349,16 +345,14 @@ class MixtralModel(nn.Module):
345
positions: torch.Tensor,
346
kv_caches: List[KVCache],
347
input_metadata: InputMetadata,
352
- cache_events: Optional[List[torch.cuda.Event]],
348
) -> SamplerOutput:
349
hidden_states = self.embed_tokens(input_ids)
350
residual = None
351
for i in range(len(self.layers)):
357
- cache_event = None if cache_events is None else cache_events[i]
352
layer = self.layers[i]
353
hidden_states, residual = layer(positions, hidden_states,
354
kv_caches[i], input_metadata,
361
- cache_event, residual)
355
+ residual)
356
hidden_states, _ = self.norm(hidden_states, residual)
357
return hidden_states
358
@@ -383,10 +377,9 @@ class MixtralForCausalLM(nn.Module):
377
positions: torch.Tensor,
378
kv_caches: List[KVCache],
379
input_metadata: InputMetadata,
386
- cache_events: Optional[List[torch.cuda.Event]],
380
) -> torch.Tensor:
381
hidden_states = self.model(input_ids, positions, kv_caches,
389
- input_metadata, cache_events)
382
+ input_metadata)
383
return hidden_states
384
385
def sample(
vllm/model_executor/models/mpt.py
+2
-10
@@ -117,7 +117,6 @@ class MPTAttention(nn.Module):
117
hidden_states: torch.Tensor,
118
kv_cache: KVCache,
119
input_metadata: InputMetadata,
120
- cache_event: Optional[torch.cuda.Event],
120
) -> torch.Tensor:
121
del position_ids # unused.
122
qkv, _ = self.Wqkv(hidden_states)
@@ -128,8 +127,7 @@ class MPTAttention(nn.Module):
127
q = self.q_ln(q)
128
k = self.k_ln(k)
129
k_cache, v_cache = kv_cache
131
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
132
- cache_event)
130
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
131
output, _ = self.out_proj(attn_output)
132
return output
133
@@ -187,7 +185,6 @@ class MPTBlock(nn.Module):
185
hidden_states: torch.Tensor,
186
kv_cache: KVCache,
187
input_metadata: InputMetadata,
190
- cache_event: Optional[torch.cuda.Event],
188
) -> torch.Tensor:
189
x = self.norm_1(hidden_states)
190
x = self.attn(
@@ -195,7 +192,6 @@ class MPTBlock(nn.Module):
192
hidden_states=x,
193
kv_cache=kv_cache,
194
input_metadata=input_metadata,
198
- cache_event=cache_event,
195
)
196
hidden_states = hidden_states + x
197
x = self.norm_2(hidden_states)
@@ -235,18 +231,15 @@ class MPTModel(nn.Module):
231
position_ids: torch.Tensor,
232
kv_caches: List[KVCache],
233
input_metadata: InputMetadata,
238
- cache_events: Optional[List[torch.cuda.Event]],
234
) -> torch.Tensor:
235
hidden_states = self.wte(input_ids)
236
for i in range(len(self.blocks)):
242
- cache_event = None if cache_events is None else cache_events[i]
237
block = self.blocks[i]
238
hidden_states = block(
239
position_ids,
240
hidden_states,
241
kv_caches[i],
242
input_metadata,
249
- cache_event,
243
)
244
hidden_states = self.norm_f(hidden_states)
245
return hidden_states
@@ -274,10 +267,9 @@ class MPTForCausalLM(nn.Module):
267
positions: torch.Tensor,
268
kv_caches: List[KVCache],
269
input_metadata: InputMetadata,
277
- cache_events: Optional[List[torch.cuda.Event]],
270
) -> torch.Tensor:
271
hidden_states = self.transformer(input_ids, positions, kv_caches,
280
- input_metadata, cache_events)
272
+ input_metadata)
273
return hidden_states
274
275
def sample(
vllm/model_executor/models/opt.py
+5
-14
@@ -98,13 +98,12 @@ class OPTAttention(nn.Module):
98
hidden_states: torch.Tensor,
99
kv_cache: KVCache,
100
input_metadata: InputMetadata,
101
- cache_event: Optional[torch.cuda.Event],
101
) -> torch.Tensor:
102
qkv, _ = self.qkv_proj(hidden_states)
103
q, k, v = qkv.chunk(chunks=3, dim=-1)
104
key_cache, value_cache = kv_cache
105
attn_output = self.attn(q, k, v, key_cache, value_cache,
107
- input_metadata, cache_event)
106
+ input_metadata)
107
output, _ = self.out_proj(attn_output)
108
return output
109
@@ -154,7 +153,6 @@ class OPTDecoderLayer(nn.Module):
153
hidden_states: torch.Tensor,
154
kv_cache: KVCache,
155
input_metadata: InputMetadata,
157
- cache_event: Optional[torch.cuda.Event],
156
) -> torch.Tensor:
157
# Self Attention
158
residual = hidden_states
@@ -163,8 +161,7 @@ class OPTDecoderLayer(nn.Module):
161
hidden_states = self.self_attn_layer_norm(hidden_states)
162
hidden_states = self.self_attn(hidden_states=hidden_states,
163
kv_cache=kv_cache,
166
- input_metadata=input_metadata,
167
- cache_event=cache_event)
164
+ input_metadata=input_metadata)
165
hidden_states = residual + hidden_states
166
# 350m applies layer norm AFTER attention
167
if not self.do_layer_norm_before:
@@ -245,7 +242,6 @@ class OPTDecoder(nn.Module):
242
positions: torch.Tensor,
243
kv_caches: List[KVCache],
244
input_metadata: InputMetadata,
248
- cache_events: Optional[List[torch.cuda.Event]],
245
) -> torch.Tensor:
246
inputs_embeds = self.embed_tokens(input_ids)
247
pos_embeds = self.embed_positions(positions)
@@ -254,10 +250,8 @@ class OPTDecoder(nn.Module):
250
hidden_states = inputs_embeds + pos_embeds
251
252
for i in range(len(self.layers)):
257
- cache_event = None if cache_events is None else cache_events[i]
253
layer = self.layers[i]
259
- hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
260
- cache_event)
254
+ hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
255
256
if self.final_layer_norm is not None:
257
hidden_states = self.final_layer_norm(hidden_states)
@@ -282,10 +276,8 @@ class OPTModel(nn.Module):
276
positions: torch.Tensor,
277
kv_caches: List[KVCache],
278
input_metadata: InputMetadata,
285
- cache_events: Optional[List[torch.cuda.Event]],
279
) -> torch.Tensor:
287
- return self.decoder(input_ids, positions, kv_caches, input_metadata,
288
- cache_events)
280
+ return self.decoder(input_ids, positions, kv_caches, input_metadata)
281
282
283
class OPTForCausalLM(nn.Module):
@@ -308,10 +300,9 @@ class OPTForCausalLM(nn.Module):
300
positions: torch.Tensor,
301
kv_caches: List[KVCache],
302
input_metadata: InputMetadata,
311
- cache_events: Optional[List[torch.cuda.Event]],
303
) -> torch.Tensor:
304
hidden_states = self.model(input_ids, positions, kv_caches,
314
- input_metadata, cache_events)
305
+ input_metadata)
306
return hidden_states
307
308
def sample(
vllm/model_executor/models/phi_1_5.py
+2
-10
@@ -135,14 +135,12 @@ class PhiAttention(nn.Module):
135
hidden_states: torch.Tensor,
136
kv_cache: KVCache,
137
input_metadata: InputMetadata,
138
- cache_event: Optional[torch.cuda.Event],
138
) -> torch.Tensor:
139
qkv, _ = self.Wqkv(hidden_states)
140
q, k, v = qkv.chunk(chunks=3, dim=-1)
141
q, k = self.rotary_emb(position_ids, q, k)
142
k_cache, v_cache = kv_cache
144
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
145
- cache_event)
143
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
144
output, _ = self.out_proj(attn_output)
145
return output
146
@@ -195,7 +193,6 @@ class PhiLayer(nn.Module):
193
hidden_states: torch.Tensor,
194
kv_cache: KVCache,
195
input_metadata: InputMetadata,
198
- cache_event: Optional[torch.cuda.Event],
196
) -> torch.Tensor:
197
residual = hidden_states
198
hidden_states = self.ln(hidden_states)
@@ -204,7 +201,6 @@ class PhiLayer(nn.Module):
201
hidden_states=hidden_states,
202
kv_cache=kv_cache,
203
input_metadata=input_metadata,
207
- cache_event=cache_event,
204
)
205
feed_forward_hidden_states = self.mlp(hidden_states)
206
hidden_states = attn_outputs + feed_forward_hidden_states + residual
@@ -231,18 +227,15 @@ class PhiModel(nn.Module):
227
positions: torch.Tensor,
228
kv_caches: List[KVCache],
229
input_metadata: InputMetadata,
234
- cache_events: Optional[List[torch.cuda.Event]],
230
) -> torch.Tensor:
231
hidden_states = self.embd(input_ids)
232
for i in range(self.config.num_hidden_layers):
238
- cache_event = None if cache_events is None else cache_events[i]
233
layer = self.h[i]
234
hidden_states = layer(
235
positions,
236
hidden_states,
237
kv_caches[i],
238
input_metadata,
245
- cache_event,
239
)
240
return hidden_states
241
@@ -277,10 +270,9 @@ class PhiForCausalLM(nn.Module):
270
positions: torch.Tensor,
271
kv_caches: List[KVCache],
272
input_metadata: InputMetadata,
280
- cache_events: Optional[List[torch.cuda.Event]],
273
) -> torch.Tensor:
274
hidden_states = self.transformer(input_ids, positions, kv_caches,
283
- input_metadata, cache_events)
275
+ input_metadata)
276
hidden_states = self.lm_head.ln(hidden_states)
277
return hidden_states
278
vllm/model_executor/models/qwen.py
+2
-10
@@ -112,14 +112,12 @@ class QWenAttention(nn.Module):
112
hidden_states: torch.Tensor,
113
kv_cache: KVCache,
114
input_metadata: InputMetadata,
115
- cache_event: Optional[torch.cuda.Event],
115
) -> torch.Tensor:
116
qkv, _ = self.c_attn(hidden_states)
117
q, k, v = qkv.chunk(chunks=3, dim=-1)
118
q, k = self.rotary_emb(positions, q, k)
119
k_cache, v_cache = kv_cache
121
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
122
- cache_event)
120
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
121
122
output, _ = self.c_proj(attn_output)
123
return output
@@ -156,7 +154,6 @@ class QWenBlock(nn.Module):
154
hidden_states: torch.Tensor,
155
kv_cache: KVCache,
156
input_metadata: InputMetadata,
159
- cache_event: Optional[torch.cuda.Event],
157
residual: Optional[torch.Tensor],
158
) -> Tuple[torch.Tensor, torch.Tensor]:
159
# Self Attention
@@ -170,7 +167,6 @@ class QWenBlock(nn.Module):
167
hidden_states=hidden_states,
168
kv_cache=kv_cache,
169
input_metadata=input_metadata,
173
- cache_event=cache_event,
170
)
171
172
# Fully Connected
@@ -206,19 +202,16 @@ class QWenModel(nn.Module):
202
positions: torch.Tensor,
203
kv_caches: List[KVCache],
204
input_metadata: InputMetadata,
209
- cache_events: Optional[List[torch.cuda.Event]],
205
) -> torch.Tensor:
206
hidden_states = self.wte(input_ids)
207
residual = None
208
for i in range(len(self.h)):
214
- cache_event = None if cache_events is None else cache_events[i]
209
layer = self.h[i]
210
hidden_states, residual = layer(
211
positions,
212
hidden_states,
213
kv_caches[i],
214
input_metadata,
221
- cache_event,
215
residual,
216
)
217
hidden_states, _ = self.ln_f(hidden_states, residual)
@@ -245,10 +238,9 @@ class QWenLMHeadModel(nn.Module):
238
positions: torch.Tensor,
239
kv_caches: List[KVCache],
240
input_metadata: InputMetadata,
248
- cache_events: Optional[List[torch.cuda.Event]],
241
) -> torch.Tensor:
242
hidden_states = self.transformer(input_ids, positions, kv_caches,
251
- input_metadata, cache_events)
243
+ input_metadata)
244
return hidden_states
245
246
def sample(
vllm/model_executor/models/yi.py
+2
-10
@@ -146,14 +146,12 @@ class YiAttention(nn.Module):
146
hidden_states: torch.Tensor,
147
kv_cache: KVCache,
148
input_metadata: InputMetadata,
149
- cache_event: Optional[torch.cuda.Event],
149
) -> torch.Tensor:
150
qkv, _ = self.qkv_proj(hidden_states)
151
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
q, k = self.rotary_emb(positions, q, k)
153
k_cache, v_cache = kv_cache
155
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
156
- cache_event)
154
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155
output, _ = self.o_proj(attn_output)
156
return output
157
@@ -195,7 +193,6 @@ class YiDecoderLayer(nn.Module):
193
hidden_states: torch.Tensor,
194
kv_cache: KVCache,
195
input_metadata: InputMetadata,
198
- cache_event: Optional[torch.cuda.Event],
196
residual: Optional[torch.Tensor],
197
) -> Tuple[torch.Tensor, torch.Tensor]:
198
# Self Attention
@@ -209,7 +206,6 @@ class YiDecoderLayer(nn.Module):
206
hidden_states=hidden_states,
207
kv_cache=kv_cache,
208
input_metadata=input_metadata,
212
- cache_event=cache_event,
209
)
210
211
# Fully Connected
@@ -245,19 +241,16 @@ class YiModel(nn.Module):
241
positions: torch.Tensor,
242
kv_caches: List[KVCache],
243
input_metadata: InputMetadata,
248
- cache_events: Optional[List[torch.cuda.Event]],
244
) -> torch.Tensor:
245
hidden_states = self.embed_tokens(input_ids)
246
residual = None
247
for i in range(len(self.layers)):
253
- cache_event = None if cache_events is None else cache_events[i]
248
layer = self.layers[i]
249
hidden_states, residual = layer(
250
positions,
251
hidden_states,
252
kv_caches[i],
253
input_metadata,
260
- cache_event,
254
residual,
255
)
256
hidden_states, _ = self.norm(hidden_states, residual)
@@ -284,10 +277,9 @@ class YiForCausalLM(nn.Module):
277
positions: torch.Tensor,
278
kv_caches: List[KVCache],
279
input_metadata: InputMetadata,
287
- cache_events: Optional[List[torch.cuda.Event]],
280
) -> torch.Tensor:
281
hidden_states = self.model(input_ids, positions, kv_caches,
290
- input_metadata, cache_events)
282
+ input_metadata)
283
return hidden_states
284
285
def sample(
vllm/model_executor/parallel_utils/communication_op.py
+8
-2
@@ -1,8 +1,10 @@
1
import torch
2
3
+from vllm.model_executor.parallel_utils import cupy_utils
4
from vllm.model_executor.parallel_utils.parallel_state import (
5
get_tensor_model_parallel_world_size,
6
get_tensor_model_parallel_group,
7
+ is_custom_nccl_enabled_for_all_reduce,
8
)
9
10
@@ -15,8 +17,12 @@ def tensor_model_parallel_all_reduce(input_):
17
if get_tensor_model_parallel_world_size() == 1:
18
return input_
19
# All-reduce.
18
- torch.distributed.all_reduce(input_,
19
- group=get_tensor_model_parallel_group())
20
+ if is_custom_nccl_enabled_for_all_reduce():
21
+ # TODO: support multiple parallel groups.
22
+ cupy_utils.all_reduce(input_)
23
+ else:
24
+ torch.distributed.all_reduce(input_,
25
+ group=get_tensor_model_parallel_group())
26
return input_
27
28
vllm/model_executor/parallel_utils/cupy_utils.py
new
+115
@@ -0,0 +1,115 @@
1
+"""CuPy utilities for all-reduce.
2
+
3
+We use CuPy all-reduce instead of torch.distributed.all_reduce when capturing
4
+CUDA graphs, because torch.distributed.all_reduce causes errors when capturing
5
+CUDA graphs.
6
+
7
+TODO: Remove this file when torch.distributed.all_reduce is fixed.
8
+"""
9
+import contextlib
10
+
11
+import torch
12
+from torch.distributed import ReduceOp
13
+
14
+try:
15
+ import cupy
16
+ from cupyx.distributed import NCCLBackend
17
+ from cupy.cuda import nccl
18
+except ImportError as e:
19
+ cupy = e
20
+ nccl = None
21
+
22
+ class NCCLBackend:
23
+ ...
24
+
25
+
26
+_OP_MAPPING = {
27
+ ReduceOp.SUM: "sum",
28
+ ReduceOp.PRODUCT: "prod",
29
+ ReduceOp.MIN: "min",
30
+ ReduceOp.MAX: "max",
31
+}
32
+
33
+
34
+class NCCLBackendWithBFloat16(NCCLBackend):
35
+ # This is enough to add bfloat16 support for most operations,
36
+ # but broadcast will fail (will require changes in compiled
37
+ # cupy code).
38
+ def _get_nccl_dtype_and_count(self, array, count=None):
39
+ nccl_dtype, count = super()._get_nccl_dtype_and_count(array, count)
40
+ torch_dtype = getattr(array, "_torch_dtype", None)
41
+ if torch_dtype is torch.bfloat16:
42
+ nccl_dtype = nccl.NCCL_BFLOAT16
43
+ return nccl_dtype, count
44
+
45
+
46
+_NCCL_BACKEND = None
47
+_WORLD_SIZE = 0
48
+
49
+
50
+def is_initialized() -> bool:
51
+ """Returns whether the NCCL backend is initialized."""
52
+ return _NCCL_BACKEND is not None
53
+
54
+
55
+@contextlib.contextmanager
56
+def set_cupy_stream(stream: torch.cuda.Stream) -> None:
57
+ """Set the cuda stream for communication"""
58
+ cupy_stream = cupy.cuda.ExternalStream(stream.cuda_stream,
59
+ stream.device_index)
60
+ with cupy_stream:
61
+ yield
62
+
63
+
64
+def init_process_group(world_size: int, rank: int, host: str,
65
+ port: int) -> None:
66
+ """Initializes the CuPy NCCL backend.
67
+
68
+ # TODO: handle NCCL timeouts.
69
+ """
70
+ assert not is_initialized()
71
+
72
+ if isinstance(cupy, Exception):
73
+ raise ImportError(
74
+ "NCCLBackend is not available. Please install cupy.") from cupy
75
+
76
+ # TODO(woosuk): Create TP and PP process groups for CuPy.
77
+ global _NCCL_BACKEND
78
+ global _WORLD_SIZE
79
+ assert world_size > 0, f"{world_size=} should be a positive integer"
80
+ assert 0 <= rank < world_size, (
81
+ f"{rank=} should be a integer between [0, {world_size})")
82
+
83
+ cupy.cuda.runtime.setDevice(torch.cuda.current_device())
84
+ _NCCL_BACKEND = NCCLBackendWithBFloat16(world_size, rank, host, port)
85
+ _WORLD_SIZE = world_size
86
+
87
+
88
+def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
89
+ """All-reduces the input tensor across the process group."""
90
+ assert input_.is_cuda, f"{input_} should be a cuda tensor"
91
+ # Hack to support bfloat16
92
+ torch_dtype = input_.dtype
93
+ if torch_dtype is torch.bfloat16:
94
+ # We need to view as float16, otherwise
95
+ # cupy will fail. This will not change
96
+ # the underlying data.
97
+ input_ = input_.view(torch.float16)
98
+ cupy_input = cupy.asarray(input_)
99
+ cupy_input._torch_dtype = torch_dtype # pylint: disable=protected-access
100
+ _NCCL_BACKEND.all_reduce(in_array=cupy_input,
101
+ out_array=cupy_input,
102
+ op=_OP_MAPPING[op])
103
+
104
+
105
+def destroy_process_group() -> None:
106
+ """Destroys the NCCL backend."""
107
+ global _NCCL_BACKEND
108
+ global _WORLD_SIZE
109
+ _NCCL_BACKEND = None
110
+ _WORLD_SIZE = 0
111
+
112
+
113
+def get_world_size() -> int:
114
+ """Returns the world size."""
115
+ return _WORLD_SIZE
vllm/model_executor/parallel_utils/parallel_state.py
+37
@@ -3,9 +3,12 @@
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
"""Tensor and pipeline parallel groups."""
6
+import contextlib
7
8
import torch
9
10
+from vllm.model_executor.parallel_utils import cupy_utils
11
+
12
# Tensor model parallel group that the current rank belongs to.
13
_TENSOR_MODEL_PARALLEL_GROUP = None
14
# Pipeline model parallel group that the current rank belongs to.
@@ -177,3 +180,37 @@ def destroy_model_parallel():
180
_PIPELINE_MODEL_PARALLEL_GROUP = None
181
global _PIPELINE_GLOBAL_RANKS
182
_PIPELINE_GLOBAL_RANKS = None
183
+
184
+ # Destroy the cupy states if any.
185
+ cupy_utils.destroy_process_group()
186
+
187
+
188
+# Whether to use cupy for nccl all reduce.
189
+# We use cupy for all reduce when using CUDA graph, because torch.distributed
190
+# is not well supported by CUDA graph.
191
+_ENABLE_CUPY_FOR_ALL_REDUCE = False
192
+
193
+
194
+@contextlib.contextmanager
195
+def with_custom_nccl_for_all_reduce():
196
+ """use custom nccl instead of torch.distributed for all reduce"""
197
+ tp_size = get_tensor_model_parallel_world_size()
198
+ if tp_size == 1:
199
+ # No-op.
200
+ # NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
201
+ yield
202
+ else:
203
+ global _ENABLE_CUPY_FOR_ALL_REDUCE
204
+ old = _ENABLE_CUPY_FOR_ALL_REDUCE
205
+ _ENABLE_CUPY_FOR_ALL_REDUCE = True
206
+
207
+ stream = torch.cuda.current_stream()
208
+ with cupy_utils.set_cupy_stream(stream):
209
+ yield
210
+ _ENABLE_CUPY_FOR_ALL_REDUCE = old
211
+
212
+
213
+def is_custom_nccl_enabled_for_all_reduce():
214
+ """check if custom nccl is enabled for all reduce"""
215
+ global _ENABLE_CUPY_FOR_ALL_REDUCE
216
+ return _ENABLE_CUPY_FOR_ALL_REDUCE
vllm/utils.py
+7
@@ -1,4 +1,5 @@
1
import enum
2
+import socket
3
import uuid
4
from platform import uname
5
@@ -52,3 +53,9 @@ def random_uuid() -> str:
53
def in_wsl() -> bool:
54
# Reference: https://github.com/microsoft/WSL/issues/4071
55
return "microsoft" in " ".join(uname()).lower()
56
+
57
+
58
+def get_open_port():
59
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
60
+ s.bind(("", 0))
61
+ return s.getsockname()[1]
vllm/worker/model_runner.py
+225
-16
@@ -1,16 +1,25 @@
1
-from typing import Dict, List, Optional, Tuple
1
+import time
2
+from typing import Dict, List, Tuple, Union
3
4
+import numpy as np
5
import torch
6
+import torch.nn as nn
7
8
from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
9
from vllm.logger import init_logger
10
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11
+from vllm.model_executor.parallel_utils.parallel_state import (
12
+ with_custom_nccl_for_all_reduce)
13
from vllm.sampling_params import SamplingParams, SamplingType
14
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
16
logger = init_logger(__name__)
17
18
+KVCache = Tuple[torch.Tensor, torch.Tensor]
19
_PAD_SLOT_ID = -1
20
+# Capture graphs for batch size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
21
+# NOTE: _get_graph_batch_size needs to be updated if this list is changed.
22
+_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [8 * i for i in range(1, 33)]
23
24
25
class ModelRunner:
@@ -32,12 +41,31 @@ class ModelRunner:
41
self.model = None
42
self.block_size = None # Set after initial profiling.
43
44
+ self.graph_runners: Dict[int, CUDAGraphRunner] = {}
45
+ self.graph_memory_pool = None # Set during graph capture.
46
+
47
+ self.max_context_len_to_capture = (
48
+ self.model_config.max_context_len_to_capture
49
+ if self.model_config is not None else 0)
50
+ # When using CUDA graph, the input block tables must be padded to
51
+ # max_context_len_to_capture. However, creating the block table in
52
+ # Python can be expensive. To optimize this, we cache the block table
53
+ # in numpy and only copy the actual input content at every iteration.
54
+ # The shape of the cached block table will be
55
+ # (max batch size to capture, max context len to capture / block size).
56
+ self.graph_block_tables = None # Set after initial profiling.
57
+
58
def load_model(self) -> None:
59
self.model = get_model(self.model_config)
60
61
def set_block_size(self, block_size: int) -> None:
62
self.block_size = block_size
63
64
+ max_num_blocks = (self.max_context_len_to_capture + block_size -
65
+ 1) // block_size
66
+ self.graph_block_tables = np.zeros(
67
+ (max(_BATCH_SIZES_TO_CAPTURE), max_num_blocks), dtype=np.int32)
68
+
69
def _prepare_prompt(
70
self,
71
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -111,6 +139,7 @@ class ModelRunner:
139
max_context_len=None,
140
context_lens=None,
141
block_tables=None,
142
+ use_cuda_graph=False,
143
)
144
return input_tokens, input_positions, input_metadata
145
@@ -154,27 +183,62 @@ class ModelRunner:
183
block_table = block_table[-sliding_window_blocks:]
184
block_tables.append(block_table)
185
186
+ batch_size = len(input_tokens)
187
+ max_context_len = max(context_lens)
188
+ use_captured_graph = (
189
+ not self.model_config.enforce_eager
190
+ and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
191
+ and max_context_len <= self.max_context_len_to_capture)
192
+ if use_captured_graph:
193
+ # Pad the input tokens, positions, and slot mapping to match the
194
+ # batch size of the captured graph.
195
+ graph_batch_size = _get_graph_batch_size(batch_size)
196
+ assert graph_batch_size >= batch_size
197
+ for _ in range(graph_batch_size - batch_size):
198
+ input_tokens.append([])
199
+ input_positions.append([])
200
+ slot_mapping.append([])
201
+ context_lens.append(1)
202
+ block_tables.append([])
203
+ batch_size = graph_batch_size
204
+
205
+ # When using CUDA graph, we don't need to make the tensors on the GPU
206
+ # because they will be eventually copied to the designated GPU buffer.
207
+ device = "cpu" if use_captured_graph else "cuda"
208
input_tokens = _make_tensor_with_pad(input_tokens,
209
max_len=1,
210
pad=0,
160
- dtype=torch.long)
211
+ dtype=torch.long,
212
+ device=device)
213
input_positions = _make_tensor_with_pad(input_positions,
214
max_len=1,
215
pad=0,
164
- dtype=torch.long)
216
+ dtype=torch.long,
217
+ device=device)
218
slot_mapping = _make_tensor_with_pad(slot_mapping,
219
max_len=1,
220
pad=_PAD_SLOT_ID,
168
- dtype=torch.long)
169
- max_context_len = max(context_lens)
221
+ dtype=torch.long,
222
+ device=device)
223
context_lens = torch.tensor(context_lens,
224
dtype=torch.int,
172
- device="cuda")
173
- max_block_table_len = max([len(t) for t in block_tables])
174
- block_tables = _make_tensor_with_pad(block_tables,
175
- max_len=max_block_table_len,
176
- pad=0,
177
- dtype=torch.int)
225
+ device=device)
226
+
227
+ if use_captured_graph:
228
+ # The shape of graph_block_tables is
229
+ # [max batch size, max context len // block size].
230
+ input_block_tables = self.graph_block_tables[:batch_size]
231
+ for i, block_table in enumerate(block_tables):
232
+ if block_table:
233
+ input_block_tables[i, :len(block_table)] = block_table
234
+ block_tables = torch.from_numpy(input_block_tables).to(device)
235
+ else:
236
+ block_tables = _make_tensor_with_pad(
237
+ block_tables,
238
+ max_len=max_context_len,
239
+ pad=0,
240
+ dtype=torch.int,
241
+ )
242
243
input_metadata = InputMetadata(
244
prompt_lens=[],
@@ -182,6 +246,7 @@ class ModelRunner:
246
max_context_len=max_context_len,
247
context_lens=context_lens,
248
block_tables=block_tables,
249
+ use_cuda_graph=use_captured_graph,
250
)
251
return input_tokens, input_positions, input_metadata
252
@@ -260,12 +325,11 @@ class ModelRunner:
325
self,
326
seq_group_metadata_list: List[SequenceGroupMetadata],
327
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
263
- cache_events: Optional[List[torch.cuda.Event]] = None,
328
) -> SamplerOutput:
329
# NOTE: We assume that all sequences in the group are all prompts or
330
# all decodes.
267
- # Prepare input tensors.
331
is_prompt = seq_group_metadata_list[0].is_prompt
332
+ # Prepare input tensors.
333
if is_prompt:
334
inputs = self._prepare_prompt(seq_group_metadata_list)
335
input_tokens, input_positions, input_metadata = inputs
@@ -276,12 +340,16 @@ class ModelRunner:
340
input_metadata.prompt_lens)
341
342
# Execute the model.
279
- hidden_states = self.model(
343
+ if input_metadata.use_cuda_graph:
344
+ graph_batch_size = input_tokens.shape[0]
345
+ model_executable = self.graph_runners[graph_batch_size]
346
+ else:
347
+ model_executable = self.model
348
+ hidden_states = model_executable(
349
input_ids=input_tokens,
350
positions=input_positions,
351
kv_caches=kv_caches,
352
input_metadata=input_metadata,
284
- cache_events=cache_events,
353
)
354
355
# Sample the next token.
@@ -319,8 +387,139 @@ class ModelRunner:
387
num_layers = self.model_config.get_num_layers(self.parallel_config)
388
kv_caches = [(None, None)] * num_layers
389
self.execute_model(seqs, kv_caches)
390
+ torch.cuda.synchronize()
391
return
392
393
+ @torch.inference_mode()
394
+ def capture_model(self, kv_caches: List[KVCache]) -> None:
395
+ assert not self.model_config.enforce_eager
396
+ logger.info("Capturing the model for CUDA graphs. This may lead to "
397
+ "unexpected consequences if the model is not static. To "
398
+ "run the model in eager mode, set 'enforce_eager=True' or "
399
+ "use '--enforce-eager' in the CLI.")
400
+ start_time = time.perf_counter()
401
+
402
+ # Prepare dummy inputs. These will be reused for all batch sizes.
403
+ max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
404
+ input_tokens = torch.zeros(max_batch_size, 1, dtype=torch.long).cuda()
405
+ input_positions = torch.zeros(max_batch_size, 1,
406
+ dtype=torch.long).cuda()
407
+ slot_mapping = torch.empty(max_batch_size, 1, dtype=torch.long).cuda()
408
+ slot_mapping.fill_(_PAD_SLOT_ID)
409
+ context_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
410
+ block_tables = torch.from_numpy(self.graph_block_tables).cuda()
411
+
412
+ # NOTE: Capturing the largest batch size first may help reduce the
413
+ # memory usage of CUDA graph.
414
+ for batch_size in reversed(_BATCH_SIZES_TO_CAPTURE):
415
+ # Create dummy input_metadata.
416
+ input_metadata = InputMetadata(
417
+ prompt_lens=[],
418
+ slot_mapping=slot_mapping[:batch_size],
419
+ max_context_len=self.max_context_len_to_capture,
420
+ context_lens=context_lens[:batch_size],
421
+ block_tables=block_tables[:batch_size],
422
+ use_cuda_graph=True,
423
+ )
424
+
425
+ graph_runner = CUDAGraphRunner(self.model)
426
+ graph_runner.capture(
427
+ input_tokens[:batch_size],
428
+ input_positions[:batch_size],
429
+ kv_caches,
430
+ input_metadata,
431
+ memory_pool=self.graph_memory_pool,
432
+ )
433
+ self.graph_memory_pool = graph_runner.graph.pool()
434
+ self.graph_runners[batch_size] = graph_runner
435
+
436
+ end_time = time.perf_counter()
437
+ elapsed_time = end_time - start_time
438
+ # This usually takes < 10 seconds.
439
+ logger.info(f"Graph capturing finished in {elapsed_time:.0f} secs.")
440
+
441
+
442
+class CUDAGraphRunner:
443
+
444
+ def __init__(self, model: nn.Module):
445
+ self.model = model
446
+ self.graph = None
447
+ self.input_buffers: Dict[str, torch.Tensor] = {}
448
+ self.output_buffers: Dict[str, torch.Tensor] = {}
449
+
450
+ def capture(
451
+ self,
452
+ input_ids: torch.Tensor,
453
+ positions: torch.Tensor,
454
+ kv_caches: List[KVCache],
455
+ input_metadata: InputMetadata,
456
+ memory_pool,
457
+ ) -> None:
458
+ assert self.graph is None
459
+ # Run the model once without capturing the graph.
460
+ # This is to make sure that the captured graph does not include the
461
+ # kernel launches for initial benchmarking (e.g., Triton autotune).
462
+ with with_custom_nccl_for_all_reduce():
463
+ self.model(
464
+ input_ids,
465
+ positions,
466
+ kv_caches,
467
+ input_metadata,
468
+ )
469
+ torch.cuda.synchronize()
470
+
471
+ # Capture the graph.
472
+ # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
473
+ # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
474
+ self.graph = torch.cuda.CUDAGraph()
475
+ with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
476
+ with with_custom_nccl_for_all_reduce():
477
+ hidden_states = self.model(
478
+ input_ids,
479
+ positions,
480
+ kv_caches,
481
+ input_metadata,
482
+ )
483
+ torch.cuda.synchronize()
484
+
485
+ # Save the input and output buffers.
486
+ self.input_buffers = {
487
+ "input_ids": input_ids,
488
+ "positions": positions,
489
+ "kv_caches": kv_caches,
490
+ "slot_mapping": input_metadata.slot_mapping,
491
+ "context_lens": input_metadata.context_lens,
492
+ "block_tables": input_metadata.block_tables,
493
+ }
494
+ self.output_buffers = {"hidden_states": hidden_states}
495
+ return
496
+
497
+ def forward(
498
+ self,
499
+ input_ids: torch.Tensor,
500
+ positions: torch.Tensor,
501
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
502
+ input_metadata: InputMetadata,
503
+ ) -> torch.Tensor:
504
+ # KV caches are fixed tensors, so we don't need to copy them.
505
+ del kv_caches
506
+
507
+ # Copy the input tensors to the input buffers.
508
+ self.input_buffers["input_ids"].copy_(input_ids)
509
+ self.input_buffers["positions"].copy_(positions)
510
+ self.input_buffers["slot_mapping"].copy_(input_metadata.slot_mapping)
511
+ self.input_buffers["context_lens"].copy_(input_metadata.context_lens)
512
+ self.input_buffers["block_tables"].copy_(input_metadata.block_tables)
513
+
514
+ # Run the graph.
515
+ self.graph.replay()
516
+
517
+ # Return the output tensor.
518
+ return self.output_buffers["hidden_states"]
519
+
520
+ def __call__(self, *args, **kwargs):
521
+ return self.forward(*args, **kwargs)
522
+
523
524
def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
525
assert len(x) <= max_len
@@ -332,6 +531,16 @@ def _make_tensor_with_pad(
531
max_len: int,
532
pad: int,
533
dtype: torch.dtype,
534
+ device: Union[str, torch.device] = "cuda",
535
) -> torch.Tensor:
536
padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
337
- return torch.tensor(padded_x, dtype=dtype, device="cuda")
537
+ return torch.tensor(padded_x, dtype=dtype, device=device)
538
+
539
+
540
+def _get_graph_batch_size(batch_size: int) -> int:
541
+ if batch_size <= 2:
542
+ return batch_size
543
+ elif batch_size <= 4:
544
+ return 4
545
+ else:
546
+ return (batch_size + 7) // 8 * 8
vllm/worker/worker.py
+40
-12
@@ -8,6 +8,7 @@ import torch.distributed
8
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig)
10
from vllm.model_executor import set_random_seed
11
+from vllm.model_executor.parallel_utils import cupy_utils
12
from vllm.model_executor.parallel_utils.parallel_state import (
13
initialize_model_parallel)
14
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -46,7 +47,7 @@ class Worker:
47
self.cache_events = None
48
self.gpu_cache = None
49
49
- def init_model(self):
50
+ def init_model(self, cupy_port: Optional[int] = None):
51
# This env var set by Ray causes exceptions with graph building.
52
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
53
# Env vars will be set by Ray.
@@ -62,7 +63,7 @@ class Worker:
63
64
# Initialize the distributed environment.
65
_init_distributed_environment(self.parallel_config, self.rank,
65
- self.distributed_init_method)
66
+ cupy_port, self.distributed_init_method)
67
68
# Initialize the model.
69
set_random_seed(self.model_config.seed)
@@ -100,10 +101,6 @@ class Worker:
101
num_gpu_blocks = max(num_gpu_blocks, 0)
102
num_cpu_blocks = max(num_cpu_blocks, 0)
103
torch.cuda.empty_cache()
103
-
104
- # Reset the seed to ensure that the random state is not affected by
105
- # the model initialization and profiling.
106
- set_random_seed(self.model_config.seed)
104
return num_gpu_blocks, num_cpu_blocks
105
106
def init_cache_engine(self, cache_config: CacheConfig) -> None:
@@ -114,6 +111,13 @@ class Worker:
111
self.gpu_cache = self.cache_engine.gpu_cache
112
self.model_runner.set_block_size(self.cache_engine.block_size)
113
114
+ def warm_up_model(self) -> None:
115
+ if not self.model_config.enforce_eager:
116
+ self.model_runner.capture_model(self.gpu_cache)
117
+ # Reset the seed to ensure that the random state is not affected by
118
+ # the model initialization and profiling.
119
+ set_random_seed(self.model_config.seed)
120
+
121
@torch.inference_mode()
122
def execute_model(
123
self,
@@ -136,21 +140,24 @@ class Worker:
140
141
cache_events = self.cache_events if issued_cache_op else None
142
143
+ # Wait for cache operations to finish.
144
+ # TODO(woosuk): Profile swapping overhead and optimize if needed.
145
+ if cache_events is not None:
146
+ for event in cache_events:
147
+ event.wait()
148
# If there is no input, we don't need to execute the model.
149
if not seq_group_metadata_list:
141
- if cache_events is not None:
142
- for event in cache_events:
143
- event.wait()
150
return {}
151
152
output = self.model_runner.execute_model(seq_group_metadata_list,
147
- self.gpu_cache, cache_events)
153
+ self.gpu_cache)
154
return output
155
156
157
def _init_distributed_environment(
158
parallel_config: ParallelConfig,
159
rank: int,
160
+ cupy_port: Optional[int],
161
distributed_init_method: Optional[str] = None,
162
) -> None:
163
"""Initialize the distributed environment."""
@@ -173,8 +180,29 @@ def _init_distributed_environment(
180
init_method=distributed_init_method,
181
)
182
176
- # A small all_reduce for warmup.
177
- torch.distributed.all_reduce(torch.zeros(1).cuda())
183
+ if cupy_utils.is_initialized():
184
+ cupy_world_size = cupy_utils.get_world_size()
185
+ if cupy_world_size != parallel_config.world_size:
186
+ raise RuntimeError(
187
+ "cupy.distributed is already initialized but the cupy world "
188
+ "size does not match parallel_config.world_size "
189
+ f"({cupy_world_size} vs. {parallel_config.world_size}).")
190
+ elif parallel_config.world_size > 1:
191
+ # NOTE(woosuk): We don't initialize CuPy process group when world size
192
+ # is 1.
193
+ # TODO(woosuk): Support multi-node connection.
194
+ cupy_utils.init_process_group(
195
+ world_size=parallel_config.world_size,
196
+ rank=rank,
197
+ host="localhost",
198
+ port=cupy_port,
199
+ )
200
+
201
+ if parallel_config.world_size > 1:
202
+ # A small all_reduce for warmup.
203
+ torch.distributed.all_reduce(torch.zeros(1).cuda())
204
+ cupy_utils.all_reduce(torch.zeros(1).cuda())
205
+
206
initialize_model_parallel(parallel_config.tensor_parallel_size,
207
parallel_config.pipeline_parallel_size)
208