@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c1376e0f82
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Change scheduler & input tensor shape (#1381)

Woosuk Kwon committed Oct 16, 2023 at 17:48 UTC c1376e0f825e88e32b5aca85c676fe547bcb03c9
13 files changed +181 -179
csrc/activation_kernels.cu
+14 -14
@@ -13,8 +13,8 @@ __device__ __forceinline__ T silu(const T& x) {
13
14 template<typename scalar_t>
15 __global__ void silu_and_mul_kernel(
16 - scalar_t* __restrict__ out, // [num_tokens, d]
17 - const scalar_t* __restrict__ input, // [num_tokens, 2, d]
16 + scalar_t* __restrict__ out, // [..., d]
17 + const scalar_t* __restrict__ input, // [..., 2, d]
18 const int d) {
19 const int token_idx = blockIdx.x;
20 for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
@@ -27,11 +27,11 @@ __global__ void silu_and_mul_kernel(
27 } // namespace vllm
28
29 void silu_and_mul(
30 - torch::Tensor& out, // [num_tokens, d]
31 - torch::Tensor& input) // [num_tokens, 2 * d]
30 + torch::Tensor& out, // [..., d]
31 + torch::Tensor& input) // [..., 2 * d]
32 {
33 - int num_tokens = input.size(0);
34 - int d = input.size(1) / 2;
33 + int num_tokens = input.numel() / input.size(-1);
34 + int d = input.size(-1) / 2;
35
36 dim3 grid(num_tokens);
37 dim3 block(std::min(d, 1024));
@@ -52,8 +52,8 @@ namespace vllm {
52 // Element-wise activation kernel template.
53 template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
54 __global__ void activation_kernel(
55 - scalar_t* __restrict__ out, // [num_tokens, d]
56 - const scalar_t* __restrict__ input, // [num_tokens, d]
55 + scalar_t* __restrict__ out, // [..., d]
56 + const scalar_t* __restrict__ input, // [..., d]
57 const int d) {
58 const int token_idx = blockIdx.x;
59 for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
@@ -66,8 +66,8 @@ __global__ void activation_kernel(
66
67 // Launch element-wise activation kernel.
68 #define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
69 - int num_tokens = input.size(0); \
70 - int d = input.size(1); \
69 + int d = input.size(-1); \
70 + int num_tokens = input.numel() / d; \
71 dim3 grid(num_tokens); \
72 dim3 block(std::min(d, 1024)); \
73 const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
@@ -100,15 +100,15 @@ __device__ __forceinline__ T gelu_fast_kernel(const T& x) {
100 } // namespace vllm
101
102 void gelu_new(
103 - torch::Tensor& out, // [num_tokens, d]
104 - torch::Tensor& input) // [num_tokens, d]
103 + torch::Tensor& out, // [..., d]
104 + torch::Tensor& input) // [..., d]
105 {
106 LAUNCH_ACTIVATION_KERNEL(vllm::gelu_new_kernel);
107 }
108
109 void gelu_fast(
110 - torch::Tensor& out, // [num_tokens, d]
111 - torch::Tensor& input) // [num_tokens, d]
110 + torch::Tensor& out, // [..., d]
111 + torch::Tensor& input) // [..., d]
112 {
113 LAUNCH_ACTIVATION_KERNEL(vllm::gelu_fast_kernel);
114 }
csrc/cache_kernels.cu
+7 -2
@@ -154,6 +154,11 @@ __global__ void reshape_and_cache_kernel(
154 const int x) {
155 const int token_idx = blockIdx.x;
156 const int slot_idx = slot_mapping[token_idx];
157 + if (slot_idx < 0) {
158 + // Padding token that should be ignored.
159 + return;
160 + }
161 +
162 const int block_idx = slot_idx / block_size;
163 const int block_offset = slot_idx % block_size;
164
@@ -176,8 +181,8 @@ __global__ void reshape_and_cache_kernel(
181 + head_idx * head_size * block_size
182 + head_offset * block_size
183 + block_offset;
179 - key_cache[tgt_key_idx] = __ldg(&key[src_key_idx]);
180 - value_cache[tgt_value_idx] = __ldg(&value[src_value_idx]);
184 + key_cache[tgt_key_idx] = key[src_key_idx];
185 + value_cache[tgt_value_idx] = value[src_value_idx];
186 }
187 }
188
csrc/layernorm_kernels.cu
+6 -6
@@ -9,8 +9,8 @@ namespace vllm {
9 // TODO(woosuk): Further optimize this kernel.
10 template<typename scalar_t>
11 __global__ void rms_norm_kernel(
12 - scalar_t* __restrict__ out, // [num_tokens, hidden_size]
13 - const scalar_t* __restrict__ input, // [num_tokens, hidden_size]
12 + scalar_t* __restrict__ out, // [..., hidden_size]
13 + const scalar_t* __restrict__ input, // [..., hidden_size]
14 const scalar_t* __restrict__ weight, // [hidden_size]
15 const float epsilon,
16 const int num_tokens,
@@ -37,12 +37,12 @@ __global__ void rms_norm_kernel(
37 } // namespace vllm
38
39 void rms_norm(
40 - torch::Tensor& out, // [num_tokens, hidden_size]
41 - torch::Tensor& input, // [num_tokens, hidden_size]
40 + torch::Tensor& out, // [..., hidden_size]
41 + torch::Tensor& input, // [..., hidden_size]
42 torch::Tensor& weight, // [hidden_size]
43 float epsilon) {
44 - int num_tokens = input.size(0);
45 - int hidden_size = input.size(1);
44 + int hidden_size = input.size(-1);
45 + int num_tokens = input.numel() / hidden_size;
46
47 dim3 grid(num_tokens);
48 dim3 block(std::min(hidden_size, 1024));
csrc/pos_encoding_kernels.cu
+11 -11
@@ -37,9 +37,9 @@ inline __device__ void apply_rotary_embedding(
37
38 template<typename scalar_t, bool IS_NEOX>
39 __global__ void rotary_embedding_kernel(
40 - const int64_t* __restrict__ positions, // [num_tokens]
41 - scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
42 - scalar_t* __restrict__ key, // [num_tokens, num_kv_heads, head_size]
40 + const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
41 + scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
42 + scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
43 const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
44 const int rot_dim,
45 const int query_stride,
@@ -78,18 +78,18 @@ __global__ void rotary_embedding_kernel(
78 } // namespace vllm
79
80 void rotary_embedding(
81 - torch::Tensor& positions, // [num_tokens]
82 - torch::Tensor& query, // [num_tokens, num_heads * head_size]
83 - torch::Tensor& key, // [num_tokens, num_kv_heads * head_size]
81 + torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
82 + torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or [num_tokens, num_heads * head_size]
83 + torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or [num_tokens, num_kv_heads * head_size]
84 int head_size,
85 torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
86 bool is_neox) {
87 - int num_tokens = query.size(0);
87 + int num_tokens = query.numel() / query.size(-1);
88 int rot_dim = cos_sin_cache.size(1);
89 - int num_heads = query.size(1) / head_size;
90 - int num_kv_heads = key.size(1) / head_size;
91 - int query_stride = query.stride(0);
92 - int key_stride = key.stride(0);
89 + int num_heads = query.size(-1) / head_size;
90 + int num_kv_heads = key.size(-1) / head_size;
91 + int query_stride = query.stride(-2);
92 + int key_stride = key.stride(-2);
93
94 dim3 grid(num_tokens);
95 dim3 block(std::min(num_heads * rot_dim / 2, 512));
vllm/config.py
+3
@@ -268,6 +268,7 @@ class SchedulerConfig:
268 iteration.
269 max_model_len: Maximum length of a sequence (including prompt
270 and generated text).
271 + max_paddings: Maximum number of paddings to be added to a batch.
272 """
273
274 def __init__(
@@ -275,6 +276,7 @@ class SchedulerConfig:
276 max_num_batched_tokens: Optional[int],
277 max_num_seqs: int,
278 max_model_len: int,
279 + max_paddings: int,
280 ) -> None:
281 if max_num_batched_tokens is not None:
282 self.max_num_batched_tokens = max_num_batched_tokens
@@ -284,6 +286,7 @@ class SchedulerConfig:
286 self.max_num_batched_tokens = max(max_model_len, 2048)
287 self.max_num_seqs = max_num_seqs
288 self.max_model_len = max_model_len
289 + self.max_paddings = max_paddings
290 self._verify_args()
291
292 def _verify_args(self) -> None:
vllm/core/scheduler.py
+11 -4
@@ -131,7 +131,8 @@ class Scheduler:
131 # requests in the generation phase.
132 num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
133 for seq_group in self.running)
134 - num_batched_tokens = 0
134 + seq_lens: List[int] = []
135 +
136 # Optimization: We do not sort the waiting queue since the preempted
137 # sequence groups are added to the front and the new sequence groups
138 # are added to the back.
@@ -157,7 +158,9 @@ class Scheduler:
158 break
159
160 # If the number of batched tokens exceeds the limit, stop.
160 - if (num_batched_tokens + num_prompt_tokens >
161 + new_seq_lens = seq_lens + [num_prompt_tokens]
162 + num_batched_tokens = len(new_seq_lens) * max(new_seq_lens)
163 + if (num_batched_tokens >
164 self.scheduler_config.max_num_batched_tokens):
165 break
166
@@ -168,10 +171,14 @@ class Scheduler:
171 self.scheduler_config.max_num_seqs):
172 break
173
174 + num_paddings = num_batched_tokens - sum(new_seq_lens)
175 + if num_paddings > self.scheduler_config.max_paddings:
176 + break
177 + seq_lens = new_seq_lens
178 +
179 seq_group = self.waiting.pop(0)
180 self._allocate(seq_group)
181 self.running.append(seq_group)
174 - num_batched_tokens += num_prompt_tokens
182 num_curr_seqs += num_new_seqs
183 scheduled.append(seq_group)
184
@@ -179,7 +186,7 @@ class Scheduler:
186 scheduler_outputs = SchedulerOutputs(
187 scheduled_seq_groups=scheduled,
188 prompt_run=True,
182 - num_batched_tokens=num_batched_tokens,
189 + num_batched_tokens=len(seq_lens) * max(seq_lens),
190 blocks_to_swap_in=blocks_to_swap_in,
191 blocks_to_swap_out=blocks_to_swap_out,
192 blocks_to_copy=blocks_to_copy,
vllm/engine/arg_utils.py
+7 -1
@@ -27,6 +27,7 @@ class EngineArgs:
27 gpu_memory_utilization: float = 0.90
28 max_num_batched_tokens: Optional[int] = None
29 max_num_seqs: int = 256
30 + max_paddings: int = 256
31 disable_log_stats: bool = False
32 revision: Optional[str] = None
33 tokenizer_revision: Optional[str] = None
@@ -156,6 +157,10 @@ class EngineArgs:
157 type=int,
158 default=EngineArgs.max_num_seqs,
159 help='maximum number of sequences per iteration')
160 + parser.add_argument('--max-paddings',
161 + type=int,
162 + default=EngineArgs.max_paddings,
163 + help='maximum number of paddings in a batch')
164 parser.add_argument('--disable-log-stats',
165 action='store_true',
166 help='disable logging statistics')
@@ -193,7 +198,8 @@ class EngineArgs:
198 self.worker_use_ray)
199 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
200 self.max_num_seqs,
196 - model_config.max_model_len)
201 + model_config.max_model_len,
202 + self.max_paddings)
203 return model_config, cache_config, parallel_config, scheduler_config
204
205
vllm/model_executor/input_metadata.py
+5 -6
@@ -39,11 +39,12 @@ class InputMetadata:
39 self.max_context_len = max_context_len
40 self.block_tables = block_tables
41
42 + self.max_prompt_len = max(prompt_lens) if prompt_lens else 0
43 self.to_cache = None
44 if sliding_window is not None:
45 # We need to keep the positions of sliding windows within
46 # the key / value tables, this is helpful to know which
46 - # elements we need to cache and where
47 + # elements we need to cache.
48 to_cache, start_idx = [], 0
49 for prompt_len in self.prompt_lens:
50 to_cache.extend(
@@ -51,16 +52,15 @@ class InputMetadata:
52 start_idx + max(0, prompt_len - sliding_window),
53 start_idx + prompt_len,
54 ))
54 - start_idx += prompt_len
55 + start_idx += self.max_prompt_len
56 to_cache.extend(range(start_idx, slot_mapping.shape[0]))
57 self.to_cache = torch.tensor(to_cache,
58 dtype=torch.int32,
59 device=self.slot_mapping.device)
60
61 self.num_prompts = len(prompt_lens)
61 - self.num_prompt_tokens = sum(prompt_lens)
62 + self.num_prompt_tokens = self.num_prompts * self.max_prompt_len
63 self.num_generation_tokens = context_lens.shape[0]
63 - self.num_valid_tokens = slot_mapping.shape[0]
64 if block_tables.numel() > 0:
65 self.max_num_blocks_per_seq = block_tables.shape[1]
66 else:
@@ -69,12 +69,11 @@ class InputMetadata:
69 assert context_lens.shape[0] == self.num_generation_tokens
70
71 # Set during the execution of the first attention op.
72 - self.attn_bias: List[AttentionBias] = []
72 + self.attn_bias: Optional[AttentionBias] = None
73
74 def __repr__(self) -> str:
75 # Print only useful metadata.
76 return (f'InputMetadata('
77 - f'num_valid_tokens={self.num_valid_tokens}, '
77 f'num_prompt_tokens={self.num_prompt_tokens}, '
78 f'num_prompts={self.num_prompts}, '
79 f'prompt_lens={self.prompt_lens}, '
vllm/model_executor/layers/activation.py
+8 -12
@@ -8,17 +8,17 @@ from vllm import activation_ops
8 class SiluAndMul(nn.Module):
9 """An activation function for SwiGLU.
10
11 - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[1] // 2.
11 + The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.
12
13 Shapes:
14 - x: (num_tokens, 2 * d)
15 - return: (num_tokens, d)
14 + x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
15 + return: (batch_size, seq_len, d) or (num_tokens, d)
16 """
17
18 def forward(self, x: torch.Tensor) -> torch.Tensor:
19 - num_tokens = x.shape[0]
20 - d = x.shape[1] // 2
21 - out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
19 + d = x.shape[-1] // 2
20 + output_shape = (x.shape[:-1] + (d, ))
21 + out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
22 activation_ops.silu_and_mul(out, x)
23 return out
24
@@ -26,9 +26,7 @@ class SiluAndMul(nn.Module):
26 class NewGELU(nn.Module):
27
28 def forward(self, x: torch.Tensor) -> torch.Tensor:
29 - num_tokens = x.shape[0]
30 - d = x.shape[1]
31 - out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
29 + out = torch.empty_like(x)
30 activation_ops.gelu_new(out, x)
31 return out
32
@@ -36,9 +34,7 @@ class NewGELU(nn.Module):
34 class FastGELU(nn.Module):
35
36 def forward(self, x: torch.Tensor) -> torch.Tensor:
39 - num_tokens = x.shape[0]
40 - d = x.shape[1]
41 - out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
37 + out = torch.empty_like(x)
38 activation_ops.gelu_fast(out, x)
39 return out
40
vllm/model_executor/layers/attention.py
+71 -95
@@ -23,25 +23,9 @@ class PagedAttention(nn.Module):
23 # pylint: disable=line-too-long
24 """GPT-style multi-head PagedAttention.
25
26 - This class takes flattened 1D query, key, and value tensors as input. The
27 - input 1D tensors can either contain prompt tokens or generation tokens, in
28 - addition to paddings.
29 -
30 - If the input tensors contain prompt tokens, the layout is as follows:
31 -
32 - |<---------------------- num_valid_tokens ---------------------->|
33 - |<--------------- num_prompt_tokens -------------->|
34 - |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|<--padding-->|
35 -
36 - Otherwise, the layout is as follows:
37 -
38 - |<------------------ num_valid_tokens ------------------->|
39 - |<------- num_generation_tokens (M) ------->|
40 - |<--generation_0-->|...|<--generation_M-1-->|<--padding-->|
41 -
42 - The prompts might have different lengths, while the generation tokens always
43 - have length 1. The paddings are appended to make the input length a multiple
44 - of 8, which is desirable for Tensor Cores.
26 + This class takes query, key, and value tensors as input. The input tensors
27 + can either contain prompt tokens or generation tokens, in addition to
28 + paddings.
29
30 The class does the following:
31 1. Perform multi_query_kv_attention for the prompts. This operation does
@@ -53,7 +37,7 @@ class PagedAttention(nn.Module):
37 4. Perform single_query_cached_kv_attention for the generation tokens.
38 This operation reads the previous key and value tensors from the KV
39 cache.
56 - 5. Output a flattened 1D tensor.
40 + 5. Return the output tensor.
41 """
42
43 def __init__(self,
@@ -85,14 +69,15 @@ class PagedAttention(nn.Module):
69 dtype: torch.dtype,
70 ) -> None:
71 del dtype # Unused.
88 - if input_metadata.attn_bias:
72 + if input_metadata.attn_bias is not None:
73 # Already set by a previous layer.
74 return
91 - prompt_lens = input_metadata.prompt_lens
75 + prompt_lens = [input_metadata.max_prompt_len
76 + ] * input_metadata.num_prompts
77 attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
78 if self.sliding_window is not None:
79 attn_bias = attn_bias.make_local_attention(self.sliding_window)
95 - input_metadata.attn_bias.append(attn_bias)
80 + input_metadata.attn_bias = attn_bias
81
82 def multi_query_kv_attention(
83 self,
@@ -111,7 +96,6 @@ class PagedAttention(nn.Module):
96 value: shape = [num_prompt_tokens, num_kv_heads, head_size]
97 input_metadata: metadata for paged attention.
98 """
114 -
99 if self.num_kv_heads != self.num_heads:
100 # Project the key and value tensors to the desired number of heads.
101 key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
@@ -124,7 +108,7 @@ class PagedAttention(nn.Module):
108 query.unsqueeze(0),
109 key.unsqueeze(0),
110 value.unsqueeze(0),
127 - attn_bias=input_metadata.attn_bias[0],
111 + attn_bias=input_metadata.attn_bias,
112 p=0.0,
113 scale=self.scale,
114 )
@@ -232,12 +216,12 @@ class PagedAttention(nn.Module):
216 """PagedAttention forward pass.
217
218 NOTE: The query, key, and value tensors must be sliced from a qkv
235 - tensor of shape [num_tokens, 3 * num_heads * head_size].
219 + tensor of shape [batch_size, seq_len, 3 * num_heads * head_size].
220
221 Args:
238 - query: shape = [num_tokens, num_heads * head_size]
239 - key: shape = [num_tokens, num_kv_heads * head_size]
240 - value: shape = [num_tokens, num_kv_heads * head_size]
222 + query: shape = [batch_size, seq_len, num_heads * head_size]
223 + key: shape = [batch_size, seq_len, num_kv_heads * head_size]
224 + value: shape = [batch_size, num_kv_heads * head_size]
225 key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
226 block_size, x]
227 value_cache: shape = [num_blocks, num_kv_heads, head_size,
@@ -246,9 +230,9 @@ class PagedAttention(nn.Module):
230 cache_event: event to wait for the cache operations to finish.
231
232 Returns:
249 - shape = [num_tokens, num_heads * head_size]
233 + shape = [batch_size, seq_len, num_heads * head_size]
234 """
251 -
235 + batch_size, seq_len, _ = query.shape
236 # Reshape the query, key, and value tensors.
237 query = query.view(-1, self.num_heads, self.head_size)
238 key = key.view(-1, self.num_kv_heads, self.head_size)
@@ -264,10 +248,10 @@ class PagedAttention(nn.Module):
248 assert input_metadata.num_generation_tokens == 0
249 self.set_attn_bias(input_metadata, dtype=query.dtype)
250 self.multi_query_kv_attention(
267 - output[:num_prompt_tokens],
268 - query[:num_prompt_tokens],
269 - key[:num_prompt_tokens],
270 - value[:num_prompt_tokens],
251 + output,
252 + query,
253 + key,
254 + value,
255 input_metadata,
256 )
257
@@ -278,13 +262,10 @@ class PagedAttention(nn.Module):
262 # Reshape the keys and values and store them in the cache.
263 # When key_cache and value_cache are not provided, the new key
264 # and value vectors will not be cached.
281 - num_valid_tokens = input_metadata.num_valid_tokens
282 - if (num_valid_tokens > 0 and key_cache is not None
283 - and value_cache is not None):
284 - # The stride is 3 because the key and value are sliced from qkv.
285 - key_to_cache = key[:num_valid_tokens]
286 - value_to_cache = value[:num_valid_tokens]
287 - slot_mapping = input_metadata.slot_mapping
265 + if key_cache is not None and value_cache is not None:
266 + key_to_cache = key
267 + value_to_cache = value
268 + slot_mapping = input_metadata.slot_mapping.view(-1)
269 if input_metadata.to_cache is not None:
270 key_to_cache = key_to_cache[input_metadata.to_cache]
271 value_to_cache = value_to_cache[input_metadata.to_cache]
@@ -305,14 +286,14 @@ class PagedAttention(nn.Module):
286 "key_cache and value_cache must be provided when "
287 "generating tokens.")
288 # Compute the attention op for generation tokens.
308 - self.single_query_cached_kv_attention(
309 - output[num_prompt_tokens:num_valid_tokens],
310 - query[num_prompt_tokens:num_valid_tokens], key_cache,
311 - value_cache, input_metadata, self.get_alibi_slopes())
289 + self.single_query_cached_kv_attention(output, query, key_cache,
290 + value_cache, input_metadata,
291 + self.get_alibi_slopes())
292
293 # Reshape the output tensor.
294 # NOTE(woosuk): The output tensor may include paddings.
315 - return output.view(-1, self.num_heads * self.head_size)
295 + return output.view(batch_size, seq_len,
296 + self.num_heads * self.head_size)
297
298
299 class PagedAttentionWithRoPE(PagedAttention):
@@ -368,10 +349,10 @@ class PagedAttentionWithRoPE(PagedAttention):
349 """ PagedAttention forward pass with rotary embedding.
350
351 Args:
371 - positions: shape = [num_tokens]
372 - query: shape = [num_tokens, num_heads * head_size]
373 - key: shape = [num_tokens, num_kv_heads * head_size]
374 - value: shape = [num_tokens, num_kv_heads * head_size]
352 + positions: shape = [batch_size, seq_len]
353 + query: shape = [batch_size, seq_len, num_heads * head_size]
354 + key: shape = [batch_size, seq_len, num_kv_heads * head_size]
355 + value: shape = [batch_size, seq_len, num_kv_heads * head_size]
356 key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
357 block_size, x]
358 value_cache: shape = [num_blocks, num_kv_heads, head_size,
@@ -380,7 +361,7 @@ class PagedAttentionWithRoPE(PagedAttention):
361 cache_event: event to wait for the cache operations to finish.
362
363 Returns:
383 - shape = [num_tokens, num_heads * head_size]
364 + shape = [batch_size, seq_len, num_heads * head_size]
365 """
366
367 # Apply rotary embedding to the query and key before passing them
@@ -414,34 +395,34 @@ class PagedAttentionWithALiBi(PagedAttention):
395
396 def set_attn_bias(self, input_metadata: InputMetadata,
397 dtype: torch.dtype) -> None:
417 - if input_metadata.attn_bias:
398 + if input_metadata.attn_bias is not None:
399 # Already set by a previous layer.
400 return
420 - # Generates ALiBi mask for each prompt.
421 - for prompt_len in input_metadata.prompt_lens:
422 - bias = torch.arange(prompt_len, dtype=dtype)
423 - # NOTE(zhuohan): HF uses
424 - # `bias = bias[None, :].repeat(prompt_len, 1)`
425 - # here. We find that both biases give the same results, but
426 - # the bias below more accurately follows the original ALiBi
427 - # paper.
428 - bias = bias[None, :] - bias[:, None]
429 - bias = bias.to(self.alibi_slopes.device)
430 -
431 - # When using custom attention bias, xformers requires the bias to
432 - # be sliced from a tensor whose length is a multiple of 8.
433 - padded_len = (prompt_len + 7) // 8 * 8
434 - bias = torch.empty(
435 - 1, # batch_size
436 - self.num_heads,
437 - prompt_len,
438 - padded_len,
439 - device=self.alibi_slopes.device,
440 - dtype=dtype,
441 - )[:, :, :, :prompt_len].copy_(bias)
442 - bias.mul_(self.alibi_slopes[:, None, None])
443 - attn_bias = LowerTriangularMaskWithTensorBias(bias)
444 - input_metadata.attn_bias.append(attn_bias)
401 + # Generates ALiBi mask based on the max prompt length.
402 + max_prompt_len = input_metadata.max_prompt_len
403 + bias = torch.arange(max_prompt_len, dtype=dtype)
404 + # NOTE(zhuohan): HF uses
405 + # `bias = bias[None, :].repeat(prompt_len, 1)`
406 + # here. We find that both biases give the same results, but
407 + # the bias below more accurately follows the original ALiBi
408 + # paper.
409 + bias = bias[None, :] - bias[:, None]
410 + bias = bias.to(self.alibi_slopes.device)
411 +
412 + # When using custom attention bias, xformers requires the bias to
413 + # be sliced from a tensor whose length is a multiple of 8.
414 + padded_len = (max_prompt_len + 7) // 8 * 8
415 + bias = torch.empty(
416 + input_metadata.num_prompts,
417 + self.num_heads,
418 + max_prompt_len,
419 + padded_len,
420 + device=self.alibi_slopes.device,
421 + dtype=dtype,
422 + )[:, :, :, :max_prompt_len].copy_(bias)
423 + bias.mul_(self.alibi_slopes[:, None, None])
424 + attn_bias = LowerTriangularMaskWithTensorBias(bias)
425 + input_metadata.attn_bias = attn_bias
426
427 def multi_query_kv_attention(
428 self,
@@ -466,24 +447,19 @@ class PagedAttentionWithALiBi(PagedAttention):
447 value = torch.repeat_interleave(value,
448 self.num_queries_per_kv,
449 dim=1)
450 + batch_size = input_metadata.num_prompts
451 + seq_len = input_metadata.max_prompt_len
452
470 - # FIXME(woosuk): Because xformers does not support dynamic sequence
471 - # lengths with custom attention bias, we process each prompt one by
472 - # one. This is inefficient, especially when we have many short prompts.
473 - start = 0
474 - for i, prompt_len in enumerate(input_metadata.prompt_lens):
475 - end = start + prompt_len
476 - out = xops.memory_efficient_attention_forward(
477 - query[None, start:end],
478 - key[None, start:end],
479 - value[None, start:end],
480 - attn_bias=input_metadata.attn_bias[i],
481 - p=0.0,
482 - scale=self.scale,
483 - )
484 - # TODO(woosuk): Unnecessary copy. Optimize.
485 - output[start:end].copy_(out.squeeze(0))
486 - start += prompt_len
453 + out = xops.memory_efficient_attention_forward(
454 + query.view(batch_size, seq_len, self.num_heads, self.head_size),
455 + key.view(batch_size, seq_len, self.num_heads, self.head_size),
456 + value.view(batch_size, seq_len, self.num_heads, self.head_size),
457 + attn_bias=input_metadata.attn_bias,
458 + p=0.0,
459 + scale=self.scale,
460 + )
461 + # TODO(woosuk): Unnecessary copy. Optimize.
462 + output.copy_(out.view(-1, self.num_heads, self.head_size))
463 return output
464
465 def get_alibi_slopes(self) -> Optional[torch.Tensor]:
vllm/model_executor/layers/quantized_linear/awq.py
+2 -2
@@ -50,7 +50,7 @@ class AWQColumnParallelLinear(ColumnParallelLinear):
50 bias: Optional[torch.Tensor],
51 ) -> torch.Tensor:
52 pack_factor = self.quant_config.pack_factor
53 - out_shape = (x.shape[-2], self.qweight.shape[-1] * pack_factor)
53 + out_shape = (x.shape[:-1] + (self.qweight.shape[-1] * pack_factor, ))
54 reshaped_x = x.reshape(-1, x.shape[-1])
55 out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
56 self.qzeros, pack_factor)
@@ -95,7 +95,7 @@ class AWQRowParallelLinear(RowParallelLinear):
95
96 def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
97 pack_factor = self.quant_config.pack_factor
98 - out_shape = (x.shape[-2], self.qweight.shape[-1] * pack_factor)
98 + out_shape = (x.shape[:-1] + (self.qweight.shape[-1] * pack_factor, ))
99 reshaped_x = x.reshape(-1, x.shape[-1])
100 out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
101 self.qzeros, pack_factor)
vllm/model_executor/layers/sampler.py
+2 -1
@@ -119,7 +119,7 @@ def _prune_hidden_states(
119 selected_token_indices.extend(
120 range(start_idx, start_idx + prompt_len - 1))
121 selected_token_indices.append(start_idx + prompt_len - 1)
122 - start_idx += prompt_len
122 + start_idx += input_metadata.max_prompt_len
123 else:
124 num_seqs = len(seq_ids)
125 selected_token_indices.extend(
@@ -129,6 +129,7 @@ def _prune_hidden_states(
129 selected_token_indices = torch.tensor(selected_token_indices,
130 dtype=torch.long,
131 device=hidden_states.device)
132 + hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
133 return hidden_states.index_select(0, selected_token_indices)
134
135
vllm/worker/worker.py
+34 -25
@@ -158,9 +158,9 @@ class Worker:
158 seq_group_metadata_list: List[SequenceGroupMetadata],
159 ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
160 seq_groups: List[Tuple[List[int], SamplingParams]] = []
161 - input_tokens: List[int] = []
162 - input_positions: List[int] = []
163 - slot_mapping: List[int] = []
161 + input_tokens: List[List[int]] = []
162 + input_positions: List[List[int]] = []
163 + slot_mapping: List[List[int]] = []
164
165 # Add prompt tokens.
166 prompt_lens: List[int] = []
@@ -180,24 +180,25 @@ class Worker:
180 prompt_len = len(prompt_tokens)
181 prompt_lens.append(prompt_len)
182
183 - input_tokens.extend(prompt_tokens)
183 + input_tokens.append(prompt_tokens)
184 # NOTE(woosuk): Here we assume that the first token in the prompt
185 # is always the first token in the sequence.
186 - input_positions.extend(range(len(prompt_tokens)))
186 + input_positions.append(list(range(prompt_len)))
187
188 if seq_group_metadata.block_tables is None:
189 # During memory profiling, the block tables are not initialized
190 # yet. In this case, we just use a dummy slot mapping.
191 - slot_mapping.extend([0] * prompt_len)
191 + slot_mapping.append([0] * prompt_len)
192 continue
193
194 # Compute the slot mapping.
195 + slot_mapping.append([])
196 block_table = seq_group_metadata.block_tables[seq_id]
197 for i in range(prompt_len):
198 block_number = block_table[i // self.block_size]
199 block_offset = i % self.block_size
200 slot = block_number * self.block_size + block_offset
200 - slot_mapping.append(slot)
201 + slot_mapping[-1].append(slot)
202
203 # Add generation tokens.
204 max_context_len = 0
@@ -215,13 +216,13 @@ class Worker:
216 for seq_id in seq_ids:
217 seq_data = seq_group_metadata.seq_data[seq_id]
218 generation_token = seq_data.get_last_token_id()
218 - input_tokens.append(generation_token)
219 + input_tokens.append([generation_token])
220
221 context_len = seq_data.get_len()
222 position = context_len - 1
223 if self.sliding_window is not None:
224 context_len = min(context_len, self.sliding_window)
224 - input_positions.append(position)
225 + input_positions.append([position])
226
227 block_table = seq_group_metadata.block_tables[seq_id]
228
@@ -233,7 +234,7 @@ class Worker:
234 block_number = block_table[position // self.block_size]
235 block_offset = position % self.block_size
236 slot = block_number * self.block_size + block_offset
236 - slot_mapping.append(slot)
237 + slot_mapping.append([slot])
238
239 if self.sliding_window is not None:
240 sliding_window_blocks = (self.sliding_window //
@@ -241,28 +242,36 @@ class Worker:
242 block_table = block_table[-sliding_window_blocks:]
243 generation_block_tables.append(block_table)
244
244 - # Optimization: Pad the input length to be a multiple of 8.
245 - # This is required for utilizing the Tensor Cores in NVIDIA GPUs.
246 - input_tokens = _pad_to_alignment(input_tokens, multiple_of=8)
247 - input_positions = _pad_to_alignment(input_positions, multiple_of=8)
245 + max_seq_len = max(prompt_lens) if prompt_lens else 1
246 + padded_input_tokens = [
247 + _pad_to_max(tokens, max_seq_len, pad=0) for tokens in input_tokens
248 + ]
249 + padded_input_positions = [
250 + _pad_to_max(positions, max_seq_len, pad=0)
251 + for positions in input_positions
252 + ]
253 + padded_slot_mapping = [
254 + _pad_to_max(mapping, max_seq_len, pad=-1)
255 + for mapping in slot_mapping
256 + ]
257 + padded_block_tables = [
258 + _pad_to_max(block_table, max_num_blocks_per_seq, pad=0)
259 + for block_table in generation_block_tables
260 + ]
261
262 # Convert to tensors.
250 - tokens_tensor = torch.tensor(input_tokens,
263 + tokens_tensor = torch.tensor(padded_input_tokens,
264 dtype=torch.long,
265 device="cuda")
253 - positions_tensor = torch.tensor(input_positions,
266 + positions_tensor = torch.tensor(padded_input_positions,
267 dtype=torch.long,
268 device="cuda")
256 - slot_mapping_tensor = torch.tensor(slot_mapping,
269 + slot_mapping_tensor = torch.tensor(padded_slot_mapping,
270 dtype=torch.int,
271 device="cuda")
272 context_lens_tensor = torch.tensor(context_lens,
273 dtype=torch.int,
274 device="cuda")
262 - padded_block_tables = [
263 - _pad_to_max(block_table, max_num_blocks_per_seq)
264 - for block_table in generation_block_tables
265 - ]
275 block_tables_tensor = torch.tensor(padded_block_tables,
276 dtype=torch.int,
277 device="cuda")
@@ -361,12 +370,12 @@ def _init_distributed_environment(
370 parallel_config.pipeline_parallel_size)
371
372
364 -def _pad_to_alignment(x: List[int], multiple_of: int) -> List[int]:
365 - return x + [0] * ((-len(x)) % multiple_of)
373 +def _pad_to_alignment(x: List[int], multiple_of: int, pad: int) -> List[int]:
374 + return x + [pad] * ((-len(x)) % multiple_of)
375
376
368 -def _pad_to_max(x: List[int], max_len: int) -> List[int]:
369 - return x + [0] * (max_len - len(x))
377 +def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
378 + return x + [pad] * (max_len - len(x))
379
380
381 def _check_if_can_support_max_seq_len(max_seq_len: int,