A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Refactor Worker & InputMetadata (#1843)
Woosuk Kwon committed
Nov 29, 2023 at 22:16 UTC
27feead2f80f906de88d64c6e69342451cf1d7f0
27 files changed
+668
-443
vllm/config.py
+6
@@ -161,6 +161,12 @@ class ModelConfig:
161
"must be divisible by pipeline parallel size "
162
f"({pipeline_parallel_size}).")
163
164
+ def get_sliding_window(self) -> Optional[int]:
165
+ return getattr(self.hf_config, "sliding_window", None)
166
+
167
+ def get_vocab_size(self) -> int:
168
+ return self.hf_config.vocab_size
169
+
170
def get_hidden_size(self) -> int:
171
return self.hf_config.hidden_size
172
vllm/engine/arg_utils.py
+4
-3
@@ -201,9 +201,10 @@ class EngineArgs:
201
self.dtype, self.seed, self.revision,
202
self.tokenizer_revision, self.max_model_len,
203
self.quantization)
204
- cache_config = CacheConfig(
205
- self.block_size, self.gpu_memory_utilization, self.swap_space,
206
- getattr(model_config.hf_config, 'sliding_window', None))
204
+ cache_config = CacheConfig(self.block_size,
205
+ self.gpu_memory_utilization,
206
+ self.swap_space,
207
+ model_config.get_sliding_window())
208
parallel_config = ParallelConfig(self.pipeline_parallel_size,
209
self.tensor_parallel_size,
210
self.worker_use_ray,
vllm/engine/llm_engine.py
-2
@@ -88,8 +88,6 @@ class LLMEngine:
88
89
self.model_config = model_config
90
self.cache_config = cache_config
91
- assert self.cache_config.sliding_window == getattr(
92
- self.model_config.hf_config, "sliding_window", None)
91
self.parallel_config = parallel_config
92
self.scheduler_config = scheduler_config
93
self.log_stats = log_stats
vllm/model_executor/__init__.py
+2
@@ -1,9 +1,11 @@
1
from vllm.model_executor.input_metadata import InputMetadata
2
from vllm.model_executor.model_loader import get_model
3
+from vllm.model_executor.sampling_metadata import SamplingMetadata
4
from vllm.model_executor.utils import set_random_seed
5
6
__all__ = [
7
"InputMetadata",
8
"get_model",
9
+ "SamplingMetadata",
10
"set_random_seed",
11
]
vllm/model_executor/input_metadata.py
+16
-65
@@ -1,91 +1,42 @@
1
-from typing import Dict, List, Optional, Tuple
1
+from typing import List, Optional
2
3
import torch
4
-from xformers.ops import AttentionBias
5
-
6
-from vllm.sampling_params import SamplingParams, SamplingType
7
-from vllm.sequence import SequenceData
4
5
6
class InputMetadata:
11
- """Metadata for input sequences. Used for PagedAttention.
7
+ """Metadata for input sequences. Used in PagedAttention.
8
9
Args:
14
- seq_groups: List of (seq_ids, sampling_params).
15
- seq_data: Seq_id -> SequenceData.
10
prompt_lens: Lengths of prompts.
11
slot_mapping: The address to write the new KV to of each token.
18
- context_lens: the length of attention context for each generation token.
12
max_context_len: The maximum context length.
13
+ context_lens: the length of attention context for each sequence.
14
block_tables: The block tables. (Seq id -> list of physical block)
15
"""
16
17
def __init__(
18
self,
25
- seq_groups: List[Tuple[List[int], SamplingParams]],
26
- seq_data: Dict[int, SequenceData],
19
prompt_lens: List[int],
20
slot_mapping: torch.Tensor,
29
- context_lens: torch.Tensor,
30
- max_context_len: int,
31
- block_tables: torch.Tensor,
32
- selected_token_indices: torch.Tensor,
33
- categorized_sample_indices: Dict[SamplingType, torch.Tensor],
34
- sliding_window: Optional[int] = None,
21
+ max_context_len: Optional[int],
22
+ context_lens: Optional[torch.Tensor],
23
+ block_tables: Optional[torch.Tensor],
24
) -> None:
36
- self.seq_groups = seq_groups
37
- self.seq_data = seq_data
25
self.prompt_lens = prompt_lens
26
+ self.max_context_len = max_context_len
27
self.slot_mapping = slot_mapping
28
self.context_lens = context_lens
41
- self.max_context_len = max_context_len
29
self.block_tables = block_tables
43
- self.selected_token_indices = selected_token_indices
44
- self.categorized_sample_indices = categorized_sample_indices
45
-
46
- self.max_prompt_len = max(prompt_lens) if prompt_lens else 0
47
- self.to_cache = None
48
- if sliding_window is not None:
49
- # We need to keep the positions of sliding windows within
50
- # the key / value tables, this is helpful to know which
51
- # elements we need to cache.
52
- to_cache, start_idx = [], 0
53
- for prompt_len in self.prompt_lens:
54
- to_cache.extend(
55
- range(
56
- start_idx + max(0, prompt_len - sliding_window),
57
- start_idx + prompt_len,
58
- ))
59
- start_idx += self.max_prompt_len
60
- to_cache.extend(range(start_idx, slot_mapping.shape[0]))
61
- self.to_cache = torch.tensor(to_cache,
62
- dtype=torch.int32,
63
- device=self.slot_mapping.device)
64
-
65
- self.num_prompts = len(prompt_lens)
66
- self.num_prompt_tokens = self.num_prompts * self.max_prompt_len
67
- self.num_generation_tokens = context_lens.shape[0]
68
- if block_tables.numel() > 0:
69
- self.max_num_blocks_per_seq = block_tables.shape[1]
70
- else:
71
- self.max_num_blocks_per_seq = 0
72
- assert block_tables.shape[0] == self.num_generation_tokens
30
31
+ self.is_prompt = len(prompt_lens) > 0
32
# Set during the execution of the first attention op.
75
- self.attn_bias: Optional[AttentionBias] = None
33
+ # FIXME(woosuk): This is a hack.
34
+ self.attn_bias = None
35
36
def __repr__(self) -> str:
78
- # Print only useful metadata.
79
- return (
80
- f'InputMetadata('
81
- f'num_prompt_tokens={self.num_prompt_tokens}, '
82
- f'num_prompts={self.num_prompts}, '
83
- f'prompt_lens={self.prompt_lens}, '
84
- f'num_generation_tokens={self.num_generation_tokens}, '
85
- f'context_lens={self.context_lens}, '
86
- f'max_context_len={self.max_context_len}), '
87
- f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
88
- f'block_tables={self.block_tables}, '
89
- f'selected_token_indices={self.selected_token_indices}, '
90
- f'categorized_sample_indices={self.categorized_sample_indices}, '
91
- f'slot_mapping={self.slot_mapping})')
37
+ return ("InputMetadata("
38
+ f"prompt_lens={self.prompt_lens}, "
39
+ f"max_context_len={self.max_context_len}, "
40
+ f"slot_mapping={self.slot_mapping}, "
41
+ f"context_lens={self.context_lens}, "
42
+ f"block_tables={self.block_tables})")
vllm/model_executor/layers/attention.py
+3
-11
@@ -101,23 +101,15 @@ class PagedAttention(nn.Module):
101
# vectors will not be cached. This happens during the initial memory
102
# profiling run.
103
if key_cache is not None and value_cache is not None:
104
- key_to_cache = key
105
- value_to_cache = value
106
- if input_metadata.to_cache is not None:
107
- key_to_cache = key_to_cache[input_metadata.to_cache]
108
- value_to_cache = value_to_cache[input_metadata.to_cache]
109
- slot_mapping = slot_mapping[input_metadata.to_cache]
110
-
104
cache_ops.reshape_and_cache(
112
- key_to_cache,
113
- value_to_cache,
105
+ key,
106
+ value,
107
key_cache,
108
value_cache,
109
slot_mapping,
110
)
111
119
- is_prompt = len(input_metadata.prompt_lens) > 0
120
- if is_prompt:
112
+ if input_metadata.is_prompt:
113
# Prompt run.
114
if self.num_kv_heads != self.num_heads:
115
# As of Nov 2023, xformers only supports MHA. For MQA/GQA,
vllm/model_executor/layers/sampler.py
+60
-55
@@ -4,9 +4,9 @@ from typing import Dict, List, Optional, Tuple
4
import torch
5
import torch.nn as nn
6
7
-from vllm.model_executor.input_metadata import InputMetadata
7
from vllm.model_executor.parallel_utils.communication_op import (
8
tensor_model_parallel_all_gather)
9
+from vllm.model_executor.sampling_metadata import SamplingMetadata
10
from vllm.sampling_params import SamplingParams, SamplingType
11
from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12
SequenceData, SequenceGroupOutput, SequenceOutput)
@@ -37,29 +37,30 @@ class Sampler(nn.Module):
37
self,
38
embedding: torch.Tensor,
39
hidden_states: torch.Tensor,
40
- input_metadata: InputMetadata,
40
+ sampling_metadata: SamplingMetadata,
41
embedding_bias: Optional[torch.Tensor] = None,
42
) -> SamplerOutput:
43
# Get the hidden states that we use for sampling.
44
- hidden_states = _prune_hidden_states(hidden_states, input_metadata)
44
+ hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)
45
46
# Get the logits for the next tokens.
47
logits = _get_logits(hidden_states, embedding, embedding_bias,
48
self.vocab_size)
49
50
# Apply logits processors (if any).
51
- logits = _apply_logits_processors(logits, input_metadata)
51
+ logits = _apply_logits_processors(logits, sampling_metadata)
52
# Apply presence and frequency penalties.
53
presence_penalties, frequency_penalties, repetition_penalties = (
54
- _get_penalties(input_metadata))
54
+ _get_penalties(sampling_metadata))
55
assert len(presence_penalties) == logits.shape[0]
56
assert len(frequency_penalties) == logits.shape[0]
57
assert len(repetition_penalties) == logits.shape[0]
58
- logits = _apply_penalties(logits, input_metadata, presence_penalties,
59
- frequency_penalties, repetition_penalties)
58
+ logits = _apply_penalties(logits, sampling_metadata,
59
+ presence_penalties, frequency_penalties,
60
+ repetition_penalties)
61
62
# Apply temperature scaling.
62
- temperatures = _get_temperatures(input_metadata)
63
+ temperatures = _get_temperatures(sampling_metadata)
64
assert len(temperatures) == logits.shape[0]
65
if any(t != 1.0 for t in temperatures):
66
t = torch.tensor(temperatures,
@@ -70,7 +71,7 @@ class Sampler(nn.Module):
71
72
# Apply top-p and top-k truncation.
73
top_ps, top_ks, min_ps = _get_top_p_top_k_min_p(
73
- input_metadata, self.vocab_size)
74
+ sampling_metadata, self.vocab_size)
75
assert len(top_ps) == len(top_ks) == logits.shape[0]
76
do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
77
do_top_k = any(k != self.vocab_size for k in top_ks)
@@ -89,11 +90,11 @@ class Sampler(nn.Module):
90
logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
91
92
# Sample the next tokens.
92
- sample_results = _sample(probs, logprobs, input_metadata)
93
+ sample_results = _sample(probs, logprobs, sampling_metadata)
94
# Get the logprobs query results.
95
prompt_logprobs, sample_logprobs = _get_logprobs(
95
- logprobs, input_metadata, sample_results)
96
- return _build_sampler_output(sample_results, input_metadata,
96
+ logprobs, sampling_metadata, sample_results)
97
+ return _build_sampler_output(sample_results, sampling_metadata,
98
prompt_logprobs, sample_logprobs)
99
100
@@ -112,29 +113,30 @@ def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
113
114
def _prune_hidden_states(
115
hidden_states: torch.Tensor,
115
- input_metadata: InputMetadata,
116
+ sampling_metadata: SamplingMetadata,
117
) -> torch.Tensor:
118
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
118
- return hidden_states.index_select(0, input_metadata.selected_token_indices)
119
+ return hidden_states.index_select(0,
120
+ sampling_metadata.selected_token_indices)
121
122
123
def _get_penalties(
122
- input_metadata: InputMetadata
124
+ sampling_metadata: SamplingMetadata
125
) -> Tuple[List[float], List[float], List[float]]:
126
# Collect the presence and frequency penalties.
127
presence_penalties: List[float] = []
128
frequency_penalties: List[float] = []
129
repetition_penalties: List[float] = []
128
- for i, seq_group in enumerate(input_metadata.seq_groups):
130
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
131
seq_ids, sampling_params = seq_group
132
p = sampling_params.presence_penalty
133
f = sampling_params.frequency_penalty
134
r = sampling_params.repetition_penalty
133
- if (i < input_metadata.num_prompts
135
+ if (i < sampling_metadata.num_prompts
136
and sampling_params.prompt_logprobs is not None):
137
# NOTE: We do not apply presence and frequency penalties for the
138
# prompt token positions where we don't sample new tokens.
137
- prompt_len = input_metadata.prompt_lens[i]
139
+ prompt_len = sampling_metadata.prompt_lens[i]
140
presence_penalties += [0] * (prompt_len - 1)
141
frequency_penalties += [0] * (prompt_len - 1)
142
repetition_penalties += [1] * (prompt_len - 1)
@@ -145,21 +147,21 @@ def _get_penalties(
147
148
149
def _get_prompt_and_output_tokens(
148
- input_metadata: InputMetadata
150
+ sampling_metadata: SamplingMetadata,
151
) -> Tuple[List[List[int]], List[List[int]]]:
152
prompt_tokens: List[List[int]] = []
153
output_tokens: List[List[int]] = []
152
- for i, seq_group in enumerate(input_metadata.seq_groups):
154
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
155
seq_ids, sampling_params = seq_group
154
- if (i < input_metadata.num_prompts
156
+ if (i < sampling_metadata.num_prompts
157
and sampling_params.prompt_logprobs is not None):
158
# NOTE: prompt token positions do not need output tokens to
159
# compute penalties.
158
- prompt_len = input_metadata.prompt_lens[i]
160
+ prompt_len = sampling_metadata.prompt_lens[i]
161
prompt_tokens.extend([] for _ in range(prompt_len - 1))
162
output_tokens.extend([] for _ in range(prompt_len - 1))
163
for seq_id in seq_ids:
162
- seq_data = input_metadata.seq_data[seq_id]
164
+ seq_data = sampling_metadata.seq_data[seq_id]
165
prompt_tokens.append(seq_data.prompt_token_ids)
166
output_tokens.append(seq_data.output_token_ids)
167
return prompt_tokens, output_tokens
@@ -191,17 +193,19 @@ def _get_bin_counts_and_mask(
193
return bin_counts, mask
194
195
194
-def _apply_logits_processors(logits: torch.Tensor,
195
- input_metadata: InputMetadata) -> torch.Tensor:
196
+def _apply_logits_processors(
197
+ logits: torch.Tensor,
198
+ sampling_metadata: SamplingMetadata,
199
+) -> torch.Tensor:
200
logits_row_idx = 0
201
found_logits_processors = False
198
- for seq_ids, sampling_params in input_metadata.seq_groups:
202
+ for seq_ids, sampling_params in sampling_metadata.seq_groups:
203
logits_processors = sampling_params.logits_processors
204
if logits_processors:
205
found_logits_processors = True
206
for seq_id in seq_ids:
207
logits_row = logits[logits_row_idx]
204
- token_ids = input_metadata.seq_data[seq_id].output_token_ids
208
+ token_ids = sampling_metadata.seq_data[seq_id].output_token_ids
209
for logits_processor in logits_processors:
210
logits_row = logits_processor(token_ids, logits_row)
211
logits[logits_row_idx] = logits_row
@@ -215,7 +219,7 @@ def _apply_logits_processors(logits: torch.Tensor,
219
220
def _apply_penalties(
221
logits: torch.Tensor,
218
- input_metadata: InputMetadata,
222
+ sampling_metadata: SamplingMetadata,
223
presence_penalties: List[float],
224
frequency_penalties: List[float],
225
repetition_penalties: List[float],
@@ -234,7 +238,7 @@ def _apply_penalties(
238
return logits
239
240
prompt_tokens, output_tokens = (
237
- _get_prompt_and_output_tokens(input_metadata))
241
+ _get_prompt_and_output_tokens(sampling_metadata))
242
assert len(prompt_tokens) == logits.shape[0]
243
assert len(output_tokens) == logits.shape[0]
244
@@ -265,10 +269,10 @@ def _apply_penalties(
269
return logits
270
271
268
-def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
272
+def _get_temperatures(sampling_metadata: SamplingMetadata) -> List[float]:
273
# Collect the temperatures for the logits.
274
temperatures: List[float] = []
271
- for i, seq_group in enumerate(input_metadata.seq_groups):
275
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
276
seq_ids, sampling_params = seq_group
277
temperature = sampling_params.temperature
278
if temperature < _SAMPLING_EPS:
@@ -276,22 +280,22 @@ def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
280
# (i.e., greedy sampling or beam search).
281
# Set the temperature to 1 to avoid division by zero.
282
temperature = 1.0
279
- if (i < input_metadata.num_prompts
283
+ if (i < sampling_metadata.num_prompts
284
and sampling_params.prompt_logprobs is not None):
281
- prompt_len = input_metadata.prompt_lens[i]
285
+ prompt_len = sampling_metadata.prompt_lens[i]
286
temperatures += [temperature] * (prompt_len - 1)
287
temperatures += [temperature] * len(seq_ids)
288
return temperatures
289
290
291
def _get_top_p_top_k_min_p(
288
- input_metadata: InputMetadata,
292
+ sampling_metadata: SamplingMetadata,
293
vocab_size: int,
294
) -> Tuple[List[float], List[int], List[float]]:
295
top_ps: List[float] = []
296
top_ks: List[int] = []
297
min_ps: List[float] = []
294
- for i, seq_group in enumerate(input_metadata.seq_groups):
298
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
299
seq_ids, sampling_params = seq_group
300
top_p = sampling_params.top_p
301
min_p = sampling_params.min_p
@@ -299,9 +303,9 @@ def _get_top_p_top_k_min_p(
303
top_k = min(sampling_params.top_k, vocab_size)
304
# k=-1 means no truncation.
305
top_k = vocab_size if top_k == -1 else top_k
302
- if (i < input_metadata.num_prompts
306
+ if (i < sampling_metadata.num_prompts
307
and sampling_params.prompt_logprobs is not None):
304
- prompt_len = input_metadata.prompt_lens[i]
308
+ prompt_len = sampling_metadata.prompt_lens[i]
309
top_ps += [top_p] * (prompt_len - 1)
310
top_ks += [top_k] * (prompt_len - 1)
311
min_ps += [min_p] * (prompt_len - 1)
@@ -471,11 +475,11 @@ def _beam_search_sample(
475
def _sample(
476
probs: torch.Tensor,
477
logprobs: torch.Tensor,
474
- input_metadata: InputMetadata,
478
+ sampling_metadata: SamplingMetadata,
479
) -> List[Tuple[List[int], List[int]]]:
480
categorized_seq_group_ids = {t: [] for t in SamplingType}
477
- categorized_sample_indices = input_metadata.categorized_sample_indices
478
- for i, seq_group in enumerate(input_metadata.seq_groups):
481
+ categorized_sample_indices = sampling_metadata.categorized_sample_indices
482
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
483
_, sampling_params = seq_group
484
sampling_type = sampling_params.sampling_type
485
categorized_seq_group_ids[sampling_type].append(i)
@@ -483,8 +487,8 @@ def _sample(
487
sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
488
for sampling_type in SamplingType:
489
seq_group_ids = categorized_seq_group_ids[sampling_type]
486
- seq_groups = [input_metadata.seq_groups[i] for i in seq_group_ids]
487
- is_prompts = [i < input_metadata.num_prompts for i in seq_group_ids]
490
+ seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
491
+ is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
492
sample_indices = categorized_sample_indices[sampling_type]
493
num_tokens = len(sample_indices)
494
if num_tokens == 0:
@@ -499,21 +503,22 @@ def _sample(
503
elif sampling_type == SamplingType.BEAM:
504
category_logprobs = logprobs[sample_indices]
505
sample_results = _beam_search_sample(seq_groups, is_prompts,
502
- input_metadata.seq_data,
506
+ sampling_metadata.seq_data,
507
category_logprobs)
508
else:
509
raise ValueError(f"Unsupported sampling type: {sampling_type}")
510
sample_results_dict.update(zip(seq_group_ids, sample_results))
511
512
sample_results = [
509
- sample_results_dict[i] for i in range(len(input_metadata.seq_groups))
513
+ sample_results_dict[i]
514
+ for i in range(len(sampling_metadata.seq_groups))
515
]
516
return sample_results
517
518
519
def _get_logprobs(
520
logprobs: torch.Tensor,
516
- input_metadata: InputMetadata,
521
+ sampling_metadata: SamplingMetadata,
522
sample_results: List[Tuple[List[int], List[int]]],
523
) -> Tuple[List[Optional[List[Optional[Dict[int, float]]]]], List[List[Dict[
524
int, float]]]]:
@@ -523,16 +528,16 @@ def _get_logprobs(
528
largest_num_logprobs = 0
529
sample_idx = 0
530
for i, (seq_group, sample_result) in enumerate(
526
- zip(input_metadata.seq_groups, sample_results)):
531
+ zip(sampling_metadata.seq_groups, sample_results)):
532
seq_ids, sampling_params = seq_group
533
next_token_ids, parent_ids = sample_result
534
num_parent_seqs = len(seq_ids)
530
- if (i < input_metadata.num_prompts
535
+ if (i < sampling_metadata.num_prompts
536
and sampling_params.prompt_logprobs is not None):
537
largest_num_logprobs = max(largest_num_logprobs,
538
sampling_params.prompt_logprobs)
534
- prompt_len = input_metadata.prompt_lens[i]
535
- prompt_tokens = input_metadata.seq_data[
539
+ prompt_len = sampling_metadata.prompt_lens[i]
540
+ prompt_tokens = sampling_metadata.seq_data[
541
seq_ids[0]].prompt_token_ids
542
batched_logprobs_query_seq_indices.extend(
543
sample_idx + j for j in range(prompt_len - 1))
@@ -570,16 +575,16 @@ def _get_logprobs(
575
sample_idx = 0
576
query_result_idx = 0
577
for i, (seq_group, sample_result) in enumerate(
573
- zip(input_metadata.seq_groups, sample_results)):
578
+ zip(sampling_metadata.seq_groups, sample_results)):
579
seq_ids, sampling_params = seq_group
580
next_token_ids, parent_ids = sample_result
581
582
# Prompt logprobs
578
- if (i < input_metadata.num_prompts
583
+ if (i < sampling_metadata.num_prompts
584
and sampling_params.prompt_logprobs is not None):
585
num_logprobs = sampling_params.prompt_logprobs
581
- prompt_len = input_metadata.prompt_lens[i]
582
- prompt_tokens = input_metadata.seq_data[
586
+ prompt_len = sampling_metadata.prompt_lens[i]
587
+ prompt_tokens = sampling_metadata.seq_data[
588
seq_ids[0]].prompt_token_ids
589
group_prompt_logprobs: PromptLogprobs = [None]
590
for token_id in prompt_tokens[1:]:
@@ -625,13 +630,13 @@ def _get_logprobs(
630
631
def _build_sampler_output(
632
sample_results: List[Tuple[List[int], List[int]]],
628
- input_metadata: InputMetadata,
633
+ sampling_metadata: SamplingMetadata,
634
prompt_logprobs: List[Optional[PromptLogprobs]],
635
sample_logprobs: List[SampleLogprobs],
636
) -> SamplerOutput:
637
sampler_output = []
638
for (seq_group, sample_result, group_prompt_logprobs,
634
- group_sample_logprobs) in zip(input_metadata.seq_groups,
639
+ group_sample_logprobs) in zip(sampling_metadata.seq_groups,
640
sample_results, prompt_logprobs,
641
sample_logprobs):
642
seq_ids, _ = seq_group
vllm/model_executor/models/aquila.py
+10
-2
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding, ParallelLMHead)
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_world_size)
42
+from vllm.model_executor.sampling_metadata import SamplingMetadata
43
from vllm.model_executor.weight_utils import (default_weight_loader,
44
hf_model_weights_iterator)
45
from vllm.sequence import SamplerOutput
@@ -296,11 +297,18 @@ class AquilaForCausalLM(nn.Module):
297
kv_caches: List[KVCache],
298
input_metadata: InputMetadata,
299
cache_events: Optional[List[torch.cuda.Event]],
299
- ) -> SamplerOutput:
300
+ ) -> torch.Tensor:
301
hidden_states = self.model(input_ids, positions, kv_caches,
302
input_metadata, cache_events)
303
+ return hidden_states
304
+
305
+ def sample(
306
+ self,
307
+ hidden_states: torch.Tensor,
308
+ sampling_metadata: SamplingMetadata,
309
+ ) -> SamplerOutput:
310
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303
- input_metadata)
311
+ sampling_metadata)
312
return next_tokens
313
314
def load_weights(self,
vllm/model_executor/models/baichuan.py
+10
-2
@@ -38,6 +38,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding, ParallelLMHead)
39
from vllm.model_executor.parallel_utils.parallel_state import (
40
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41
+from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.model_executor.weight_utils import (default_weight_loader,
43
hf_model_weights_iterator)
44
from vllm.sequence import SamplerOutput
@@ -311,11 +312,18 @@ class BaiChuanBaseForCausalLM(nn.Module):
312
kv_caches: List[KVCache],
313
input_metadata: InputMetadata,
314
cache_events: Optional[List[torch.cuda.Event]],
314
- ) -> SamplerOutput:
315
+ ) -> torch.Tensor:
316
hidden_states = self.model(input_ids, positions, kv_caches,
317
input_metadata, cache_events)
318
+ return hidden_states
319
+
320
+ def sample(
321
+ self,
322
+ hidden_states: torch.Tensor,
323
+ sampling_metadata: SamplingMetadata,
324
+ ) -> SamplerOutput:
325
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
318
- input_metadata)
326
+ sampling_metadata)
327
return next_tokens
328
329
def load_weights(self,
vllm/model_executor/models/bloom.py
+10
-2
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35
VocabParallelEmbedding)
36
from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
+from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
@@ -288,11 +289,18 @@ class BloomForCausalLM(nn.Module):
289
kv_caches: List[KVCache],
290
input_metadata: InputMetadata,
291
cache_events: Optional[List[torch.cuda.Event]],
291
- ) -> SamplerOutput:
292
+ ) -> torch.Tensor:
293
hidden_states = self.transformer(input_ids, positions, kv_caches,
294
input_metadata, cache_events)
295
+ return hidden_states
296
+
297
+ def sample(
298
+ self,
299
+ hidden_states: torch.Tensor,
300
+ sampling_metadata: SamplingMetadata,
301
+ ) -> SamplerOutput:
302
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
295
- input_metadata)
303
+ sampling_metadata)
304
return next_tokens
305
306
def load_weights(self,
vllm/model_executor/models/chatglm.py
+10
-2
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
22
VocabParallelEmbedding, ParallelLMHead)
23
from vllm.model_executor.parallel_utils.parallel_state import (
24
get_tensor_model_parallel_world_size)
25
+from vllm.model_executor.sampling_metadata import SamplingMetadata
26
from vllm.model_executor.weight_utils import (default_weight_loader,
27
hf_model_weights_iterator)
28
from vllm.sequence import SamplerOutput
@@ -350,11 +351,18 @@ class ChatGLMForCausalLM(nn.Module):
351
kv_caches: List[KVCache],
352
input_metadata: InputMetadata,
353
cache_events: Optional[List[torch.cuda.Event]],
353
- ) -> SamplerOutput:
354
+ ) -> torch.Tensor:
355
hidden_states = self.transformer(input_ids, positions, kv_caches,
356
input_metadata, cache_events)
357
+ return hidden_states
358
+
359
+ def sample(
360
+ self,
361
+ hidden_states: torch.Tensor,
362
+ sampling_metadata: SamplingMetadata,
363
+ ) -> SamplerOutput:
364
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
357
- input_metadata)
365
+ sampling_metadata)
366
return next_tokens
367
368
def load_weights(self,
vllm/model_executor/models/falcon.py
+10
-3
@@ -41,6 +41,7 @@ from vllm.model_executor.parallel_utils.communication_op import (
41
tensor_model_parallel_all_reduce)
42
from vllm.model_executor.parallel_utils.parallel_state import (
43
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.model_executor.weight_utils import (default_weight_loader,
46
hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
@@ -389,7 +390,7 @@ class FalconForCausalLM(nn.Module):
390
kv_caches: List[KVCache],
391
input_metadata: InputMetadata,
392
cache_events: Optional[List[torch.cuda.Event]],
392
- ) -> SamplerOutput:
393
+ ) -> torch.Tensor:
394
hidden_states = self.transformer(
395
input_ids,
396
positions,
@@ -397,9 +398,15 @@ class FalconForCausalLM(nn.Module):
398
input_metadata,
399
cache_events,
400
)
400
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
401
- input_metadata)
401
+ return hidden_states
402
403
+ def sample(
404
+ self,
405
+ hidden_states: torch.Tensor,
406
+ sampling_metadata: SamplingMetadata,
407
+ ) -> SamplerOutput:
408
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
409
+ sampling_metadata)
410
return next_tokens
411
412
def load_weights(self,
vllm/model_executor/models/gpt2.py
+10
-2
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35
VocabParallelEmbedding)
36
from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_world_size)
38
+from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
@@ -232,11 +233,18 @@ class GPT2LMHeadModel(nn.Module):
233
kv_caches: List[KVCache],
234
input_metadata: InputMetadata,
235
cache_events: Optional[List[torch.cuda.Event]],
235
- ) -> SamplerOutput:
236
+ ) -> torch.Tensor:
237
hidden_states = self.transformer(input_ids, positions, kv_caches,
238
input_metadata, cache_events)
239
+ return hidden_states
240
+
241
+ def sample(
242
+ self,
243
+ hidden_states: torch.Tensor,
244
+ sampling_metadata: SamplingMetadata,
245
+ ) -> SamplerOutput:
246
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
239
- input_metadata)
247
+ sampling_metadata)
248
return next_tokens
249
250
def load_weights(self,
vllm/model_executor/models/gpt_bigcode.py
+10
-2
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding)
37
from vllm.model_executor.parallel_utils.parallel_state import (
38
get_tensor_model_parallel_world_size)
39
+from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.model_executor.weight_utils import (default_weight_loader,
41
hf_model_weights_iterator)
42
from vllm.sequence import SamplerOutput
@@ -251,11 +252,18 @@ class GPTBigCodeForCausalLM(nn.Module):
252
kv_caches: List[KVCache],
253
input_metadata: InputMetadata,
254
cache_events: Optional[List[torch.cuda.Event]],
254
- ) -> SamplerOutput:
255
+ ) -> torch.Tensor:
256
hidden_states = self.transformer(input_ids, positions, kv_caches,
257
input_metadata, cache_events)
258
+ return hidden_states
259
+
260
+ def sample(
261
+ self,
262
+ hidden_states: torch.Tensor,
263
+ sampling_metadata: SamplingMetadata,
264
+ ) -> SamplerOutput:
265
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
258
- input_metadata)
266
+ sampling_metadata)
267
return next_tokens
268
269
def load_weights(self,
vllm/model_executor/models/gpt_j.py
+10
-2
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35
VocabParallelEmbedding, ParallelLMHead)
36
from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_world_size)
38
+from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
@@ -238,11 +239,18 @@ class GPTJForCausalLM(nn.Module):
239
kv_caches: List[KVCache],
240
input_metadata: InputMetadata,
241
cache_events: Optional[List[torch.cuda.Event]],
241
- ) -> SamplerOutput:
242
+ ) -> torch.Tensor:
243
hidden_states = self.transformer(input_ids, positions, kv_caches,
244
input_metadata, cache_events)
245
+ return hidden_states
246
+
247
+ def sample(
248
+ self,
249
+ hidden_states: torch.Tensor,
250
+ sampling_metadata: SamplingMetadata,
251
+ ) -> SamplerOutput:
252
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
245
- input_metadata, self.lm_head.bias)
253
+ sampling_metadata, self.lm_head.bias)
254
return next_tokens
255
256
def load_weights(self,
vllm/model_executor/models/gpt_neox.py
+10
-2
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35
VocabParallelEmbedding, ParallelLMHead)
36
from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_world_size)
38
+from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
@@ -251,11 +252,18 @@ class GPTNeoXForCausalLM(nn.Module):
252
kv_caches: List[KVCache],
253
input_metadata: InputMetadata,
254
cache_events: Optional[List[torch.cuda.Event]],
254
- ) -> SamplerOutput:
255
+ ) -> torch.Tensor:
256
hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
257
input_metadata, cache_events)
258
+ return hidden_states
259
+
260
+ def sample(
261
+ self,
262
+ hidden_states: torch.Tensor,
263
+ sampling_metadata: SamplingMetadata,
264
+ ) -> SamplerOutput:
265
next_tokens = self.sampler(self.embed_out.weight, hidden_states,
258
- input_metadata)
266
+ sampling_metadata)
267
return next_tokens
268
269
def load_weights(self,
vllm/model_executor/models/internlm.py
+10
-2
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
19
VocabParallelEmbedding, ParallelLMHead)
20
from vllm.model_executor.parallel_utils.parallel_state import (
21
get_tensor_model_parallel_world_size)
22
+from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.model_executor.weight_utils import (default_weight_loader,
24
hf_model_weights_iterator)
25
from vllm.sequence import SamplerOutput
@@ -250,11 +251,18 @@ class InternLMForCausalLM(nn.Module):
251
kv_caches: List[KVCache],
252
input_metadata: InputMetadata,
253
cache_events: Optional[List[torch.cuda.Event]],
253
- ) -> SamplerOutput:
254
+ ) -> torch.Tensor:
255
hidden_states = self.model(input_ids, positions, kv_caches,
256
input_metadata, cache_events)
257
+ return hidden_states
258
+
259
+ def sample(
260
+ self,
261
+ hidden_states: torch.Tensor,
262
+ sampling_metadata: SamplingMetadata,
263
+ ) -> SamplerOutput:
264
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
257
- input_metadata)
265
+ sampling_metadata)
266
return next_tokens
267
268
def load_weights(self,
vllm/model_executor/models/llama.py
+10
-2
@@ -41,6 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41
VocabParallelEmbedding, ParallelLMHead)
42
from vllm.model_executor.parallel_utils.parallel_state import (
43
get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.model_executor.weight_utils import (default_weight_loader,
46
hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
@@ -289,11 +290,18 @@ class LlamaForCausalLM(nn.Module):
290
kv_caches: List[KVCache],
291
input_metadata: InputMetadata,
292
cache_events: Optional[List[torch.cuda.Event]],
292
- ) -> SamplerOutput:
293
+ ) -> torch.Tensor:
294
hidden_states = self.model(input_ids, positions, kv_caches,
295
input_metadata, cache_events)
296
+ return hidden_states
297
+
298
+ def sample(
299
+ self,
300
+ hidden_states: torch.Tensor,
301
+ sampling_metadata: SamplingMetadata,
302
+ ) -> SamplerOutput:
303
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
296
- input_metadata)
304
+ sampling_metadata)
305
return next_tokens
306
307
def load_weights(self,
vllm/model_executor/models/mistral.py
+10
-2
@@ -41,6 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41
VocabParallelEmbedding, ParallelLMHead)
42
from vllm.model_executor.parallel_utils.parallel_state import (
43
get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.model_executor.weight_utils import (default_weight_loader,
46
hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
@@ -285,11 +286,18 @@ class MistralForCausalLM(nn.Module):
286
kv_caches: List[KVCache],
287
input_metadata: InputMetadata,
288
cache_events: Optional[List[torch.cuda.Event]],
288
- ) -> SamplerOutput:
289
+ ) -> torch.Tensor:
290
hidden_states = self.model(input_ids, positions, kv_caches,
291
input_metadata, cache_events)
292
+ return hidden_states
293
+
294
+ def sample(
295
+ self,
296
+ hidden_states: torch.Tensor,
297
+ sampling_metadata: SamplingMetadata,
298
+ ) -> SamplerOutput:
299
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
292
- input_metadata)
300
+ sampling_metadata)
301
return next_tokens
302
303
def load_weights(self,
vllm/model_executor/models/mpt.py
+10
-2
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
18
VocabParallelEmbedding)
19
from vllm.model_executor.parallel_utils.parallel_state import (
20
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
21
+from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.model_executor.weight_utils import (default_weight_loader,
23
hf_model_weights_iterator)
24
from vllm.sequence import SamplerOutput
@@ -256,11 +257,18 @@ class MPTForCausalLM(nn.Module):
257
kv_caches: List[KVCache],
258
input_metadata: InputMetadata,
259
cache_events: Optional[List[torch.cuda.Event]],
259
- ) -> SamplerOutput:
260
+ ) -> torch.Tensor:
261
hidden_states = self.transformer(input_ids, positions, kv_caches,
262
input_metadata, cache_events)
263
+ return hidden_states
264
+
265
+ def sample(
266
+ self,
267
+ hidden_states: torch.Tensor,
268
+ sampling_metadata: SamplingMetadata,
269
+ ) -> SamplerOutput:
270
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
263
- input_metadata)
271
+ sampling_metadata)
272
return next_tokens
273
274
def load_weights(self,
vllm/model_executor/models/opt.py
+10
-2
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding)
37
from vllm.model_executor.parallel_utils.parallel_state import (
38
get_tensor_model_parallel_world_size)
39
+from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.model_executor.weight_utils import (default_weight_loader,
41
hf_model_weights_iterator)
42
from vllm.sequence import SamplerOutput
@@ -308,11 +309,18 @@ class OPTForCausalLM(nn.Module):
309
kv_caches: List[KVCache],
310
input_metadata: InputMetadata,
311
cache_events: Optional[List[torch.cuda.Event]],
311
- ) -> SamplerOutput:
312
+ ) -> torch.Tensor:
313
hidden_states = self.model(input_ids, positions, kv_caches,
314
input_metadata, cache_events)
315
+ return hidden_states
316
+
317
+ def sample(
318
+ self,
319
+ hidden_states: torch.Tensor,
320
+ sampling_metadata: SamplingMetadata,
321
+ ) -> SamplerOutput:
322
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
315
- input_metadata)
323
+ sampling_metadata)
324
return next_tokens
325
326
def load_weights(self,
vllm/model_executor/models/phi_1_5.py
+27
-26
@@ -54,6 +54,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
54
VocabParallelEmbedding, ParallelLMHead)
55
from vllm.model_executor.parallel_utils.parallel_state import (
56
get_tensor_model_parallel_world_size)
57
+from vllm.model_executor.sampling_metadata import SamplingMetadata
58
from vllm.model_executor.weight_utils import (default_weight_loader,
59
hf_model_weights_iterator)
60
from vllm.sequence import SamplerOutput
@@ -210,28 +211,6 @@ class PhiLayer(nn.Module):
211
return hidden_states
212
213
213
-class PhiCausalLMHead(nn.Module):
214
-
215
- def __init__(self, config: PretrainedConfig):
216
- super().__init__()
217
- self.ln = nn.LayerNorm(config.hidden_size,
218
- eps=config.layer_norm_epsilon)
219
- self.linear = ParallelLMHead(config.vocab_size,
220
- config.hidden_size,
221
- bias=True)
222
- self.sampler = Sampler(config.vocab_size)
223
-
224
- def forward(
225
- self,
226
- hidden_states: torch.Tensor,
227
- input_metadata: InputMetadata,
228
- ):
229
- hidden_states = self.ln(hidden_states)
230
- next_tokens = self.sampler(self.linear.weight, hidden_states,
231
- input_metadata, self.linear.bias)
232
- return next_tokens
233
-
234
-
214
class PhiModel(nn.Module):
215
216
def __init__(self,
@@ -253,7 +232,7 @@ class PhiModel(nn.Module):
232
kv_caches: List[KVCache],
233
input_metadata: InputMetadata,
234
cache_events: Optional[List[torch.cuda.Event]],
256
- ) -> SamplerOutput:
235
+ ) -> torch.Tensor:
236
hidden_states = self.embd(input_ids)
237
for i in range(self.config.num_hidden_layers):
238
cache_event = None if cache_events is None else cache_events[i]
@@ -268,6 +247,17 @@ class PhiModel(nn.Module):
247
return hidden_states
248
249
250
+class PhiCausalLMHead(nn.Module):
251
+
252
+ def __init__(self, config: PretrainedConfig):
253
+ super().__init__()
254
+ self.ln = nn.LayerNorm(config.hidden_size,
255
+ eps=config.layer_norm_epsilon)
256
+ self.linear = ParallelLMHead(config.vocab_size,
257
+ config.hidden_size,
258
+ bias=True)
259
+
260
+
261
class PhiForCausalLM(nn.Module):
262
263
def __init__(self,
@@ -279,6 +269,7 @@ class PhiForCausalLM(nn.Module):
269
270
self.transformer = PhiModel(config, linear_method)
271
self.lm_head = PhiCausalLMHead(config)
272
+ self.sampler = Sampler(config.vocab_size)
273
274
def forward(
275
self,
@@ -287,11 +278,21 @@ class PhiForCausalLM(nn.Module):
278
kv_caches: List[KVCache],
279
input_metadata: InputMetadata,
280
cache_events: Optional[List[torch.cuda.Event]],
290
- ) -> SamplerOutput:
281
+ ) -> torch.Tensor:
282
hidden_states = self.transformer(input_ids, positions, kv_caches,
283
input_metadata, cache_events)
293
- lm_logits = self.lm_head(hidden_states, input_metadata)
294
- return lm_logits
284
+ hidden_states = self.lm_head.ln(hidden_states)
285
+ return hidden_states
286
+
287
+ def sample(
288
+ self,
289
+ hidden_states: torch.Tensor,
290
+ sampling_metadata: SamplingMetadata,
291
+ ) -> SamplerOutput:
292
+ head = self.lm_head.linear
293
+ next_tokens = self.sampler(head.weight, hidden_states,
294
+ sampling_metadata, head.bias)
295
+ return next_tokens
296
297
def load_weights(self,
298
model_name_or_path: str,
vllm/model_executor/models/qwen.py
+10
-2
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23
VocabParallelEmbedding, ParallelLMHead)
24
from vllm.model_executor.parallel_utils.parallel_state import (
25
get_tensor_model_parallel_world_size)
26
+from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.model_executor.weight_utils import (default_weight_loader,
28
hf_model_weights_iterator)
29
from vllm.sequence import SamplerOutput
@@ -246,11 +247,18 @@ class QWenLMHeadModel(nn.Module):
247
kv_caches: List[KVCache],
248
input_metadata: InputMetadata,
249
cache_events: Optional[List[torch.cuda.Event]],
249
- ) -> SamplerOutput:
250
+ ) -> torch.Tensor:
251
hidden_states = self.transformer(input_ids, positions, kv_caches,
252
input_metadata, cache_events)
253
+ return hidden_states
254
+
255
+ def sample(
256
+ self,
257
+ hidden_states: torch.Tensor,
258
+ sampling_metadata: SamplingMetadata,
259
+ ) -> SamplerOutput:
260
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
253
- input_metadata)
261
+ sampling_metadata)
262
return next_tokens
263
264
def load_weights(self,
vllm/model_executor/models/yi.py
+10
-2
@@ -41,6 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41
VocabParallelEmbedding, ParallelLMHead)
42
from vllm.model_executor.parallel_utils.parallel_state import (
43
get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.model_executor.weight_utils import (default_weight_loader,
46
hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
@@ -284,11 +285,18 @@ class YiForCausalLM(nn.Module):
285
kv_caches: List[KVCache],
286
input_metadata: InputMetadata,
287
cache_events: Optional[List[torch.cuda.Event]],
287
- ) -> SamplerOutput:
288
+ ) -> torch.Tensor:
289
hidden_states = self.model(input_ids, positions, kv_caches,
290
input_metadata, cache_events)
291
+ return hidden_states
292
+
293
+ def sample(
294
+ self,
295
+ hidden_states: torch.Tensor,
296
+ sampling_metadata: SamplingMetadata,
297
+ ) -> SamplerOutput:
298
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291
- input_metadata)
299
+ sampling_metadata)
300
return next_tokens
301
302
def load_weights(self,
vllm/model_executor/sampling_metadata.py
new
+43
@@ -0,0 +1,43 @@
1
+from typing import Dict, List, Tuple
2
+
3
+import torch
4
+
5
+from vllm.sampling_params import SamplingParams, SamplingType
6
+from vllm.sequence import SequenceData
7
+
8
+
9
+class SamplingMetadata:
10
+ """Metadata for input sequences. Used in sampler.
11
+
12
+ Args:
13
+ seq_groups: List of (seq_ids, sampling_params).
14
+ seq_data: Seq_id -> SequenceData.
15
+ prompt_lens: Lengths of prompts.
16
+ selected_token_indices: Token indices selected for sampling.
17
+ categorized_sample_indices: SamplingType -> token indicies to sample.
18
+ """
19
+
20
+ def __init__(
21
+ self,
22
+ seq_groups: List[Tuple[List[int], SamplingParams]],
23
+ seq_data: Dict[int, SequenceData],
24
+ prompt_lens: List[int],
25
+ selected_token_indices: torch.Tensor,
26
+ categorized_sample_indices: Dict[SamplingType, torch.Tensor],
27
+ ) -> None:
28
+ self.seq_groups = seq_groups
29
+ self.seq_data = seq_data
30
+ self.prompt_lens = prompt_lens
31
+ self.selected_token_indices = selected_token_indices
32
+ self.categorized_sample_indices = categorized_sample_indices
33
+
34
+ self.num_prompts = len(prompt_lens)
35
+
36
+ def __repr__(self) -> str:
37
+ return (
38
+ "SamplingMetadata("
39
+ f"seq_groups={self.seq_groups}, "
40
+ f"seq_data={self.seq_data}, "
41
+ f"prompt_lens={self.prompt_lens}, "
42
+ f"selected_token_indices={self.selected_token_indices}, "
43
+ f"categorized_sample_indices={self.categorized_sample_indices})")
vllm/worker/model_runner.py
new
+334
@@ -0,0 +1,334 @@
1
+from typing import Dict, List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
6
+from vllm.logger import init_logger
7
+from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
8
+from vllm.sampling_params import SamplingParams, SamplingType
9
+from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
10
+
11
+logger = init_logger(__name__)
12
+
13
+_PAD_SLOT_ID = -1
14
+
15
+
16
+class ModelRunner:
17
+
18
+ def __init__(
19
+ self,
20
+ model_config: ModelConfig,
21
+ parallel_config: ParallelConfig,
22
+ scheduler_config: SchedulerConfig,
23
+ ):
24
+ self.model_config = model_config
25
+ self.parallel_config = parallel_config
26
+ self.scheduler_config = scheduler_config
27
+
28
+ self.sliding_window = model_config.get_sliding_window()
29
+ self.model = None
30
+ self.block_size = None # Set after initial profiling.
31
+
32
+ def load_model(self) -> None:
33
+ self.model = get_model(self.model_config)
34
+
35
+ def set_block_size(self, block_size: int) -> None:
36
+ self.block_size = block_size
37
+
38
+ def _prepare_prompt(
39
+ self,
40
+ seq_group_metadata_list: List[SequenceGroupMetadata],
41
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
42
+ assert len(seq_group_metadata_list) > 0
43
+ input_tokens: List[List[int]] = []
44
+ input_positions: List[List[int]] = []
45
+ slot_mapping: List[List[int]] = []
46
+
47
+ prompt_lens: List[int] = []
48
+ for seq_group_metadata in seq_group_metadata_list:
49
+ assert seq_group_metadata.is_prompt
50
+ seq_ids = list(seq_group_metadata.seq_data.keys())
51
+ assert len(seq_ids) == 1
52
+ seq_id = seq_ids[0]
53
+
54
+ seq_data = seq_group_metadata.seq_data[seq_id]
55
+ prompt_tokens = seq_data.get_token_ids()
56
+ prompt_len = len(prompt_tokens)
57
+ prompt_lens.append(prompt_len)
58
+
59
+ input_tokens.append(prompt_tokens)
60
+ # NOTE(woosuk): Here we assume that the first token in the prompt
61
+ # is always the first token in the sequence.
62
+ input_positions.append(list(range(prompt_len)))
63
+
64
+ if seq_group_metadata.block_tables is None:
65
+ # During memory profiling, the block tables are not initialized
66
+ # yet. In this case, we just use a dummy slot mapping.
67
+ slot_mapping.append([_PAD_SLOT_ID] * prompt_len)
68
+ continue
69
+
70
+ # Compute the slot mapping.
71
+ slot_mapping.append([])
72
+ block_table = seq_group_metadata.block_tables[seq_id]
73
+ # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
74
+ # where start_idx is max(0, prompt_len - sliding_window).
75
+ # For example, if the prompt len is 10, sliding window is 8, and
76
+ # block size is 4, the first two tokens are masked and the slot
77
+ # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
78
+ start_idx = 0
79
+ if self.sliding_window is not None:
80
+ start_idx = max(0, prompt_len - self.sliding_window)
81
+ for i in range(prompt_len):
82
+ if i < start_idx:
83
+ slot_mapping[-1].append(_PAD_SLOT_ID)
84
+ continue
85
+
86
+ block_number = block_table[i // self.block_size]
87
+ block_offset = i % self.block_size
88
+ slot = block_number * self.block_size + block_offset
89
+ slot_mapping[-1].append(slot)
90
+
91
+ max_prompt_len = max(prompt_lens)
92
+ input_tokens = _make_tensor_with_pad(input_tokens,
93
+ max_prompt_len,
94
+ pad=0,
95
+ dtype=torch.long)
96
+ input_positions = _make_tensor_with_pad(input_positions,
97
+ max_prompt_len,
98
+ pad=0,
99
+ dtype=torch.long)
100
+ slot_mapping = _make_tensor_with_pad(slot_mapping,
101
+ max_prompt_len,
102
+ pad=_PAD_SLOT_ID,
103
+ dtype=torch.long)
104
+
105
+ input_metadata = InputMetadata(
106
+ prompt_lens=prompt_lens,
107
+ slot_mapping=slot_mapping,
108
+ max_context_len=None,
109
+ context_lens=None,
110
+ block_tables=None,
111
+ )
112
+ return input_tokens, input_positions, input_metadata
113
+
114
+ def _prepare_decode(
115
+ self,
116
+ seq_group_metadata_list: List[SequenceGroupMetadata],
117
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
118
+ assert len(seq_group_metadata_list) > 0
119
+ input_tokens: List[List[int]] = []
120
+ input_positions: List[List[int]] = []
121
+ slot_mapping: List[List[int]] = []
122
+ context_lens: List[int] = []
123
+ block_tables: List[List[int]] = []
124
+
125
+ for seq_group_metadata in seq_group_metadata_list:
126
+ assert not seq_group_metadata.is_prompt
127
+
128
+ seq_ids = list(seq_group_metadata.seq_data.keys())
129
+ for seq_id in seq_ids:
130
+ seq_data = seq_group_metadata.seq_data[seq_id]
131
+ generation_token = seq_data.get_last_token_id()
132
+ input_tokens.append([generation_token])
133
+
134
+ context_len = seq_data.get_len()
135
+ if self.sliding_window is not None:
136
+ context_len = min(context_len, self.sliding_window)
137
+ context_lens.append(context_len)
138
+
139
+ position = context_len - 1
140
+ input_positions.append([position])
141
+
142
+ block_table = seq_group_metadata.block_tables[seq_id]
143
+ block_number = block_table[position // self.block_size]
144
+ block_offset = position % self.block_size
145
+ slot = block_number * self.block_size + block_offset
146
+ slot_mapping.append([slot])
147
+
148
+ if self.sliding_window is not None:
149
+ sliding_window_blocks = (self.sliding_window //
150
+ self.block_size)
151
+ block_table = block_table[-sliding_window_blocks:]
152
+ block_tables.append(block_table)
153
+
154
+ input_tokens = _make_tensor_with_pad(input_tokens,
155
+ max_len=1,
156
+ pad=0,
157
+ dtype=torch.long)
158
+ input_positions = _make_tensor_with_pad(input_positions,
159
+ max_len=1,
160
+ pad=0,
161
+ dtype=torch.long)
162
+ slot_mapping = _make_tensor_with_pad(slot_mapping,
163
+ max_len=1,
164
+ pad=_PAD_SLOT_ID,
165
+ dtype=torch.long)
166
+ max_context_len = max(context_lens)
167
+ context_lens = torch.tensor(context_lens,
168
+ dtype=torch.int,
169
+ device="cuda")
170
+ max_block_table_len = max([len(t) for t in block_tables])
171
+ block_tables = _make_tensor_with_pad(block_tables,
172
+ max_len=max_block_table_len,
173
+ pad=0,
174
+ dtype=torch.int)
175
+
176
+ input_metadata = InputMetadata(
177
+ prompt_lens=[],
178
+ slot_mapping=slot_mapping,
179
+ max_context_len=max_context_len,
180
+ context_lens=context_lens,
181
+ block_tables=block_tables,
182
+ )
183
+ return input_tokens, input_positions, input_metadata
184
+
185
+ def _prepare_sample(
186
+ self,
187
+ seq_group_metadata_list: List[SequenceGroupMetadata],
188
+ prompt_lens: List[int],
189
+ ) -> SamplingMetadata:
190
+ seq_groups: List[Tuple[List[int], SamplingParams]] = []
191
+ selected_token_indices: List[int] = []
192
+ selected_token_start_idx = 0
193
+ categorized_sample_indices = {t: [] for t in SamplingType}
194
+ categorized_sample_indices_start_idx = 0
195
+
196
+ max_prompt_len = max(prompt_lens) if prompt_lens else 1
197
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
198
+ seq_ids = list(seq_group_metadata.seq_data.keys())
199
+ sampling_params = seq_group_metadata.sampling_params
200
+ seq_groups.append((seq_ids, sampling_params))
201
+
202
+ if seq_group_metadata.is_prompt:
203
+ assert len(seq_ids) == 1
204
+ prompt_len = prompt_lens[i]
205
+ if sampling_params.prompt_logprobs is not None:
206
+ # NOTE: prompt token positions do not need sample, skip
207
+ categorized_sample_indices_start_idx += prompt_len - 1
208
+
209
+ categorized_sample_indices[
210
+ sampling_params.sampling_type].append(
211
+ categorized_sample_indices_start_idx)
212
+ categorized_sample_indices_start_idx += 1
213
+
214
+ if sampling_params.prompt_logprobs is not None:
215
+ selected_token_indices.extend(
216
+ range(selected_token_start_idx,
217
+ selected_token_start_idx + prompt_len - 1))
218
+ selected_token_indices.append(selected_token_start_idx +
219
+ prompt_len - 1)
220
+ selected_token_start_idx += max_prompt_len
221
+ else:
222
+ num_seqs = len(seq_ids)
223
+ selected_token_indices.extend(
224
+ range(selected_token_start_idx,
225
+ selected_token_start_idx + num_seqs))
226
+ selected_token_start_idx += num_seqs
227
+
228
+ categorized_sample_indices[
229
+ sampling_params.sampling_type].extend(
230
+ range(categorized_sample_indices_start_idx,
231
+ categorized_sample_indices_start_idx + num_seqs))
232
+ categorized_sample_indices_start_idx += num_seqs
233
+
234
+ selected_token_indices = torch.tensor(selected_token_indices,
235
+ dtype=torch.long,
236
+ device="cuda")
237
+ categorized_sample_indices = {
238
+ t: torch.tensor(seq_ids, dtype=torch.int, device="cuda")
239
+ for t, seq_ids in categorized_sample_indices.items()
240
+ }
241
+
242
+ seq_data: Dict[int, SequenceData] = {}
243
+ for seq_group_metadata in seq_group_metadata_list:
244
+ seq_data.update(seq_group_metadata.seq_data)
245
+
246
+ sampling_metadata = SamplingMetadata(
247
+ seq_groups=seq_groups,
248
+ seq_data=seq_data,
249
+ prompt_lens=prompt_lens,
250
+ selected_token_indices=selected_token_indices,
251
+ categorized_sample_indices=categorized_sample_indices,
252
+ )
253
+ return sampling_metadata
254
+
255
+ @torch.inference_mode()
256
+ def execute_model(
257
+ self,
258
+ seq_group_metadata_list: List[SequenceGroupMetadata],
259
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
260
+ cache_events: Optional[List[torch.cuda.Event]] = None,
261
+ ) -> SamplerOutput:
262
+ # NOTE: We assume that all sequences in the group are all prompts or
263
+ # all decodes.
264
+ # Prepare input tensors.
265
+ is_prompt = seq_group_metadata_list[0].is_prompt
266
+ if is_prompt:
267
+ inputs = self._prepare_prompt(seq_group_metadata_list)
268
+ input_tokens, input_positions, input_metadata = inputs
269
+ else:
270
+ inputs = self._prepare_decode(seq_group_metadata_list)
271
+ input_tokens, input_positions, input_metadata = inputs
272
+ sampling_metadata = self._prepare_sample(seq_group_metadata_list,
273
+ input_metadata.prompt_lens)
274
+
275
+ # Execute the model.
276
+ hidden_states = self.model(
277
+ input_ids=input_tokens,
278
+ positions=input_positions,
279
+ kv_caches=kv_caches,
280
+ input_metadata=input_metadata,
281
+ cache_events=cache_events,
282
+ )
283
+
284
+ # Sample the next token.
285
+ output = self.model.sample(
286
+ hidden_states=hidden_states,
287
+ sampling_metadata=sampling_metadata,
288
+ )
289
+ return output
290
+
291
+ @torch.inference_mode()
292
+ def profile_run(self) -> None:
293
+ # Enable top-k sampling to reflect the accurate memory usage.
294
+ vocab_size = self.model_config.get_vocab_size()
295
+ sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
296
+ max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
297
+ max_num_seqs = self.scheduler_config.max_num_seqs
298
+
299
+ # Profile memory usage with max_num_sequences sequences and the total
300
+ # number of tokens equal to max_num_batched_tokens.
301
+ seqs: List[SequenceGroupMetadata] = []
302
+ for group_id in range(max_num_seqs):
303
+ seq_len = (max_num_batched_tokens // max_num_seqs +
304
+ (group_id < max_num_batched_tokens % max_num_seqs))
305
+ seq_data = SequenceData([0] * seq_len)
306
+ seq = SequenceGroupMetadata(
307
+ request_id=str(group_id),
308
+ is_prompt=True,
309
+ seq_data={group_id: seq_data},
310
+ sampling_params=sampling_params,
311
+ block_tables=None,
312
+ )
313
+ seqs.append(seq)
314
+
315
+ # Run the model with the dummy inputs.
316
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
317
+ kv_caches = [(None, None)] * num_layers
318
+ self.execute_model(seqs, kv_caches)
319
+ return
320
+
321
+
322
+def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
323
+ assert len(x) <= max_len
324
+ return x + [pad] * (max_len - len(x))
325
+
326
+
327
+def _make_tensor_with_pad(
328
+ x: List[List[int]],
329
+ max_len: int,
330
+ pad: int,
331
+ dtype: torch.dtype,
332
+) -> torch.Tensor:
333
+ padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
334
+ return torch.tensor(padded_x, dtype=dtype, device="cuda")
vllm/worker/worker.py
+13
-248
@@ -1,18 +1,18 @@
1
"""A GPU worker class."""
2
import os
3
-from typing import Dict, List, Tuple, Optional
3
+from typing import Dict, List, Optional, Tuple
4
5
import torch
6
import torch.distributed
7
8
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig)
10
-from vllm.model_executor import get_model, InputMetadata, set_random_seed
10
+from vllm.model_executor import set_random_seed
11
from vllm.model_executor.parallel_utils.parallel_state import (
12
initialize_model_parallel)
13
-from vllm.sampling_params import SamplingParams, SamplingType
14
-from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
13
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
14
from vllm.worker.cache_engine import CacheEngine
15
+from vllm.worker.model_runner import ModelRunner
16
from vllm.utils import get_gpu_memory
17
18
@@ -38,11 +38,11 @@ class Worker:
38
self.rank = rank
39
self.distributed_init_method = distributed_init_method
40
41
+ self.model_runner = ModelRunner(model_config, parallel_config,
42
+ scheduler_config)
43
# Uninitialized cache engine. Will be initialized by
44
# self.init_cache_engine().
45
self.cache_config = None
44
- self.block_size = None
45
- self.sliding_window = None
46
self.cache_engine = None
47
self.cache_events = None
48
self.gpu_cache = None
@@ -69,7 +69,7 @@ class Worker:
69
set_random_seed(self.model_config.seed)
70
71
def load_model(self):
72
- self.model = get_model(self.model_config)
72
+ self.model_runner.load_model()
73
74
@torch.inference_mode()
75
def profile_num_available_blocks(
@@ -83,40 +83,9 @@ class Worker:
83
torch.cuda.empty_cache()
84
torch.cuda.reset_peak_memory_stats()
85
86
- # Profile memory usage with max_num_sequences sequences and the total
87
- # number of tokens equal to max_num_batched_tokens.
88
-
89
- # Enable top-k sampling to reflect the accurate memory usage.
90
- vocab_size = self.model.config.vocab_size
91
- sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
92
- max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
93
- max_num_seqs = self.scheduler_config.max_num_seqs
94
- seqs = []
95
- for group_id in range(max_num_seqs):
96
- seq_len = (max_num_batched_tokens // max_num_seqs +
97
- (group_id < max_num_batched_tokens % max_num_seqs))
98
- seq_data = SequenceData([0] * seq_len)
99
- seq = SequenceGroupMetadata(
100
- request_id=str(group_id),
101
- is_prompt=True,
102
- seq_data={group_id: seq_data},
103
- sampling_params=sampling_params,
104
- block_tables=None,
105
- )
106
- seqs.append(seq)
107
-
108
- input_tokens, input_positions, input_metadata = self._prepare_inputs(
109
- seqs)
110
-
111
- # Execute the model.
112
- num_layers = self.model_config.get_num_layers(self.parallel_config)
113
- self.model(
114
- input_ids=input_tokens,
115
- positions=input_positions,
116
- kv_caches=[(None, None)] * num_layers,
117
- input_metadata=input_metadata,
118
- cache_events=None,
119
- )
86
+ # Execute a forward pass with dummy inputs to profile the memory usage
87
+ # of the model.
88
+ self.model_runner.profile_run()
89
90
# Calculate the number of blocks that can be allocated with the
91
# profiled peak memory.
@@ -140,197 +109,11 @@ class Worker:
109
110
def init_cache_engine(self, cache_config: CacheConfig) -> None:
111
self.cache_config = cache_config
143
- self.block_size = cache_config.block_size
144
- self.sliding_window = cache_config.sliding_window
145
-
112
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
113
self.parallel_config)
114
self.cache_events = self.cache_engine.events
115
self.gpu_cache = self.cache_engine.gpu_cache
150
-
151
- def _prepare_inputs(
152
- self,
153
- seq_group_metadata_list: List[SequenceGroupMetadata],
154
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
155
- seq_groups: List[Tuple[List[int], SamplingParams]] = []
156
- input_tokens: List[List[int]] = []
157
- input_positions: List[List[int]] = []
158
- slot_mapping: List[List[int]] = []
159
- selected_token_indices: List[int] = []
160
- selected_token_start_idx = 0
161
- categorized_sample_indices = {t: [] for t in SamplingType}
162
- categorized_sample_indices_start_idx = 0
163
-
164
- # Add prompt tokens.
165
- prompt_lens: List[int] = []
166
- for seq_group_metadata in seq_group_metadata_list:
167
- if not seq_group_metadata.is_prompt:
168
- continue
169
-
170
- seq_ids = list(seq_group_metadata.seq_data.keys())
171
- sampling_params = seq_group_metadata.sampling_params
172
- seq_groups.append((seq_ids, sampling_params))
173
-
174
- # Use any sequence in the group.
175
- seq_id = seq_ids[0]
176
-
177
- seq_data = seq_group_metadata.seq_data[seq_id]
178
- prompt_tokens = seq_data.get_token_ids()
179
- prompt_len = len(prompt_tokens)
180
- prompt_lens.append(prompt_len)
181
-
182
- if sampling_params.prompt_logprobs is not None:
183
- # NOTE: prompt token positions do not need sample, skip
184
- categorized_sample_indices_start_idx += prompt_len - 1
185
-
186
- categorized_sample_indices[sampling_params.sampling_type].append(
187
- categorized_sample_indices_start_idx)
188
- categorized_sample_indices_start_idx += 1
189
-
190
- input_tokens.append(prompt_tokens)
191
- # NOTE(woosuk): Here we assume that the first token in the prompt
192
- # is always the first token in the sequence.
193
- input_positions.append(list(range(prompt_len)))
194
-
195
- if seq_group_metadata.block_tables is None:
196
- # During memory profiling, the block tables are not initialized
197
- # yet. In this case, we just use a dummy slot mapping.
198
- slot_mapping.append([0] * prompt_len)
199
- continue
200
-
201
- # Compute the slot mapping.
202
- slot_mapping.append([])
203
- block_table = seq_group_metadata.block_tables[seq_id]
204
- for i in range(prompt_len):
205
- block_number = block_table[i // self.block_size]
206
- block_offset = i % self.block_size
207
- slot = block_number * self.block_size + block_offset
208
- slot_mapping[-1].append(slot)
209
-
210
- # Add generation tokens.
211
- max_context_len = 0
212
- max_num_blocks_per_seq = 0
213
- context_lens: List[int] = []
214
- generation_block_tables: List[List[int]] = []
215
- max_seq_len = max(prompt_lens) if prompt_lens else 1
216
- for i, seq_group_metadata in enumerate(seq_group_metadata_list):
217
- if seq_group_metadata.is_prompt:
218
- # We need to do this in this loop as we need to know max_seq_len
219
- assert len(
220
- seq_ids) == 1, "Prompt input should have only one seq."
221
- sampling_params = seq_group_metadata.sampling_params
222
- assert len(prompt_lens) == len(seq_group_metadata_list)
223
- prompt_len = prompt_lens[i]
224
- if sampling_params.prompt_logprobs is not None:
225
- selected_token_indices.extend(
226
- range(selected_token_start_idx,
227
- selected_token_start_idx + prompt_len - 1))
228
- selected_token_indices.append(selected_token_start_idx +
229
- prompt_len - 1)
230
- selected_token_start_idx += max_seq_len
231
- continue
232
-
233
- seq_ids = list(seq_group_metadata.seq_data.keys())
234
- sampling_params = seq_group_metadata.sampling_params
235
- seq_groups.append((seq_ids, sampling_params))
236
-
237
- num_seqs = len(seq_ids)
238
- selected_token_indices.extend(
239
- range(selected_token_start_idx,
240
- selected_token_start_idx + num_seqs))
241
- selected_token_start_idx += num_seqs
242
-
243
- categorized_sample_indices[sampling_params.sampling_type].extend(
244
- range(categorized_sample_indices_start_idx,
245
- categorized_sample_indices_start_idx + num_seqs))
246
- categorized_sample_indices_start_idx += num_seqs
247
-
248
- for seq_id in seq_ids:
249
- seq_data = seq_group_metadata.seq_data[seq_id]
250
- generation_token = seq_data.get_last_token_id()
251
- input_tokens.append([generation_token])
252
-
253
- context_len = seq_data.get_len()
254
- position = context_len - 1
255
- if self.sliding_window is not None:
256
- context_len = min(context_len, self.sliding_window)
257
- input_positions.append([position])
258
-
259
- block_table = seq_group_metadata.block_tables[seq_id]
260
-
261
- max_context_len = max(max_context_len, context_len)
262
- max_num_blocks_per_seq = max(max_num_blocks_per_seq,
263
- len(block_table))
264
- context_lens.append(context_len)
265
-
266
- block_number = block_table[position // self.block_size]
267
- block_offset = position % self.block_size
268
- slot = block_number * self.block_size + block_offset
269
- slot_mapping.append([slot])
270
-
271
- if self.sliding_window is not None:
272
- sliding_window_blocks = (self.sliding_window //
273
- self.block_size)
274
- block_table = block_table[-sliding_window_blocks:]
275
- generation_block_tables.append(block_table)
276
-
277
- padded_input_tokens = [
278
- _pad_to_max(tokens, max_seq_len, pad=0) for tokens in input_tokens
279
- ]
280
- padded_input_positions = [
281
- _pad_to_max(positions, max_seq_len, pad=0)
282
- for positions in input_positions
283
- ]
284
- padded_slot_mapping = [
285
- _pad_to_max(mapping, max_seq_len, pad=-1)
286
- for mapping in slot_mapping
287
- ]
288
- padded_block_tables = [
289
- _pad_to_max(block_table, max_num_blocks_per_seq, pad=0)
290
- for block_table in generation_block_tables
291
- ]
292
-
293
- # Convert to tensors.
294
- tokens_tensor = torch.tensor(padded_input_tokens,
295
- dtype=torch.long,
296
- device="cuda")
297
- positions_tensor = torch.tensor(padded_input_positions,
298
- dtype=torch.long,
299
- device="cuda")
300
- slot_mapping_tensor = torch.tensor(padded_slot_mapping,
301
- dtype=torch.long,
302
- device="cuda")
303
- context_lens_tensor = torch.tensor(context_lens,
304
- dtype=torch.int,
305
- device="cuda")
306
- selected_token_indices = torch.tensor(selected_token_indices,
307
- dtype=torch.long,
308
- device="cuda")
309
- categorized_sample_indices = {
310
- t: torch.tensor(seq_ids, dtype=torch.int, device="cuda")
311
- for t, seq_ids in categorized_sample_indices.items()
312
- }
313
- block_tables_tensor = torch.tensor(padded_block_tables,
314
- dtype=torch.int,
315
- device="cuda")
316
-
317
- seq_data: Dict[int, SequenceData] = {}
318
- for seq_group_metadata in seq_group_metadata_list:
319
- seq_data.update(seq_group_metadata.seq_data)
320
-
321
- input_metadata = InputMetadata(
322
- seq_groups=seq_groups,
323
- seq_data=seq_data,
324
- prompt_lens=prompt_lens,
325
- slot_mapping=slot_mapping_tensor,
326
- context_lens=context_lens_tensor,
327
- max_context_len=max_context_len,
328
- block_tables=block_tables_tensor,
329
- selected_token_indices=selected_token_indices,
330
- categorized_sample_indices=categorized_sample_indices,
331
- sliding_window=self.sliding_window,
332
- )
333
- return tokens_tensor, positions_tensor, input_metadata
116
+ self.model_runner.set_block_size(self.cache_engine.block_size)
117
118
@torch.inference_mode()
119
def execute_model(
@@ -361,18 +144,8 @@ class Worker:
144
event.wait()
145
return {}
146
364
- # Prepare input tensors.
365
- input_tokens, input_positions, input_metadata = self._prepare_inputs(
366
- seq_group_metadata_list)
367
-
368
- # Execute the model.
369
- output = self.model(
370
- input_ids=input_tokens,
371
- positions=input_positions,
372
- kv_caches=self.gpu_cache,
373
- input_metadata=input_metadata,
374
- cache_events=cache_events,
375
- )
147
+ output = self.model_runner.execute_model(seq_group_metadata_list,
148
+ self.gpu_cache, cache_events)
149
return output
150
151
@@ -407,14 +180,6 @@ def _init_distributed_environment(
180
parallel_config.pipeline_parallel_size)
181
182
410
-def _pad_to_alignment(x: List[int], multiple_of: int, pad: int) -> List[int]:
411
- return x + [pad] * ((-len(x)) % multiple_of)
412
-
413
-
414
-def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
415
- return x + [pad] * (max_len - len(x))
416
-
417
-
183
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
184
# Check if the GPU supports the dtype.
185
if torch_dtype == torch.bfloat16: