@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 27feead2f8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Refactor Worker & InputMetadata (#1843)

Woosuk Kwon committed Nov 29, 2023 at 22:16 UTC 27feead2f80f906de88d64c6e69342451cf1d7f0
27 files changed +668 -443
vllm/config.py
+6
@@ -161,6 +161,12 @@ class ModelConfig:
161 "must be divisible by pipeline parallel size "
162 f"({pipeline_parallel_size}).")
163
164 + def get_sliding_window(self) -> Optional[int]:
165 + return getattr(self.hf_config, "sliding_window", None)
166 +
167 + def get_vocab_size(self) -> int:
168 + return self.hf_config.vocab_size
169 +
170 def get_hidden_size(self) -> int:
171 return self.hf_config.hidden_size
172
vllm/engine/arg_utils.py
+4 -3
@@ -201,9 +201,10 @@ class EngineArgs:
201 self.dtype, self.seed, self.revision,
202 self.tokenizer_revision, self.max_model_len,
203 self.quantization)
204 - cache_config = CacheConfig(
205 - self.block_size, self.gpu_memory_utilization, self.swap_space,
206 - getattr(model_config.hf_config, 'sliding_window', None))
204 + cache_config = CacheConfig(self.block_size,
205 + self.gpu_memory_utilization,
206 + self.swap_space,
207 + model_config.get_sliding_window())
208 parallel_config = ParallelConfig(self.pipeline_parallel_size,
209 self.tensor_parallel_size,
210 self.worker_use_ray,
vllm/engine/llm_engine.py
-2
@@ -88,8 +88,6 @@ class LLMEngine:
88
89 self.model_config = model_config
90 self.cache_config = cache_config
91 - assert self.cache_config.sliding_window == getattr(
92 - self.model_config.hf_config, "sliding_window", None)
91 self.parallel_config = parallel_config
92 self.scheduler_config = scheduler_config
93 self.log_stats = log_stats
vllm/model_executor/__init__.py
+2
@@ -1,9 +1,11 @@
1 from vllm.model_executor.input_metadata import InputMetadata
2 from vllm.model_executor.model_loader import get_model
3 +from vllm.model_executor.sampling_metadata import SamplingMetadata
4 from vllm.model_executor.utils import set_random_seed
5
6 __all__ = [
7 "InputMetadata",
8 "get_model",
9 + "SamplingMetadata",
10 "set_random_seed",
11 ]
vllm/model_executor/input_metadata.py
+16 -65
@@ -1,91 +1,42 @@
1 -from typing import Dict, List, Optional, Tuple
1 +from typing import List, Optional
2
3 import torch
4 -from xformers.ops import AttentionBias
5 -
6 -from vllm.sampling_params import SamplingParams, SamplingType
7 -from vllm.sequence import SequenceData
4
5
6 class InputMetadata:
11 - """Metadata for input sequences. Used for PagedAttention.
7 + """Metadata for input sequences. Used in PagedAttention.
8
9 Args:
14 - seq_groups: List of (seq_ids, sampling_params).
15 - seq_data: Seq_id -> SequenceData.
10 prompt_lens: Lengths of prompts.
11 slot_mapping: The address to write the new KV to of each token.
18 - context_lens: the length of attention context for each generation token.
12 max_context_len: The maximum context length.
13 + context_lens: the length of attention context for each sequence.
14 block_tables: The block tables. (Seq id -> list of physical block)
15 """
16
17 def __init__(
18 self,
25 - seq_groups: List[Tuple[List[int], SamplingParams]],
26 - seq_data: Dict[int, SequenceData],
19 prompt_lens: List[int],
20 slot_mapping: torch.Tensor,
29 - context_lens: torch.Tensor,
30 - max_context_len: int,
31 - block_tables: torch.Tensor,
32 - selected_token_indices: torch.Tensor,
33 - categorized_sample_indices: Dict[SamplingType, torch.Tensor],
34 - sliding_window: Optional[int] = None,
21 + max_context_len: Optional[int],
22 + context_lens: Optional[torch.Tensor],
23 + block_tables: Optional[torch.Tensor],
24 ) -> None:
36 - self.seq_groups = seq_groups
37 - self.seq_data = seq_data
25 self.prompt_lens = prompt_lens
26 + self.max_context_len = max_context_len
27 self.slot_mapping = slot_mapping
28 self.context_lens = context_lens
41 - self.max_context_len = max_context_len
29 self.block_tables = block_tables
43 - self.selected_token_indices = selected_token_indices
44 - self.categorized_sample_indices = categorized_sample_indices
45 -
46 - self.max_prompt_len = max(prompt_lens) if prompt_lens else 0
47 - self.to_cache = None
48 - if sliding_window is not None:
49 - # We need to keep the positions of sliding windows within
50 - # the key / value tables, this is helpful to know which
51 - # elements we need to cache.
52 - to_cache, start_idx = [], 0
53 - for prompt_len in self.prompt_lens:
54 - to_cache.extend(
55 - range(
56 - start_idx + max(0, prompt_len - sliding_window),
57 - start_idx + prompt_len,
58 - ))
59 - start_idx += self.max_prompt_len
60 - to_cache.extend(range(start_idx, slot_mapping.shape[0]))
61 - self.to_cache = torch.tensor(to_cache,
62 - dtype=torch.int32,
63 - device=self.slot_mapping.device)
64 -
65 - self.num_prompts = len(prompt_lens)
66 - self.num_prompt_tokens = self.num_prompts * self.max_prompt_len
67 - self.num_generation_tokens = context_lens.shape[0]
68 - if block_tables.numel() > 0:
69 - self.max_num_blocks_per_seq = block_tables.shape[1]
70 - else:
71 - self.max_num_blocks_per_seq = 0
72 - assert block_tables.shape[0] == self.num_generation_tokens
30
31 + self.is_prompt = len(prompt_lens) > 0
32 # Set during the execution of the first attention op.
75 - self.attn_bias: Optional[AttentionBias] = None
33 + # FIXME(woosuk): This is a hack.
34 + self.attn_bias = None
35
36 def __repr__(self) -> str:
78 - # Print only useful metadata.
79 - return (
80 - f'InputMetadata('
81 - f'num_prompt_tokens={self.num_prompt_tokens}, '
82 - f'num_prompts={self.num_prompts}, '
83 - f'prompt_lens={self.prompt_lens}, '
84 - f'num_generation_tokens={self.num_generation_tokens}, '
85 - f'context_lens={self.context_lens}, '
86 - f'max_context_len={self.max_context_len}), '
87 - f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
88 - f'block_tables={self.block_tables}, '
89 - f'selected_token_indices={self.selected_token_indices}, '
90 - f'categorized_sample_indices={self.categorized_sample_indices}, '
91 - f'slot_mapping={self.slot_mapping})')
37 + return ("InputMetadata("
38 + f"prompt_lens={self.prompt_lens}, "
39 + f"max_context_len={self.max_context_len}, "
40 + f"slot_mapping={self.slot_mapping}, "
41 + f"context_lens={self.context_lens}, "
42 + f"block_tables={self.block_tables})")
vllm/model_executor/layers/attention.py
+3 -11
@@ -101,23 +101,15 @@ class PagedAttention(nn.Module):
101 # vectors will not be cached. This happens during the initial memory
102 # profiling run.
103 if key_cache is not None and value_cache is not None:
104 - key_to_cache = key
105 - value_to_cache = value
106 - if input_metadata.to_cache is not None:
107 - key_to_cache = key_to_cache[input_metadata.to_cache]
108 - value_to_cache = value_to_cache[input_metadata.to_cache]
109 - slot_mapping = slot_mapping[input_metadata.to_cache]
110 -
104 cache_ops.reshape_and_cache(
112 - key_to_cache,
113 - value_to_cache,
105 + key,
106 + value,
107 key_cache,
108 value_cache,
109 slot_mapping,
110 )
111
119 - is_prompt = len(input_metadata.prompt_lens) > 0
120 - if is_prompt:
112 + if input_metadata.is_prompt:
113 # Prompt run.
114 if self.num_kv_heads != self.num_heads:
115 # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
vllm/model_executor/layers/sampler.py
+60 -55
@@ -4,9 +4,9 @@ from typing import Dict, List, Optional, Tuple
4 import torch
5 import torch.nn as nn
6
7 -from vllm.model_executor.input_metadata import InputMetadata
7 from vllm.model_executor.parallel_utils.communication_op import (
8 tensor_model_parallel_all_gather)
9 +from vllm.model_executor.sampling_metadata import SamplingMetadata
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12 SequenceData, SequenceGroupOutput, SequenceOutput)
@@ -37,29 +37,30 @@ class Sampler(nn.Module):
37 self,
38 embedding: torch.Tensor,
39 hidden_states: torch.Tensor,
40 - input_metadata: InputMetadata,
40 + sampling_metadata: SamplingMetadata,
41 embedding_bias: Optional[torch.Tensor] = None,
42 ) -> SamplerOutput:
43 # Get the hidden states that we use for sampling.
44 - hidden_states = _prune_hidden_states(hidden_states, input_metadata)
44 + hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)
45
46 # Get the logits for the next tokens.
47 logits = _get_logits(hidden_states, embedding, embedding_bias,
48 self.vocab_size)
49
50 # Apply logits processors (if any).
51 - logits = _apply_logits_processors(logits, input_metadata)
51 + logits = _apply_logits_processors(logits, sampling_metadata)
52 # Apply presence and frequency penalties.
53 presence_penalties, frequency_penalties, repetition_penalties = (
54 - _get_penalties(input_metadata))
54 + _get_penalties(sampling_metadata))
55 assert len(presence_penalties) == logits.shape[0]
56 assert len(frequency_penalties) == logits.shape[0]
57 assert len(repetition_penalties) == logits.shape[0]
58 - logits = _apply_penalties(logits, input_metadata, presence_penalties,
59 - frequency_penalties, repetition_penalties)
58 + logits = _apply_penalties(logits, sampling_metadata,
59 + presence_penalties, frequency_penalties,
60 + repetition_penalties)
61
62 # Apply temperature scaling.
62 - temperatures = _get_temperatures(input_metadata)
63 + temperatures = _get_temperatures(sampling_metadata)
64 assert len(temperatures) == logits.shape[0]
65 if any(t != 1.0 for t in temperatures):
66 t = torch.tensor(temperatures,
@@ -70,7 +71,7 @@ class Sampler(nn.Module):
71
72 # Apply top-p and top-k truncation.
73 top_ps, top_ks, min_ps = _get_top_p_top_k_min_p(
73 - input_metadata, self.vocab_size)
74 + sampling_metadata, self.vocab_size)
75 assert len(top_ps) == len(top_ks) == logits.shape[0]
76 do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
77 do_top_k = any(k != self.vocab_size for k in top_ks)
@@ -89,11 +90,11 @@ class Sampler(nn.Module):
90 logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
91
92 # Sample the next tokens.
92 - sample_results = _sample(probs, logprobs, input_metadata)
93 + sample_results = _sample(probs, logprobs, sampling_metadata)
94 # Get the logprobs query results.
95 prompt_logprobs, sample_logprobs = _get_logprobs(
95 - logprobs, input_metadata, sample_results)
96 - return _build_sampler_output(sample_results, input_metadata,
96 + logprobs, sampling_metadata, sample_results)
97 + return _build_sampler_output(sample_results, sampling_metadata,
98 prompt_logprobs, sample_logprobs)
99
100
@@ -112,29 +113,30 @@ def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
113
114 def _prune_hidden_states(
115 hidden_states: torch.Tensor,
115 - input_metadata: InputMetadata,
116 + sampling_metadata: SamplingMetadata,
117 ) -> torch.Tensor:
118 hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
118 - return hidden_states.index_select(0, input_metadata.selected_token_indices)
119 + return hidden_states.index_select(0,
120 + sampling_metadata.selected_token_indices)
121
122
123 def _get_penalties(
122 - input_metadata: InputMetadata
124 + sampling_metadata: SamplingMetadata
125 ) -> Tuple[List[float], List[float], List[float]]:
126 # Collect the presence and frequency penalties.
127 presence_penalties: List[float] = []
128 frequency_penalties: List[float] = []
129 repetition_penalties: List[float] = []
128 - for i, seq_group in enumerate(input_metadata.seq_groups):
130 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
131 seq_ids, sampling_params = seq_group
132 p = sampling_params.presence_penalty
133 f = sampling_params.frequency_penalty
134 r = sampling_params.repetition_penalty
133 - if (i < input_metadata.num_prompts
135 + if (i < sampling_metadata.num_prompts
136 and sampling_params.prompt_logprobs is not None):
137 # NOTE: We do not apply presence and frequency penalties for the
138 # prompt token positions where we don't sample new tokens.
137 - prompt_len = input_metadata.prompt_lens[i]
139 + prompt_len = sampling_metadata.prompt_lens[i]
140 presence_penalties += [0] * (prompt_len - 1)
141 frequency_penalties += [0] * (prompt_len - 1)
142 repetition_penalties += [1] * (prompt_len - 1)
@@ -145,21 +147,21 @@ def _get_penalties(
147
148
149 def _get_prompt_and_output_tokens(
148 - input_metadata: InputMetadata
150 + sampling_metadata: SamplingMetadata,
151 ) -> Tuple[List[List[int]], List[List[int]]]:
152 prompt_tokens: List[List[int]] = []
153 output_tokens: List[List[int]] = []
152 - for i, seq_group in enumerate(input_metadata.seq_groups):
154 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
155 seq_ids, sampling_params = seq_group
154 - if (i < input_metadata.num_prompts
156 + if (i < sampling_metadata.num_prompts
157 and sampling_params.prompt_logprobs is not None):
158 # NOTE: prompt token positions do not need output tokens to
159 # compute penalties.
158 - prompt_len = input_metadata.prompt_lens[i]
160 + prompt_len = sampling_metadata.prompt_lens[i]
161 prompt_tokens.extend([] for _ in range(prompt_len - 1))
162 output_tokens.extend([] for _ in range(prompt_len - 1))
163 for seq_id in seq_ids:
162 - seq_data = input_metadata.seq_data[seq_id]
164 + seq_data = sampling_metadata.seq_data[seq_id]
165 prompt_tokens.append(seq_data.prompt_token_ids)
166 output_tokens.append(seq_data.output_token_ids)
167 return prompt_tokens, output_tokens
@@ -191,17 +193,19 @@ def _get_bin_counts_and_mask(
193 return bin_counts, mask
194
195
194 -def _apply_logits_processors(logits: torch.Tensor,
195 - input_metadata: InputMetadata) -> torch.Tensor:
196 +def _apply_logits_processors(
197 + logits: torch.Tensor,
198 + sampling_metadata: SamplingMetadata,
199 +) -> torch.Tensor:
200 logits_row_idx = 0
201 found_logits_processors = False
198 - for seq_ids, sampling_params in input_metadata.seq_groups:
202 + for seq_ids, sampling_params in sampling_metadata.seq_groups:
203 logits_processors = sampling_params.logits_processors
204 if logits_processors:
205 found_logits_processors = True
206 for seq_id in seq_ids:
207 logits_row = logits[logits_row_idx]
204 - token_ids = input_metadata.seq_data[seq_id].output_token_ids
208 + token_ids = sampling_metadata.seq_data[seq_id].output_token_ids
209 for logits_processor in logits_processors:
210 logits_row = logits_processor(token_ids, logits_row)
211 logits[logits_row_idx] = logits_row
@@ -215,7 +219,7 @@ def _apply_logits_processors(logits: torch.Tensor,
219
220 def _apply_penalties(
221 logits: torch.Tensor,
218 - input_metadata: InputMetadata,
222 + sampling_metadata: SamplingMetadata,
223 presence_penalties: List[float],
224 frequency_penalties: List[float],
225 repetition_penalties: List[float],
@@ -234,7 +238,7 @@ def _apply_penalties(
238 return logits
239
240 prompt_tokens, output_tokens = (
237 - _get_prompt_and_output_tokens(input_metadata))
241 + _get_prompt_and_output_tokens(sampling_metadata))
242 assert len(prompt_tokens) == logits.shape[0]
243 assert len(output_tokens) == logits.shape[0]
244
@@ -265,10 +269,10 @@ def _apply_penalties(
269 return logits
270
271
268 -def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
272 +def _get_temperatures(sampling_metadata: SamplingMetadata) -> List[float]:
273 # Collect the temperatures for the logits.
274 temperatures: List[float] = []
271 - for i, seq_group in enumerate(input_metadata.seq_groups):
275 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
276 seq_ids, sampling_params = seq_group
277 temperature = sampling_params.temperature
278 if temperature < _SAMPLING_EPS:
@@ -276,22 +280,22 @@ def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
280 # (i.e., greedy sampling or beam search).
281 # Set the temperature to 1 to avoid division by zero.
282 temperature = 1.0
279 - if (i < input_metadata.num_prompts
283 + if (i < sampling_metadata.num_prompts
284 and sampling_params.prompt_logprobs is not None):
281 - prompt_len = input_metadata.prompt_lens[i]
285 + prompt_len = sampling_metadata.prompt_lens[i]
286 temperatures += [temperature] * (prompt_len - 1)
287 temperatures += [temperature] * len(seq_ids)
288 return temperatures
289
290
291 def _get_top_p_top_k_min_p(
288 - input_metadata: InputMetadata,
292 + sampling_metadata: SamplingMetadata,
293 vocab_size: int,
294 ) -> Tuple[List[float], List[int], List[float]]:
295 top_ps: List[float] = []
296 top_ks: List[int] = []
297 min_ps: List[float] = []
294 - for i, seq_group in enumerate(input_metadata.seq_groups):
298 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
299 seq_ids, sampling_params = seq_group
300 top_p = sampling_params.top_p
301 min_p = sampling_params.min_p
@@ -299,9 +303,9 @@ def _get_top_p_top_k_min_p(
303 top_k = min(sampling_params.top_k, vocab_size)
304 # k=-1 means no truncation.
305 top_k = vocab_size if top_k == -1 else top_k
302 - if (i < input_metadata.num_prompts
306 + if (i < sampling_metadata.num_prompts
307 and sampling_params.prompt_logprobs is not None):
304 - prompt_len = input_metadata.prompt_lens[i]
308 + prompt_len = sampling_metadata.prompt_lens[i]
309 top_ps += [top_p] * (prompt_len - 1)
310 top_ks += [top_k] * (prompt_len - 1)
311 min_ps += [min_p] * (prompt_len - 1)
@@ -471,11 +475,11 @@ def _beam_search_sample(
475 def _sample(
476 probs: torch.Tensor,
477 logprobs: torch.Tensor,
474 - input_metadata: InputMetadata,
478 + sampling_metadata: SamplingMetadata,
479 ) -> List[Tuple[List[int], List[int]]]:
480 categorized_seq_group_ids = {t: [] for t in SamplingType}
477 - categorized_sample_indices = input_metadata.categorized_sample_indices
478 - for i, seq_group in enumerate(input_metadata.seq_groups):
481 + categorized_sample_indices = sampling_metadata.categorized_sample_indices
482 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
483 _, sampling_params = seq_group
484 sampling_type = sampling_params.sampling_type
485 categorized_seq_group_ids[sampling_type].append(i)
@@ -483,8 +487,8 @@ def _sample(
487 sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
488 for sampling_type in SamplingType:
489 seq_group_ids = categorized_seq_group_ids[sampling_type]
486 - seq_groups = [input_metadata.seq_groups[i] for i in seq_group_ids]
487 - is_prompts = [i < input_metadata.num_prompts for i in seq_group_ids]
490 + seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
491 + is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
492 sample_indices = categorized_sample_indices[sampling_type]
493 num_tokens = len(sample_indices)
494 if num_tokens == 0:
@@ -499,21 +503,22 @@ def _sample(
503 elif sampling_type == SamplingType.BEAM:
504 category_logprobs = logprobs[sample_indices]
505 sample_results = _beam_search_sample(seq_groups, is_prompts,
502 - input_metadata.seq_data,
506 + sampling_metadata.seq_data,
507 category_logprobs)
508 else:
509 raise ValueError(f"Unsupported sampling type: {sampling_type}")
510 sample_results_dict.update(zip(seq_group_ids, sample_results))
511
512 sample_results = [
509 - sample_results_dict[i] for i in range(len(input_metadata.seq_groups))
513 + sample_results_dict[i]
514 + for i in range(len(sampling_metadata.seq_groups))
515 ]
516 return sample_results
517
518
519 def _get_logprobs(
520 logprobs: torch.Tensor,
516 - input_metadata: InputMetadata,
521 + sampling_metadata: SamplingMetadata,
522 sample_results: List[Tuple[List[int], List[int]]],
523 ) -> Tuple[List[Optional[List[Optional[Dict[int, float]]]]], List[List[Dict[
524 int, float]]]]:
@@ -523,16 +528,16 @@ def _get_logprobs(
528 largest_num_logprobs = 0
529 sample_idx = 0
530 for i, (seq_group, sample_result) in enumerate(
526 - zip(input_metadata.seq_groups, sample_results)):
531 + zip(sampling_metadata.seq_groups, sample_results)):
532 seq_ids, sampling_params = seq_group
533 next_token_ids, parent_ids = sample_result
534 num_parent_seqs = len(seq_ids)
530 - if (i < input_metadata.num_prompts
535 + if (i < sampling_metadata.num_prompts
536 and sampling_params.prompt_logprobs is not None):
537 largest_num_logprobs = max(largest_num_logprobs,
538 sampling_params.prompt_logprobs)
534 - prompt_len = input_metadata.prompt_lens[i]
535 - prompt_tokens = input_metadata.seq_data[
539 + prompt_len = sampling_metadata.prompt_lens[i]
540 + prompt_tokens = sampling_metadata.seq_data[
541 seq_ids[0]].prompt_token_ids
542 batched_logprobs_query_seq_indices.extend(
543 sample_idx + j for j in range(prompt_len - 1))
@@ -570,16 +575,16 @@ def _get_logprobs(
575 sample_idx = 0
576 query_result_idx = 0
577 for i, (seq_group, sample_result) in enumerate(
573 - zip(input_metadata.seq_groups, sample_results)):
578 + zip(sampling_metadata.seq_groups, sample_results)):
579 seq_ids, sampling_params = seq_group
580 next_token_ids, parent_ids = sample_result
581
582 # Prompt logprobs
578 - if (i < input_metadata.num_prompts
583 + if (i < sampling_metadata.num_prompts
584 and sampling_params.prompt_logprobs is not None):
585 num_logprobs = sampling_params.prompt_logprobs
581 - prompt_len = input_metadata.prompt_lens[i]
582 - prompt_tokens = input_metadata.seq_data[
586 + prompt_len = sampling_metadata.prompt_lens[i]
587 + prompt_tokens = sampling_metadata.seq_data[
588 seq_ids[0]].prompt_token_ids
589 group_prompt_logprobs: PromptLogprobs = [None]
590 for token_id in prompt_tokens[1:]:
@@ -625,13 +630,13 @@ def _get_logprobs(
630
631 def _build_sampler_output(
632 sample_results: List[Tuple[List[int], List[int]]],
628 - input_metadata: InputMetadata,
633 + sampling_metadata: SamplingMetadata,
634 prompt_logprobs: List[Optional[PromptLogprobs]],
635 sample_logprobs: List[SampleLogprobs],
636 ) -> SamplerOutput:
637 sampler_output = []
638 for (seq_group, sample_result, group_prompt_logprobs,
634 - group_sample_logprobs) in zip(input_metadata.seq_groups,
639 + group_sample_logprobs) in zip(sampling_metadata.seq_groups,
640 sample_results, prompt_logprobs,
641 sample_logprobs):
642 seq_ids, _ = seq_group
vllm/model_executor/models/aquila.py
+10 -2
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding, ParallelLMHead)
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_world_size)
42 +from vllm.model_executor.sampling_metadata import SamplingMetadata
43 from vllm.model_executor.weight_utils import (default_weight_loader,
44 hf_model_weights_iterator)
45 from vllm.sequence import SamplerOutput
@@ -296,11 +297,18 @@ class AquilaForCausalLM(nn.Module):
297 kv_caches: List[KVCache],
298 input_metadata: InputMetadata,
299 cache_events: Optional[List[torch.cuda.Event]],
299 - ) -> SamplerOutput:
300 + ) -> torch.Tensor:
301 hidden_states = self.model(input_ids, positions, kv_caches,
302 input_metadata, cache_events)
303 + return hidden_states
304 +
305 + def sample(
306 + self,
307 + hidden_states: torch.Tensor,
308 + sampling_metadata: SamplingMetadata,
309 + ) -> SamplerOutput:
310 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303 - input_metadata)
311 + sampling_metadata)
312 return next_tokens
313
314 def load_weights(self,
vllm/model_executor/models/baichuan.py
+10 -2
@@ -38,6 +38,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding, ParallelLMHead)
39 from vllm.model_executor.parallel_utils.parallel_state import (
40 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41 +from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.model_executor.weight_utils import (default_weight_loader,
43 hf_model_weights_iterator)
44 from vllm.sequence import SamplerOutput
@@ -311,11 +312,18 @@ class BaiChuanBaseForCausalLM(nn.Module):
312 kv_caches: List[KVCache],
313 input_metadata: InputMetadata,
314 cache_events: Optional[List[torch.cuda.Event]],
314 - ) -> SamplerOutput:
315 + ) -> torch.Tensor:
316 hidden_states = self.model(input_ids, positions, kv_caches,
317 input_metadata, cache_events)
318 + return hidden_states
319 +
320 + def sample(
321 + self,
322 + hidden_states: torch.Tensor,
323 + sampling_metadata: SamplingMetadata,
324 + ) -> SamplerOutput:
325 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
318 - input_metadata)
326 + sampling_metadata)
327 return next_tokens
328
329 def load_weights(self,
vllm/model_executor/models/bloom.py
+10 -2
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35 VocabParallelEmbedding)
36 from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 +from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
@@ -288,11 +289,18 @@ class BloomForCausalLM(nn.Module):
289 kv_caches: List[KVCache],
290 input_metadata: InputMetadata,
291 cache_events: Optional[List[torch.cuda.Event]],
291 - ) -> SamplerOutput:
292 + ) -> torch.Tensor:
293 hidden_states = self.transformer(input_ids, positions, kv_caches,
294 input_metadata, cache_events)
295 + return hidden_states
296 +
297 + def sample(
298 + self,
299 + hidden_states: torch.Tensor,
300 + sampling_metadata: SamplingMetadata,
301 + ) -> SamplerOutput:
302 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
295 - input_metadata)
303 + sampling_metadata)
304 return next_tokens
305
306 def load_weights(self,
vllm/model_executor/models/chatglm.py
+10 -2
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
22 VocabParallelEmbedding, ParallelLMHead)
23 from vllm.model_executor.parallel_utils.parallel_state import (
24 get_tensor_model_parallel_world_size)
25 +from vllm.model_executor.sampling_metadata import SamplingMetadata
26 from vllm.model_executor.weight_utils import (default_weight_loader,
27 hf_model_weights_iterator)
28 from vllm.sequence import SamplerOutput
@@ -350,11 +351,18 @@ class ChatGLMForCausalLM(nn.Module):
351 kv_caches: List[KVCache],
352 input_metadata: InputMetadata,
353 cache_events: Optional[List[torch.cuda.Event]],
353 - ) -> SamplerOutput:
354 + ) -> torch.Tensor:
355 hidden_states = self.transformer(input_ids, positions, kv_caches,
356 input_metadata, cache_events)
357 + return hidden_states
358 +
359 + def sample(
360 + self,
361 + hidden_states: torch.Tensor,
362 + sampling_metadata: SamplingMetadata,
363 + ) -> SamplerOutput:
364 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
357 - input_metadata)
365 + sampling_metadata)
366 return next_tokens
367
368 def load_weights(self,
vllm/model_executor/models/falcon.py
+10 -3
@@ -41,6 +41,7 @@ from vllm.model_executor.parallel_utils.communication_op import (
41 tensor_model_parallel_all_reduce)
42 from vllm.model_executor.parallel_utils.parallel_state import (
43 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.model_executor.weight_utils import (default_weight_loader,
46 hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
@@ -389,7 +390,7 @@ class FalconForCausalLM(nn.Module):
390 kv_caches: List[KVCache],
391 input_metadata: InputMetadata,
392 cache_events: Optional[List[torch.cuda.Event]],
392 - ) -> SamplerOutput:
393 + ) -> torch.Tensor:
394 hidden_states = self.transformer(
395 input_ids,
396 positions,
@@ -397,9 +398,15 @@ class FalconForCausalLM(nn.Module):
398 input_metadata,
399 cache_events,
400 )
400 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
401 - input_metadata)
401 + return hidden_states
402
403 + def sample(
404 + self,
405 + hidden_states: torch.Tensor,
406 + sampling_metadata: SamplingMetadata,
407 + ) -> SamplerOutput:
408 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
409 + sampling_metadata)
410 return next_tokens
411
412 def load_weights(self,
vllm/model_executor/models/gpt2.py
+10 -2
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35 VocabParallelEmbedding)
36 from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_world_size)
38 +from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
@@ -232,11 +233,18 @@ class GPT2LMHeadModel(nn.Module):
233 kv_caches: List[KVCache],
234 input_metadata: InputMetadata,
235 cache_events: Optional[List[torch.cuda.Event]],
235 - ) -> SamplerOutput:
236 + ) -> torch.Tensor:
237 hidden_states = self.transformer(input_ids, positions, kv_caches,
238 input_metadata, cache_events)
239 + return hidden_states
240 +
241 + def sample(
242 + self,
243 + hidden_states: torch.Tensor,
244 + sampling_metadata: SamplingMetadata,
245 + ) -> SamplerOutput:
246 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
239 - input_metadata)
247 + sampling_metadata)
248 return next_tokens
249
250 def load_weights(self,
vllm/model_executor/models/gpt_bigcode.py
+10 -2
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding)
37 from vllm.model_executor.parallel_utils.parallel_state import (
38 get_tensor_model_parallel_world_size)
39 +from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.model_executor.weight_utils import (default_weight_loader,
41 hf_model_weights_iterator)
42 from vllm.sequence import SamplerOutput
@@ -251,11 +252,18 @@ class GPTBigCodeForCausalLM(nn.Module):
252 kv_caches: List[KVCache],
253 input_metadata: InputMetadata,
254 cache_events: Optional[List[torch.cuda.Event]],
254 - ) -> SamplerOutput:
255 + ) -> torch.Tensor:
256 hidden_states = self.transformer(input_ids, positions, kv_caches,
257 input_metadata, cache_events)
258 + return hidden_states
259 +
260 + def sample(
261 + self,
262 + hidden_states: torch.Tensor,
263 + sampling_metadata: SamplingMetadata,
264 + ) -> SamplerOutput:
265 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
258 - input_metadata)
266 + sampling_metadata)
267 return next_tokens
268
269 def load_weights(self,
vllm/model_executor/models/gpt_j.py
+10 -2
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35 VocabParallelEmbedding, ParallelLMHead)
36 from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_world_size)
38 +from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
@@ -238,11 +239,18 @@ class GPTJForCausalLM(nn.Module):
239 kv_caches: List[KVCache],
240 input_metadata: InputMetadata,
241 cache_events: Optional[List[torch.cuda.Event]],
241 - ) -> SamplerOutput:
242 + ) -> torch.Tensor:
243 hidden_states = self.transformer(input_ids, positions, kv_caches,
244 input_metadata, cache_events)
245 + return hidden_states
246 +
247 + def sample(
248 + self,
249 + hidden_states: torch.Tensor,
250 + sampling_metadata: SamplingMetadata,
251 + ) -> SamplerOutput:
252 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
245 - input_metadata, self.lm_head.bias)
253 + sampling_metadata, self.lm_head.bias)
254 return next_tokens
255
256 def load_weights(self,
vllm/model_executor/models/gpt_neox.py
+10 -2
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
35 VocabParallelEmbedding, ParallelLMHead)
36 from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_world_size)
38 +from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
@@ -251,11 +252,18 @@ class GPTNeoXForCausalLM(nn.Module):
252 kv_caches: List[KVCache],
253 input_metadata: InputMetadata,
254 cache_events: Optional[List[torch.cuda.Event]],
254 - ) -> SamplerOutput:
255 + ) -> torch.Tensor:
256 hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
257 input_metadata, cache_events)
258 + return hidden_states
259 +
260 + def sample(
261 + self,
262 + hidden_states: torch.Tensor,
263 + sampling_metadata: SamplingMetadata,
264 + ) -> SamplerOutput:
265 next_tokens = self.sampler(self.embed_out.weight, hidden_states,
258 - input_metadata)
266 + sampling_metadata)
267 return next_tokens
268
269 def load_weights(self,
vllm/model_executor/models/internlm.py
+10 -2
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
19 VocabParallelEmbedding, ParallelLMHead)
20 from vllm.model_executor.parallel_utils.parallel_state import (
21 get_tensor_model_parallel_world_size)
22 +from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.model_executor.weight_utils import (default_weight_loader,
24 hf_model_weights_iterator)
25 from vllm.sequence import SamplerOutput
@@ -250,11 +251,18 @@ class InternLMForCausalLM(nn.Module):
251 kv_caches: List[KVCache],
252 input_metadata: InputMetadata,
253 cache_events: Optional[List[torch.cuda.Event]],
253 - ) -> SamplerOutput:
254 + ) -> torch.Tensor:
255 hidden_states = self.model(input_ids, positions, kv_caches,
256 input_metadata, cache_events)
257 + return hidden_states
258 +
259 + def sample(
260 + self,
261 + hidden_states: torch.Tensor,
262 + sampling_metadata: SamplingMetadata,
263 + ) -> SamplerOutput:
264 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
257 - input_metadata)
265 + sampling_metadata)
266 return next_tokens
267
268 def load_weights(self,
vllm/model_executor/models/llama.py
+10 -2
@@ -41,6 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41 VocabParallelEmbedding, ParallelLMHead)
42 from vllm.model_executor.parallel_utils.parallel_state import (
43 get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.model_executor.weight_utils import (default_weight_loader,
46 hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
@@ -289,11 +290,18 @@ class LlamaForCausalLM(nn.Module):
290 kv_caches: List[KVCache],
291 input_metadata: InputMetadata,
292 cache_events: Optional[List[torch.cuda.Event]],
292 - ) -> SamplerOutput:
293 + ) -> torch.Tensor:
294 hidden_states = self.model(input_ids, positions, kv_caches,
295 input_metadata, cache_events)
296 + return hidden_states
297 +
298 + def sample(
299 + self,
300 + hidden_states: torch.Tensor,
301 + sampling_metadata: SamplingMetadata,
302 + ) -> SamplerOutput:
303 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
296 - input_metadata)
304 + sampling_metadata)
305 return next_tokens
306
307 def load_weights(self,
vllm/model_executor/models/mistral.py
+10 -2
@@ -41,6 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41 VocabParallelEmbedding, ParallelLMHead)
42 from vllm.model_executor.parallel_utils.parallel_state import (
43 get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.model_executor.weight_utils import (default_weight_loader,
46 hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
@@ -285,11 +286,18 @@ class MistralForCausalLM(nn.Module):
286 kv_caches: List[KVCache],
287 input_metadata: InputMetadata,
288 cache_events: Optional[List[torch.cuda.Event]],
288 - ) -> SamplerOutput:
289 + ) -> torch.Tensor:
290 hidden_states = self.model(input_ids, positions, kv_caches,
291 input_metadata, cache_events)
292 + return hidden_states
293 +
294 + def sample(
295 + self,
296 + hidden_states: torch.Tensor,
297 + sampling_metadata: SamplingMetadata,
298 + ) -> SamplerOutput:
299 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
292 - input_metadata)
300 + sampling_metadata)
301 return next_tokens
302
303 def load_weights(self,
vllm/model_executor/models/mpt.py
+10 -2
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
18 VocabParallelEmbedding)
19 from vllm.model_executor.parallel_utils.parallel_state import (
20 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
21 +from vllm.model_executor.sampling_metadata import SamplingMetadata
22 from vllm.model_executor.weight_utils import (default_weight_loader,
23 hf_model_weights_iterator)
24 from vllm.sequence import SamplerOutput
@@ -256,11 +257,18 @@ class MPTForCausalLM(nn.Module):
257 kv_caches: List[KVCache],
258 input_metadata: InputMetadata,
259 cache_events: Optional[List[torch.cuda.Event]],
259 - ) -> SamplerOutput:
260 + ) -> torch.Tensor:
261 hidden_states = self.transformer(input_ids, positions, kv_caches,
262 input_metadata, cache_events)
263 + return hidden_states
264 +
265 + def sample(
266 + self,
267 + hidden_states: torch.Tensor,
268 + sampling_metadata: SamplingMetadata,
269 + ) -> SamplerOutput:
270 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
263 - input_metadata)
271 + sampling_metadata)
272 return next_tokens
273
274 def load_weights(self,
vllm/model_executor/models/opt.py
+10 -2
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding)
37 from vllm.model_executor.parallel_utils.parallel_state import (
38 get_tensor_model_parallel_world_size)
39 +from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.model_executor.weight_utils import (default_weight_loader,
41 hf_model_weights_iterator)
42 from vllm.sequence import SamplerOutput
@@ -308,11 +309,18 @@ class OPTForCausalLM(nn.Module):
309 kv_caches: List[KVCache],
310 input_metadata: InputMetadata,
311 cache_events: Optional[List[torch.cuda.Event]],
311 - ) -> SamplerOutput:
312 + ) -> torch.Tensor:
313 hidden_states = self.model(input_ids, positions, kv_caches,
314 input_metadata, cache_events)
315 + return hidden_states
316 +
317 + def sample(
318 + self,
319 + hidden_states: torch.Tensor,
320 + sampling_metadata: SamplingMetadata,
321 + ) -> SamplerOutput:
322 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
315 - input_metadata)
323 + sampling_metadata)
324 return next_tokens
325
326 def load_weights(self,
vllm/model_executor/models/phi_1_5.py
+27 -26
@@ -54,6 +54,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
54 VocabParallelEmbedding, ParallelLMHead)
55 from vllm.model_executor.parallel_utils.parallel_state import (
56 get_tensor_model_parallel_world_size)
57 +from vllm.model_executor.sampling_metadata import SamplingMetadata
58 from vllm.model_executor.weight_utils import (default_weight_loader,
59 hf_model_weights_iterator)
60 from vllm.sequence import SamplerOutput
@@ -210,28 +211,6 @@ class PhiLayer(nn.Module):
211 return hidden_states
212
213
213 -class PhiCausalLMHead(nn.Module):
214 -
215 - def __init__(self, config: PretrainedConfig):
216 - super().__init__()
217 - self.ln = nn.LayerNorm(config.hidden_size,
218 - eps=config.layer_norm_epsilon)
219 - self.linear = ParallelLMHead(config.vocab_size,
220 - config.hidden_size,
221 - bias=True)
222 - self.sampler = Sampler(config.vocab_size)
223 -
224 - def forward(
225 - self,
226 - hidden_states: torch.Tensor,
227 - input_metadata: InputMetadata,
228 - ):
229 - hidden_states = self.ln(hidden_states)
230 - next_tokens = self.sampler(self.linear.weight, hidden_states,
231 - input_metadata, self.linear.bias)
232 - return next_tokens
233 -
234 -
214 class PhiModel(nn.Module):
215
216 def __init__(self,
@@ -253,7 +232,7 @@ class PhiModel(nn.Module):
232 kv_caches: List[KVCache],
233 input_metadata: InputMetadata,
234 cache_events: Optional[List[torch.cuda.Event]],
256 - ) -> SamplerOutput:
235 + ) -> torch.Tensor:
236 hidden_states = self.embd(input_ids)
237 for i in range(self.config.num_hidden_layers):
238 cache_event = None if cache_events is None else cache_events[i]
@@ -268,6 +247,17 @@ class PhiModel(nn.Module):
247 return hidden_states
248
249
250 +class PhiCausalLMHead(nn.Module):
251 +
252 + def __init__(self, config: PretrainedConfig):
253 + super().__init__()
254 + self.ln = nn.LayerNorm(config.hidden_size,
255 + eps=config.layer_norm_epsilon)
256 + self.linear = ParallelLMHead(config.vocab_size,
257 + config.hidden_size,
258 + bias=True)
259 +
260 +
261 class PhiForCausalLM(nn.Module):
262
263 def __init__(self,
@@ -279,6 +269,7 @@ class PhiForCausalLM(nn.Module):
269
270 self.transformer = PhiModel(config, linear_method)
271 self.lm_head = PhiCausalLMHead(config)
272 + self.sampler = Sampler(config.vocab_size)
273
274 def forward(
275 self,
@@ -287,11 +278,21 @@ class PhiForCausalLM(nn.Module):
278 kv_caches: List[KVCache],
279 input_metadata: InputMetadata,
280 cache_events: Optional[List[torch.cuda.Event]],
290 - ) -> SamplerOutput:
281 + ) -> torch.Tensor:
282 hidden_states = self.transformer(input_ids, positions, kv_caches,
283 input_metadata, cache_events)
293 - lm_logits = self.lm_head(hidden_states, input_metadata)
294 - return lm_logits
284 + hidden_states = self.lm_head.ln(hidden_states)
285 + return hidden_states
286 +
287 + def sample(
288 + self,
289 + hidden_states: torch.Tensor,
290 + sampling_metadata: SamplingMetadata,
291 + ) -> SamplerOutput:
292 + head = self.lm_head.linear
293 + next_tokens = self.sampler(head.weight, hidden_states,
294 + sampling_metadata, head.bias)
295 + return next_tokens
296
297 def load_weights(self,
298 model_name_or_path: str,
vllm/model_executor/models/qwen.py
+10 -2
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23 VocabParallelEmbedding, ParallelLMHead)
24 from vllm.model_executor.parallel_utils.parallel_state import (
25 get_tensor_model_parallel_world_size)
26 +from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.model_executor.weight_utils import (default_weight_loader,
28 hf_model_weights_iterator)
29 from vllm.sequence import SamplerOutput
@@ -246,11 +247,18 @@ class QWenLMHeadModel(nn.Module):
247 kv_caches: List[KVCache],
248 input_metadata: InputMetadata,
249 cache_events: Optional[List[torch.cuda.Event]],
249 - ) -> SamplerOutput:
250 + ) -> torch.Tensor:
251 hidden_states = self.transformer(input_ids, positions, kv_caches,
252 input_metadata, cache_events)
253 + return hidden_states
254 +
255 + def sample(
256 + self,
257 + hidden_states: torch.Tensor,
258 + sampling_metadata: SamplingMetadata,
259 + ) -> SamplerOutput:
260 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
253 - input_metadata)
261 + sampling_metadata)
262 return next_tokens
263
264 def load_weights(self,
vllm/model_executor/models/yi.py
+10 -2
@@ -41,6 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41 VocabParallelEmbedding, ParallelLMHead)
42 from vllm.model_executor.parallel_utils.parallel_state import (
43 get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.model_executor.weight_utils import (default_weight_loader,
46 hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
@@ -284,11 +285,18 @@ class YiForCausalLM(nn.Module):
285 kv_caches: List[KVCache],
286 input_metadata: InputMetadata,
287 cache_events: Optional[List[torch.cuda.Event]],
287 - ) -> SamplerOutput:
288 + ) -> torch.Tensor:
289 hidden_states = self.model(input_ids, positions, kv_caches,
290 input_metadata, cache_events)
291 + return hidden_states
292 +
293 + def sample(
294 + self,
295 + hidden_states: torch.Tensor,
296 + sampling_metadata: SamplingMetadata,
297 + ) -> SamplerOutput:
298 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291 - input_metadata)
299 + sampling_metadata)
300 return next_tokens
301
302 def load_weights(self,
vllm/model_executor/sampling_metadata.py new
+43
@@ -0,0 +1,43 @@
1 +from typing import Dict, List, Tuple
2 +
3 +import torch
4 +
5 +from vllm.sampling_params import SamplingParams, SamplingType
6 +from vllm.sequence import SequenceData
7 +
8 +
9 +class SamplingMetadata:
10 + """Metadata for input sequences. Used in sampler.
11 +
12 + Args:
13 + seq_groups: List of (seq_ids, sampling_params).
14 + seq_data: Seq_id -> SequenceData.
15 + prompt_lens: Lengths of prompts.
16 + selected_token_indices: Token indices selected for sampling.
17 + categorized_sample_indices: SamplingType -> token indicies to sample.
18 + """
19 +
20 + def __init__(
21 + self,
22 + seq_groups: List[Tuple[List[int], SamplingParams]],
23 + seq_data: Dict[int, SequenceData],
24 + prompt_lens: List[int],
25 + selected_token_indices: torch.Tensor,
26 + categorized_sample_indices: Dict[SamplingType, torch.Tensor],
27 + ) -> None:
28 + self.seq_groups = seq_groups
29 + self.seq_data = seq_data
30 + self.prompt_lens = prompt_lens
31 + self.selected_token_indices = selected_token_indices
32 + self.categorized_sample_indices = categorized_sample_indices
33 +
34 + self.num_prompts = len(prompt_lens)
35 +
36 + def __repr__(self) -> str:
37 + return (
38 + "SamplingMetadata("
39 + f"seq_groups={self.seq_groups}, "
40 + f"seq_data={self.seq_data}, "
41 + f"prompt_lens={self.prompt_lens}, "
42 + f"selected_token_indices={self.selected_token_indices}, "
43 + f"categorized_sample_indices={self.categorized_sample_indices})")
vllm/worker/model_runner.py new
+334
@@ -0,0 +1,334 @@
1 +from typing import Dict, List, Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
6 +from vllm.logger import init_logger
7 +from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
8 +from vllm.sampling_params import SamplingParams, SamplingType
9 +from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
10 +
11 +logger = init_logger(__name__)
12 +
13 +_PAD_SLOT_ID = -1
14 +
15 +
16 +class ModelRunner:
17 +
18 + def __init__(
19 + self,
20 + model_config: ModelConfig,
21 + parallel_config: ParallelConfig,
22 + scheduler_config: SchedulerConfig,
23 + ):
24 + self.model_config = model_config
25 + self.parallel_config = parallel_config
26 + self.scheduler_config = scheduler_config
27 +
28 + self.sliding_window = model_config.get_sliding_window()
29 + self.model = None
30 + self.block_size = None # Set after initial profiling.
31 +
32 + def load_model(self) -> None:
33 + self.model = get_model(self.model_config)
34 +
35 + def set_block_size(self, block_size: int) -> None:
36 + self.block_size = block_size
37 +
38 + def _prepare_prompt(
39 + self,
40 + seq_group_metadata_list: List[SequenceGroupMetadata],
41 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
42 + assert len(seq_group_metadata_list) > 0
43 + input_tokens: List[List[int]] = []
44 + input_positions: List[List[int]] = []
45 + slot_mapping: List[List[int]] = []
46 +
47 + prompt_lens: List[int] = []
48 + for seq_group_metadata in seq_group_metadata_list:
49 + assert seq_group_metadata.is_prompt
50 + seq_ids = list(seq_group_metadata.seq_data.keys())
51 + assert len(seq_ids) == 1
52 + seq_id = seq_ids[0]
53 +
54 + seq_data = seq_group_metadata.seq_data[seq_id]
55 + prompt_tokens = seq_data.get_token_ids()
56 + prompt_len = len(prompt_tokens)
57 + prompt_lens.append(prompt_len)
58 +
59 + input_tokens.append(prompt_tokens)
60 + # NOTE(woosuk): Here we assume that the first token in the prompt
61 + # is always the first token in the sequence.
62 + input_positions.append(list(range(prompt_len)))
63 +
64 + if seq_group_metadata.block_tables is None:
65 + # During memory profiling, the block tables are not initialized
66 + # yet. In this case, we just use a dummy slot mapping.
67 + slot_mapping.append([_PAD_SLOT_ID] * prompt_len)
68 + continue
69 +
70 + # Compute the slot mapping.
71 + slot_mapping.append([])
72 + block_table = seq_group_metadata.block_tables[seq_id]
73 + # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
74 + # where start_idx is max(0, prompt_len - sliding_window).
75 + # For example, if the prompt len is 10, sliding window is 8, and
76 + # block size is 4, the first two tokens are masked and the slot
77 + # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
78 + start_idx = 0
79 + if self.sliding_window is not None:
80 + start_idx = max(0, prompt_len - self.sliding_window)
81 + for i in range(prompt_len):
82 + if i < start_idx:
83 + slot_mapping[-1].append(_PAD_SLOT_ID)
84 + continue
85 +
86 + block_number = block_table[i // self.block_size]
87 + block_offset = i % self.block_size
88 + slot = block_number * self.block_size + block_offset
89 + slot_mapping[-1].append(slot)
90 +
91 + max_prompt_len = max(prompt_lens)
92 + input_tokens = _make_tensor_with_pad(input_tokens,
93 + max_prompt_len,
94 + pad=0,
95 + dtype=torch.long)
96 + input_positions = _make_tensor_with_pad(input_positions,
97 + max_prompt_len,
98 + pad=0,
99 + dtype=torch.long)
100 + slot_mapping = _make_tensor_with_pad(slot_mapping,
101 + max_prompt_len,
102 + pad=_PAD_SLOT_ID,
103 + dtype=torch.long)
104 +
105 + input_metadata = InputMetadata(
106 + prompt_lens=prompt_lens,
107 + slot_mapping=slot_mapping,
108 + max_context_len=None,
109 + context_lens=None,
110 + block_tables=None,
111 + )
112 + return input_tokens, input_positions, input_metadata
113 +
114 + def _prepare_decode(
115 + self,
116 + seq_group_metadata_list: List[SequenceGroupMetadata],
117 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
118 + assert len(seq_group_metadata_list) > 0
119 + input_tokens: List[List[int]] = []
120 + input_positions: List[List[int]] = []
121 + slot_mapping: List[List[int]] = []
122 + context_lens: List[int] = []
123 + block_tables: List[List[int]] = []
124 +
125 + for seq_group_metadata in seq_group_metadata_list:
126 + assert not seq_group_metadata.is_prompt
127 +
128 + seq_ids = list(seq_group_metadata.seq_data.keys())
129 + for seq_id in seq_ids:
130 + seq_data = seq_group_metadata.seq_data[seq_id]
131 + generation_token = seq_data.get_last_token_id()
132 + input_tokens.append([generation_token])
133 +
134 + context_len = seq_data.get_len()
135 + if self.sliding_window is not None:
136 + context_len = min(context_len, self.sliding_window)
137 + context_lens.append(context_len)
138 +
139 + position = context_len - 1
140 + input_positions.append([position])
141 +
142 + block_table = seq_group_metadata.block_tables[seq_id]
143 + block_number = block_table[position // self.block_size]
144 + block_offset = position % self.block_size
145 + slot = block_number * self.block_size + block_offset
146 + slot_mapping.append([slot])
147 +
148 + if self.sliding_window is not None:
149 + sliding_window_blocks = (self.sliding_window //
150 + self.block_size)
151 + block_table = block_table[-sliding_window_blocks:]
152 + block_tables.append(block_table)
153 +
154 + input_tokens = _make_tensor_with_pad(input_tokens,
155 + max_len=1,
156 + pad=0,
157 + dtype=torch.long)
158 + input_positions = _make_tensor_with_pad(input_positions,
159 + max_len=1,
160 + pad=0,
161 + dtype=torch.long)
162 + slot_mapping = _make_tensor_with_pad(slot_mapping,
163 + max_len=1,
164 + pad=_PAD_SLOT_ID,
165 + dtype=torch.long)
166 + max_context_len = max(context_lens)
167 + context_lens = torch.tensor(context_lens,
168 + dtype=torch.int,
169 + device="cuda")
170 + max_block_table_len = max([len(t) for t in block_tables])
171 + block_tables = _make_tensor_with_pad(block_tables,
172 + max_len=max_block_table_len,
173 + pad=0,
174 + dtype=torch.int)
175 +
176 + input_metadata = InputMetadata(
177 + prompt_lens=[],
178 + slot_mapping=slot_mapping,
179 + max_context_len=max_context_len,
180 + context_lens=context_lens,
181 + block_tables=block_tables,
182 + )
183 + return input_tokens, input_positions, input_metadata
184 +
185 + def _prepare_sample(
186 + self,
187 + seq_group_metadata_list: List[SequenceGroupMetadata],
188 + prompt_lens: List[int],
189 + ) -> SamplingMetadata:
190 + seq_groups: List[Tuple[List[int], SamplingParams]] = []
191 + selected_token_indices: List[int] = []
192 + selected_token_start_idx = 0
193 + categorized_sample_indices = {t: [] for t in SamplingType}
194 + categorized_sample_indices_start_idx = 0
195 +
196 + max_prompt_len = max(prompt_lens) if prompt_lens else 1
197 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
198 + seq_ids = list(seq_group_metadata.seq_data.keys())
199 + sampling_params = seq_group_metadata.sampling_params
200 + seq_groups.append((seq_ids, sampling_params))
201 +
202 + if seq_group_metadata.is_prompt:
203 + assert len(seq_ids) == 1
204 + prompt_len = prompt_lens[i]
205 + if sampling_params.prompt_logprobs is not None:
206 + # NOTE: prompt token positions do not need sample, skip
207 + categorized_sample_indices_start_idx += prompt_len - 1
208 +
209 + categorized_sample_indices[
210 + sampling_params.sampling_type].append(
211 + categorized_sample_indices_start_idx)
212 + categorized_sample_indices_start_idx += 1
213 +
214 + if sampling_params.prompt_logprobs is not None:
215 + selected_token_indices.extend(
216 + range(selected_token_start_idx,
217 + selected_token_start_idx + prompt_len - 1))
218 + selected_token_indices.append(selected_token_start_idx +
219 + prompt_len - 1)
220 + selected_token_start_idx += max_prompt_len
221 + else:
222 + num_seqs = len(seq_ids)
223 + selected_token_indices.extend(
224 + range(selected_token_start_idx,
225 + selected_token_start_idx + num_seqs))
226 + selected_token_start_idx += num_seqs
227 +
228 + categorized_sample_indices[
229 + sampling_params.sampling_type].extend(
230 + range(categorized_sample_indices_start_idx,
231 + categorized_sample_indices_start_idx + num_seqs))
232 + categorized_sample_indices_start_idx += num_seqs
233 +
234 + selected_token_indices = torch.tensor(selected_token_indices,
235 + dtype=torch.long,
236 + device="cuda")
237 + categorized_sample_indices = {
238 + t: torch.tensor(seq_ids, dtype=torch.int, device="cuda")
239 + for t, seq_ids in categorized_sample_indices.items()
240 + }
241 +
242 + seq_data: Dict[int, SequenceData] = {}
243 + for seq_group_metadata in seq_group_metadata_list:
244 + seq_data.update(seq_group_metadata.seq_data)
245 +
246 + sampling_metadata = SamplingMetadata(
247 + seq_groups=seq_groups,
248 + seq_data=seq_data,
249 + prompt_lens=prompt_lens,
250 + selected_token_indices=selected_token_indices,
251 + categorized_sample_indices=categorized_sample_indices,
252 + )
253 + return sampling_metadata
254 +
255 + @torch.inference_mode()
256 + def execute_model(
257 + self,
258 + seq_group_metadata_list: List[SequenceGroupMetadata],
259 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
260 + cache_events: Optional[List[torch.cuda.Event]] = None,
261 + ) -> SamplerOutput:
262 + # NOTE: We assume that all sequences in the group are all prompts or
263 + # all decodes.
264 + # Prepare input tensors.
265 + is_prompt = seq_group_metadata_list[0].is_prompt
266 + if is_prompt:
267 + inputs = self._prepare_prompt(seq_group_metadata_list)
268 + input_tokens, input_positions, input_metadata = inputs
269 + else:
270 + inputs = self._prepare_decode(seq_group_metadata_list)
271 + input_tokens, input_positions, input_metadata = inputs
272 + sampling_metadata = self._prepare_sample(seq_group_metadata_list,
273 + input_metadata.prompt_lens)
274 +
275 + # Execute the model.
276 + hidden_states = self.model(
277 + input_ids=input_tokens,
278 + positions=input_positions,
279 + kv_caches=kv_caches,
280 + input_metadata=input_metadata,
281 + cache_events=cache_events,
282 + )
283 +
284 + # Sample the next token.
285 + output = self.model.sample(
286 + hidden_states=hidden_states,
287 + sampling_metadata=sampling_metadata,
288 + )
289 + return output
290 +
291 + @torch.inference_mode()
292 + def profile_run(self) -> None:
293 + # Enable top-k sampling to reflect the accurate memory usage.
294 + vocab_size = self.model_config.get_vocab_size()
295 + sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
296 + max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
297 + max_num_seqs = self.scheduler_config.max_num_seqs
298 +
299 + # Profile memory usage with max_num_sequences sequences and the total
300 + # number of tokens equal to max_num_batched_tokens.
301 + seqs: List[SequenceGroupMetadata] = []
302 + for group_id in range(max_num_seqs):
303 + seq_len = (max_num_batched_tokens // max_num_seqs +
304 + (group_id < max_num_batched_tokens % max_num_seqs))
305 + seq_data = SequenceData([0] * seq_len)
306 + seq = SequenceGroupMetadata(
307 + request_id=str(group_id),
308 + is_prompt=True,
309 + seq_data={group_id: seq_data},
310 + sampling_params=sampling_params,
311 + block_tables=None,
312 + )
313 + seqs.append(seq)
314 +
315 + # Run the model with the dummy inputs.
316 + num_layers = self.model_config.get_num_layers(self.parallel_config)
317 + kv_caches = [(None, None)] * num_layers
318 + self.execute_model(seqs, kv_caches)
319 + return
320 +
321 +
322 +def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
323 + assert len(x) <= max_len
324 + return x + [pad] * (max_len - len(x))
325 +
326 +
327 +def _make_tensor_with_pad(
328 + x: List[List[int]],
329 + max_len: int,
330 + pad: int,
331 + dtype: torch.dtype,
332 +) -> torch.Tensor:
333 + padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
334 + return torch.tensor(padded_x, dtype=dtype, device="cuda")
vllm/worker/worker.py
+13 -248
@@ -1,18 +1,18 @@
1 """A GPU worker class."""
2 import os
3 -from typing import Dict, List, Tuple, Optional
3 +from typing import Dict, List, Optional, Tuple
4
5 import torch
6 import torch.distributed
7
8 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig)
10 -from vllm.model_executor import get_model, InputMetadata, set_random_seed
10 +from vllm.model_executor import set_random_seed
11 from vllm.model_executor.parallel_utils.parallel_state import (
12 initialize_model_parallel)
13 -from vllm.sampling_params import SamplingParams, SamplingType
14 -from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
13 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
14 from vllm.worker.cache_engine import CacheEngine
15 +from vllm.worker.model_runner import ModelRunner
16 from vllm.utils import get_gpu_memory
17
18
@@ -38,11 +38,11 @@ class Worker:
38 self.rank = rank
39 self.distributed_init_method = distributed_init_method
40
41 + self.model_runner = ModelRunner(model_config, parallel_config,
42 + scheduler_config)
43 # Uninitialized cache engine. Will be initialized by
44 # self.init_cache_engine().
45 self.cache_config = None
44 - self.block_size = None
45 - self.sliding_window = None
46 self.cache_engine = None
47 self.cache_events = None
48 self.gpu_cache = None
@@ -69,7 +69,7 @@ class Worker:
69 set_random_seed(self.model_config.seed)
70
71 def load_model(self):
72 - self.model = get_model(self.model_config)
72 + self.model_runner.load_model()
73
74 @torch.inference_mode()
75 def profile_num_available_blocks(
@@ -83,40 +83,9 @@ class Worker:
83 torch.cuda.empty_cache()
84 torch.cuda.reset_peak_memory_stats()
85
86 - # Profile memory usage with max_num_sequences sequences and the total
87 - # number of tokens equal to max_num_batched_tokens.
88 -
89 - # Enable top-k sampling to reflect the accurate memory usage.
90 - vocab_size = self.model.config.vocab_size
91 - sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
92 - max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
93 - max_num_seqs = self.scheduler_config.max_num_seqs
94 - seqs = []
95 - for group_id in range(max_num_seqs):
96 - seq_len = (max_num_batched_tokens // max_num_seqs +
97 - (group_id < max_num_batched_tokens % max_num_seqs))
98 - seq_data = SequenceData([0] * seq_len)
99 - seq = SequenceGroupMetadata(
100 - request_id=str(group_id),
101 - is_prompt=True,
102 - seq_data={group_id: seq_data},
103 - sampling_params=sampling_params,
104 - block_tables=None,
105 - )
106 - seqs.append(seq)
107 -
108 - input_tokens, input_positions, input_metadata = self._prepare_inputs(
109 - seqs)
110 -
111 - # Execute the model.
112 - num_layers = self.model_config.get_num_layers(self.parallel_config)
113 - self.model(
114 - input_ids=input_tokens,
115 - positions=input_positions,
116 - kv_caches=[(None, None)] * num_layers,
117 - input_metadata=input_metadata,
118 - cache_events=None,
119 - )
86 + # Execute a forward pass with dummy inputs to profile the memory usage
87 + # of the model.
88 + self.model_runner.profile_run()
89
90 # Calculate the number of blocks that can be allocated with the
91 # profiled peak memory.
@@ -140,197 +109,11 @@ class Worker:
109
110 def init_cache_engine(self, cache_config: CacheConfig) -> None:
111 self.cache_config = cache_config
143 - self.block_size = cache_config.block_size
144 - self.sliding_window = cache_config.sliding_window
145 -
112 self.cache_engine = CacheEngine(self.cache_config, self.model_config,
113 self.parallel_config)
114 self.cache_events = self.cache_engine.events
115 self.gpu_cache = self.cache_engine.gpu_cache
150 -
151 - def _prepare_inputs(
152 - self,
153 - seq_group_metadata_list: List[SequenceGroupMetadata],
154 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
155 - seq_groups: List[Tuple[List[int], SamplingParams]] = []
156 - input_tokens: List[List[int]] = []
157 - input_positions: List[List[int]] = []
158 - slot_mapping: List[List[int]] = []
159 - selected_token_indices: List[int] = []
160 - selected_token_start_idx = 0
161 - categorized_sample_indices = {t: [] for t in SamplingType}
162 - categorized_sample_indices_start_idx = 0
163 -
164 - # Add prompt tokens.
165 - prompt_lens: List[int] = []
166 - for seq_group_metadata in seq_group_metadata_list:
167 - if not seq_group_metadata.is_prompt:
168 - continue
169 -
170 - seq_ids = list(seq_group_metadata.seq_data.keys())
171 - sampling_params = seq_group_metadata.sampling_params
172 - seq_groups.append((seq_ids, sampling_params))
173 -
174 - # Use any sequence in the group.
175 - seq_id = seq_ids[0]
176 -
177 - seq_data = seq_group_metadata.seq_data[seq_id]
178 - prompt_tokens = seq_data.get_token_ids()
179 - prompt_len = len(prompt_tokens)
180 - prompt_lens.append(prompt_len)
181 -
182 - if sampling_params.prompt_logprobs is not None:
183 - # NOTE: prompt token positions do not need sample, skip
184 - categorized_sample_indices_start_idx += prompt_len - 1
185 -
186 - categorized_sample_indices[sampling_params.sampling_type].append(
187 - categorized_sample_indices_start_idx)
188 - categorized_sample_indices_start_idx += 1
189 -
190 - input_tokens.append(prompt_tokens)
191 - # NOTE(woosuk): Here we assume that the first token in the prompt
192 - # is always the first token in the sequence.
193 - input_positions.append(list(range(prompt_len)))
194 -
195 - if seq_group_metadata.block_tables is None:
196 - # During memory profiling, the block tables are not initialized
197 - # yet. In this case, we just use a dummy slot mapping.
198 - slot_mapping.append([0] * prompt_len)
199 - continue
200 -
201 - # Compute the slot mapping.
202 - slot_mapping.append([])
203 - block_table = seq_group_metadata.block_tables[seq_id]
204 - for i in range(prompt_len):
205 - block_number = block_table[i // self.block_size]
206 - block_offset = i % self.block_size
207 - slot = block_number * self.block_size + block_offset
208 - slot_mapping[-1].append(slot)
209 -
210 - # Add generation tokens.
211 - max_context_len = 0
212 - max_num_blocks_per_seq = 0
213 - context_lens: List[int] = []
214 - generation_block_tables: List[List[int]] = []
215 - max_seq_len = max(prompt_lens) if prompt_lens else 1
216 - for i, seq_group_metadata in enumerate(seq_group_metadata_list):
217 - if seq_group_metadata.is_prompt:
218 - # We need to do this in this loop as we need to know max_seq_len
219 - assert len(
220 - seq_ids) == 1, "Prompt input should have only one seq."
221 - sampling_params = seq_group_metadata.sampling_params
222 - assert len(prompt_lens) == len(seq_group_metadata_list)
223 - prompt_len = prompt_lens[i]
224 - if sampling_params.prompt_logprobs is not None:
225 - selected_token_indices.extend(
226 - range(selected_token_start_idx,
227 - selected_token_start_idx + prompt_len - 1))
228 - selected_token_indices.append(selected_token_start_idx +
229 - prompt_len - 1)
230 - selected_token_start_idx += max_seq_len
231 - continue
232 -
233 - seq_ids = list(seq_group_metadata.seq_data.keys())
234 - sampling_params = seq_group_metadata.sampling_params
235 - seq_groups.append((seq_ids, sampling_params))
236 -
237 - num_seqs = len(seq_ids)
238 - selected_token_indices.extend(
239 - range(selected_token_start_idx,
240 - selected_token_start_idx + num_seqs))
241 - selected_token_start_idx += num_seqs
242 -
243 - categorized_sample_indices[sampling_params.sampling_type].extend(
244 - range(categorized_sample_indices_start_idx,
245 - categorized_sample_indices_start_idx + num_seqs))
246 - categorized_sample_indices_start_idx += num_seqs
247 -
248 - for seq_id in seq_ids:
249 - seq_data = seq_group_metadata.seq_data[seq_id]
250 - generation_token = seq_data.get_last_token_id()
251 - input_tokens.append([generation_token])
252 -
253 - context_len = seq_data.get_len()
254 - position = context_len - 1
255 - if self.sliding_window is not None:
256 - context_len = min(context_len, self.sliding_window)
257 - input_positions.append([position])
258 -
259 - block_table = seq_group_metadata.block_tables[seq_id]
260 -
261 - max_context_len = max(max_context_len, context_len)
262 - max_num_blocks_per_seq = max(max_num_blocks_per_seq,
263 - len(block_table))
264 - context_lens.append(context_len)
265 -
266 - block_number = block_table[position // self.block_size]
267 - block_offset = position % self.block_size
268 - slot = block_number * self.block_size + block_offset
269 - slot_mapping.append([slot])
270 -
271 - if self.sliding_window is not None:
272 - sliding_window_blocks = (self.sliding_window //
273 - self.block_size)
274 - block_table = block_table[-sliding_window_blocks:]
275 - generation_block_tables.append(block_table)
276 -
277 - padded_input_tokens = [
278 - _pad_to_max(tokens, max_seq_len, pad=0) for tokens in input_tokens
279 - ]
280 - padded_input_positions = [
281 - _pad_to_max(positions, max_seq_len, pad=0)
282 - for positions in input_positions
283 - ]
284 - padded_slot_mapping = [
285 - _pad_to_max(mapping, max_seq_len, pad=-1)
286 - for mapping in slot_mapping
287 - ]
288 - padded_block_tables = [
289 - _pad_to_max(block_table, max_num_blocks_per_seq, pad=0)
290 - for block_table in generation_block_tables
291 - ]
292 -
293 - # Convert to tensors.
294 - tokens_tensor = torch.tensor(padded_input_tokens,
295 - dtype=torch.long,
296 - device="cuda")
297 - positions_tensor = torch.tensor(padded_input_positions,
298 - dtype=torch.long,
299 - device="cuda")
300 - slot_mapping_tensor = torch.tensor(padded_slot_mapping,
301 - dtype=torch.long,
302 - device="cuda")
303 - context_lens_tensor = torch.tensor(context_lens,
304 - dtype=torch.int,
305 - device="cuda")
306 - selected_token_indices = torch.tensor(selected_token_indices,
307 - dtype=torch.long,
308 - device="cuda")
309 - categorized_sample_indices = {
310 - t: torch.tensor(seq_ids, dtype=torch.int, device="cuda")
311 - for t, seq_ids in categorized_sample_indices.items()
312 - }
313 - block_tables_tensor = torch.tensor(padded_block_tables,
314 - dtype=torch.int,
315 - device="cuda")
316 -
317 - seq_data: Dict[int, SequenceData] = {}
318 - for seq_group_metadata in seq_group_metadata_list:
319 - seq_data.update(seq_group_metadata.seq_data)
320 -
321 - input_metadata = InputMetadata(
322 - seq_groups=seq_groups,
323 - seq_data=seq_data,
324 - prompt_lens=prompt_lens,
325 - slot_mapping=slot_mapping_tensor,
326 - context_lens=context_lens_tensor,
327 - max_context_len=max_context_len,
328 - block_tables=block_tables_tensor,
329 - selected_token_indices=selected_token_indices,
330 - categorized_sample_indices=categorized_sample_indices,
331 - sliding_window=self.sliding_window,
332 - )
333 - return tokens_tensor, positions_tensor, input_metadata
116 + self.model_runner.set_block_size(self.cache_engine.block_size)
117
118 @torch.inference_mode()
119 def execute_model(
@@ -361,18 +144,8 @@ class Worker:
144 event.wait()
145 return {}
146
364 - # Prepare input tensors.
365 - input_tokens, input_positions, input_metadata = self._prepare_inputs(
366 - seq_group_metadata_list)
367 -
368 - # Execute the model.
369 - output = self.model(
370 - input_ids=input_tokens,
371 - positions=input_positions,
372 - kv_caches=self.gpu_cache,
373 - input_metadata=input_metadata,
374 - cache_events=cache_events,
375 - )
147 + output = self.model_runner.execute_model(seq_group_metadata_list,
148 + self.gpu_cache, cache_events)
149 return output
150
151
@@ -407,14 +180,6 @@ def _init_distributed_environment(
180 parallel_config.pipeline_parallel_size)
181
182
410 -def _pad_to_alignment(x: List[int], multiple_of: int, pad: int) -> List[int]:
411 - return x + [pad] * ((-len(x)) % multiple_of)
412 -
413 -
414 -def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
415 - return x + [pad] * (max_len - len(x))
416 -
417 -
183 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
184 # Check if the GPU supports the dtype.
185 if torch_dtype == torch.bfloat16: