@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c5af09b39
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[V1] Implement vLLM V1 [1/N] (#9289)

Woosuk Kwon committed Oct 22, 2024 at 01:24 UTC 6c5af09b3969721da2e3a32d612a0fdd5cb077d6
27 files changed +3058 -180
vllm/attention/selector.py
+8
@@ -17,6 +17,7 @@ logger = init_logger(__name__)
17
18 class _Backend(enum.Enum):
19 FLASH_ATTN = enum.auto()
20 + FLASH_ATTN_VLLM_V1 = enum.auto()
21 XFORMERS = enum.auto()
22 ROCM_FLASH = enum.auto()
23 TORCH_SDPA = enum.auto()
@@ -110,6 +111,10 @@ def get_attn_backend(
111 from vllm.attention.backends.flash_attn import ( # noqa: F401
112 FlashAttentionBackend)
113 return FlashAttentionBackend
114 + if backend == _Backend.FLASH_ATTN_VLLM_V1:
115 + from vllm.v1.attention.backends.flash_attn import ( # noqa: F401
116 + FlashAttentionBackend as FlashAttentionBackendV1)
117 + return FlashAttentionBackendV1
118 if backend == _Backend.XFORMERS:
119 logger.info("Using XFormers backend.")
120 from vllm.attention.backends.xformers import ( # noqa: F401
@@ -215,6 +220,9 @@ def which_attn_to_use(
220 logger.info("%s is not supported in AMD GPUs.", selected_backend)
221 return _Backend.ROCM_FLASH
222
223 + if envs.VLLM_USE_V1:
224 + return _Backend.FLASH_ATTN_VLLM_V1
225 +
226 # FlashAttn in NVIDIA GPUs.
227 if selected_backend == _Backend.FLASH_ATTN:
228 if not current_platform.has_device_capability(80):
vllm/engine/multiprocessing/engine.py
+17 -10
@@ -8,7 +8,7 @@ from typing import Iterator, List, Optional, Union
8 import cloudpickle
9 import zmq
10
11 -from vllm import AsyncEngineArgs, LLMEngine, SamplingParams
11 +from vllm import AsyncEngineArgs, SamplingParams
12 from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
13 ParallelConfig, SchedulerConfig)
14 # yapf conflicts with isort for this block
@@ -21,12 +21,17 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
21 RPCStartupRequest, RPCStartupResponse,
22 RPCUProfileRequest)
23 # yapf: enable
24 -from vllm.envs import VLLM_RPC_TIMEOUT
24 +from vllm.envs import VLLM_RPC_TIMEOUT, VLLM_USE_V1
25 from vllm.executor.gpu_executor import GPUExecutor
26 from vllm.logger import init_logger
27 from vllm.outputs import RequestOutput
28 from vllm.usage.usage_lib import UsageContext
29
30 +if VLLM_USE_V1:
31 + from vllm.v1.engine.llm_engine import LLMEngine
32 +else:
33 + from vllm.engine.llm_engine import LLMEngine
34 +
35 CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
36 SchedulerConfig, LoRAConfig]
37
@@ -136,14 +141,16 @@ class MQLLMEngine:
141
142 executor_class = LLMEngine._get_executor_cls(engine_config)
143
139 - return cls(
140 - ipc_path=ipc_path,
141 - use_async_sockets=engine_config.model_config.use_async_output_proc,
142 - **engine_config.to_dict(),
143 - executor_class=executor_class,
144 - log_requests=not engine_args.disable_log_requests,
145 - log_stats=not engine_args.disable_log_stats,
146 - usage_context=usage_context)
144 + use_async_sockets = (engine_config.model_config.use_async_output_proc
145 + and not VLLM_USE_V1)
146 +
147 + return cls(ipc_path=ipc_path,
148 + use_async_sockets=use_async_sockets,
149 + **engine_config.to_dict(),
150 + executor_class=executor_class,
151 + log_requests=not engine_args.disable_log_requests,
152 + log_stats=not engine_args.disable_log_stats,
153 + usage_context=usage_context)
154
155 def start(self):
156 try:
vllm/entrypoints/llm.py
+6 -1
@@ -6,10 +6,10 @@ from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
6
7 from tqdm import tqdm
8
9 +from vllm import envs
10 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
11 BeamSearchSequence, get_beam_search_score)
12 from vllm.engine.arg_utils import EngineArgs, TaskOption
12 -from vllm.engine.llm_engine import LLMEngine
13 from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
14 apply_hf_chat_template,
15 apply_mistral_chat_template,
@@ -31,6 +31,11 @@ from vllm.transformers_utils.tokenizer_group import TokenizerGroup
31 from vllm.usage.usage_lib import UsageContext
32 from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
33
34 +if envs.VLLM_USE_V1:
35 + from vllm.v1.engine.llm_engine import LLMEngine # type: ignore
36 +else:
37 + from vllm.engine.llm_engine import LLMEngine # type: ignore
38 +
39 logger = init_logger(__name__)
40
41
vllm/envs.py
+5
@@ -68,6 +68,7 @@ if TYPE_CHECKING:
68 VLLM_TORCH_COMPILE_LEVEL: int = 0
69 VLLM_CUSTOM_OPS: List[str] = []
70 VLLM_DISABLED_KERNELS: List[str] = []
71 + VLLM_USE_V1: bool = False
72
73
74 def get_default_cache_root():
@@ -450,6 +451,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
451 "VLLM_DISABLED_KERNELS":
452 lambda: [] if "VLLM_DISABLED_KERNELS" not in os.environ else os.environ[
453 "VLLM_DISABLED_KERNELS"].split(","),
454 +
455 + # If set, use the V1 code path.
456 + "VLLM_USE_V1":
457 + lambda: bool(int(os.getenv("VLLM_USE_V1", "0"))),
458 }
459
460 # end-env-vars-definition
vllm/model_executor/layers/logits_processor.py
+6 -4
@@ -48,14 +48,15 @@ class LogitsProcessor(nn.Module):
48 self,
49 lm_head: VocabParallelEmbedding,
50 hidden_states: torch.Tensor,
51 - sampling_metadata: SamplingMetadata,
51 + sampling_metadata: Optional[SamplingMetadata] = None,
52 embedding_bias: Optional[torch.Tensor] = None,
53 ) -> Optional[torch.Tensor]:
54 if self.logits_as_input:
55 logits = hidden_states
56 else:
57 - hidden_states = _prune_hidden_states(hidden_states,
58 - sampling_metadata)
57 + if sampling_metadata is not None:
58 + hidden_states = _prune_hidden_states(hidden_states,
59 + sampling_metadata)
60
61 # Get the logits for the next tokens.
62 logits = self._get_logits(hidden_states, lm_head, embedding_bias)
@@ -69,7 +70,8 @@ class LogitsProcessor(nn.Module):
70 logits *= self.scale
71
72 # Apply logits processors (if any).
72 - logits = _apply_logits_processors(logits, sampling_metadata)
73 + if sampling_metadata is not None:
74 + logits = _apply_logits_processors(logits, sampling_metadata)
75
76 return logits
77
vllm/transformers_utils/detokenizer.py
+3 -165
@@ -1,8 +1,10 @@
1 -from typing import Dict, List, Optional, Tuple
1 +from typing import Dict, List, Optional
2
3 from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Logprob, SamplingParams,
4 Sequence, SequenceGroup)
5
6 +from .detokenizer_utils import (convert_prompt_ids_to_tokens,
7 + detokenize_incrementally)
8 from .tokenizer import AnyTokenizer
9 from .tokenizer_group import BaseTokenizerGroup
10
@@ -161,167 +163,3 @@ class Detokenizer:
163 seq.output_text += new_decoded_token_text
164
165 return len(new_decoded_token_text)
164 -
165 -
166 -def _replace_none_with_empty(tokens: List[Optional[str]]):
167 - for i, token in enumerate(tokens):
168 - if token is None:
169 - tokens[i] = ""
170 -
171 -
172 -def _convert_tokens_to_string_with_added_encoders(
173 - tokenizer: AnyTokenizer,
174 - output_tokens: List[str],
175 - skip_special_tokens: bool,
176 - spaces_between_special_tokens: bool,
177 -) -> str:
178 - # Adapted from
179 - # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
180 - # NOTE(woosuk): The following code is slow because it runs a for loop over
181 - # the output_tokens. In Python, running a for loop over a list can be slow
182 - # even when the loop body is very simple.
183 - sub_texts: List[str] = []
184 - current_sub_text: List[str] = []
185 - all_special_tokens = set(tokenizer.all_special_tokens)
186 - for token in output_tokens:
187 - if skip_special_tokens and token in all_special_tokens:
188 - continue
189 - if token in tokenizer.get_added_vocab():
190 - if current_sub_text:
191 - sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
192 - sub_texts.append(sub_text)
193 - current_sub_text = []
194 - sub_texts.append(token)
195 - else:
196 - current_sub_text.append(token)
197 - if current_sub_text:
198 - sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
199 - sub_texts.append(sub_text)
200 - if spaces_between_special_tokens:
201 - return " ".join(sub_texts)
202 - else:
203 - return "".join(sub_texts)
204 -
205 -
206 -# 5 is an arbitrary value that should work for all
207 -# tokenizers (bigger = more conservative).
208 -INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
209 -
210 -
211 -def convert_prompt_ids_to_tokens(
212 - tokenizer: AnyTokenizer,
213 - prompt_ids: List[int],
214 - skip_special_tokens: bool = False,
215 -) -> Tuple[List[str], int, int]:
216 - """Converts the prompt ids to tokens and returns the tokens and offsets
217 - for incremental detokenization.
218 -
219 - Note that not all tokens are converted to strings. Only the tokens that
220 - are necessary for incremental detokenization are converted to strings.
221 - """
222 - # We do not need to convert the whole prompt to tokens.
223 - # Offset a little more in case we have special tokens.
224 - new_tokens = tokenizer.convert_ids_to_tokens(
225 - prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2:],
226 - skip_special_tokens=skip_special_tokens)
227 - read_offset = len(new_tokens)
228 - prefix_offset = max(
229 - read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
230 - # This is required to guard against out-of-vocab prompt token ids
231 - _replace_none_with_empty(new_tokens) # type: ignore[arg-type]
232 - return new_tokens, prefix_offset, read_offset
233 -
234 -
235 -# Based on
236 -# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
237 -# under Apache 2.0 license
238 -def detokenize_incrementally(
239 - tokenizer: AnyTokenizer,
240 - all_input_ids: List[int],
241 - prev_tokens: Optional[List[str]],
242 - prefix_offset: int,
243 - read_offset: int,
244 - skip_special_tokens: bool = False,
245 - spaces_between_special_tokens: bool = True,
246 -) -> Tuple[List[str], str, int, int]:
247 - """Detokenizes the input ids incrementally and returns the new tokens
248 - and the new text.
249 -
250 - If `prev_tokens` is None, this function will convert the input ids to
251 - tokens and return the tokens and the new text. Otherwise, it will return the
252 - new tokens and the new text.
253 -
254 - This function will also return the new prefix offset and the new read
255 - offset to be used in the next iteration.
256 -
257 - The offsets are necessary to defeat cleanup algorithms in the decode which
258 - decide to add a space or not depending on the surrounding ids.
259 -
260 - Args:
261 - tokenizer: The tokenizer to use.
262 - all_input_ids: The input ids. The last id is the new token id.
263 - prev_tokens: The previous tokens. If None, this function will convert
264 - the input ids to tokens and return the tokens and the new text.
265 - prefix_offset: The prefix offset.
266 - read_offset: The read offset.
267 - skip_special_tokens: Whether to skip special tokens.
268 - spaces_between_special_tokens: Whether to add spaces between special
269 - tokens.
270 - """
271 - new_token_id = all_input_ids[-1]
272 - # This is the first iteration for this sequence
273 - is_first_iter = prev_tokens is None
274 - if is_first_iter:
275 - (prev_tokens, prefix_offset,
276 - read_offset) = convert_prompt_ids_to_tokens(
277 - tokenizer,
278 - all_input_ids[:-1],
279 - skip_special_tokens=skip_special_tokens)
280 - assert prev_tokens is not None
281 -
282 - # If the new token id is out of bounds, return an empty string.
283 - if 0 <= new_token_id < len(tokenizer):
284 - # Put new_token_id in a list so skip_special_tokens is respected
285 - new_tokens = tokenizer.convert_ids_to_tokens(
286 - [new_token_id], skip_special_tokens=skip_special_tokens)
287 - if isinstance(new_tokens, str):
288 - new_tokens = [new_tokens]
289 - else:
290 - new_tokens = [""]
291 - output_tokens = prev_tokens + new_tokens
292 -
293 - # If this is the first iteration, return all tokens.
294 - if is_first_iter:
295 - new_tokens = output_tokens
296 -
297 - # The prefix text is necessary only to defeat cleanup algorithms in
298 - # the decode which decide to add a space or not depending on the
299 - # surrounding ids.
300 - if tokenizer.is_fast or not tokenizer.get_added_vocab():
301 - prefix_text = tokenizer.convert_tokens_to_string(
302 - output_tokens[prefix_offset:read_offset])
303 - new_text = tokenizer.convert_tokens_to_string(
304 - output_tokens[prefix_offset:])
305 - else:
306 - prefix_text = _convert_tokens_to_string_with_added_encoders(
307 - tokenizer,
308 - output_tokens[prefix_offset:read_offset],
309 - skip_special_tokens=skip_special_tokens,
310 - spaces_between_special_tokens=spaces_between_special_tokens,
311 - )
312 - new_text = _convert_tokens_to_string_with_added_encoders(
313 - tokenizer,
314 - output_tokens[prefix_offset:],
315 - skip_special_tokens=skip_special_tokens,
316 - spaces_between_special_tokens=spaces_between_special_tokens,
317 - )
318 -
319 - if len(new_text) <= len(prefix_text) or new_text.endswith("�"):
320 - # utf-8 char at the end means it's a potential unfinished byte sequence
321 - # from byte fallback tokenization.
322 - # If it's in the middle, it's probably a real invalid id generated
323 - # by the model
324 - return new_tokens, "", prefix_offset, read_offset
325 -
326 - new_text = new_text[len(prefix_text):]
327 - return new_tokens, new_text, read_offset, len(output_tokens)
vllm/transformers_utils/detokenizer_utils.py new
+167
@@ -0,0 +1,167 @@
1 +from typing import List, Optional, Tuple
2 +
3 +from .tokenizer import AnyTokenizer
4 +
5 +
6 +def _replace_none_with_empty(tokens: List[Optional[str]]):
7 + for i, token in enumerate(tokens):
8 + if token is None:
9 + tokens[i] = ""
10 +
11 +
12 +def _convert_tokens_to_string_with_added_encoders(
13 + tokenizer: AnyTokenizer,
14 + output_tokens: List[str],
15 + skip_special_tokens: bool,
16 + spaces_between_special_tokens: bool,
17 +) -> str:
18 + # Adapted from
19 + # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
20 + # NOTE(woosuk): The following code is slow because it runs a for loop over
21 + # the output_tokens. In Python, running a for loop over a list can be slow
22 + # even when the loop body is very simple.
23 + sub_texts: List[str] = []
24 + current_sub_text: List[str] = []
25 + all_special_tokens = set(tokenizer.all_special_tokens)
26 + for token in output_tokens:
27 + if skip_special_tokens and token in all_special_tokens:
28 + continue
29 + if token in tokenizer.get_added_vocab():
30 + if current_sub_text:
31 + sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
32 + sub_texts.append(sub_text)
33 + current_sub_text = []
34 + sub_texts.append(token)
35 + else:
36 + current_sub_text.append(token)
37 + if current_sub_text:
38 + sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
39 + sub_texts.append(sub_text)
40 + if spaces_between_special_tokens:
41 + return " ".join(sub_texts)
42 + else:
43 + return "".join(sub_texts)
44 +
45 +
46 +# 5 is an arbitrary value that should work for all
47 +# tokenizers (bigger = more conservative).
48 +INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
49 +
50 +
51 +def convert_prompt_ids_to_tokens(
52 + tokenizer: AnyTokenizer,
53 + prompt_ids: List[int],
54 + skip_special_tokens: bool = False,
55 +) -> Tuple[List[str], int, int]:
56 + """Converts the prompt ids to tokens and returns the tokens and offsets
57 + for incremental detokenization.
58 +
59 + Note that not all tokens are converted to strings. Only the tokens that
60 + are necessary for incremental detokenization are converted to strings.
61 + """
62 + # We do not need to convert the whole prompt to tokens.
63 + # Offset a little more in case we have special tokens.
64 + new_tokens = tokenizer.convert_ids_to_tokens(
65 + prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2:],
66 + skip_special_tokens=skip_special_tokens)
67 + read_offset = len(new_tokens)
68 + prefix_offset = max(
69 + read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
70 + # This is required to guard against out-of-vocab prompt token ids
71 + _replace_none_with_empty(new_tokens) # type: ignore[arg-type]
72 + return new_tokens, prefix_offset, read_offset
73 +
74 +
75 +# Based on
76 +# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
77 +# under Apache 2.0 license
78 +def detokenize_incrementally(
79 + tokenizer: AnyTokenizer,
80 + all_input_ids: List[int],
81 + prev_tokens: Optional[List[str]],
82 + prefix_offset: int,
83 + read_offset: int,
84 + skip_special_tokens: bool = False,
85 + spaces_between_special_tokens: bool = True,
86 +) -> Tuple[List[str], str, int, int]:
87 + """Detokenizes the input ids incrementally and returns the new tokens
88 + and the new text.
89 +
90 + If `prev_tokens` is None, this function will convert the input ids to
91 + tokens and return the tokens and the new text. Otherwise, it will return the
92 + new tokens and the new text.
93 +
94 + This function will also return the new prefix offset and the new read
95 + offset to be used in the next iteration.
96 +
97 + The offsets are necessary to defeat cleanup algorithms in the decode which
98 + decide to add a space or not depending on the surrounding ids.
99 +
100 + Args:
101 + tokenizer: The tokenizer to use.
102 + all_input_ids: The input ids. The last id is the new token id.
103 + prev_tokens: The previous tokens. If None, this function will convert
104 + the input ids to tokens and return the tokens and the new text.
105 + prefix_offset: The prefix offset.
106 + read_offset: The read offset.
107 + skip_special_tokens: Whether to skip special tokens.
108 + spaces_between_special_tokens: Whether to add spaces between special
109 + tokens.
110 + """
111 + new_token_id = all_input_ids[-1]
112 + # This is the first iteration for this sequence
113 + is_first_iter = prev_tokens is None
114 + if is_first_iter:
115 + (prev_tokens, prefix_offset,
116 + read_offset) = convert_prompt_ids_to_tokens(
117 + tokenizer,
118 + all_input_ids[:-1],
119 + skip_special_tokens=skip_special_tokens)
120 + assert prev_tokens is not None
121 +
122 + # If the new token id is out of bounds, return an empty string.
123 + if 0 <= new_token_id < len(tokenizer):
124 + # Put new_token_id in a list so skip_special_tokens is respected
125 + new_tokens = tokenizer.convert_ids_to_tokens(
126 + [new_token_id], skip_special_tokens=skip_special_tokens)
127 + if isinstance(new_tokens, str):
128 + new_tokens = [new_tokens]
129 + else:
130 + new_tokens = [""]
131 + output_tokens = prev_tokens + new_tokens
132 +
133 + # If this is the first iteration, return all tokens.
134 + if is_first_iter:
135 + new_tokens = output_tokens
136 +
137 + # The prefix text is necessary only to defeat cleanup algorithms in
138 + # the decode which decide to add a space or not depending on the
139 + # surrounding ids.
140 + if tokenizer.is_fast or not tokenizer.get_added_vocab():
141 + prefix_text = tokenizer.convert_tokens_to_string(
142 + output_tokens[prefix_offset:read_offset])
143 + new_text = tokenizer.convert_tokens_to_string(
144 + output_tokens[prefix_offset:])
145 + else:
146 + prefix_text = _convert_tokens_to_string_with_added_encoders(
147 + tokenizer,
148 + output_tokens[prefix_offset:read_offset],
149 + skip_special_tokens=skip_special_tokens,
150 + spaces_between_special_tokens=spaces_between_special_tokens,
151 + )
152 + new_text = _convert_tokens_to_string_with_added_encoders(
153 + tokenizer,
154 + output_tokens[prefix_offset:],
155 + skip_special_tokens=skip_special_tokens,
156 + spaces_between_special_tokens=spaces_between_special_tokens,
157 + )
158 +
159 + if len(new_text) <= len(prefix_text) or new_text.endswith("�"):
160 + # utf-8 char at the end means it's a potential unfinished byte sequence
161 + # from byte fallback tokenization.
162 + # If it's in the middle, it's probably a real invalid id generated
163 + # by the model
164 + return new_tokens, "", prefix_offset, read_offset
165 +
166 + new_text = new_text[len(prefix_text):]
167 + return new_tokens, new_text, read_offset, len(output_tokens)
vllm/v1/attention/__init__.py
vllm/v1/attention/backends/__init__.py
vllm/v1/attention/backends/flash_attn.py new
+241
@@ -0,0 +1,241 @@
1 +"""Attention layer with FlashAttention."""
2 +from dataclasses import dataclass
3 +from typing import Any, Dict, List, Optional, Tuple, Type
4 +
5 +import torch
6 +
7 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8 + AttentionMetadata, AttentionType)
9 +from vllm.forward_context import get_forward_context
10 +from vllm.vllm_flash_attn import flash_attn_varlen_func
11 +
12 +
13 +class FlashAttentionBackend(AttentionBackend):
14 +
15 + @staticmethod
16 + def get_supported_head_sizes() -> List[int]:
17 + return [32, 64, 96, 128, 160, 192, 224, 256]
18 +
19 + @staticmethod
20 + def get_name() -> str:
21 + return "flash-attn-vllm-v1"
22 +
23 + @staticmethod
24 + def get_impl_cls() -> Type["FlashAttentionImpl"]:
25 + return FlashAttentionImpl
26 +
27 + @staticmethod
28 + def get_metadata_cls() -> Type["AttentionMetadata"]:
29 + return FlashAttentionMetadata
30 +
31 + @staticmethod
32 + def get_kv_cache_shape(
33 + num_blocks: int,
34 + block_size: int,
35 + num_kv_heads: int,
36 + head_size: int,
37 + ) -> Tuple[int, ...]:
38 + if block_size % 16 != 0:
39 + raise ValueError("Block size must be a multiple of 16.")
40 + return (2, num_blocks, block_size, num_kv_heads, head_size)
41 +
42 +
43 +@dataclass
44 +class FlashAttentionMetadata:
45 + # NOTE(sang): Definition of context_len, query_len, and seq_len.
46 + # |---------- N-1 iteration --------|
47 + # |---------------- N iteration ---------------------|
48 + # |- tokenA -|......................|-- newTokens ---|
49 + # |---------- context_len ----------|
50 + # |-------------------- seq_len ---------------------|
51 + # |-- query_len ---|
52 +
53 + max_query_len: int
54 + query_start_loc: torch.Tensor
55 + max_seq_len: int
56 + seq_start_loc: torch.Tensor
57 + block_table: torch.Tensor
58 + slot_mapping: torch.Tensor
59 +
60 +
61 +class FlashAttentionImpl(AttentionImpl):
62 +
63 + def __init__(
64 + self,
65 + num_heads: int,
66 + head_size: int,
67 + scale: float,
68 + num_kv_heads: int,
69 + alibi_slopes: Optional[List[float]],
70 + sliding_window: Optional[int],
71 + kv_cache_dtype: str,
72 + blocksparse_params: Optional[Dict[str, Any]] = None,
73 + logits_soft_cap: Optional[float] = None,
74 + ) -> None:
75 + if blocksparse_params is not None:
76 + raise ValueError(
77 + "FlashAttention does not support block-sparse attention.")
78 + self.num_heads = num_heads
79 + self.head_size = head_size
80 + self.scale = float(scale)
81 + self.num_kv_heads = num_kv_heads
82 + if alibi_slopes is not None:
83 + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
84 + self.alibi_slopes = alibi_slopes
85 + self.sliding_window = ((sliding_window, sliding_window)
86 + if sliding_window is not None else (-1, -1))
87 + self.kv_cache_dtype = kv_cache_dtype
88 + if logits_soft_cap is None:
89 + # In flash-attn, setting logits_soft_cap as 0 means no soft cap.
90 + logits_soft_cap = 0
91 + self.logits_soft_cap = logits_soft_cap
92 +
93 + assert self.num_heads % self.num_kv_heads == 0
94 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
95 +
96 + if sliding_window is not None:
97 + # NOTE(woosuk): flash-attn's sliding window does not work with
98 + # paged KV cache.
99 + raise ValueError(
100 + "Sliding window is not supported in FlashAttention.")
101 +
102 + support_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
103 + if head_size not in support_head_sizes:
104 + raise ValueError(
105 + f"Head size {head_size} is not supported by FlashAttention. "
106 + f"Supported head sizes are: {support_head_sizes}.")
107 +
108 + def forward(
109 + self,
110 + query: torch.Tensor,
111 + key: torch.Tensor,
112 + value: torch.Tensor,
113 + kv_cache: torch.Tensor,
114 + attn_metadata: FlashAttentionMetadata,
115 + k_scale: float = 1.0,
116 + v_scale: float = 1.0,
117 + attn_type: AttentionType = AttentionType.DECODER,
118 + ) -> torch.Tensor:
119 + """Forward pass with FlashAttention.
120 +
121 + Args:
122 + query: shape = [num_tokens, num_heads * head_size]
123 + key: shape = [num_tokens, num_kv_heads * head_size]
124 + value: shape = [num_tokens, num_kv_heads * head_size]
125 + kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
126 + attn_metadata: Metadata for attention.
127 + Returns:
128 + shape = [num_tokens, num_heads * head_size]
129 + """
130 + if attn_type != AttentionType.DECODER:
131 + raise NotImplementedError("Encoder self-attention and "
132 + "encoder/decoder cross-attention "
133 + "are not implemented for "
134 + "FlashAttentionImpl")
135 +
136 + # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
137 + assert k_scale == 1.0 and v_scale == 1.0, (
138 + "key/v_scale is not supported in FlashAttention.")
139 +
140 + output = torch.ops.vllm.unified_flash_attention(
141 + query,
142 + key,
143 + value,
144 + self.num_heads,
145 + self.head_size,
146 + self.num_kv_heads,
147 + kv_cache,
148 + self.kv_cache_dtype,
149 + k_scale,
150 + v_scale,
151 + self.scale,
152 + self.sliding_window,
153 + self.alibi_slopes,
154 + self.logits_soft_cap,
155 + )
156 + return output
157 +
158 +
159 +@torch.library.custom_op("vllm::unified_flash_attention",
160 + mutates_args=["kv_cache"])
161 +def unified_flash_attention(
162 + query: torch.Tensor,
163 + key: torch.Tensor,
164 + value: torch.Tensor,
165 + num_heads: int,
166 + head_size: int,
167 + num_kv_heads: int,
168 + kv_cache: torch.Tensor,
169 + kv_cache_dtype: str,
170 + k_scale: float,
171 + v_scale: float,
172 + softmax_scale: float,
173 + window_size: Optional[List[int]] = None,
174 + alibi_slopes: Optional[torch.Tensor] = None,
175 + logits_soft_cap: Optional[float] = None,
176 +) -> torch.Tensor:
177 + current_metadata = get_forward_context()
178 + if current_metadata is None:
179 + # Profiling run.
180 + return torch.empty_like(query)
181 +
182 + assert current_metadata is not None
183 + assert isinstance(current_metadata, FlashAttentionMetadata)
184 + attn_metadata: FlashAttentionMetadata = current_metadata
185 +
186 + num_tokens, hidden_size = query.shape
187 + # Reshape the query, key, and value tensors.
188 + query = query.view(-1, num_heads, head_size)
189 + key = key.view(-1, num_kv_heads, head_size)
190 + value = value.view(-1, num_kv_heads, head_size)
191 +
192 + # Reshape the input keys and values and store them in the cache.
193 + key_cache = kv_cache[0]
194 + value_cache = kv_cache[1]
195 + torch.ops._C_cache_ops.reshape_and_cache_flash(
196 + key,
197 + value,
198 + kv_cache[0],
199 + kv_cache[1],
200 + attn_metadata.slot_mapping,
201 + kv_cache_dtype,
202 + k_scale,
203 + v_scale,
204 + )
205 +
206 + output = flash_attn_varlen_func(
207 + q=query,
208 + k=key_cache,
209 + v=value_cache,
210 + cu_seqlens_q=attn_metadata.query_start_loc,
211 + max_seqlen_q=attn_metadata.max_query_len,
212 + cu_seqlens_k=attn_metadata.seq_start_loc,
213 + max_seqlen_k=attn_metadata.max_seq_len,
214 + softmax_scale=softmax_scale,
215 + causal=True,
216 + alibi_slopes=alibi_slopes,
217 + window_size=window_size,
218 + block_table=attn_metadata.block_table,
219 + softcap=logits_soft_cap,
220 + )
221 + return output.view(num_tokens, hidden_size)
222 +
223 +
224 +@unified_flash_attention.register_fake
225 +def _(
226 + query: torch.Tensor,
227 + key: torch.Tensor,
228 + value: torch.Tensor,
229 + num_heads: int,
230 + head_size: int,
231 + num_kv_heads: int,
232 + kv_cache: torch.Tensor,
233 + kv_cache_dtype: str,
234 + k_scale: float,
235 + v_scale: float,
236 + softmax_scale: float,
237 + window_size: Optional[List[int]] = None,
238 + alibi_slopes: Optional[torch.Tensor] = None,
239 + logits_soft_cap: Optional[float] = None,
240 +) -> torch.Tensor:
241 + return torch.empty_like(query)
vllm/v1/core/__init__.py
vllm/v1/core/kv_cache_manager.py new
+108
@@ -0,0 +1,108 @@
1 +from typing import Dict, List, Optional
2 +
3 +import numpy as np
4 +
5 +from vllm.logger import init_logger
6 +from vllm.utils import cdiv
7 +from vllm.v1.request import Request
8 +
9 +logger = init_logger(__name__)
10 +
11 +
12 +class KVCacheManager:
13 +
14 + def __init__(
15 + self,
16 + block_size: int,
17 + num_gpu_blocks: int,
18 + sliding_window: Optional[int] = None,
19 + enable_caching: bool = True,
20 + num_preallocate_tokens: int = 64,
21 + ) -> None:
22 + self.block_size = block_size
23 + self.num_gpu_blocks = num_gpu_blocks
24 + self.sliding_window = sliding_window
25 + self.enable_caching = enable_caching
26 + # NOTE(woosuk): To avoid frequent block allocation, we preallocate some
27 + # blocks for each request. For example, when a request reaches the end
28 + # of its block table, we preallocate N blocks in advance. This way, we
29 + # reduce the overhead of updating free_block_ids and ref_cnts for each
30 + # request every step (at the cost of some memory waste).
31 + # NOTE(woosuk): This is different from the "lookahead" slots since this
32 + # does not guarantee that the request always has N empty blocks. After
33 + # the request gets N empty blocks, it starts to use the blocks without
34 + # further allocation. When it uses up all the N empty blocks, it gets
35 + # N new empty blocks.
36 + self.num_preallocate_tokens = num_preallocate_tokens
37 + self.num_preallocate_blocks = cdiv(num_preallocate_tokens, block_size)
38 +
39 + self.free_block_ids = list(range(num_gpu_blocks))
40 + self.req_to_block_ids: Dict[str, List[int]] = {}
41 + self.ref_cnts = np.zeros(num_gpu_blocks, dtype=np.int32)
42 +
43 + def get_computed_blocks(self, request: Request) -> List[int]:
44 + if not self.enable_caching:
45 + # No prefix caching.
46 + return []
47 + # TODO(woosuk): Implement hash-based caching.
48 + return []
49 +
50 + def append_slots(
51 + self,
52 + request: Request,
53 + num_tokens: int,
54 + ) -> Optional[List[int]]:
55 + num_required_blocks = cdiv(request.num_computed_tokens + num_tokens,
56 + self.block_size)
57 + req_block_ids = self.req_to_block_ids[request.request_id]
58 + if num_required_blocks <= len(req_block_ids):
59 + # No new block is needed.
60 + return []
61 +
62 + num_new_blocks = num_required_blocks - len(req_block_ids)
63 + num_free_blocks = len(self.free_block_ids)
64 + if num_new_blocks > num_free_blocks:
65 + # Cannot allocate new blocks.
66 + return None
67 +
68 + # Allocate new blocks.
69 + num_new_blocks = min(num_new_blocks + self.num_preallocate_blocks,
70 + num_free_blocks)
71 + new_block_ids = self._get_new_blocks(num_new_blocks)
72 + req_block_ids.extend(new_block_ids)
73 + self.ref_cnts[new_block_ids] += 1
74 + return new_block_ids
75 +
76 + def allocate_slots(
77 + self,
78 + request: Request,
79 + num_tokens: int,
80 + computed_block_ids: List[int],
81 + ) -> Optional[List[int]]:
82 + num_required_blocks = cdiv(num_tokens, self.block_size)
83 + num_free_blocks = len(self.free_block_ids)
84 + if num_required_blocks > num_free_blocks:
85 + # Cannot allocate new blocks.
86 + return None
87 +
88 + num_new_blocks = min(num_required_blocks + self.num_preallocate_blocks,
89 + num_free_blocks)
90 + new_block_ids = self._get_new_blocks(num_new_blocks)
91 + block_ids = computed_block_ids + new_block_ids
92 + self.req_to_block_ids[request.request_id] = block_ids
93 + self.ref_cnts[block_ids] += 1
94 + return new_block_ids
95 +
96 + def free(self, request: Request) -> None:
97 + block_ids = self.req_to_block_ids.pop(request.request_id)
98 + self.ref_cnts[block_ids] -= 1
99 + for block_id in block_ids:
100 + ref_cnt = self.ref_cnts[block_id]
101 + if ref_cnt == 0:
102 + self.free_block_ids.append(block_id)
103 +
104 + def _get_new_blocks(self, num_blocks: int) -> List[int]:
105 + assert num_blocks <= len(self.free_block_ids)
106 + new_block_ids = self.free_block_ids[-num_blocks:]
107 + self.free_block_ids = self.free_block_ids[:-num_blocks]
108 + return new_block_ids
vllm/v1/core/scheduler.py new
+412
@@ -0,0 +1,412 @@
1 +from collections import deque
2 +from dataclasses import dataclass
3 +from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
4 +
5 +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
6 +from vllm.logger import init_logger
7 +from vllm.multimodal import MultiModalDataDict
8 +from vllm.sampling_params import SamplingParams
9 +from vllm.v1.core.kv_cache_manager import KVCacheManager
10 +from vllm.v1.outputs import ModelRunnerOutput
11 +from vllm.v1.request import Request, RequestStatus
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +class Scheduler:
17 +
18 + def __init__(
19 + self,
20 + scheduler_config: SchedulerConfig,
21 + cache_config: CacheConfig,
22 + lora_config: Optional[LoRAConfig],
23 + ) -> None:
24 + self.scheduler_config = scheduler_config
25 + self.cache_config = cache_config
26 + self.lora_config = lora_config
27 + # TODO: Support LoRA.
28 + assert lora_config is None, "V1 does not support LoRA yet."
29 +
30 + num_gpu_blocks = cache_config.num_gpu_blocks
31 + assert isinstance(num_gpu_blocks, int) and num_gpu_blocks > 0
32 + # Create the block space manager.
33 + self.kv_cache_manager = KVCacheManager(
34 + block_size=self.cache_config.block_size,
35 + num_gpu_blocks=num_gpu_blocks,
36 + sliding_window=self.cache_config.sliding_window,
37 + enable_caching=True)
38 + self.block_size = self.cache_config.block_size
39 +
40 + # Scheduling constraints.
41 + self.max_num_running_reqs = self.scheduler_config.max_num_seqs
42 + self.max_num_scheduled_tokens = \
43 + self.scheduler_config.max_num_batched_tokens
44 + self.max_model_len = self.scheduler_config.max_model_len
45 +
46 + # req_id -> Request
47 + self.requests: Dict[str, Request] = {}
48 + # Priority queues for requests.
49 + self.waiting: Deque[Request] = deque()
50 + self.running: List[Request] = []
51 +
52 + # The request IDs that are finished in between the previous and the
53 + # current steps. This is used to notify the workers about the finished
54 + # requests so that they can free the cached states for those requests.
55 + # This is flushed at the end of each scheduling step.
56 + self.finished_req_ids: Set[str] = set()
57 +
58 + # OPTIMIZATION: Cache the RunningRequestData objects to avoid creating
59 + # them at each scheduling step.
60 + # Request id -> RunningRequestData
61 + self.running_reqs_data: Dict[str, RunningRequestData] = {}
62 +
63 + def schedule(self) -> "SchedulerOutput":
64 + scheduled_new_reqs: List[Request] = []
65 + scheduled_resumed_reqs: List[Request] = []
66 + scheduled_running_reqs: List[Request] = []
67 + preempted_reqs: List[Request] = []
68 +
69 + # NOTE(woosuk) on the scheduling algorithm:
70 + # There's no "decoding phase" nor "prefill phase" in the scheduler.
71 + # Each request just has the num_computed_tokens and num_tokens,
72 + # which is equal to len(prompt_token_ids) + len(output_token_ids).
73 + # At each step, the scheduler tries to assign tokens to the requests
74 + # so that each request's num_computed_tokens can catch up its
75 + # num_tokens. This is general enough to cover chunked prefills,
76 + # prefix caching, and the "jump forward" optimization in the future.
77 +
78 + req_to_new_block_ids: Dict[str, List[int]] = {}
79 + num_scheduled_tokens: Dict[str, int] = {}
80 + token_budget = self.max_num_scheduled_tokens
81 +
82 + # First, schedule the RUNNING requests.
83 + req_index = 0
84 + while req_index < len(self.running):
85 + if token_budget == 0:
86 + break
87 +
88 + request = self.running[req_index]
89 + num_new_tokens = request.num_tokens - request.num_computed_tokens
90 + num_new_tokens = min(num_new_tokens, token_budget)
91 + assert num_new_tokens > 0
92 +
93 + while True:
94 + new_block_ids = self.kv_cache_manager.append_slots(
95 + request, num_new_tokens)
96 + if new_block_ids is None:
97 + # The request cannot be scheduled.
98 + # Preempt the lowest-priority request.
99 + preempted_req = self.running.pop()
100 + self.kv_cache_manager.free(preempted_req)
101 + preempted_req.status = RequestStatus.PREEMPTED
102 + preempted_req.num_computed_tokens = 0
103 +
104 + self.waiting.appendleft(preempted_req)
105 + preempted_reqs.append(preempted_req)
106 + if preempted_req == request:
107 + # No more request to preempt.
108 + break
109 + else:
110 + # The request can be scheduled.
111 + scheduled_running_reqs.append(request)
112 +
113 + req_to_new_block_ids[request.request_id] = new_block_ids
114 + num_scheduled_tokens[request.request_id] = num_new_tokens
115 + token_budget -= num_new_tokens
116 + req_index += 1
117 + break
118 +
119 + # Next, schedule the WAITING requests.
120 + if not preempted_reqs:
121 + while self.waiting:
122 + if len(self.running) == self.max_num_running_reqs:
123 + break
124 + if token_budget == 0:
125 + break
126 +
127 + request = self.waiting[0]
128 + # Get already-cached tokens.
129 + computed_block_ids = self.kv_cache_manager.get_computed_blocks(
130 + request)
131 + # NOTE(woosuk): Since incomplete blocks are not eligible for
132 + # sharing, `num_computed_tokens` is always a multiple of
133 + # `block_size`.
134 + num_computed_tokens = len(computed_block_ids) * self.block_size
135 + # Number of tokens to be scheduled.
136 + # We use `request.num_tokens` instead of
137 + # `request.num_prompt_tokens` to consider the resumed requests,
138 + # which have output tokens.
139 + num_new_tokens = request.num_tokens - num_computed_tokens
140 + num_new_tokens = min(num_new_tokens, token_budget)
141 + assert num_new_tokens > 0
142 + new_block_ids = self.kv_cache_manager.allocate_slots(
143 + request, num_new_tokens, computed_block_ids)
144 + if new_block_ids is None:
145 + # The request cannot be scheduled.
146 + break
147 + request.num_computed_tokens = num_computed_tokens
148 +
149 + self.waiting.popleft()
150 + self.running.append(request)
151 + if request.status == RequestStatus.WAITING:
152 + scheduled_new_reqs.append(request)
153 + elif request.status == RequestStatus.PREEMPTED:
154 + scheduled_resumed_reqs.append(request)
155 + else:
156 + raise RuntimeError(
157 + f"Invalid request status: {request.status}")
158 +
159 + req_to_new_block_ids[request.request_id] = (
160 + computed_block_ids + new_block_ids)
161 + num_scheduled_tokens[request.request_id] = num_new_tokens
162 + token_budget -= num_new_tokens
163 + request.status = RequestStatus.RUNNING
164 +
165 + # Check if the scheduling constraints are satisfied.
166 + total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
167 + assert total_num_scheduled_tokens <= self.max_num_scheduled_tokens
168 + assert token_budget >= 0
169 + assert len(self.running) <= self.max_num_running_reqs
170 + assert (len(scheduled_new_reqs) + len(scheduled_resumed_reqs) +
171 + len(scheduled_running_reqs) == len(self.running))
172 +
173 + # Construct the scheduler output.
174 + new_reqs_data = [
175 + NewRequestData.from_request(req,
176 + req_to_new_block_ids[req.request_id],
177 + req.num_computed_tokens)
178 + for req in scheduled_new_reqs
179 + ]
180 + resumed_reqs_data = [
181 + ResumedRequestData.from_request(
182 + req, req_to_new_block_ids[req.request_id],
183 + req.num_computed_tokens) for req in scheduled_resumed_reqs
184 + ]
185 + running_reqs_data = [
186 + self._make_running_request_data(
187 + req, req_to_new_block_ids[req.request_id],
188 + req.num_computed_tokens) for req in scheduled_running_reqs
189 + ]
190 + preempted_req_ids = {req.request_id for req in preempted_reqs}
191 + scheduler_output = SchedulerOutput(
192 + scheduled_new_reqs=new_reqs_data,
193 + scheduled_resumed_reqs=resumed_reqs_data,
194 + scheduled_running_reqs=running_reqs_data,
195 + num_scheduled_tokens=num_scheduled_tokens,
196 + total_num_scheduled_tokens=total_num_scheduled_tokens,
197 + preempted_req_ids=preempted_req_ids,
198 + # finished_req_ids is an existing state in the scheduler,
199 + # instead of being newly scheduled in this step.
200 + # It contains the request IDs that are finished in between
201 + # the previous and the current steps.
202 + finished_req_ids=self.finished_req_ids,
203 + )
204 +
205 + self.finished_req_ids = set()
206 + return scheduler_output
207 +
208 + def _make_running_request_data(
209 + self,
210 + request: Request,
211 + new_block_ids: List[int],
212 + num_computed_tokens: int,
213 + ) -> "RunningRequestData":
214 + # OPTIMIZATION: Cache the RunningRequestData objects to avoid creating
215 + # them at each scheduling step.
216 + if request.request_id in self.running_reqs_data:
217 + req_data = self.running_reqs_data[request.request_id]
218 + req_data.new_block_ids = new_block_ids
219 + req_data.num_computed_tokens = num_computed_tokens
220 + else:
221 + req_data = RunningRequestData.from_request(request, new_block_ids,
222 + num_computed_tokens)
223 + self.running_reqs_data[request.request_id] = req_data
224 + return req_data
225 +
226 + def update_from_output(
227 + self,
228 + scheduler_output: "SchedulerOutput",
229 + model_runner_output: "ModelRunnerOutput",
230 + ) -> List[Tuple[Request, int]]:
231 + # NOTE(woosuk): This method doesn't consider speculative decoding.
232 + sampled_token_ids = model_runner_output.sampled_token_ids_cpu.tolist()
233 + num_scheduled_tokens = scheduler_output.num_scheduled_tokens
234 + new_running: List[Request] = []
235 + # (request, num_sampled_tokens)
236 + sampled: List[Tuple[Request, int]] = []
237 + for request in self.running:
238 + req_id = request.request_id
239 + request.num_computed_tokens += num_scheduled_tokens[req_id]
240 + # When the request's num_computed_tokens catches up its num_tokens,
241 + # the request generates output tokens. Otherwise, we ignore the
242 + # sampler output for the request.
243 + assert request.num_computed_tokens <= request.num_tokens
244 + if request.num_computed_tokens == request.num_tokens:
245 + req_index = model_runner_output.req_id_to_index[req_id]
246 + # NOTE(woosuk): Currently, we assume that each request
247 + # generates at most one token at each step.
248 + token_id = sampled_token_ids[req_index]
249 + request.output_token_ids.append(token_id)
250 + sampled.append((request, 1))
251 + # TODO: Update the KV cache manager for prefix caching.
252 +
253 + # Check if the request is finished.
254 + stopped = self._check_stop(request)
255 + if stopped:
256 + continue
257 +
258 + new_running.append(request)
259 + self.running = new_running
260 + return sampled
261 +
262 + def _check_stop(self, request: Request) -> bool:
263 + if (request.num_tokens >= self.max_model_len
264 + or request.num_output_tokens >= request.max_tokens):
265 + request.status = RequestStatus.FINISHED_LENGTH_CAPPED
266 + self._free_request(request)
267 + return True
268 +
269 + sampling_params = request.sampling_params
270 + last_token_id = request.output_token_ids[-1]
271 + if (not sampling_params.ignore_eos
272 + and last_token_id == request.eos_token_id):
273 + request.status = RequestStatus.FINISHED_STOPPED
274 + self._free_request(request)
275 + return True
276 +
277 + if last_token_id in (sampling_params.stop_token_ids or ()):
278 + request.status = RequestStatus.FINISHED_STOPPED
279 + request.stop_reason = last_token_id
280 + self._free_request(request)
281 + return True
282 + return False
283 +
284 + def add_request(self, request: Request) -> None:
285 + self.waiting.append(request)
286 + self.requests[request.request_id] = request
287 +
288 + def finish_requests(
289 + self,
290 + request_ids: Union[str, Iterable[str]],
291 + finished_status: RequestStatus,
292 + ) -> None:
293 + """Handles the finish signal from outside the scheduler.
294 +
295 + For example, the API server can abort a request when the client
296 + disconnects.
297 + """
298 + assert RequestStatus.is_finished(finished_status)
299 + if isinstance(request_ids, str):
300 + request_ids = (request_ids, )
301 + request_ids = set(request_ids)
302 +
303 + for req_id in request_ids:
304 + request = self.requests.get(req_id)
305 + if request is None:
306 + # Invalid request ID.
307 + continue
308 +
309 + if request.status == RequestStatus.RUNNING:
310 + self.running.remove(request)
311 + else:
312 + self.waiting.remove(request)
313 + request.status = finished_status
314 + self._free_request(request)
315 +
316 + def _free_request(self, request: Request) -> None:
317 + assert request.is_finished()
318 + self.kv_cache_manager.free(request)
319 + self.running_reqs_data.pop(request.request_id, None)
320 + del self.requests[request.request_id]
321 + self.finished_req_ids.add(request.request_id)
322 +
323 + def get_num_unfinished_requests(self) -> int:
324 + return len(self.waiting) + len(self.running)
325 +
326 + def has_unfinished_requests(self) -> bool:
327 + return self.get_num_unfinished_requests() > 0
328 +
329 +
330 +@dataclass
331 +class NewRequestData:
332 +
333 + req_id: str
334 + prompt_token_ids: List[int]
335 + prompt: Optional[str]
336 + multi_modal_data: Optional[MultiModalDataDict]
337 + sampling_params: SamplingParams
338 + block_ids: List[int]
339 + num_computed_tokens: int
340 +
341 + @classmethod
342 + def from_request(
343 + cls,
344 + request: Request,
345 + block_ids: List[int],
346 + num_computed_tokens: int,
347 + ) -> "NewRequestData":
348 + return cls(
349 + req_id=request.request_id,
350 + prompt_token_ids=request.inputs["prompt_token_ids"],
351 + prompt=request.inputs.get("prompt"),
352 + multi_modal_data=request.inputs.get("multi_modal_data"),
353 + sampling_params=request.sampling_params,
354 + block_ids=block_ids,
355 + num_computed_tokens=num_computed_tokens,
356 + )
357 +
358 +
359 +@dataclass
360 +class ResumedRequestData:
361 +
362 + req_id: str
363 + block_ids: List[int]
364 + num_computed_tokens: int
365 +
366 + @classmethod
367 + def from_request(
368 + cls,
369 + request: Request,
370 + block_ids: List[int],
371 + num_computed_tokens: int,
372 + ) -> "ResumedRequestData":
373 + return cls(
374 + req_id=request.request_id,
375 + block_ids=block_ids,
376 + num_computed_tokens=num_computed_tokens,
377 + )
378 +
379 +
380 +@dataclass
381 +class RunningRequestData:
382 +
383 + req_id: str
384 + new_block_ids: List[int]
385 + num_computed_tokens: int
386 +
387 + @classmethod
388 + def from_request(
389 + cls,
390 + request: Request,
391 + new_block_ids: List[int],
392 + num_computed_tokens: int,
393 + ) -> "RunningRequestData":
394 + return cls(
395 + req_id=request.request_id,
396 + new_block_ids=new_block_ids,
397 + num_computed_tokens=num_computed_tokens,
398 + )
399 +
400 +
401 +@dataclass
402 +class SchedulerOutput:
403 +
404 + scheduled_new_reqs: List[NewRequestData]
405 + scheduled_resumed_reqs: List[ResumedRequestData]
406 + scheduled_running_reqs: List[RunningRequestData]
407 +
408 + num_scheduled_tokens: Dict[str, int]
409 + total_num_scheduled_tokens: int
410 +
411 + preempted_req_ids: Set[str]
412 + finished_req_ids: Set[str]
vllm/v1/engine/__init__.py
vllm/v1/engine/llm_engine.py new
+523
@@ -0,0 +1,523 @@
1 +import time
2 +from typing import (Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type,
3 + Union)
4 +
5 +from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
6 + EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
7 + ObservabilityConfig, ParallelConfig,
8 + PromptAdapterConfig, SchedulerConfig,
9 + SpeculativeConfig)
10 +from vllm.engine.arg_utils import EngineArgs
11 +from vllm.engine.metrics_types import StatLoggerBase
12 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
13 + EncoderDecoderLLMInputs, InputRegistry, PromptType)
14 +from vllm.inputs.preprocess import InputPreprocessor
15 +from vllm.logger import init_logger
16 +from vllm.lora.request import LoRARequest
17 +from vllm.outputs import CompletionOutput, RequestOutput
18 +from vllm.pooling_params import PoolingParams
19 +from vllm.prompt_adapter.request import PromptAdapterRequest
20 +from vllm.sampling_params import RequestOutputKind, SamplingParams
21 +from vllm.transformers_utils.config import try_get_generation_config
22 +from vllm.transformers_utils.tokenizer_group import (
23 + BaseTokenizerGroup, init_tokenizer_from_configs)
24 +from vllm.usage.usage_lib import UsageContext
25 +from vllm.v1.core.scheduler import Scheduler
26 +from vllm.v1.executor.gpu_executor import GPUExecutor
27 +from vllm.v1.request import Request, RequestStatus
28 +from vllm.v1.tokenizer.detokenizer import Detokenizer, DetokenizerInputs
29 +from vllm.version import __version__ as VLLM_VERSION
30 +
31 +logger = init_logger(__name__)
32 +
33 +
34 +class LLMEngine:
35 +
36 + def __init__(
37 + self,
38 + model_config: ModelConfig,
39 + cache_config: CacheConfig,
40 + parallel_config: ParallelConfig,
41 + scheduler_config: SchedulerConfig,
42 + device_config: DeviceConfig,
43 + load_config: LoadConfig,
44 + lora_config: Optional[LoRAConfig],
45 + speculative_config: Optional[SpeculativeConfig],
46 + decoding_config: Optional[DecodingConfig],
47 + observability_config: Optional[ObservabilityConfig],
48 + prompt_adapter_config: Optional[PromptAdapterConfig],
49 + executor_class: Type[GPUExecutor],
50 + log_stats: bool,
51 + usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
52 + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
53 + input_registry: InputRegistry = INPUT_REGISTRY,
54 + use_cached_outputs: bool = False,
55 + ) -> None:
56 + # Override the configs for V1.
57 + # FIXME
58 + if usage_context == UsageContext.LLM_CLASS:
59 + scheduler_config.max_num_seqs = 1024
60 + scheduler_config.max_num_batched_tokens = 8192
61 + elif usage_context == UsageContext.OPENAI_API_SERVER:
62 + scheduler_config.max_num_seqs = 1024
63 + scheduler_config.max_num_batched_tokens = 2048
64 +
65 + logger.info(
66 + "Initializing an LLM engine (v%s) with config: "
67 + "model=%r, speculative_config=%r, tokenizer=%r, "
68 + "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
69 + "override_neuron_config=%s, "
70 + "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
71 + "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
72 + "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
73 + "pipeline_parallel_size=%d, "
74 + "disable_custom_all_reduce=%s, quantization=%s, "
75 + "enforce_eager=%s, kv_cache_dtype=%s, "
76 + "quantization_param_path=%s, device_config=%s, "
77 + "decoding_config=%r, observability_config=%r, "
78 + "seed=%d, served_model_name=%s, "
79 + "num_scheduler_steps=%d, enable_prefix_caching=%s, "
80 + "use_async_output_proc=%s, mm_processor_kwargs=%s)",
81 + VLLM_VERSION,
82 + model_config.model,
83 + speculative_config,
84 + model_config.tokenizer,
85 + model_config.skip_tokenizer_init,
86 + model_config.tokenizer_mode,
87 + model_config.revision,
88 + model_config.override_neuron_config,
89 + model_config.rope_scaling,
90 + model_config.rope_theta,
91 + model_config.tokenizer_revision,
92 + model_config.trust_remote_code,
93 + model_config.dtype,
94 + model_config.max_model_len,
95 + load_config.download_dir,
96 + load_config.load_format,
97 + parallel_config.tensor_parallel_size,
98 + parallel_config.pipeline_parallel_size,
99 + parallel_config.disable_custom_all_reduce,
100 + model_config.quantization,
101 + model_config.enforce_eager,
102 + cache_config.cache_dtype,
103 + model_config.quantization_param_path,
104 + device_config.device,
105 + decoding_config,
106 + observability_config,
107 + model_config.seed,
108 + model_config.served_model_name,
109 + scheduler_config.num_scheduler_steps,
110 + cache_config.enable_prefix_caching,
111 + model_config.use_async_output_proc,
112 + model_config.mm_processor_kwargs,
113 + )
114 +
115 + self.model_config = model_config
116 + self.cache_config = cache_config
117 + self.lora_config = lora_config
118 + self.parallel_config = parallel_config
119 + self.scheduler_config = scheduler_config
120 + self.device_config = device_config
121 + self.speculative_config = speculative_config
122 + self.load_config = load_config
123 + self.decoding_config = decoding_config or DecodingConfig()
124 + self.prompt_adapter_config = prompt_adapter_config
125 + self.observability_config = observability_config or ObservabilityConfig(
126 + )
127 + self.log_stats = log_stats
128 +
129 + assert not self.model_config.skip_tokenizer_init
130 + self.tokenizer = self._init_tokenizer()
131 + if self.tokenizer:
132 + # Ping the tokenizer to ensure liveness if it runs in a
133 + # different process.
134 + self.tokenizer.ping()
135 + self.detokenizer = Detokenizer(self.model_config.tokenizer)
136 +
137 + self.generation_config_fields = _load_generation_config_dict(
138 + model_config)
139 + self.input_preprocessor = InputPreprocessor(model_config,
140 + self.tokenizer)
141 + self.input_registry = input_registry
142 + self.input_processor = input_registry.create_input_processor(
143 + model_config)
144 +
145 + # Request id -> Request
146 + self.requests: Dict[str, Request] = {}
147 + # NOTE(woosuk): Now that the detokenizer works asynchronously, we need
148 + # to keep track of how many steps each request has been lagged behind
149 + # in terms of detokenization.
150 + # Request id -> how many detokenizer steps the request should wait for.
151 + self.num_lagged_steps: Dict[str, int] = {}
152 + # OPTIMIZATION: Cache the request output and update it incrementally.
153 + # This is used to avoid creating a new RequestOutput object every step.
154 + # Request id -> RequestOutput
155 + self.request_outputs: Dict[str, RequestOutput] = {}
156 +
157 + self.model_executor = executor_class(
158 + model_config=model_config,
159 + cache_config=cache_config,
160 + parallel_config=parallel_config,
161 + scheduler_config=scheduler_config,
162 + device_config=device_config,
163 + lora_config=lora_config,
164 + speculative_config=speculative_config,
165 + load_config=load_config,
166 + prompt_adapter_config=prompt_adapter_config,
167 + observability_config=self.observability_config,
168 + )
169 + assert self.model_config.task != "embedding"
170 + self._initialize_kv_caches()
171 +
172 + # Create the scheduler.
173 + # NOTE: the cache_config here have been updated with the numbers of
174 + # GPU and CPU blocks, which are profiled in the distributed executor.
175 + self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
176 +
177 + def _initialize_kv_caches(self) -> None:
178 + num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
179 + )
180 +
181 + if self.cache_config.num_gpu_blocks_override is not None:
182 + num_gpu_blocks_override = self.cache_config.num_gpu_blocks_override
183 + logger.info(
184 + "Overriding num_gpu_blocks=%d with "
185 + "num_gpu_blocks_override=%d", num_gpu_blocks,
186 + num_gpu_blocks_override)
187 + num_gpu_blocks = num_gpu_blocks_override
188 +
189 + self.cache_config.num_gpu_blocks = num_gpu_blocks
190 + self.cache_config.num_cpu_blocks = 0
191 + self.model_executor.initialize_cache(num_gpu_blocks)
192 +
193 + @classmethod
194 + def from_engine_args(
195 + cls,
196 + engine_args: EngineArgs,
197 + usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
198 + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
199 + ) -> "LLMEngine":
200 + """Creates an LLM engine from the engine arguments."""
201 + # Create the engine configs.
202 + engine_config = engine_args.create_engine_config()
203 + executor_class = cls._get_executor_cls(engine_config)
204 + # Create the LLM engine.
205 + engine = cls(
206 + **engine_config.to_dict(),
207 + executor_class=executor_class,
208 + log_stats=not engine_args.disable_log_stats,
209 + usage_context=usage_context,
210 + stat_loggers=stat_loggers,
211 + )
212 + return engine
213 +
214 + def _init_tokenizer(self) -> BaseTokenizerGroup:
215 + return init_tokenizer_from_configs(
216 + model_config=self.model_config,
217 + scheduler_config=self.scheduler_config,
218 + parallel_config=self.parallel_config,
219 + enable_lora=bool(self.lora_config))
220 +
221 + def _verify_args(self) -> None:
222 + self.model_config.verify_with_parallel_config(self.parallel_config)
223 + self.cache_config.verify_with_parallel_config(self.parallel_config)
224 + if self.lora_config:
225 + self.lora_config.verify_with_model_config(self.model_config)
226 + self.lora_config.verify_with_scheduler_config(
227 + self.scheduler_config)
228 + if self.prompt_adapter_config:
229 + self.prompt_adapter_config.verify_with_model_config(
230 + self.model_config)
231 +
232 + def _add_processed_request(
233 + self,
234 + request_id: str,
235 + processed_inputs: Union[DecoderOnlyInputs, EncoderDecoderLLMInputs],
236 + params: Union[SamplingParams, PoolingParams],
237 + arrival_time: float,
238 + lora_request: Optional[LoRARequest],
239 + prompt_adapter_request: Optional[PromptAdapterRequest],
240 + trace_headers: Optional[Mapping[str, str]] = None,
241 + ) -> None:
242 + assert prompt_adapter_request is None
243 + assert trace_headers is None
244 + self._validate_model_inputs(processed_inputs)
245 + eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
246 +
247 + # TODO(woosuk): Support embedding mode.
248 + assert isinstance(params, SamplingParams)
249 + sampling_params = params.clone()
250 + sampling_params.update_from_generation_config(
251 + self.generation_config_fields, eos_token_id)
252 +
253 + # TODO(woosuk): Check max_logprobs
254 + # TODO(woosuk): Support encoder-decoder models.
255 + req = Request(request_id, processed_inputs, params, eos_token_id,
256 + arrival_time)
257 + self.requests[request_id] = req
258 + self.num_lagged_steps[request_id] = 0
259 + self.scheduler.add_request(req)
260 +
261 + def stop_remote_worker_execution_loop(self) -> None:
262 + raise NotImplementedError("TP not implemented yet.")
263 +
264 + def add_request(
265 + self,
266 + request_id: str,
267 + prompt: PromptType,
268 + params: Union[SamplingParams, PoolingParams],
269 + arrival_time: Optional[float] = None,
270 + lora_request: Optional[LoRARequest] = None,
271 + trace_headers: Optional[Mapping[str, str]] = None,
272 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
273 + priority: int = 0,
274 + ) -> None:
275 + if lora_request is not None and not self.lora_config:
276 + raise ValueError(f"Got lora_request {lora_request} but LoRA is "
277 + "not enabled!")
278 + if arrival_time is None:
279 + arrival_time = time.time()
280 + assert priority == 0, "vLLM V1 does not support priority at the moment."
281 +
282 + preprocessed_inputs = self.input_preprocessor.preprocess(
283 + prompt,
284 + request_id=request_id,
285 + lora_request=lora_request,
286 + prompt_adapter_request=prompt_adapter_request,
287 + )
288 + processed_inputs = self.input_processor(preprocessed_inputs)
289 +
290 + self._add_processed_request(
291 + request_id=request_id,
292 + processed_inputs=processed_inputs,
293 + params=params,
294 + arrival_time=arrival_time,
295 + lora_request=lora_request,
296 + prompt_adapter_request=prompt_adapter_request,
297 + trace_headers=trace_headers,
298 + )
299 +
300 + def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
301 + self.scheduler.finish_requests(request_id,
302 + RequestStatus.FINISHED_ABORTED)
303 +
304 + def get_num_unfinished_requests(self) -> int:
305 + """Gets the number of unfinished requests."""
306 + return len(self.requests)
307 +
308 + def has_unfinished_requests(self) -> bool:
309 + """Returns True if there are unfinished requests."""
310 + return len(self.requests) > 0
311 +
312 + def step(self) -> List[RequestOutput]:
313 + # NOTE(woosuk): This method may return an empty list when the
314 + # detokenizer is still processing the outputs. This should not be
315 + # considered as the end of the generation process.
316 + # FIXME(woosuk): Currently, the step method is inefficient because it
317 + # creates RequestOutput objects for all running requests, while they
318 + # may not be needed unless the output is streamed to the client.
319 + if self.scheduler.has_unfinished_requests():
320 + scheduler_output = self.scheduler.schedule()
321 + output = self.model_executor.execute_model(scheduler_output)
322 + sampled = self.scheduler.update_from_output(
323 + scheduler_output, output)
324 + self.send_to_detokenizer(sampled)
325 + req_outputs = self.recv_from_detokenizer()
326 + return req_outputs
327 +
328 + def send_to_detokenizer(self, sampled: List[Tuple[Request, int]]) -> None:
329 + inputs = DetokenizerInputs(
330 + req_ids=[],
331 + prompt_token_ids=[],
332 + new_token_ids=[],
333 + skip_special_tokens=[],
334 + spaces_between_special_tokens=[],
335 + free_req_ids=[], # TODO(woosuk): Implement freeing.
336 + )
337 + for req, num_tokens in sampled:
338 + inputs.req_ids.append(req.request_id)
339 + if len(req.output_token_ids) == num_tokens:
340 + # The request is first detokenized.
341 + inputs.prompt_token_ids.append(req.prompt_token_ids)
342 + else:
343 + # The prompt token ids are already cached in the detokenizer.
344 + inputs.prompt_token_ids.append([])
345 + inputs.new_token_ids.append(req.output_token_ids[-num_tokens:])
346 + inputs.skip_special_tokens.append(
347 + req.sampling_params.skip_special_tokens)
348 + inputs.spaces_between_special_tokens.append(
349 + req.sampling_params.spaces_between_special_tokens)
350 +
351 + # Update the number of lagged steps.
352 + self.num_lagged_steps[req.request_id] += 1
353 + self.detokenizer.send(inputs)
354 +
355 + def recv_from_detokenizer(self) -> List[RequestOutput]:
356 + detokenizer_output = self.detokenizer.recv()
357 + if detokenizer_output is None:
358 + return []
359 +
360 + req_outputs: List[RequestOutput] = []
361 + num_reqs = len(detokenizer_output.req_ids)
362 + for i in range(num_reqs):
363 + req_id = detokenizer_output.req_ids[i]
364 + req = self.requests[req_id]
365 + req.output_text += detokenizer_output.detokenized_texts[i]
366 +
367 + self.num_lagged_steps[req_id] -= 1
368 + finished = (self.num_lagged_steps[req_id] == 0
369 + and req.is_finished())
370 + req_output = self._make_request_output(
371 + req, detokenizer_output.num_output_token_ids[i],
372 + detokenizer_output.detokenized_texts[i], finished)
373 + req_outputs.append(req_output)
374 +
375 + if finished:
376 + del self.requests[req_id]
377 + del self.num_lagged_steps[req_id]
378 + del self.request_outputs[req_id]
379 + return req_outputs
380 +
381 + def terminate_detokenizer(self) -> None:
382 + self.detokenizer.terminate()
383 +
384 + def _make_request_output(
385 + self,
386 + request: Request,
387 + num_output_tokens: int,
388 + new_output_text: str,
389 + finished: bool,
390 + ) -> RequestOutput:
391 + req_output = self.request_outputs.get(request.request_id)
392 + if req_output is None:
393 + # TODO: Support `n` > 1.
394 + completion_output = CompletionOutput(
395 + index=0,
396 + text="",
397 + token_ids=[],
398 + cumulative_logprob=None,
399 + logprobs=None, # TODO
400 + finish_reason=None,
401 + stop_reason=None,
402 + lora_request=None,
403 + )
404 + req_output = RequestOutput(
405 + request_id=request.request_id,
406 + prompt=request.prompt,
407 + prompt_token_ids=request.prompt_token_ids,
408 + prompt_logprobs=None, # TODO
409 + outputs=[completion_output],
410 + finished=False,
411 + metrics=None,
412 + lora_request=None,
413 + encoder_prompt=None,
414 + encoder_prompt_token_ids=None,
415 + )
416 + self.request_outputs[request.request_id] = req_output
417 +
418 + completion_output = req_output.outputs[0]
419 + if request.sampling_params.output_kind == RequestOutputKind.CUMULATIVE:
420 + completion_output.text += new_output_text
421 + completion_output.token_ids = (
422 + request.output_token_ids[:num_output_tokens])
423 + elif request.sampling_params.output_kind == RequestOutputKind.DELTA:
424 + completion_output.text = new_output_text
425 + num_prev_tokens = len(completion_output.token_ids)
426 + completion_output.token_ids = request.output_token_ids[
427 + num_prev_tokens:num_output_tokens]
428 + elif (request.sampling_params.output_kind ==
429 + RequestOutputKind.FINAL_ONLY):
430 + if finished:
431 + completion_output.text = request.output_text
432 + completion_output.token_ids = request.output_token_ids
433 + else:
434 + completion_output.text = ""
435 + completion_output.token_ids = []
436 +
437 + if finished:
438 + completion_output.finish_reason = request.get_finished_reason()
439 + completion_output.stop_reason = request.stop_reason
440 + req_output.finished = finished
441 + return req_output
442 +
443 + def check_health(self) -> None:
444 + if self.tokenizer:
445 + self.tokenizer.check_health()
446 + self.model_executor.check_health()
447 +
448 + def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
449 + EncoderDecoderLLMInputs]):
450 + prompt_ids = inputs.get("prompt_token_ids")
451 + if prompt_ids is None or len(prompt_ids) == 0:
452 + raise ValueError("Prompt cannot be empty")
453 +
454 + if self.model_config.is_multimodal_model:
455 + max_prompt_len = self.model_config.max_model_len
456 +
457 + if len(prompt_ids) > max_prompt_len:
458 + raise ValueError(
459 + f"The prompt (total length {len(prompt_ids)}) is too long "
460 + f"to fit into the model (context length {max_prompt_len}). "
461 + "Make sure that `max_model_len` is no smaller than the "
462 + "number of text tokens plus multimodal tokens. For image "
463 + "inputs, the number of image tokens depends on the number "
464 + "of images, and possibly their aspect ratios as well.")
465 +
466 + @classmethod
467 + def validate_outputs(cls, outputs, output_type):
468 + return outputs
469 +
470 + def get_model_config(self) -> ModelConfig:
471 + """Gets the model configuration."""
472 + return self.model_config
473 +
474 + def get_parallel_config(self) -> ParallelConfig:
475 + """Gets the parallel configuration."""
476 + return self.parallel_config
477 +
478 + def get_decoding_config(self) -> DecodingConfig:
479 + """Gets the decoding configuration."""
480 + return self.decoding_config
481 +
482 + def get_scheduler_config(self) -> SchedulerConfig:
483 + """Gets the scheduler configuration."""
484 + return self.scheduler_config
485 +
486 + def get_lora_config(self) -> LoRAConfig:
487 + """Gets the LoRA configuration."""
488 + return self.lora_config
489 +
490 + @classmethod
491 + def _get_executor_cls(cls, engine_config: EngineConfig):
492 + return GPUExecutor
493 +
494 + def is_tracing_enabled(self) -> bool:
495 + return False
496 +
497 + def do_log_stats(self, *args, **kwargs) -> None:
498 + pass
499 +
500 + def is_encoder_decoder_model(self) -> bool:
501 + return False
502 +
503 + def start_profile(self) -> None:
504 + pass
505 +
506 + def stop_profile(self) -> None:
507 + pass
508 +
509 + def get_tokenizer_group(self, *args, **kwargs):
510 + return self.tokenizer
511 +
512 +
513 +def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
514 + config = try_get_generation_config(
515 + model_config.model,
516 + trust_remote_code=model_config.trust_remote_code,
517 + revision=model_config.revision,
518 + )
519 +
520 + if config is None:
521 + return {}
522 +
523 + return config.to_diff_dict()
vllm/v1/executor/__init__.py
vllm/v1/executor/gpu_executor.py new
+100
@@ -0,0 +1,100 @@
1 +import os
2 +from typing import Optional, Tuple
3 +
4 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5 + ModelConfig, ObservabilityConfig, ParallelConfig,
6 + PromptAdapterConfig, SchedulerConfig,
7 + SpeculativeConfig)
8 +from vllm.logger import init_logger
9 +from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10 +from vllm.v1.outputs import ModelRunnerOutput
11 +from vllm.v1.worker.gpu_worker import Worker
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +class GPUExecutor:
17 +
18 + def __init__(
19 + self,
20 + model_config: ModelConfig,
21 + cache_config: CacheConfig,
22 + parallel_config: ParallelConfig,
23 + scheduler_config: SchedulerConfig,
24 + device_config: DeviceConfig,
25 + load_config: LoadConfig,
26 + lora_config: Optional[LoRAConfig],
27 + speculative_config: Optional[SpeculativeConfig],
28 + prompt_adapter_config: Optional[PromptAdapterConfig],
29 + observability_config: Optional[ObservabilityConfig],
30 + ) -> None:
31 + self.model_config = model_config
32 + self.cache_config = cache_config
33 + self.lora_config = lora_config
34 + self.load_config = load_config
35 + self.parallel_config = parallel_config
36 + self.scheduler_config = scheduler_config
37 + self.device_config = device_config
38 + self.speculative_config = speculative_config
39 + self.prompt_adapter_config = prompt_adapter_config
40 + self.observability_config = observability_config
41 +
42 + self.worker = self._create_worker()
43 + self.worker.initialize()
44 + self.worker.load_model()
45 +
46 + def _create_worker(
47 + self,
48 + local_rank: int = 0,
49 + rank: int = 0,
50 + distributed_init_method: Optional[str] = None) -> Worker:
51 + """Return worker init args for a given rank."""
52 + # see https://github.com/NVIDIA/nccl/issues/1234
53 + os.environ['NCCL_CUMEM_ENABLE'] = '0'
54 +
55 + if distributed_init_method is None:
56 + distributed_init_method = get_distributed_init_method(
57 + get_ip(), get_open_port())
58 + return Worker(
59 + model_config=self.model_config,
60 + parallel_config=self.parallel_config,
61 + scheduler_config=self.scheduler_config,
62 + device_config=self.device_config,
63 + cache_config=self.cache_config,
64 + load_config=self.load_config,
65 + local_rank=local_rank,
66 + rank=rank,
67 + distributed_init_method=distributed_init_method,
68 + lora_config=self.lora_config,
69 + speculative_config=self.speculative_config,
70 + prompt_adapter_config=self.prompt_adapter_config,
71 + observability_config=self.observability_config,
72 + )
73 +
74 + def determine_num_available_blocks(self) -> Tuple[int, int]:
75 + """Determine the number of available KV blocks by invoking the
76 + underlying worker.
77 + """
78 + return self.worker.determine_num_available_blocks()
79 +
80 + def initialize_cache(self, num_gpu_blocks: int) -> None:
81 + """Initialize the KV cache by invoking the underlying worker.
82 + """
83 + # NOTE: This is logged in the executor because there can be >1 worker
84 + # with other executors. We could log in the engine level, but work
85 + # remains to abstract away the device for non-GPU configurations.
86 + logger.info("# GPU blocks: %d", num_gpu_blocks)
87 + self.worker.initialize_cache(num_gpu_blocks)
88 + self.worker.compile_or_warm_up_model()
89 +
90 + def execute_model(
91 + self,
92 + scheduler_output,
93 + ) -> ModelRunnerOutput:
94 + output = self.worker.execute_model(scheduler_output)
95 + return output
96 +
97 + def check_health(self) -> None:
98 + # GPUExecutor will always be healthy as long as
99 + # it's running.
100 + return
vllm/v1/outputs.py new
+37
@@ -0,0 +1,37 @@
1 +from dataclasses import dataclass
2 +from typing import Dict, List, Optional
3 +
4 +import torch
5 +
6 +
7 +@dataclass
8 +class SamplerOutput:
9 +
10 + # [num_reqs]
11 + sampled_token_ids: torch.Tensor
12 +
13 + # [num_reqs, max_num_logprobs + 1]
14 + logprob_token_ids: Optional[torch.Tensor]
15 + # [num_reqs, max_num_logprobs + 1]
16 + logprobs: Optional[torch.Tensor]
17 +
18 + # TODO: Support prompt logprobs.
19 + prompt_logprob_token_ids: Optional[torch.Tensor]
20 + prompt_logprobs: Optional[torch.Tensor]
21 +
22 +
23 +@dataclass
24 +class ModelRunnerOutput:
25 +
26 + # [num_reqs]
27 + req_ids: List[str]
28 + # req_id -> index
29 + req_id_to_index: Dict[str, int]
30 +
31 + # [num_reqs]
32 + sampled_token_ids_cpu: torch.Tensor
33 +
34 + # [num_reqs, max_num_logprobs + 1]
35 + logprob_token_ids_cpu: Optional[torch.Tensor]
36 + # [num_reqs, max_num_logprobs + 1]
37 + logprobs_cpu: Optional[torch.Tensor]
vllm/v1/request.py new
+92
@@ -0,0 +1,92 @@
1 +import enum
2 +from typing import TYPE_CHECKING, List, Optional, Union
3 +
4 +from vllm.lora.request import LoRARequest
5 +from vllm.sampling_params import SamplingParams
6 +from vllm.sequence import RequestMetrics
7 +
8 +if TYPE_CHECKING:
9 + from vllm.inputs import DecoderOnlyInputs
10 +
11 +
12 +class Request:
13 +
14 + def __init__(
15 + self,
16 + request_id: str,
17 + inputs: "DecoderOnlyInputs",
18 + sampling_params: SamplingParams,
19 + eos_token_id: Optional[int],
20 + arrival_time: float,
21 + lora_request: Optional[LoRARequest] = None,
22 + ) -> None:
23 + self.request_id = request_id
24 + self.inputs = inputs
25 + self.sampling_params = sampling_params
26 + # Because of LoRA, the eos token id can be different for each request.
27 + self.eos_token_id = eos_token_id
28 + self.metrics = RequestMetrics(arrival_time=arrival_time,
29 + last_token_time=arrival_time,
30 + first_scheduled_time=None,
31 + first_token_time=None,
32 + time_in_queue=None)
33 + self.lora_request = lora_request
34 +
35 + self.status = RequestStatus.WAITING
36 + self.stop_reason: Union[int, str, None] = None
37 + assert sampling_params.max_tokens is not None
38 + self.max_tokens = sampling_params.max_tokens
39 +
40 + self.prompt = inputs.get("prompt")
41 + self.prompt_token_ids = inputs["prompt_token_ids"]
42 + self.num_prompt_tokens = len(self.prompt_token_ids)
43 + self.output_token_ids: List[int] = []
44 + self.output_text = ""
45 + self.num_computed_tokens = 0
46 +
47 + @property
48 + def num_tokens(self) -> int:
49 + return self.num_prompt_tokens + len(self.output_token_ids)
50 +
51 + @property
52 + def num_output_tokens(self) -> int:
53 + return len(self.output_token_ids)
54 +
55 + def is_finished(self) -> bool:
56 + return RequestStatus.is_finished(self.status)
57 +
58 + def get_finished_reason(self) -> Union[str, None]:
59 + return RequestStatus.get_finished_reason(self.status)
60 +
61 +
62 +class RequestStatus(enum.IntEnum):
63 + """Status of a sequence."""
64 + WAITING = 0
65 + RUNNING = 1
66 + PREEMPTED = 2
67 + # Note: anything after PREEMPTED (2) will be considered
68 + # as a finished status.
69 + FINISHED_STOPPED = 3
70 + FINISHED_LENGTH_CAPPED = 4
71 + FINISHED_ABORTED = 5
72 + FINISHED_IGNORED = 6
73 +
74 + @staticmethod
75 + def is_finished(status: "RequestStatus") -> bool:
76 + return status > RequestStatus.PREEMPTED
77 +
78 + @staticmethod
79 + def get_finished_reason(status: "RequestStatus") -> Union[str, None]:
80 + return _FINISHED_REASON_MAP.get(status)
81 +
82 +
83 +# Mapping of finished statuses to their finish reasons.
84 +# NOTE: The ignored sequences are the sequences whose prompt lengths
85 +# are longer than the model's length cap. Therefore, the stop
86 +# reason should also be "length" as in OpenAI API.
87 +_FINISHED_REASON_MAP = {
88 + RequestStatus.FINISHED_STOPPED: "stop",
89 + RequestStatus.FINISHED_LENGTH_CAPPED: "length",
90 + RequestStatus.FINISHED_ABORTED: "abort",
91 + RequestStatus.FINISHED_IGNORED: "length",
92 +}
vllm/v1/sample/__init__.py
vllm/v1/sample/metadata.py new
+22
@@ -0,0 +1,22 @@
1 +from dataclasses import dataclass
2 +from typing import List, Optional
3 +
4 +import torch
5 +
6 +
7 +@dataclass
8 +class SamplingMetadata:
9 +
10 + temperature: torch.Tensor
11 + all_greedy: bool
12 + all_random: bool
13 +
14 + top_p: torch.Tensor
15 + top_k: torch.Tensor
16 + no_top_p: bool
17 + no_top_k: bool
18 +
19 + generators: List[Optional[torch.Generator]]
20 + no_generator: bool
21 +
22 + max_num_logprobs: int
vllm/v1/sample/sampler.py new
+161
@@ -0,0 +1,161 @@
1 +"""A layer that samples the next tokens from the model's outputs."""
2 +from typing import List, Optional
3 +
4 +import torch
5 +import torch.nn as nn
6 +
7 +from vllm.v1.outputs import SamplerOutput
8 +from vllm.v1.sample.metadata import SamplingMetadata
9 +
10 +_SAMPLING_EPS = 1e-5
11 +
12 +
13 +class Sampler(nn.Module):
14 +
15 + def forward(
16 + self,
17 + logits: torch.Tensor,
18 + sampling_metadata: SamplingMetadata,
19 + ) -> SamplerOutput:
20 + logits = self.apply_temperature(logits, sampling_metadata.temperature)
21 + logits = self.apply_top_k_top_p(logits, sampling_metadata)
22 +
23 + probs = self.get_probs(logits)
24 + sampled = self.sample(probs, sampling_metadata)
25 + # Use int32 to reduce the tensor size.
26 + sampled = sampled.to(torch.int32)
27 +
28 + if sampling_metadata.max_num_logprobs > 0:
29 + logprobs = self.get_logprobs(logits)
30 + # FIXME: Mask the sampled token_id, get topk logprobs,
31 + # and concatenate the topk with the sampled token_id.
32 + topk_logprobs, topk_indices = torch.topk(
33 + logprobs, sampling_metadata.max_num_logprobs, dim=-1)
34 + # Use int32 to reduce the tensor size.
35 + topk_indices = topk_indices.to(torch.int32)
36 + else:
37 + topk_logprobs = None
38 + topk_indices = None
39 +
40 + sampler_output = SamplerOutput(
41 + sampled_token_ids=sampled,
42 + logprob_token_ids=topk_indices,
43 + logprobs=topk_logprobs,
44 + prompt_logprob_token_ids=None,
45 + prompt_logprobs=None,
46 + )
47 + return sampler_output
48 +
49 + def apply_temperature(
50 + self,
51 + logits: torch.Tensor,
52 + temp: torch.Tensor,
53 + ) -> torch.Tensor:
54 + # Use float32 to apply temperature scaling.
55 + logits = logits.to(torch.float32)
56 + # Avoid division by zero.
57 + temp = torch.where(temp < _SAMPLING_EPS, 1.0, temp)
58 + # Use in-place division to avoid creating a new tensor.
59 + logits.div_(temp.unsqueeze(dim=1))
60 + return logits
61 +
62 + def apply_top_k_top_p(
63 + self,
64 + logits: torch.Tensor,
65 + sampling_metadata: SamplingMetadata,
66 + ) -> torch.Tensor:
67 + return _apply_top_k_top_p(
68 + logits,
69 + sampling_metadata.no_top_k,
70 + sampling_metadata.top_k,
71 + sampling_metadata.no_top_p,
72 + sampling_metadata.top_p,
73 + )
74 +
75 + def get_probs(self, logits: torch.Tensor) -> torch.Tensor:
76 + return torch.softmax(logits, dim=-1, dtype=torch.float32)
77 +
78 + def get_logprobs(self, logits: torch.Tensor) -> torch.Tensor:
79 + return torch.log_softmax(logits, dim=-1, dtype=torch.float32)
80 +
81 + def greedy_sample(self, probs: torch.Tensor) -> torch.Tensor:
82 + return probs.argmax(dim=-1).view(-1)
83 +
84 + def random_sample(
85 + self,
86 + probs: torch.Tensor,
87 + generators: List[Optional[torch.Generator]],
88 + no_generator: bool,
89 + ) -> torch.Tensor:
90 + q = torch.empty_like(probs)
91 + # NOTE(woosuk): To batch-process the requests without their own seeds,
92 + # which is the common case, we first assume that every request does
93 + # not have its own seed. Then, we overwrite the values for the requests
94 + # that have their own seeds.
95 + q.exponential_()
96 + if not no_generator:
97 + assert len(generators) == probs.shape[0]
98 + # TODO(woosuk): This can be slow because we handle each request
99 + # one by one. Optimize this.
100 + for i, generator in enumerate(generators):
101 + if generator is not None:
102 + q[i].exponential_(generator=generator)
103 + return probs.div_(q).argmax(dim=-1).view(-1)
104 +
105 + def sample(
106 + self,
107 + probs: torch.Tensor,
108 + sampling_metadata: SamplingMetadata,
109 + ) -> torch.Tensor:
110 + assert not (sampling_metadata.all_greedy
111 + and sampling_metadata.all_random)
112 + if sampling_metadata.all_greedy:
113 + return self.greedy_sample(probs)
114 + if sampling_metadata.all_random:
115 + return self.random_sample(probs, sampling_metadata.generators,
116 + sampling_metadata.no_generator)
117 +
118 + greedy_sampled = self.greedy_sample(probs)
119 + random_sampled = self.random_sample(probs,
120 + sampling_metadata.generators,
121 + sampling_metadata.no_generator)
122 + sampled = torch.where(
123 + sampling_metadata.temperature < _SAMPLING_EPS,
124 + greedy_sampled,
125 + random_sampled,
126 + )
127 + return sampled
128 +
129 +
130 +# TODO(woosuk): Optimize this with a custom kernel.
131 +def _apply_top_k_top_p(
132 + logits: torch.Tensor,
133 + no_top_k: bool,
134 + k: torch.Tensor,
135 + no_top_p: bool,
136 + p: torch.Tensor,
137 +) -> torch.Tensor:
138 + if no_top_k and no_top_p:
139 + return logits
140 + logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
141 +
142 + if not no_top_k:
143 + # Apply top-k.
144 + top_k_mask = logits_sort.size(1) - k.to(torch.long)
145 + # Get all the top_k values.
146 + top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1))
147 + top_k_mask = logits_sort < top_k_mask
148 + logits_sort.masked_fill_(top_k_mask, -float("inf"))
149 +
150 + if not no_top_p:
151 + # Apply top-p.
152 + probs_sort = logits_sort.softmax(dim=-1)
153 + probs_sum = probs_sort.cumsum(dim=-1)
154 + top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1)
155 + # at least one
156 + top_p_mask[:, -1] = False
157 + logits_sort.masked_fill_(top_p_mask, -float("inf"))
158 +
159 + # Re-sort the probabilities.
160 + logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
161 + return logits
vllm/v1/tokenizer/__init__.py
vllm/v1/tokenizer/detokenizer.py new
+215
@@ -0,0 +1,215 @@
1 +import multiprocessing
2 +from dataclasses import dataclass
3 +from typing import Dict, List, Optional
4 +
5 +import msgspec
6 +import zmq
7 +from msgspec import msgpack
8 +
9 +from vllm.transformers_utils.detokenizer_utils import (
10 + convert_prompt_ids_to_tokens, detokenize_incrementally)
11 +from vllm.transformers_utils.tokenizer import get_tokenizer
12 +from vllm.utils import get_open_port
13 +
14 +
15 +class DetokenizerInputs(msgspec.Struct):
16 +
17 + # [num_reqs]
18 + req_ids: List[str]
19 + # A request's prompt token ids is sent to the detokenizer only when
20 + # the request is first detokenized. Otherwise, an empty list is sent.
21 + prompt_token_ids: List[List[int]]
22 + new_token_ids: List[List[int]]
23 + skip_special_tokens: List[bool]
24 + spaces_between_special_tokens: List[bool]
25 +
26 + # [num_free_reqs]
27 + free_req_ids: List[str]
28 +
29 +
30 +class DetokenizerOutputs(msgspec.Struct):
31 +
32 + # [num_reqs]
33 + req_ids: List[str]
34 + detokenized_texts: List[str]
35 + # NOTE(woosuk): The number of the output token ids of each request
36 + # at the time of detokenization. The detokenizer returns this to the engine
37 + # because the request state (including the output token ids) is
38 + # asynchronously updated in the engine, while RequestOutput requires the
39 + # output token ids to be consistent with the detokenized text.
40 + num_output_token_ids: List[int]
41 +
42 +
43 +class Detokenizer:
44 +
45 + def __init__(self, tokenizer_name: str):
46 + # FIXME(woosuk): Currently, the detokenizer is just a hacky prototype.
47 + # For example, it does not terminate properly. We need to improve this.
48 + self.push_port = get_open_port()
49 + self.pull_port = get_open_port()
50 + self.detokenizer = DetokenizerProc(tokenizer_name, self.push_port,
51 + self.pull_port)
52 + self.detokenizer.start()
53 +
54 + self.zmq_context = zmq.Context()
55 + self.push_socket = self.zmq_context.socket(zmq.PUSH)
56 + self.push_socket.connect(f"tcp://localhost:{self.push_port}")
57 + self.pull_socket = self.zmq_context.socket(zmq.PULL)
58 + self.pull_socket.connect(f"tcp://localhost:{self.pull_port}")
59 + self.poller = zmq.Poller()
60 + self.poller.register(self.pull_socket, zmq.POLLIN)
61 + self.msgpack_encoder = msgpack.Encoder()
62 + self.msgpack_decoder = msgpack.Decoder(DetokenizerOutputs)
63 +
64 + def send(self, inputs: DetokenizerInputs) -> None:
65 + self.push_socket.send(self.msgpack_encoder.encode(inputs),
66 + flags=zmq.NOBLOCK)
67 +
68 + def recv(self) -> Optional[DetokenizerOutputs]:
69 + socks = dict(self.poller.poll(timeout=0))
70 + if self.pull_socket in socks and socks[self.pull_socket] == zmq.POLLIN:
71 + msg = self.pull_socket.recv()
72 + return self.msgpack_decoder.decode(msg)
73 + return None
74 +
75 + def terminate(self) -> None:
76 + self.push_socket.send(b"", flags=zmq.NOBLOCK)
77 + self.detokenizer.join()
78 +
79 +
80 +class DetokenizerProc(multiprocessing.Process):
81 +
82 + def __init__(
83 + self,
84 + tokenizer_name: str,
85 + pull_port: int,
86 + push_port: int,
87 + ):
88 + super().__init__()
89 + self.tokenizer_name = tokenizer_name
90 + # NOTE: The pull_port of the detokenizer should be the same as the
91 + # push_port of the engine. Vice versa.
92 + self.pull_port = pull_port
93 + self.push_port = push_port
94 +
95 + def run(self):
96 + # Initialize these objects after the process is forked since they are
97 + # not picklable.
98 + self.msgpack_encoder = msgpack.Encoder()
99 + self.msgpack_decoder = msgpack.Decoder(DetokenizerInputs)
100 + self.tokenizer = get_tokenizer(self.tokenizer_name)
101 + # req_id -> RequestState
102 + self.request_states: Dict[str, RequestState] = {}
103 +
104 + self.zmq_context = zmq.Context()
105 + self.pull_socket = self.zmq_context.socket(zmq.PULL)
106 + self.pull_socket.bind(f"tcp://*:{self.pull_port}")
107 + self.push_socket = self.zmq_context.socket(zmq.PUSH)
108 + self.push_socket.bind(f"tcp://*:{self.push_port}")
109 +
110 + while True:
111 + message = self.pull_socket.recv()
112 + if message == b"":
113 + # Terminate signal.
114 + break
115 + inputs = self.msgpack_decoder.decode(message)
116 +
117 + for req_id in inputs.free_req_ids:
118 + self.free(req_id)
119 +
120 + detokenized_texts: List[str] = []
121 + num_output_token_ids: List[int] = []
122 + num_reqs = len(inputs.req_ids)
123 + for i in range(num_reqs):
124 + req_id = inputs.req_ids[i]
125 + if req_id not in self.request_states:
126 + self.add_request(
127 + request_id=req_id,
128 + prompt_token_ids=inputs.prompt_token_ids[i],
129 + skip_special_tokens=inputs.skip_special_tokens[i],
130 + spaces_between_special_tokens=inputs.
131 + spaces_between_special_tokens[i],
132 + )
133 + new_str = self.detokenize(req_id, inputs.new_token_ids[i])
134 + detokenized_texts.append(new_str)
135 + req_state = self.request_states[req_id]
136 + num_output_token_ids.append(
137 + len(req_state.token_ids) - req_state.num_prompt_tokens)
138 +
139 + detokenized = DetokenizerOutputs(
140 + req_ids=inputs.req_ids,
141 + detokenized_texts=detokenized_texts,
142 + num_output_token_ids=num_output_token_ids,
143 + )
144 + self.push_socket.send(self.msgpack_encoder.encode(detokenized),
145 + flags=zmq.NOBLOCK)
146 +
147 + def add_request(
148 + self,
149 + request_id: str,
150 + prompt_token_ids: List[int],
151 + skip_special_tokens: bool,
152 + spaces_between_special_tokens: bool,
153 + ) -> None:
154 + tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens(
155 + tokenizer=self.tokenizer,
156 + prompt_ids=prompt_token_ids,
157 + skip_special_tokens=skip_special_tokens,
158 + )
159 + self.request_states[request_id] = RequestState(
160 + req_id=request_id,
161 + token_ids=prompt_token_ids,
162 + tokens=tokens,
163 + num_prompt_tokens=len(prompt_token_ids),
164 + prefix_offset=prefix_offset,
165 + read_offset=read_offset,
166 + skip_special_tokens=skip_special_tokens,
167 + spaces_between_special_tokens=spaces_between_special_tokens,
168 + )
169 +
170 + def free(self, request_id: str) -> None:
171 + del self.request_states[request_id]
172 +
173 + def detokenize(self, request_id: str, new_token_ids: List[int]) -> str:
174 + # TODO(woosuk): This method becomes very inefficient when the number of
175 + # new_token_ids is more than 1. We need to optimize this.
176 + req_state = self.request_states[request_id]
177 + decoded_text = ""
178 + for new_token_id in new_token_ids:
179 + req_state.token_ids.append(new_token_id)
180 + (new_tokens, new_decoded_token_text, prefix_offset,
181 + read_offset) = detokenize_incrementally(
182 + tokenizer=self.tokenizer,
183 + all_input_ids=req_state.token_ids,
184 + prev_tokens=req_state.tokens,
185 + prefix_offset=req_state.prefix_offset,
186 + read_offset=req_state.read_offset,
187 + skip_special_tokens=req_state.skip_special_tokens,
188 + spaces_between_special_tokens=req_state.
189 + spaces_between_special_tokens,
190 + )
191 +
192 + req_state.tokens.extend(new_tokens)
193 + req_state.prefix_offset = prefix_offset
194 + req_state.read_offset = read_offset
195 + req_state.output_text += new_decoded_token_text
196 + decoded_text += new_decoded_token_text
197 + return decoded_text
198 +
199 +
200 +@dataclass
201 +class RequestState:
202 +
203 + req_id: str
204 +
205 + token_ids: List[int]
206 + tokens: List[str]
207 + num_prompt_tokens: int
208 +
209 + prefix_offset: int
210 + read_offset: int
211 +
212 + skip_special_tokens: bool
213 + spaces_between_special_tokens: bool
214 +
215 + output_text: str = ""
vllm/v1/worker/__init__.py
vllm/v1/worker/gpu_model_runner.py new
+690
@@ -0,0 +1,690 @@
1 +from dataclasses import dataclass
2 +from typing import TYPE_CHECKING, Dict, List, Optional, Set
3 +from unittest.mock import patch
4 +
5 +import numpy as np
6 +import torch
7 +import torch.distributed
8 +import torch.nn as nn
9 +
10 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 + ModelConfig, ObservabilityConfig, ParallelConfig,
12 + PromptAdapterConfig, SchedulerConfig)
13 +from vllm.forward_context import set_forward_context
14 +from vllm.logger import init_logger
15 +from vllm.model_executor.model_loader import get_model
16 +from vllm.multimodal import MultiModalDataDict
17 +from vllm.sampling_params import SamplingParams, SamplingType
18 +from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
19 + is_pin_memory_available)
20 +from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21 + FlashAttentionMetadata)
22 +from vllm.v1.outputs import ModelRunnerOutput
23 +from vllm.v1.sample.metadata import SamplingMetadata
24 +from vllm.v1.sample.sampler import Sampler
25 +
26 +if TYPE_CHECKING:
27 + from vllm.v1.core.scheduler import SchedulerOutput
28 +
29 +logger = init_logger(__name__)
30 +
31 +
32 +class GPUModelRunner:
33 +
34 + def __init__(
35 + self,
36 + model_config: ModelConfig,
37 + parallel_config: ParallelConfig,
38 + scheduler_config: SchedulerConfig,
39 + device_config: DeviceConfig,
40 + cache_config: CacheConfig,
41 + load_config: LoadConfig,
42 + lora_config: Optional[LoRAConfig] = None,
43 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
44 + observability_config: Optional[ObservabilityConfig] = None,
45 + ):
46 + self.model_config = model_config
47 + self.parallel_config = parallel_config
48 + self.scheduler_config = scheduler_config
49 + self.device_config = device_config
50 + self.cache_config = cache_config
51 + self.lora_config = lora_config
52 + self.load_config = load_config
53 + self.prompt_adapter_config = prompt_adapter_config
54 + self.observability_config = observability_config
55 +
56 + self.device = self.device_config.device
57 + self.pin_memory = is_pin_memory_available()
58 + self.dtype = self.model_config.dtype
59 + if cache_config.cache_dtype == "auto":
60 + self.kv_cache_dtype = self.dtype
61 + else:
62 + self.kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
63 + cache_config.cache_dtype]
64 +
65 + self.sliding_window = model_config.get_sliding_window()
66 + self.block_size = cache_config.block_size
67 + self.max_model_len = model_config.max_model_len
68 + self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size)
69 + self.max_num_tokens = scheduler_config.max_num_batched_tokens
70 +
71 + # Model-related.
72 + self.num_attn_layers = model_config.get_num_attention_layers(
73 + parallel_config)
74 + self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
75 + self.head_size = model_config.get_head_size()
76 +
77 + # Lazy initialization
78 + # self.model: nn.Module # Set after load_model
79 + self.kv_caches: List[torch.Tensor] = []
80 +
81 + # Request states.
82 + self.requests: Dict[str, CachedRequestState] = {}
83 + # Persistent batch.
84 + self.input_batch = InputBatch(
85 + max_num_reqs=self.scheduler_config.max_num_seqs,
86 + max_model_len=self.max_model_len,
87 + max_num_blocks_per_req=self.max_num_blocks_per_req,
88 + device=self.device,
89 + pin_memory=self.pin_memory,
90 + )
91 +
92 + def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
93 + # Remove stopped requests from the cached states.
94 + # Keep the states of the pre-empted requests.
95 + for req_id in scheduler_output.finished_req_ids:
96 + self.requests.pop(req_id, None)
97 +
98 + # Remove the requests from the persistent batch.
99 + stopped_req_ids = set().union(
100 + scheduler_output.preempted_req_ids,
101 + scheduler_output.finished_req_ids,
102 + )
103 + removed_req_indices: List[int] = []
104 + for req_id in stopped_req_ids:
105 + req_index = self.input_batch.remove_request(req_id)
106 + if req_index is not None:
107 + removed_req_indices.append(req_index)
108 +
109 + # Update the states of the running requests.
110 + for req_data in scheduler_output.scheduled_running_reqs:
111 + req_id = req_data.req_id
112 + req_state = self.requests[req_id]
113 + req_index = self.input_batch.req_id_to_index[req_id]
114 +
115 + # Update the num_computed_tokens.
116 + req_state.num_computed_tokens = req_data.num_computed_tokens
117 + self.input_batch.num_computed_tokens_cpu[req_index] = (
118 + req_data.num_computed_tokens)
119 +
120 + # Update the block table.
121 + num_new_blocks = len(req_data.new_block_ids)
122 + if num_new_blocks == 0:
123 + continue
124 + start_index = len(req_state.block_ids)
125 + end_index = start_index + num_new_blocks
126 + req_state.block_ids.extend(req_data.new_block_ids)
127 + self.input_batch.block_table_cpu[
128 + req_index, start_index:end_index] = req_data.new_block_ids
129 +
130 + req_ids_to_add: List[str] = []
131 + # Add new requests to the cached states.
132 + for req_data in scheduler_output.scheduled_new_reqs:
133 + req_id = req_data.req_id
134 + self.requests[req_id] = CachedRequestState(
135 + req_id=req_id,
136 + prompt_token_ids=req_data.prompt_token_ids,
137 + prompt=req_data.prompt,
138 + multi_modal_data=req_data.multi_modal_data,
139 + sampling_params=req_data.sampling_params,
140 + generator=None, # TODO
141 + block_ids=req_data.block_ids,
142 + num_computed_tokens=req_data.num_computed_tokens,
143 + output_token_ids=[],
144 + )
145 + req_ids_to_add.append(req_id)
146 +
147 + # Update the cached states of the resumed requests.
148 + for req_data in scheduler_output.scheduled_resumed_reqs:
149 + req_id = req_data.req_id
150 + req_state = self.requests[req_id]
151 +
152 + req_state.block_ids = req_data.block_ids
153 + req_state.num_computed_tokens = req_data.num_computed_tokens
154 + req_ids_to_add.append(req_id)
155 +
156 + # Add the new or resumed requests to the persistent batch.
157 + # The smaller empty indices are filled first.
158 + removed_req_indices = sorted(removed_req_indices, reverse=True)
159 + for req_id in req_ids_to_add:
160 + req_state = self.requests[req_id]
161 + if removed_req_indices:
162 + # Fill the empty index.
163 + req_index = removed_req_indices.pop()
164 + else:
165 + # Append to the end.
166 + req_index = None
167 + self.input_batch.add_request(req_state, req_index)
168 +
169 + # Condense the batched states if there are empty indices.
170 + if removed_req_indices:
171 + self.input_batch.condense(removed_req_indices)
172 +
173 + def _prepare_inputs(self, scheduler_output: "SchedulerOutput"):
174 + total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
175 + assert total_num_scheduled_tokens > 0
176 + num_reqs = self.input_batch.num_reqs
177 + assert num_reqs > 0
178 +
179 + # OPTIMIZATION: Start copying the block table first.
180 + # This way, we can overlap the copy with the following CPU operations.
181 + self.input_batch.block_table[:num_reqs].copy_(
182 + self.input_batch.block_table_cpu_tensor[:num_reqs],
183 + non_blocking=True)
184 +
185 + # Get the number of scheduled tokens for each request.
186 + # TODO: The Python loop can be slow. Optimize.
187 + num_scheduled_tokens = []
188 + max_num_scheduled_tokens = 0
189 + for req_id in self.input_batch.req_ids[:num_reqs]:
190 + num_tokens = scheduler_output.num_scheduled_tokens[req_id]
191 + num_scheduled_tokens.append(num_tokens)
192 + max_num_scheduled_tokens = max(max_num_scheduled_tokens,
193 + num_tokens)
194 + num_scheduled_tokens = np.array(num_scheduled_tokens, dtype=np.int32)
195 + assert max_num_scheduled_tokens > 0
196 +
197 + # Get request indices.
198 + # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]
199 + indices = np.arange(num_reqs)
200 + req_indices = np.repeat(indices, num_scheduled_tokens)
201 +
202 + # Get batched arange.
203 + # E.g., [2, 5, 3] -> [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
204 + arange_matrix = np.tile(np.arange(max_num_scheduled_tokens),
205 + (num_reqs, 1))
206 + mask = arange_matrix < num_scheduled_tokens[:, np.newaxis]
207 + arange = arange_matrix[mask]
208 +
209 + # Get positions.
210 + positions = torch.empty((total_num_scheduled_tokens, ),
211 + dtype=torch.int32,
212 + device="cpu",
213 + pin_memory=self.pin_memory)
214 + positions_np = positions.numpy()
215 + np.add(self.input_batch.num_computed_tokens_cpu[req_indices],
216 + arange,
217 + out=positions_np)
218 +
219 + # Get token indices.
220 + # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
221 + # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
222 + # where M is the max_model_len.
223 + token_indices = positions_np + req_indices * self.max_model_len
224 + token_indices = torch.from_numpy(token_indices)
225 + input_ids = torch.empty((total_num_scheduled_tokens, ),
226 + dtype=torch.int32,
227 + device="cpu",
228 + pin_memory=self.pin_memory)
229 + torch.index_select(torch.from_numpy(
230 + self.input_batch.token_ids_cpu).flatten(),
231 + 0,
232 + token_indices,
233 + out=input_ids)
234 +
235 + # Calculate the slot mapping.
236 + block_numbers = self.input_batch.block_table_cpu_tensor.flatten()[
237 + token_indices // self.block_size]
238 + block_offsets = token_indices % self.block_size
239 + slot_mapping = torch.empty((total_num_scheduled_tokens, ),
240 + dtype=torch.int32,
241 + device="cpu",
242 + pin_memory=self.pin_memory)
243 + torch.add(block_numbers * self.block_size,
244 + block_offsets,
245 + out=slot_mapping)
246 +
247 + # Prepare the attention metadata.
248 + query_start_loc = torch.empty((num_reqs + 1, ),
249 + dtype=torch.int32,
250 + device="cpu",
251 + pin_memory=self.pin_memory)
252 + query_start_loc_np = query_start_loc.numpy()
253 + query_start_loc_np[0] = 0
254 + np.cumsum(num_scheduled_tokens, out=query_start_loc_np[1:])
255 +
256 + seq_lens = (self.input_batch.num_computed_tokens_cpu[:num_reqs] +
257 + num_scheduled_tokens)
258 + max_seq_len = seq_lens.max()
259 + seq_start_loc = torch.empty((num_reqs + 1, ),
260 + dtype=torch.int32,
261 + device="cpu",
262 + pin_memory=self.pin_memory)
263 + seq_start_loc_np = seq_start_loc.numpy()
264 + seq_start_loc_np[0] = 0
265 + np.cumsum(seq_lens, out=seq_start_loc_np[1:])
266 +
267 + input_ids = input_ids.to(self.device, non_blocking=True)
268 + positions = positions.to(self.device, non_blocking=True).long()
269 + query_start_loc = query_start_loc.to(self.device, non_blocking=True)
270 + seq_start_loc = seq_start_loc.to(self.device, non_blocking=True)
271 + slot_mapping = slot_mapping.to(self.device, non_blocking=True).long()
272 + attn_metadata = FlashAttentionMetadata(
273 + max_query_len=max_num_scheduled_tokens,
274 + query_start_loc=query_start_loc,
275 + max_seq_len=max_seq_len,
276 + seq_start_loc=seq_start_loc,
277 + block_table=self.input_batch.block_table[:num_reqs],
278 + slot_mapping=slot_mapping,
279 + )
280 + # NOTE(woosuk): Due to chunked prefills, there can be at most 1 partial
281 + # request in the batch. While we should not sample any token from this
282 + # partial request, we do so for simplicity. We will ignore the sampled
283 + # token from the partial request.
284 + # TODO: Support prompt logprobs.
285 + logits_indices = query_start_loc[1:] - 1
286 + return input_ids, positions, attn_metadata, logits_indices
287 +
288 + def _prepare_sampling(
289 + self,
290 + scheduler_output: "SchedulerOutput",
291 + ) -> SamplingMetadata:
292 + skip_copy = True
293 + if (scheduler_output.finished_req_ids
294 + or scheduler_output.preempted_req_ids):
295 + skip_copy = False
296 + if (scheduler_output.scheduled_new_reqs
297 + or scheduler_output.scheduled_resumed_reqs):
298 + skip_copy = False
299 + # Create the sampling metadata.
300 + sampling_metadata = self.input_batch.make_sampling_metadata(skip_copy)
301 + return sampling_metadata
302 +
303 + @torch.inference_mode()
304 + def execute_model(
305 + self,
306 + scheduler_output: "SchedulerOutput",
307 + ) -> ModelRunnerOutput:
308 + self._update_states(scheduler_output)
309 + inputs = self._prepare_inputs(scheduler_output)
310 + input_ids, positions, attn_metadata, logits_indices = inputs
311 +
312 + with set_forward_context(attn_metadata):
313 + hidden_states = self.model(
314 + input_ids=input_ids,
315 + positions=positions,
316 + kv_caches=self.kv_caches,
317 + attn_metadata=attn_metadata,
318 + )
319 + hidden_states = hidden_states[logits_indices]
320 + logits = self.model.compute_logits(hidden_states, None)
321 +
322 + # Sample the next token and get logprobs if needed.
323 + sampling_metadata = self._prepare_sampling(scheduler_output)
324 + sampler_output = self.model.sample(
325 + logits=logits,
326 + sampling_metadata=sampling_metadata,
327 + )
328 +
329 + # NOTE: CPU-GPU synchronization happens here.
330 + sampled_token_ids = sampler_output.sampled_token_ids.cpu()
331 + sampled_token_ids_list = sampled_token_ids.tolist()
332 + # TODO(woosuk): The following loop can be slow since it iterates over
333 + # the requests one by one. Optimize.
334 + num_reqs = self.input_batch.num_reqs
335 + for i, req_id in enumerate(self.input_batch.req_ids[:num_reqs]):
336 + req_state = self.requests[req_id]
337 + seq_len = (req_state.num_computed_tokens +
338 + scheduler_output.num_scheduled_tokens[req_id])
339 + assert seq_len <= req_state.num_tokens
340 + if seq_len == req_state.num_tokens:
341 + # Append the sampled token to the output token ids.
342 + token_id = sampled_token_ids_list[i]
343 + self.input_batch.token_ids_cpu[i, seq_len] = token_id
344 + req_state.output_token_ids.append(token_id)
345 + else:
346 + # Ignore the sampled token from the partial request.
347 + # Rewind the generator state as if the token was not sampled.
348 + generator = self.input_batch.generators[i]
349 + if generator is not None:
350 + offset = generator.get_offset()
351 + generator = generator.set_offset(offset - 1)
352 + self.input_batch.generators[i] = generator
353 +
354 + if sampler_output.logprob_token_ids is None:
355 + logprob_token_ids = None
356 + else:
357 + logprob_token_ids = sampler_output.logprob_token_ids.cpu()
358 + if sampler_output.logprobs is None:
359 + logprobs = None
360 + else:
361 + logprobs = sampler_output.logprobs.cpu()
362 + model_runner_output = ModelRunnerOutput(
363 + req_ids=self.input_batch.req_ids[:num_reqs],
364 + req_id_to_index=self.input_batch.req_id_to_index,
365 + sampled_token_ids_cpu=sampled_token_ids,
366 + logprob_token_ids_cpu=logprob_token_ids,
367 + logprobs_cpu=logprobs,
368 + )
369 + return model_runner_output
370 +
371 + def load_model(self) -> None:
372 + logger.info("Starting to load model %s...", self.model_config.model)
373 + with DeviceMemoryProfiler() as m: # noqa: SIM117
374 + with patch("vllm.model_executor.layers.sampler.Sampler", Sampler):
375 + self.model = get_model(model_config=self.model_config,
376 + device_config=self.device_config,
377 + load_config=self.load_config,
378 + lora_config=self.lora_config,
379 + parallel_config=self.parallel_config,
380 + scheduler_config=self.scheduler_config,
381 + cache_config=self.cache_config)
382 +
383 + self.model_memory_usage = m.consumed_memory
384 + logger.info("Loading model weights took %.4f GB",
385 + self.model_memory_usage / float(2**30))
386 +
387 + def _dummy_run(self, model: nn.Module, num_tokens: int) -> None:
388 + input_ids = torch.zeros(num_tokens,
389 + dtype=torch.int32,
390 + device=self.device)
391 + positions = torch.zeros(num_tokens,
392 + dtype=torch.long,
393 + device=self.device)
394 + kv_caches = [None for _ in range(self.num_attn_layers)]
395 + model(input_ids, positions, kv_caches, attn_metadata=None)
396 + return
397 +
398 + @torch.inference_mode()
399 + def profile_run(self) -> None:
400 + self._dummy_run(self.model, self.max_num_tokens)
401 + torch.cuda.synchronize()
402 + return
403 +
404 + @torch.inference_mode()
405 + def capture_model(self) -> None:
406 + # TODO: Implement CUDA graph support.
407 + return
408 +
409 + def initialize_kv_cache(self, num_blocks: int) -> None:
410 + assert len(self.kv_caches) == 0
411 + kv_cache_shape = FlashAttentionBackend.get_kv_cache_shape(
412 + num_blocks, self.block_size, self.num_kv_heads, self.head_size)
413 + for _ in range(self.num_attn_layers):
414 + self.kv_caches.append(
415 + torch.zeros(kv_cache_shape,
416 + dtype=self.kv_cache_dtype,
417 + device=self.device))
418 +
419 +
420 +@dataclass
421 +class CachedRequestState:
422 +
423 + req_id: str
424 + prompt_token_ids: List[int]
425 + prompt: Optional[str]
426 + multi_modal_data: Optional["MultiModalDataDict"]
427 + sampling_params: SamplingParams
428 + generator: Optional[torch.Generator]
429 +
430 + block_ids: List[int]
431 + num_computed_tokens: int
432 + output_token_ids: List[int]
433 +
434 + @property
435 + def num_tokens(self) -> int:
436 + return len(self.prompt_token_ids) + len(self.output_token_ids)
437 +
438 +
439 +class InputBatch:
440 +
441 + def __init__(
442 + self,
443 + max_num_reqs: int,
444 + max_model_len: int,
445 + max_num_blocks_per_req: int,
446 + device: torch.device,
447 + pin_memory: bool,
448 + ):
449 + self.max_num_reqs = max_num_reqs
450 + self.max_model_len = max_model_len
451 + self.max_num_blocks_per_req = max_num_blocks_per_req
452 + self.device = device
453 + self.pin_memory = pin_memory
454 +
455 + self.req_ids: List[Optional[str]] = [None] * max_num_reqs
456 + self.req_id_to_index: Dict[str, int] = {}
457 +
458 + self.token_ids_cpu = np.empty((max_num_reqs, max_model_len),
459 + dtype=np.int32)
460 + self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
461 +
462 + # Attention-related.
463 + self.block_table = torch.zeros((max_num_reqs, max_num_blocks_per_req),
464 + device=self.device,
465 + dtype=torch.int32)
466 + self.block_table_cpu_tensor = torch.zeros(
467 + (max_num_reqs, max_num_blocks_per_req),
468 + device="cpu",
469 + dtype=torch.int32,
470 + pin_memory=pin_memory,
471 + )
472 + self.block_table_cpu = self.block_table_cpu_tensor.numpy()
473 +
474 + # Sampling-related.
475 + self.temperature = torch.empty((max_num_reqs, ),
476 + dtype=torch.float32,
477 + device=device)
478 + self.temperature_cpu_tensor = torch.empty((max_num_reqs, ),
479 + dtype=torch.float32,
480 + device="cpu",
481 + pin_memory=pin_memory)
482 + self.temperature_cpu = self.temperature_cpu_tensor.numpy()
483 + self.greedy_reqs: Set[str] = set()
484 + self.random_reqs: Set[str] = set()
485 +
486 + self.top_p = torch.empty((max_num_reqs, ),
487 + dtype=torch.float32,
488 + device=device)
489 + self.top_p_cpu_tensor = torch.empty((max_num_reqs, ),
490 + dtype=torch.float32,
491 + device="cpu",
492 + pin_memory=pin_memory)
493 + self.top_p_cpu = self.top_p_cpu_tensor.numpy()
494 + self.top_p_reqs: Set[str] = set()
495 +
496 + self.top_k = torch.empty((max_num_reqs, ),
497 + dtype=torch.int32,
498 + device=device)
499 + self.top_k_cpu_tensor = torch.empty((max_num_reqs, ),
500 + dtype=torch.int32,
501 + device="cpu",
502 + pin_memory=pin_memory)
503 + self.top_k_cpu = self.top_k_cpu_tensor.numpy()
504 + self.top_k_reqs: Set[str] = set()
505 +
506 + self.generators: List[Optional[torch.Generator]] = [None
507 + ] * max_num_reqs
508 +
509 + self.num_logprobs: Dict[str, int] = {}
510 + self.prompt_logprob_reqs: Set[str] = set()
511 +
512 + def add_request(
513 + self,
514 + request: "CachedRequestState",
515 + req_index: Optional[int] = None,
516 + ) -> None:
517 + if req_index is None:
518 + req_index = self.num_reqs
519 + assert req_index < self.max_num_reqs
520 +
521 + self.req_ids[req_index] = request.req_id
522 + self.req_id_to_index[request.req_id] = req_index
523 +
524 + # Copy the prompt token ids and output token ids.
525 + num_prompt_tokens = len(request.prompt_token_ids)
526 + self.token_ids_cpu[
527 + req_index, :num_prompt_tokens] = request.prompt_token_ids
528 + start_idx = num_prompt_tokens
529 + end_idx = start_idx + len(request.output_token_ids)
530 + self.token_ids_cpu[req_index,
531 + start_idx:end_idx] = request.output_token_ids
532 +
533 + self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
534 + num_blocks = len(request.block_ids)
535 + self.block_table_cpu[req_index, :num_blocks] = request.block_ids
536 +
537 + sampling_params = request.sampling_params
538 + self.temperature_cpu[req_index] = sampling_params.temperature
539 + if sampling_params.sampling_type == SamplingType.GREEDY:
540 + self.greedy_reqs.add(req_index)
541 + elif sampling_params.sampling_type == SamplingType.RANDOM:
542 + self.random_reqs.add(req_index)
543 + elif sampling_params.sampling_type == SamplingType.RANDOM_SEED:
544 + # TODO(woosuk): Support per-request random seed.
545 + raise NotImplementedError("Per-request seed is not supported yet.")
546 +
547 + self.top_p_cpu[req_index] = sampling_params.top_p
548 + if sampling_params.top_p < 1:
549 + self.top_p_reqs.add(req_index)
550 + self.top_k_cpu[req_index] = sampling_params.top_k
551 + if sampling_params.top_k > 0:
552 + self.top_k_reqs.add(req_index)
553 +
554 + self.generators[req_index] = request.generator
555 +
556 + num_logprobs = sampling_params.logprobs
557 + if num_logprobs is not None and num_logprobs > 0:
558 + self.num_logprobs[request.req_id] = num_logprobs
559 + if sampling_params.prompt_logprobs:
560 + self.prompt_logprob_reqs.add(req_index)
561 +
562 + def remove_request(self, req_id: str) -> Optional[int]:
563 + req_index = self.req_id_to_index.pop(req_id, None)
564 + if req_index is None:
565 + return None
566 + self.req_ids[req_index] = None
567 +
568 + self.greedy_reqs.discard(req_id)
569 + self.random_reqs.discard(req_id)
570 + self.top_p_reqs.discard(req_id)
571 + self.top_k_reqs.discard(req_id)
572 + self.generators[req_index] = None
573 + self.num_logprobs.pop(req_id, None)
574 + self.prompt_logprob_reqs.discard(req_id)
575 + return req_index
576 +
577 + def clear(self) -> None:
578 + self.req_ids = [None] * self.max_num_reqs
579 + self.req_id_to_index.clear()
580 + self.greedy_reqs.clear()
581 + self.random_reqs.clear()
582 + self.top_p_reqs.clear()
583 + self.top_k_reqs.clear()
584 + self.generators.clear()
585 + self.num_logprobs.clear()
586 + self.prompt_logprob_reqs.clear()
587 +
588 + def condense(self, empty_req_indices: List[int]) -> None:
589 + if self.num_reqs == 0:
590 + # The batched states are empty.
591 + return
592 +
593 + # NOTE(woosuk): This function assumes that the empty_req_indices
594 + # is sorted in descending order.
595 + last_req_index = self.num_reqs + len(empty_req_indices) - 1
596 + while empty_req_indices:
597 + # Find the largest non-empty index.
598 + while last_req_index in empty_req_indices:
599 + last_req_index -= 1
600 +
601 + # Find the smallest empty index.
602 + empty_index = empty_req_indices.pop()
603 + if empty_index >= last_req_index:
604 + break
605 +
606 + # Swap the states.
607 + req_id = self.req_ids[last_req_index]
608 + self.req_ids[empty_index] = req_id
609 + self.req_ids[last_req_index] = None
610 + self.req_id_to_index[req_id] = empty_index
611 +
612 + # TODO(woosuk): Optimize the copy of token_ids_cpu and
613 + # block_table_cpu.
614 + self.token_ids_cpu[empty_index] = self.token_ids_cpu[
615 + last_req_index]
616 + self.num_computed_tokens_cpu[
617 + empty_index] = self.num_computed_tokens_cpu[last_req_index]
618 + self.block_table_cpu[empty_index] = self.block_table_cpu[
619 + last_req_index]
620 + self.temperature_cpu[empty_index] = self.temperature_cpu[
621 + last_req_index]
622 + self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
623 + self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
624 + self.generators[empty_index] = self.generators[last_req_index]
625 +
626 + # Decrement last_req_index since it is now empty.
627 + last_req_index -= 1
628 +
629 + def make_sampling_metadata(
630 + self,
631 + skip_copy: bool = False,
632 + ) -> SamplingMetadata:
633 + if not skip_copy:
634 + self.temperature[:self.num_reqs].copy_(
635 + self.temperature_cpu_tensor[:self.num_reqs], non_blocking=True)
636 + self.top_p[:self.num_reqs].copy_(
637 + self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
638 + self.top_k[:self.num_reqs].copy_(
639 + self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
640 + return SamplingMetadata(
641 + temperature=self.temperature[:self.num_reqs],
642 + all_greedy=self.all_greedy,
643 + all_random=self.all_random,
644 + top_p=self.top_p[:self.num_reqs],
645 + top_k=self.top_k[:self.num_reqs],
646 + no_top_p=self.no_top_p,
647 + no_top_k=self.no_top_k,
648 + generators=self.generators[:self.num_reqs],
649 + no_generator=self.no_generator,
650 + max_num_logprobs=self.max_num_logprobs,
651 + )
652 +
653 + @property
654 + def num_reqs(self) -> int:
655 + return len(self.req_id_to_index)
656 +
657 + @property
658 + def all_greedy(self) -> bool:
659 + return len(self.random_reqs) == 0
660 +
661 + @property
662 + def all_random(self) -> bool:
663 + return len(self.greedy_reqs) == 0
664 +
665 + @property
666 + def no_top_p(self) -> bool:
667 + return len(self.top_p_reqs) == 0
668 +
669 + @property
670 + def no_top_k(self) -> bool:
671 + return len(self.top_k_reqs) == 0
672 +
673 + @property
674 + def no_generator(self) -> bool:
675 + return len(self.generators) == 0
676 +
677 + @property
678 + def max_num_logprobs(self) -> int:
679 + if self.num_logprobs:
680 + return max(self.num_logprobs.values())
681 + else:
682 + return 0
683 +
684 + @property
685 + def no_logprob(self) -> bool:
686 + return len(self.num_logprobs) == 0
687 +
688 + @property
689 + def no_prompt_logprob(self) -> bool:
690 + return len(self.prompt_logprob_reqs) == 0
vllm/v1/worker/gpu_worker.py new
+245
@@ -0,0 +1,245 @@
1 +"""A GPU worker class."""
2 +import gc
3 +import os
4 +from typing import TYPE_CHECKING, Optional, Tuple
5 +
6 +import torch
7 +import torch.distributed
8 +
9 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 + ModelConfig, ObservabilityConfig, ParallelConfig,
11 + PromptAdapterConfig, SchedulerConfig,
12 + SpeculativeConfig)
13 +from vllm.distributed import (ensure_model_parallel_initialized,
14 + init_distributed_environment,
15 + set_custom_all_reduce)
16 +from vllm.logger import init_logger
17 +from vllm.model_executor import set_random_seed
18 +from vllm.platforms import current_platform
19 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
20 +from vllm.v1.outputs import ModelRunnerOutput
21 +from vllm.v1.worker.gpu_model_runner import GPUModelRunner
22 +
23 +logger = init_logger(__name__)
24 +
25 +if TYPE_CHECKING:
26 + from vllm.v1.core.scheduler import SchedulerOutput
27 +
28 +
29 +class Worker:
30 +
31 + def __init__(
32 + self,
33 + model_config: ModelConfig,
34 + parallel_config: ParallelConfig,
35 + scheduler_config: SchedulerConfig,
36 + device_config: DeviceConfig,
37 + cache_config: CacheConfig,
38 + load_config: LoadConfig,
39 + local_rank: int,
40 + rank: int,
41 + distributed_init_method: str,
42 + speculative_config: Optional[SpeculativeConfig] = None,
43 + lora_config: Optional[LoRAConfig] = None,
44 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
45 + observability_config: Optional[ObservabilityConfig] = None,
46 + ):
47 + self.model_config = model_config
48 + self.parallel_config = parallel_config
49 + self.scheduler_config = scheduler_config
50 + self.device_config = device_config
51 + self.cache_config = cache_config
52 + self.load_config = load_config
53 + self.local_rank = local_rank
54 + self.rank = rank
55 + self.distributed_init_method = distributed_init_method
56 + self.lora_config = lora_config
57 + self.speculative_config = speculative_config
58 + self.prompt_adapter_config = prompt_adapter_config
59 + self.observability_config = observability_config
60 +
61 + if self.model_config.trust_remote_code:
62 + # note: lazy import to avoid importing torch before initializing
63 + from vllm.utils import init_cached_hf_modules
64 + init_cached_hf_modules()
65 +
66 + self.model_runner = GPUModelRunner(
67 + model_config,
68 + parallel_config,
69 + scheduler_config,
70 + device_config,
71 + cache_config,
72 + load_config,
73 + lora_config=lora_config,
74 + )
75 +
76 + def initialize(self):
77 + if self.device_config.device.type == "cuda":
78 + # torch.distributed.all_reduce does not free the input tensor until
79 + # the synchronization point. This causes the memory usage to grow
80 + # as the number of all_reduce calls increases. This env var disables
81 + # this behavior.
82 + # Related issue:
83 + # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
84 + os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
85 +
86 + # This env var set by Ray causes exceptions with graph building.
87 + os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
88 + self.device = torch.device(f"cuda:{self.local_rank}")
89 + torch.cuda.set_device(self.device)
90 +
91 + _check_if_gpu_supports_dtype(self.model_config.dtype)
92 + gc.collect()
93 + torch.cuda.empty_cache()
94 + self.init_gpu_memory = torch.cuda.mem_get_info()[0]
95 + else:
96 + raise RuntimeError(
97 + f"Not support device type: {self.device_config.device}")
98 + # Initialize the distributed environment.
99 + init_worker_distributed_environment(self.parallel_config, self.rank,
100 + self.distributed_init_method,
101 + self.local_rank)
102 + # Set random seed.
103 + set_random_seed(self.model_config.seed)
104 +
105 + def load_model(self) -> None:
106 + self.model_runner.load_model()
107 +
108 + @torch.inference_mode()
109 + def determine_num_available_blocks(self) -> Tuple[int, int]:
110 + """Profiles the peak memory usage of the model to determine how many
111 + KV blocks may be allocated without OOMs.
112 +
113 + The engine will first conduct a profiling of the existing memory usage.
114 + Then, it calculate the maximum possible number of GPU and CPU blocks
115 + that can be allocated with the remaining free memory.
116 +
117 + .. tip::
118 + You may limit the usage of GPU memory
119 + by adjusting the `gpu_memory_utilization` parameter.
120 + """
121 + # Profile the memory usage of the model and get the maximum number of
122 + # cache blocks that can be allocated with the remaining free memory.
123 + torch.cuda.empty_cache()
124 +
125 + # Execute a forward pass with dummy inputs to profile the memory usage
126 + # of the model.
127 + self.model_runner.profile_run()
128 +
129 + # Calculate the number of blocks that can be allocated with the
130 + # profiled peak memory.
131 + torch.cuda.synchronize()
132 + free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info()
133 + # NOTE(woosuk): Here we assume that the other processes using the same
134 + # GPU did not change their memory usage during the profiling.
135 + peak_memory = self.init_gpu_memory - free_gpu_memory
136 + assert peak_memory > 0, (
137 + "Error in memory profiling. "
138 + f"Initial free memory {self.init_gpu_memory}, current free memory"
139 + f" {free_gpu_memory}. This happens when the GPU memory was "
140 + "not properly cleaned up before initializing the vLLM instance.")
141 +
142 + cache_block_size = _get_cache_block_size(self.cache_config,
143 + self.model_config,
144 + self.parallel_config)
145 + num_gpu_blocks = int(
146 + (total_gpu_memory * self.cache_config.gpu_memory_utilization -
147 + peak_memory) // cache_block_size)
148 + num_gpu_blocks = max(num_gpu_blocks, 0)
149 + # if self.model_runner.lora_manager:
150 + # self.model_runner.remove_all_loras()
151 + gc.collect()
152 + torch.cuda.empty_cache()
153 + return num_gpu_blocks, 0
154 +
155 + def initialize_cache(self, num_gpu_blocks: int) -> None:
156 + """Allocate GPU and CPU KV cache with the specified number of blocks."""
157 + if num_gpu_blocks <= 0:
158 + raise ValueError("No available memory for the cache blocks. "
159 + "Try increasing `gpu_memory_utilization` when "
160 + "initializing the engine.")
161 +
162 + max_seq_len = self.cache_config.block_size * num_gpu_blocks
163 + max_model_len = self.model_config.max_model_len
164 + if max_model_len > max_seq_len:
165 + raise ValueError(
166 + f"The model's max seq len ({max_model_len}) "
167 + "is larger than the maximum number of tokens that can be "
168 + f"stored in KV cache ({max_seq_len}). Try increasing "
169 + "`gpu_memory_utilization` or decreasing `max_model_len` when "
170 + "initializing the engine.")
171 +
172 + self.model_runner.initialize_kv_cache(num_gpu_blocks)
173 +
174 + def compile_or_warm_up_model(self) -> None:
175 + if not self.model_config.enforce_eager:
176 + self.model_runner.capture_model()
177 + # Reset the seed to ensure that the random state is not affected by
178 + # the model initialization and profiling.
179 + set_random_seed(self.model_config.seed)
180 +
181 + @torch.inference_mode()
182 + def execute_model(
183 + self,
184 + scheduler_output: "SchedulerOutput",
185 + ) -> ModelRunnerOutput:
186 + output = self.model_runner.execute_model(scheduler_output)
187 + # TODO(woosuk): Send the output to the engine process.
188 + return output
189 +
190 +
191 +def init_worker_distributed_environment(
192 + parallel_config: ParallelConfig,
193 + rank: int,
194 + distributed_init_method: Optional[str] = None,
195 + local_rank: int = -1,
196 +) -> None:
197 + """Initialize the distributed environment."""
198 + set_custom_all_reduce(not parallel_config.disable_custom_all_reduce)
199 +
200 + init_distributed_environment(parallel_config.world_size, rank,
201 + distributed_init_method, local_rank)
202 +
203 + ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
204 + parallel_config.pipeline_parallel_size)
205 +
206 +
207 +def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
208 + # Check if the GPU supports the dtype.
209 + if torch_dtype == torch.bfloat16: # noqa: SIM102
210 + if not current_platform.has_device_capability(80):
211 + capability = current_platform.get_device_capability()
212 + gpu_name = current_platform.get_device_name()
213 +
214 + if capability is None:
215 + compute_str = "does not have a compute capability"
216 + else:
217 + version_str = capability.as_version_str()
218 + compute_str = f"has compute capability {version_str}"
219 +
220 + raise ValueError(
221 + "Bfloat16 is only supported on GPUs with compute capability "
222 + f"of at least 8.0. Your {gpu_name} GPU {compute_str}. "
223 + "You can use float16 instead by explicitly setting the"
224 + "`dtype` flag in CLI, for example: --dtype=half.")
225 +
226 +
227 +def _get_cache_block_size(
228 + cache_config: CacheConfig,
229 + model_config: ModelConfig,
230 + parallel_config: ParallelConfig,
231 +) -> int:
232 + head_size = model_config.get_head_size()
233 + num_heads = model_config.get_num_kv_heads(parallel_config)
234 + num_attention_layers = model_config.get_num_attention_layers(
235 + parallel_config)
236 +
237 + key_cache_block = cache_config.block_size * num_heads * head_size
238 + value_cache_block = key_cache_block
239 + total = num_attention_layers * (key_cache_block + value_cache_block)
240 + if cache_config.cache_dtype == "auto":
241 + dtype = model_config.dtype
242 + else:
243 + dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
244 + dtype_size = get_dtype_size(dtype)
245 + return dtype_size * total