@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bbd3e86926
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[V1] Support VLMs with fine-grained scheduling (#9871)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Roger Wang <ywang@roblox.com>

Woosuk Kwon committed Nov 12, 2024 at 20:53 UTC bbd3e86926f15e59e4c62246b4b3185e71fe7ff2
12 files changed +542 -96
vllm/model_executor/models/gpt2.py
+9 -2
@@ -216,9 +216,11 @@ class GPT2Model(nn.Module):
216 kv_caches: List[torch.Tensor],
217 attn_metadata: AttentionMetadata,
218 intermediate_tensors: Optional[IntermediateTensors],
219 + inputs_embeds: Optional[torch.Tensor],
220 ) -> Union[torch.Tensor, IntermediateTensors]:
221 if get_pp_group().is_first_rank:
221 - inputs_embeds = self.wte(input_ids)
222 + if inputs_embeds is None:
223 + inputs_embeds = self.wte(input_ids)
224 position_embeds = self.wpe(position_ids)
225 hidden_states = inputs_embeds + position_embeds
226 else:
@@ -263,6 +265,9 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
265 self.make_empty_intermediate_tensors = (
266 self.transformer.make_empty_intermediate_tensors)
267
268 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
269 + return self.transformer.wte(input_ids)
270 +
271 def forward(
272 self,
273 input_ids: torch.Tensor,
@@ -270,9 +275,11 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
275 kv_caches: List[torch.Tensor],
276 attn_metadata: AttentionMetadata,
277 intermediate_tensors: Optional[IntermediateTensors] = None,
278 + inputs_embeds: Optional[torch.Tensor] = None,
279 ) -> Union[torch.Tensor, IntermediateTensors]:
280 hidden_states = self.transformer(input_ids, positions, kv_caches,
275 - attn_metadata, intermediate_tensors)
281 + attn_metadata, intermediate_tensors,
282 + inputs_embeds)
283 return hidden_states
284
285 def compute_logits(
vllm/model_executor/models/llama.py
+6 -1
@@ -538,6 +538,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
538 normalize=False,
539 softmax=False)
540
541 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
542 + return self.model.get_input_embeddings(input_ids)
543 +
544 def forward(
545 self,
546 input_ids: torch.Tensor,
@@ -545,9 +548,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
548 kv_caches: List[torch.Tensor],
549 attn_metadata: AttentionMetadata,
550 intermediate_tensors: Optional[IntermediateTensors] = None,
551 + inputs_embeds: Optional[torch.Tensor] = None,
552 ) -> Union[torch.Tensor, IntermediateTensors]:
553 model_output = self.model(input_ids, positions, kv_caches,
550 - attn_metadata, intermediate_tensors)
554 + attn_metadata, intermediate_tensors,
555 + inputs_embeds)
556 return model_output
557
558 def compute_logits(
vllm/model_executor/models/llava.py
+28 -18
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
20 +from vllm.multimodal.base import NestedTensors
21 from vllm.sequence import IntermediateTensors
22 from vllm.utils import is_list_of
23
@@ -448,6 +449,25 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
449 image_features = self._process_image_pixels(image_input)
450 return self.multi_modal_projector(image_features)
451
452 + def process_mm_inputs(self, **kwargs):
453 + image_input = self._parse_and_validate_image_input(**kwargs)
454 + if image_input is None:
455 + return None
456 + vision_embeddings = self._process_image_input(image_input)
457 + return vision_embeddings
458 +
459 + def get_input_embeddings(
460 + self,
461 + input_ids: torch.Tensor,
462 + vision_embeddings: Optional[NestedTensors] = None,
463 + ) -> torch.Tensor:
464 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
465 + if vision_embeddings is not None:
466 + inputs_embeds = merge_multimodal_embeddings(
467 + input_ids, inputs_embeds, vision_embeddings,
468 + self.config.image_token_index)
469 + return inputs_embeds
470 +
471 def forward(
472 self,
473 input_ids: torch.Tensor,
@@ -455,6 +475,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
475 kv_caches: List[torch.Tensor],
476 attn_metadata: AttentionMetadata,
477 intermediate_tensors: Optional[IntermediateTensors] = None,
478 + inputs_embeds: Optional[torch.Tensor] = None,
479 **kwargs: object,
480 ) -> Union[torch.Tensor, IntermediateTensors]:
481 """Run forward pass for LLaVA-1.5.
@@ -494,24 +515,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
515 """
516 if intermediate_tensors is not None:
517 inputs_embeds = None
497 - else:
498 - image_input = self._parse_and_validate_image_input(**kwargs)
499 - if image_input is not None:
500 - vision_embeddings = self._process_image_input(image_input)
501 - inputs_embeds = self.language_model.model.get_input_embeddings(
502 - input_ids)
503 -
504 - inputs_embeds = merge_multimodal_embeddings(
505 - input_ids, inputs_embeds, vision_embeddings,
506 - self.config.image_token_index)
507 - else:
508 - inputs_embeds = self.language_model.model.get_input_embeddings(
509 - input_ids)
510 -
511 - # always pass the input via `inputs_embeds`
512 - # to make sure the computation graph is consistent
513 - # for `torch.compile` integration
514 - input_ids = None
518 + elif inputs_embeds is None:
519 + vision_embeddings = self.process_mm_inputs(**kwargs)
520 + # always pass the input via `inputs_embeds`
521 + # to make sure the computation graph is consistent
522 + inputs_embeds = self.get_input_embeddings(input_ids,
523 + vision_embeddings)
524 + input_ids = None
525
526 hidden_states = self.language_model.model(input_ids,
527 positions,
vllm/model_executor/models/opt.py
+6 -1
@@ -360,6 +360,9 @@ class OPTForCausalLM(nn.Module, SupportsPP):
360 self.make_empty_intermediate_tensors = (
361 self.model.make_empty_intermediate_tensors)
362
363 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
364 + return self.model.get_input_embeddings(input_ids)
365 +
366 def forward(
367 self,
368 input_ids: torch.Tensor,
@@ -367,9 +370,11 @@ class OPTForCausalLM(nn.Module, SupportsPP):
370 kv_caches: List[torch.Tensor],
371 attn_metadata: AttentionMetadata,
372 intermediate_tensors: Optional[IntermediateTensors] = None,
373 + inputs_embeds: Optional[torch.Tensor] = None,
374 ) -> Union[torch.Tensor, IntermediateTensors]:
375 hidden_states = self.model(input_ids, positions, kv_caches,
372 - attn_metadata, intermediate_tensors)
376 + attn_metadata, intermediate_tensors,
377 + inputs_embeds)
378 return hidden_states
379
380 def compute_logits(
vllm/model_executor/models/phi3v.py
+40 -23
@@ -39,6 +39,7 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
39 from vllm.model_executor.pooling_metadata import PoolingMetadata
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 from vllm.multimodal import MULTIMODAL_REGISTRY
42 +from vllm.multimodal.base import NestedTensors, PlaceholderRange
43 from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
44 from vllm.sequence import IntermediateTensors, PoolerOutput
45 from vllm.utils import is_list_of
@@ -500,15 +501,20 @@ def input_processor_for_phi3v(ctx: InputContext,
501
502 # TODO: Move this to utils or integrate with clip.
503 new_token_ids: List[int] = []
504 + placeholder_ranges: List[PlaceholderRange] = []
505 placeholder_idx = 0
506 while merged_token_ids:
507 token_id = merged_token_ids.pop(0)
508 if token_id == _IMAGE_TOKEN_ID:
507 - new_token_ids.extend(
508 - repeat_and_pad_token(
509 - _IMAGE_TOKEN_ID,
510 - repeat_count=image_feature_size[placeholder_idx],
511 - ))
509 + replacement_ids = repeat_and_pad_token(
510 + _IMAGE_TOKEN_ID,
511 + repeat_count=image_feature_size[placeholder_idx],
512 + )
513 + placeholder_ranges.append({
514 + "offset": len(new_token_ids),
515 + "length": len(replacement_ids)
516 + })
517 + new_token_ids.extend(replacement_ids)
518 placeholder_idx += 1
519 else:
520 new_token_ids.append(token_id)
@@ -516,7 +522,8 @@ def input_processor_for_phi3v(ctx: InputContext,
522 # NOTE: Create a defensive copy of the original inputs
523 return token_inputs(prompt_token_ids=new_token_ids,
524 prompt=new_prompt,
519 - multi_modal_data=multi_modal_data)
525 + multi_modal_data=multi_modal_data,
526 + multi_modal_placeholders={"image": placeholder_ranges})
527
528
529 @MULTIMODAL_REGISTRY.register_image_input_mapper()
@@ -669,32 +676,42 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
676
677 return image_embeds
678
679 + def process_mm_inputs(self, **kwargs):
680 + image_input = self._parse_and_validate_image_input(**kwargs)
681 + if image_input is None:
682 + return None
683 + vision_embeddings = self._process_image_input(image_input)
684 + return vision_embeddings
685 +
686 + def get_input_embeddings(
687 + self,
688 + input_ids: torch.Tensor,
689 + vision_embeddings: Optional[NestedTensors] = None,
690 + ) -> torch.Tensor:
691 + inputs_embeds = self.embed_tokens(input_ids)
692 + if vision_embeddings is not None:
693 + inputs_embeds = merge_multimodal_embeddings(
694 + input_ids, inputs_embeds, vision_embeddings,
695 + self.image_token_id)
696 + return inputs_embeds
697 +
698 def forward(self,
699 input_ids: torch.Tensor,
700 positions: torch.Tensor,
701 kv_caches: List[torch.Tensor],
702 attn_metadata: AttentionMetadata,
703 intermediate_tensors: Optional[IntermediateTensors] = None,
704 + inputs_embeds: Optional[torch.Tensor] = None,
705 **kwargs: object):
706 if intermediate_tensors is not None:
707 inputs_embeds = None
681 - else:
682 - image_input = self._parse_and_validate_image_input(**kwargs)
683 -
684 - if image_input is not None:
685 - vision_embeddings = self._process_image_input(image_input)
686 - inputs_embeds = self.embed_tokens(input_ids)
687 - inputs_embeds = merge_multimodal_embeddings(
688 - input_ids, inputs_embeds, vision_embeddings,
689 - self.image_token_id)
690 - else:
691 - inputs_embeds = self.language_model.model.embed_tokens(
692 - input_ids)
693 -
694 - # always pass the input via `inputs_embeds`
695 - # to make sure the computation graph is consistent
696 - # for `torch.compile` integration
697 - input_ids = None
708 + elif inputs_embeds is None:
709 + vision_embeddings = self.process_mm_inputs(**kwargs)
710 + # always pass the input via `inputs_embeds`
711 + # to make sure the computation graph is consistent
712 + inputs_embeds = self.get_input_embeddings(input_ids,
713 + vision_embeddings)
714 + input_ids = None
715
716 hidden_states = self.language_model.model(input_ids,
717 positions,
vllm/model_executor/models/qwen2.py
+6 -1
@@ -441,6 +441,9 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
441 self.make_empty_intermediate_tensors = (
442 self.model.make_empty_intermediate_tensors)
443
444 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
445 + return self.model.get_input_embeddings(input_ids)
446 +
447 def forward(
448 self,
449 input_ids: torch.Tensor,
@@ -448,9 +451,11 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
451 kv_caches: List[torch.Tensor],
452 attn_metadata: AttentionMetadata,
453 intermediate_tensors: Optional[IntermediateTensors] = None,
454 + inputs_embeds: Optional[torch.Tensor] = None,
455 ) -> Union[torch.Tensor, IntermediateTensors]:
456 hidden_states = self.model(input_ids, positions, kv_caches,
453 - attn_metadata, intermediate_tensors)
457 + attn_metadata, intermediate_tensors,
458 + inputs_embeds)
459 return hidden_states
460
461 def compute_logits(
vllm/v1/core/encoder_cache_manager.py new
+48
@@ -0,0 +1,48 @@
1 +from typing import Dict, List, Set, Tuple
2 +
3 +from vllm.v1.request import Request
4 +
5 +
6 +class EncoderCacheManager:
7 +
8 + def __init__(self, cache_size: int):
9 + self.cache_size = cache_size
10 + self.num_free_slots = cache_size
11 + # req_id -> cached input ids
12 + self.cached: Dict[str, Set[int]] = {}
13 + # List of [req_id, input_id]
14 + self.freed: List[Tuple[str, int]] = []
15 +
16 + def has_cache(self, request: Request, input_id: int) -> bool:
17 + req_id = request.request_id
18 + return req_id in self.cached and input_id in self.cached[req_id]
19 +
20 + def can_allocate(self, request: Request, input_id: int) -> bool:
21 + num_tokens = request.get_num_encoder_tokens(input_id)
22 + return num_tokens <= self.num_free_slots
23 +
24 + def allocate(self, request: Request, input_id: int) -> None:
25 + req_id = request.request_id
26 + if req_id not in self.cached:
27 + self.cached[req_id] = set()
28 + self.cached[req_id].add(input_id)
29 + self.num_free_slots -= request.get_num_encoder_tokens(input_id)
30 +
31 + def get_cached_input_ids(self, request: Request) -> Set[int]:
32 + return self.cached.get(request.request_id, set())
33 +
34 + def free(self, request: Request, input_id: int) -> None:
35 + req_id = request.request_id
36 + if req_id not in self.cached:
37 + return
38 +
39 + self.cached[req_id].discard(input_id)
40 + if len(self.cached[req_id]) == 0:
41 + del self.cached[req_id]
42 + self.num_free_slots += request.get_num_encoder_tokens(input_id)
43 + self.freed.append((req_id, input_id))
44 +
45 + def get_freed_ids(self) -> List[Tuple[str, int]]:
46 + freed = self.freed
47 + self.freed = []
48 + return freed
vllm/v1/core/scheduler.py
+181 -24
@@ -1,16 +1,21 @@
1 from collections import deque
2 from dataclasses import dataclass
3 -from typing import Deque, Dict, Iterable, List, Optional, Set, Union
3 +from typing import (TYPE_CHECKING, Deque, Dict, Iterable, List, Optional, Set,
4 + Tuple, Union)
5
6 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7 from vllm.logger import init_logger
7 -from vllm.multimodal import MultiModalDataDict
8 from vllm.sampling_params import SamplingParams
9 +from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
10 from vllm.v1.core.kv_cache_manager import KVCacheManager
11 from vllm.v1.engine import EngineCoreOutput
12 from vllm.v1.outputs import ModelRunnerOutput
13 from vllm.v1.request import Request, RequestStatus
14
15 +if TYPE_CHECKING:
16 + from vllm.multimodal import MultiModalKwargs
17 + from vllm.multimodal.base import PlaceholderRange
18 +
19 logger = init_logger(__name__)
20
21
@@ -61,12 +66,20 @@ class Scheduler:
66 # Request id -> RunningRequestData
67 self.running_reqs_data: Dict[str, RunningRequestData] = {}
68
64 - def schedule(self) -> "SchedulerOutput":
65 - scheduled_new_reqs: List[Request] = []
66 - scheduled_resumed_reqs: List[Request] = []
67 - scheduled_running_reqs: List[Request] = []
68 - preempted_reqs: List[Request] = []
69 + # Encoder-related.
70 + # NOTE(woosuk): Here, "encoder" includes the vision encoder (and
71 + # projector if needed). Currently, we assume that the encoder also
72 + # has the Transformer architecture (e.g., ViT).
73 + # FIXME(woosuk): Below are placeholder values. We need to calculate the
74 + # actual values from the configurations.
75 + self.max_num_encoder_input_tokens = 2048
76 + # NOTE(woosuk): For the models without encoder (e.g., text-only models),
77 + # the encoder cache will not be initialized and used, regardless of
78 + # the cache size. This is because the memory space for the encoder cache
79 + # is preallocated in the profiling run.
80 + self.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
81
82 + def schedule(self) -> "SchedulerOutput":
83 # NOTE(woosuk) on the scheduling algorithm:
84 # There's no "decoding phase" nor "prefill phase" in the scheduler.
85 # Each request just has the num_computed_tokens and num_tokens,
@@ -74,23 +87,45 @@ class Scheduler:
87 # At each step, the scheduler tries to assign tokens to the requests
88 # so that each request's num_computed_tokens can catch up its
89 # num_tokens. This is general enough to cover chunked prefills,
77 - # prefix caching, and the "jump forward" optimization in the future.
90 + # prefix caching, and the "jump decoding" optimization in the future.
91 +
92 + scheduled_new_reqs: List[Request] = []
93 + scheduled_resumed_reqs: List[Request] = []
94 + scheduled_running_reqs: List[Request] = []
95 + preempted_reqs: List[Request] = []
96
97 req_to_new_block_ids: Dict[str, List[int]] = {}
98 num_scheduled_tokens: Dict[str, int] = {}
99 token_budget = self.max_num_scheduled_tokens
100 + # Encoder-related.
101 + scheduled_encoder_inputs: Dict[str, List[int]] = {}
102 + encoder_budget = self.max_num_encoder_input_tokens
103
104 # First, schedule the RUNNING requests.
105 + # NOTE(woosuk): At most 1 request in the RUNNING queue is allowed to be
106 + # in the "partial" state, where the request has some tokens computed
107 + # but not all. The constraint is due to the persistent batch in the
108 + # V1 model runner.
109 + # TODO(woosuk): Remove this constraint after refactoring model runner.
110 + has_partial_request = False
111 req_index = 0
112 while req_index < len(self.running):
86 - if token_budget == 0:
87 - break
88 -
113 + # Only the last request in the RUNNING queue can be "partial".
114 + assert not has_partial_request
115 + assert token_budget > 0
116 request = self.running[req_index]
117 num_new_tokens = request.num_tokens - request.num_computed_tokens
118 num_new_tokens = min(num_new_tokens, token_budget)
119 assert num_new_tokens > 0
120
121 + # Schedule encoder inputs.
122 + encoder_inputs_to_schedule, num_new_tokens, new_encoder_budget = (
123 + self._try_schedule_encoder_inputs(request,
124 + request.num_computed_tokens,
125 + num_new_tokens,
126 + encoder_budget))
127 + assert num_new_tokens > 0
128 +
129 while True:
130 new_blocks = self.kv_cache_manager.append_slots(
131 request, num_new_tokens)
@@ -106,22 +141,40 @@ class Scheduler:
141 preempted_reqs.append(preempted_req)
142 if preempted_req == request:
143 # No more request to preempt.
144 + can_schedule = False
145 break
146 else:
147 # The request can be scheduled.
112 - scheduled_running_reqs.append(request)
113 -
114 - req_to_new_block_ids[request.request_id] = [
115 - b.block_id for b in new_blocks
116 - ]
117 - num_scheduled_tokens[request.request_id] = num_new_tokens
118 - token_budget -= num_new_tokens
119 - req_index += 1
148 + can_schedule = True
149 break
150 + if not can_schedule:
151 + break
152 +
153 + # Schedule the request.
154 + scheduled_running_reqs.append(request)
155 + req_to_new_block_ids[request.request_id] = [
156 + b.block_id for b in new_blocks
157 + ]
158 + num_scheduled_tokens[request.request_id] = num_new_tokens
159 + token_budget -= num_new_tokens
160 + req_index += 1
161 + has_partial_request = (request.num_computed_tokens + num_new_tokens
162 + < request.num_tokens)
163 +
164 + # Encoder-related.
165 + if encoder_inputs_to_schedule:
166 + scheduled_encoder_inputs[request.request_id] = (
167 + encoder_inputs_to_schedule)
168 + # Allocate the encoder cache.
169 + for i in encoder_inputs_to_schedule:
170 + self.encoder_cache_manager.allocate(request, i)
171 + encoder_budget = new_encoder_budget
172
173 # Next, schedule the WAITING requests.
174 if not preempted_reqs:
175 while self.waiting:
176 + if has_partial_request:
177 + break
178 if len(self.running) == self.max_num_running_reqs:
179 break
180 if token_budget == 0:
@@ -149,12 +202,21 @@ class Scheduler:
202 computed_blocks.pop()
203 num_new_tokens = min(num_new_tokens, token_budget)
204 assert num_new_tokens > 0
205 +
206 + # Schedule encoder inputs.
207 + (encoder_inputs_to_schedule, num_new_tokens,
208 + new_encoder_budget) = self._try_schedule_encoder_inputs(
209 + request, num_computed_tokens, num_new_tokens,
210 + encoder_budget)
211 + if num_new_tokens == 0:
212 + # The request cannot be scheduled.
213 + break
214 +
215 new_blocks = self.kv_cache_manager.allocate_slots(
216 request, num_new_tokens, computed_blocks)
217 if new_blocks is None:
218 # The request cannot be scheduled.
219 break
157 - request.num_computed_tokens = num_computed_tokens
220
221 self.waiting.popleft()
222 self.running.append(request)
@@ -172,6 +234,18 @@ class Scheduler:
234 num_scheduled_tokens[request.request_id] = num_new_tokens
235 token_budget -= num_new_tokens
236 request.status = RequestStatus.RUNNING
237 + request.num_computed_tokens = num_computed_tokens
238 + has_partial_request = (num_computed_tokens + num_new_tokens <
239 + request.num_tokens)
240 +
241 + # Encoder-related.
242 + if encoder_inputs_to_schedule:
243 + scheduled_encoder_inputs[request.request_id] = (
244 + encoder_inputs_to_schedule)
245 + # Allocate the encoder cache.
246 + for i in encoder_inputs_to_schedule:
247 + self.encoder_cache_manager.allocate(request, i)
248 + encoder_budget = new_encoder_budget
249
250 # Check if the scheduling constraints are satisfied.
251 total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
@@ -205,12 +279,14 @@ class Scheduler:
279 scheduled_running_reqs=running_reqs_data,
280 num_scheduled_tokens=num_scheduled_tokens,
281 total_num_scheduled_tokens=total_num_scheduled_tokens,
282 + scheduled_encoder_inputs=scheduled_encoder_inputs,
283 preempted_req_ids=preempted_req_ids,
284 # finished_req_ids is an existing state in the scheduler,
285 # instead of being newly scheduled in this step.
286 # It contains the request IDs that are finished in between
287 # the previous and the current steps.
288 finished_req_ids=self.finished_req_ids,
289 + free_encoder_input_ids=self.encoder_cache_manager.get_freed_ids(),
290 )
291
292 self.finished_req_ids = set()
@@ -234,6 +310,72 @@ class Scheduler:
310 self.running_reqs_data[request.request_id] = req_data
311 return req_data
312
313 + def _try_schedule_encoder_inputs(
314 + self,
315 + request: Request,
316 + num_computed_tokens: int,
317 + num_new_tokens: int,
318 + encoder_budget: int,
319 + ) -> Tuple[List[int], int, int]:
320 + """
321 + Determine which encoder inputs need to be scheduled in the current step,
322 + and update `num_new_tokens` and encoder token budget accordingly.
323 +
324 + An encoder input will be scheduled if:
325 + - Its output tokens overlap with the range of tokens being computed
326 + in this step, i.e.,
327 + [num_computed_tokens, num_computed_tokens + num_new_tokens).
328 + - It is not already computed and stored in the encoder cache.
329 + - There is sufficient encoder token budget to process it.
330 + - The encoder cache has space to store it.
331 +
332 + If an encoder input cannot be scheduled due to cache or budget
333 + limitations, the method adjusts `num_new_tokens` to schedule only the
334 + decoder tokens up to just before the unschedulable encoder input.
335 + """
336 + if not request.has_encoder_inputs():
337 + return [], num_new_tokens, encoder_budget
338 +
339 + encoder_inputs_to_schedule: List[int] = []
340 + mm_positions = request.mm_positions
341 + assert mm_positions is not None
342 + assert len(mm_positions) > 0
343 + for i, pos_info in enumerate(mm_positions):
344 + start_pos = pos_info["offset"]
345 + num_encoder_tokens = pos_info["length"]
346 +
347 + # The encoder output is needed if the two ranges overlap:
348 + # [num_computed_tokens, num_computed_tokens + num_new_tokens) and
349 + # [start_pos, start_pos + num_encoder_tokens)
350 + if start_pos >= num_computed_tokens + num_new_tokens:
351 + # The encoder input is not needed in this step.
352 + break
353 + if start_pos + num_encoder_tokens <= num_computed_tokens:
354 + # The encoder input is already computed and stored
355 + # in the decoder's KV cache.
356 + continue
357 +
358 + if self.encoder_cache_manager.has_cache(request, i):
359 + # The encoder input is already computed and cached.
360 + continue
361 + if not self.encoder_cache_manager.can_allocate(request, i):
362 + # The encoder cache is full. We can only schedule the decoder
363 + # tokens just before the encoder input.
364 + num_new_tokens = start_pos - num_computed_tokens
365 + break
366 + if num_encoder_tokens > encoder_budget:
367 + # The encoder budget is exhausted. We can only schedule the
368 + # decoder tokens up until the encoder input.
369 + # NOTE(woosuk): We assume that the encoder tokens should be
370 + # processed altogether, as the encoder usually uses
371 + # bidirectional attention.
372 + num_new_tokens = start_pos - num_computed_tokens
373 + break
374 +
375 + encoder_budget -= num_encoder_tokens
376 + encoder_inputs_to_schedule.append(i)
377 + return encoder_inputs_to_schedule, num_new_tokens, encoder_budget
378 +
379 def update_from_output(
380 self,
381 scheduler_output: "SchedulerOutput",
@@ -251,6 +393,17 @@ class Scheduler:
393 # the request generates output tokens. Otherwise, we ignore the
394 # sampler output for the request.
395 assert request.num_computed_tokens <= request.num_tokens
396 +
397 + cached_encoder_input_ids = (
398 + self.encoder_cache_manager.get_cached_input_ids(request))
399 + for input_id in list(cached_encoder_input_ids):
400 + start_pos = request.mm_positions[input_id]["offset"]
401 + num_tokens = request.mm_positions[input_id]["length"]
402 + if start_pos + num_tokens <= request.num_computed_tokens:
403 + # The encoder output is already processed and stored
404 + # in the decoder's KV cache.
405 + self.encoder_cache_manager.free(request, input_id)
406 +
407 if request.num_computed_tokens == request.num_tokens:
408 req_index = model_runner_output.req_id_to_index[req_id]
409 # NOTE(woosuk): Currently, we assume that each request
@@ -355,7 +508,8 @@ class NewRequestData:
508 req_id: str
509 prompt_token_ids: List[int]
510 prompt: Optional[str]
358 - multi_modal_data: Optional[MultiModalDataDict]
511 + mm_inputs: List["MultiModalKwargs"]
512 + mm_positions: List["PlaceholderRange"]
513 sampling_params: SamplingParams
514 block_ids: List[int]
515 num_computed_tokens: int
@@ -369,9 +523,10 @@ class NewRequestData:
523 ) -> "NewRequestData":
524 return cls(
525 req_id=request.request_id,
372 - prompt_token_ids=request.inputs["prompt_token_ids"],
373 - prompt=request.inputs.get("prompt"),
374 - multi_modal_data=request.inputs.get("multi_modal_data"),
526 + prompt_token_ids=request.prompt_token_ids,
527 + prompt=request.prompt,
528 + mm_inputs=request.mm_inputs,
529 + mm_positions=request.mm_positions,
530 sampling_params=request.sampling_params,
531 block_ids=block_ids,
532 num_computed_tokens=num_computed_tokens,
@@ -429,6 +584,8 @@ class SchedulerOutput:
584
585 num_scheduled_tokens: Dict[str, int]
586 total_num_scheduled_tokens: int
587 + scheduled_encoder_inputs: Dict[str, List[int]]
588
589 preempted_req_ids: Set[str]
590 finished_req_ids: Set[str]
591 + free_encoder_input_ids: List[Tuple[str, int]]
vllm/v1/engine/core.py
+10
@@ -17,6 +17,7 @@ from vllm.usage.usage_lib import UsageContext
17 from vllm.v1.core.scheduler import Scheduler
18 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
19 EngineCoreRequest, EngineCoreRequestType)
20 +from vllm.v1.engine.mm_input_mapper import MMInputMapper
21 from vllm.v1.executor.gpu_executor import GPUExecutor
22 from vllm.v1.request import Request, RequestStatus
23 from vllm.v1.serial_utils import PickleEncoder
@@ -65,6 +66,9 @@ class EngineCore:
66 vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
67 vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks
68
69 + # Set up multimodal input mapper (e.g., convert PIL images to tensors).
70 + self.mm_input_mapper = MMInputMapper(vllm_config.model_config)
71 +
72 # Setup scheduler.
73 self.scheduler = Scheduler(vllm_config.scheduler_config,
74 vllm_config.cache_config,
@@ -93,6 +97,12 @@ class EngineCore:
97 """Add request to the scheduler."""
98
99 req = Request.from_engine_core_request(request)
100 + # FIXME(woosuk): The input mapping (e.g., PIL images to tensors) may
101 + # take 10-50 ms, which can cause a spike in the latency. We should
102 + # consider moving this to a separate thread.
103 + if req.mm_data:
104 + req.mm_inputs = self.mm_input_mapper.process_inputs(
105 + req.mm_data, req.mm_processor_kwargs)
106 self.scheduler.add_request(req)
107
108 def abort_requests(self, request_ids: List[str]):
vllm/v1/engine/mm_input_mapper.py new
+39
@@ -0,0 +1,39 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +from vllm.config import ModelConfig
4 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
5 + MultiModalKwargs, MultiModalRegistry)
6 +
7 +
8 +class MMInputMapper:
9 +
10 + def __init__(
11 + self,
12 + model_config: ModelConfig,
13 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
14 + ):
15 + self.mm_registry = mm_registry
16 + self.multi_modal_input_mapper = mm_registry.create_input_mapper(
17 + model_config)
18 + self.mm_registry.init_mm_limits_per_prompt(model_config)
19 +
20 + def process_inputs(
21 + self,
22 + mm_data: MultiModalDataDict,
23 + mm_processor_kwargs: Optional[Dict[str, Any]],
24 + ) -> List[MultiModalKwargs]:
25 + image_inputs = mm_data["image"]
26 + if not isinstance(image_inputs, list):
27 + image_inputs = [image_inputs]
28 +
29 + # Process each image input separately so that later we can schedule
30 + # them in a fine-grained manner.
31 + mm_inputs: List[MultiModalKwargs] = []
32 + num_images = len(image_inputs)
33 + for i in range(num_images):
34 + mm_input = self.multi_modal_input_mapper(
35 + {"image": [image_inputs[i]]},
36 + mm_processor_kwargs=mm_processor_kwargs,
37 + )
38 + mm_inputs.append(mm_input)
39 + return mm_inputs
vllm/v1/request.py
+35 -6
@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
3
4 from vllm.inputs.data import DecoderOnlyInputs
5 from vllm.lora.request import LoRARequest
6 +from vllm.multimodal import MultiModalKwargs
7 from vllm.sampling_params import SamplingParams
8 from vllm.sequence import RequestMetrics
9 from vllm.v1.engine import EngineCoreRequest
@@ -47,14 +48,30 @@ class Request:
48 self._all_token_ids: List[int] = self.prompt_token_ids.copy()
49 self.num_computed_tokens = 0
50
51 + # Raw multimodal data before the mm input mapper (e.g., PIL images).
52 + self.mm_data = inputs.get("multi_modal_data")
53 + self.mm_processor_kwargs = inputs.get("mm_processor_kwargs")
54 + mm_positions = inputs.get("multi_modal_placeholders")
55 + if mm_positions:
56 + # FIXME(woosuk): Support other modalities.
57 + self.mm_positions = mm_positions.get("image", [])
58 + else:
59 + self.mm_positions = []
60 + # Output of the mm input mapper (e.g., image tensors).
61 + self.mm_inputs: List[MultiModalKwargs] = []
62 +
63 @classmethod
64 def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
52 -
65 return cls(
66 request_id=request.request_id,
55 - inputs=DecoderOnlyInputs(type="token",
56 - prompt_token_ids=request.prompt_token_ids,
57 - prompt=request.prompt),
67 + inputs=DecoderOnlyInputs(
68 + type="token",
69 + prompt_token_ids=request.prompt_token_ids,
70 + prompt=request.prompt,
71 + multi_modal_data=request.mm_data,
72 + multi_modal_placeholders=request.mm_placeholders,
73 + mm_processor_kwargs=request.mm_processor_kwargs,
74 + ),
75 sampling_params=request.sampling_params,
76 eos_token_id=request.eos_token_id,
77 arrival_time=request.arrival_time,
@@ -96,9 +113,21 @@ class Request:
113 def get_finished_reason(self) -> Union[str, None]:
114 return RequestStatus.get_finished_reason(self.status)
115
116 + def has_encoder_inputs(self) -> bool:
117 + return self.mm_data is not None
118 +
119 + @property
120 + def num_encoder_inputs(self) -> int:
121 + return len(self.mm_positions)
122 +
123 + def get_num_encoder_tokens(self, input_id: int) -> int:
124 + assert input_id < len(self.mm_positions)
125 + num_tokens = self.mm_positions[input_id]["length"]
126 + return num_tokens
127 +
128
129 class RequestStatus(enum.IntEnum):
101 - """Status of a sequence."""
130 + """Status of a request."""
131 WAITING = 0
132 RUNNING = 1
133 PREEMPTED = 2
@@ -119,7 +148,7 @@ class RequestStatus(enum.IntEnum):
148
149
150 # Mapping of finished statuses to their finish reasons.
122 -# NOTE: The ignored sequences are the sequences whose prompt lengths
151 +# NOTE: The ignored requests are the requests whose prompt lengths
152 # are longer than the model's length cap. Therefore, the stop
153 # reason should also be "length" as in OpenAI API.
154 _FINISHED_REASON_MAP = {
vllm/v1/worker/gpu_model_runner.py
+134 -20
@@ -1,7 +1,7 @@
1 import os
2 import time
3 from dataclasses import dataclass
4 -from typing import TYPE_CHECKING, Dict, List, Optional, Set
4 +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
5
6 import numpy as np
7 import torch
@@ -14,9 +14,10 @@ from vllm.compilation.config import CompilationConfig
14 from vllm.compilation.levels import CompilationLevel
15 from vllm.config import VllmConfig
16 from vllm.forward_context import set_forward_context
17 +from vllm.inputs import INPUT_REGISTRY, InputRegistry
18 from vllm.logger import init_logger
19 from vllm.model_executor.model_loader import get_model
19 -from vllm.multimodal import MultiModalDataDict
20 +from vllm.multimodal import MultiModalKwargs
21 from vllm.plugins import set_compilation_config
22 from vllm.sampling_params import SamplingParams, SamplingType
23 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
@@ -27,6 +28,7 @@ from vllm.v1.outputs import ModelRunnerOutput
28 from vllm.v1.sample.metadata import SamplingMetadata
29
30 if TYPE_CHECKING:
31 + from vllm.multimodal.base import PlaceholderRange
32 from vllm.v1.core.scheduler import SchedulerOutput
33
34 logger = init_logger(__name__)
@@ -37,8 +39,8 @@ class GPUModelRunner:
39 def __init__(
40 self,
41 vllm_config: VllmConfig,
42 + input_registry: InputRegistry = INPUT_REGISTRY,
43 ):
41 - # TODO: use ModelRunnerBase.__init__(self, vllm_config=vllm_config)
44 self.vllm_config = vllm_config
45 self.model_config = vllm_config.model_config
46 self.cache_config = vllm_config.cache_config
@@ -75,10 +77,16 @@ class GPUModelRunner:
77 parallel_config)
78 self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
79 self.head_size = model_config.get_head_size()
80 + self.hidden_size = model_config.get_hidden_size()
81 +
82 + # Multi-modal data support
83 + self.input_registry = input_registry
84
85 # Lazy initialization
86 # self.model: nn.Module # Set after load_model
87 self.kv_caches: List[torch.Tensor] = []
88 + # req_id -> (input_id -> encoder_output)
89 + self.encoder_cache: Dict[str, Dict[int, torch.Tensor]] = {}
90
91 # Request states.
92 self.requests: Dict[str, CachedRequestState] = {}
@@ -96,18 +104,28 @@ class GPUModelRunner:
104 and not self.model_config.enforce_eager)
105 # TODO(woosuk): Provide an option to tune the max cudagraph batch size.
106 self.cudagraph_batch_sizes = [1, 2, 4] + [i for i in range(8, 513, 8)]
99 - self.input_ids = torch.zeros(self.max_num_tokens,
100 - dtype=torch.int32,
101 - device=self.device)
107 self.positions = torch.zeros(self.max_num_tokens,
108 dtype=torch.int64,
109 device=self.device)
110 + self.inputs_embeds = torch.zeros(
111 + (self.max_num_tokens, self.hidden_size),
112 + dtype=self.dtype,
113 + device=self.device)
114
115 def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
116 # Remove stopped requests from the cached states.
117 # Keep the states of the pre-empted requests.
118 for req_id in scheduler_output.finished_req_ids:
119 self.requests.pop(req_id, None)
120 + self.encoder_cache.pop(req_id, None)
121 +
122 + # Free the cached encoder outputs.
123 + for req_id, input_id in scheduler_output.free_encoder_input_ids:
124 + encoder_outputs = self.encoder_cache.get(req_id)
125 + if encoder_outputs is not None:
126 + encoder_outputs.pop(input_id, None)
127 + if not encoder_outputs:
128 + self.encoder_cache.pop(req_id, None)
129
130 # Remove the requests from the persistent batch.
131 stopped_req_ids = set().union(
@@ -156,7 +174,8 @@ class GPUModelRunner:
174 req_id=req_id,
175 prompt_token_ids=req_data.prompt_token_ids,
176 prompt=req_data.prompt,
159 - multi_modal_data=req_data.multi_modal_data,
177 + mm_inputs=req_data.mm_inputs,
178 + mm_positions=req_data.mm_positions,
179 sampling_params=sampling_params,
180 generator=generator,
181 block_ids=req_data.block_ids,
@@ -285,11 +304,9 @@ class GPUModelRunner:
304 seq_start_loc_np[0] = 0
305 np.cumsum(seq_lens, out=seq_start_loc_np[1:])
306
288 - self.input_ids[:total_num_scheduled_tokens].copy_(input_ids,
289 - non_blocking=True)
307 + input_ids = input_ids.to(self.device, non_blocking=True)
308 self.positions[:total_num_scheduled_tokens].copy_(positions,
309 non_blocking=True)
292 -
310 query_start_loc = query_start_loc.to(self.device, non_blocking=True)
311 seq_start_loc = seq_start_loc.to(self.device, non_blocking=True)
312 slot_mapping = slot_mapping.to(self.device, non_blocking=True).long()
@@ -308,7 +325,7 @@ class GPUModelRunner:
325 # token from the partial request.
326 # TODO: Support prompt logprobs.
327 logits_indices = query_start_loc[1:] - 1
311 - return attn_metadata, logits_indices
328 + return input_ids, attn_metadata, logits_indices
329
330 def _prepare_sampling(
331 self,
@@ -325,13 +342,91 @@ class GPUModelRunner:
342 sampling_metadata = self.input_batch.make_sampling_metadata(skip_copy)
343 return sampling_metadata
344
345 + def _execute_encoder(self, scheduler_output: "SchedulerOutput"):
346 + scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
347 + if not scheduled_encoder_inputs:
348 + return
349 +
350 + # Batch the multi-modal inputs.
351 + mm_inputs: List[MultiModalKwargs] = []
352 + req_input_ids: List[Tuple[int, int]] = []
353 + for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
354 + req_state = self.requests[req_id]
355 + for input_id in encoder_input_ids:
356 + mm_inputs.append(req_state.mm_inputs[input_id])
357 + req_input_ids.append((req_id, input_id))
358 + batched_mm_inputs = MultiModalKwargs.batch(mm_inputs)
359 + batched_mm_inputs = MultiModalKwargs.as_kwargs(batched_mm_inputs,
360 + device=self.device)
361 +
362 + # Run the encoder.
363 + # `encoder_outputs` is either of the following:
364 + # 1. A tensor of shape [num_images, feature_size, hidden_size]
365 + # in case when feature_size is fixed across all images.
366 + # 2. A list (length: num_images) of tensors, each of shape
367 + # [feature_size, hidden_size] in case when the feature size is
368 + # dynamic depending on input images.
369 + encoder_outputs = self.model.process_mm_inputs(**batched_mm_inputs)
370 +
371 + # Cache the encoder outputs.
372 + for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):
373 + if req_id not in self.encoder_cache:
374 + self.encoder_cache[req_id] = {}
375 + self.encoder_cache[req_id][input_id] = output
376 +
377 + def _gather_encoder_outputs(
378 + self,
379 + scheduler_output: "SchedulerOutput",
380 + ) -> List[torch.Tensor]:
381 + encoder_outputs: List[torch.Tensor] = []
382 + num_reqs = self.input_batch.num_reqs
383 + for req_id in self.input_batch.req_ids[:num_reqs]:
384 + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
385 + req_id]
386 + req_state = self.requests[req_id]
387 + num_computed_tokens = req_state.num_computed_tokens
388 + mm_positions = req_state.mm_positions
389 + for i, pos_info in enumerate(mm_positions):
390 + start_pos = pos_info["offset"]
391 + num_encoder_tokens = pos_info["length"]
392 +
393 + # The encoder output is needed if the two ranges overlap:
394 + # [num_computed_tokens,
395 + # num_computed_tokens + num_scheduled_tokens) and
396 + # [start_pos, start_pos + num_encoder_tokens)
397 + if start_pos >= num_computed_tokens + num_scheduled_tokens:
398 + # The encoder output is not needed in this step.
399 + break
400 + if start_pos + num_encoder_tokens <= num_computed_tokens:
401 + # The encoder output is already processed and stored
402 + # in the decoder's KV cache.
403 + continue
404 +
405 + start_idx = max(num_computed_tokens - start_pos, 0)
406 + end_idx = min(
407 + num_computed_tokens - start_pos + num_scheduled_tokens,
408 + num_encoder_tokens)
409 + assert start_idx < end_idx
410 + assert req_id in self.encoder_cache
411 + assert i in self.encoder_cache[req_id]
412 + encoder_output = self.encoder_cache[req_id][i]
413 + encoder_outputs.append(encoder_output[start_idx:end_idx])
414 + return encoder_outputs
415 +
416 @torch.inference_mode()
417 def execute_model(
418 self,
419 scheduler_output: "SchedulerOutput",
420 ) -> ModelRunnerOutput:
421 self._update_states(scheduler_output)
334 - attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)
422 +
423 + # Run the encoder.
424 + self._execute_encoder(scheduler_output)
425 + encoder_outputs = self._gather_encoder_outputs(scheduler_output)
426 +
427 + # Prepare the decoder inputs.
428 + input_ids, attn_metadata, logits_indices = self._prepare_inputs(
429 + scheduler_output)
430 num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
431 if (self.use_cuda_graph
432 and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
@@ -343,12 +438,26 @@ class GPUModelRunner:
438 # Eager mode.
439 num_input_tokens = num_scheduled_tokens
440
441 + # Get the inputs embeds.
442 + if encoder_outputs:
443 + inputs_embeds = self.model.get_input_embeddings(
444 + input_ids, encoder_outputs)
445 + else:
446 + inputs_embeds = self.model.get_input_embeddings(input_ids)
447 + # NOTE(woosuk): To unify token ids and soft tokens (vision embeddings),
448 + # always use embeddings (rather than token ids) as input to the model.
449 + # TODO(woosuk): Avoid the copy. Optimize.
450 + self.inputs_embeds[:num_scheduled_tokens].copy_(inputs_embeds)
451 +
452 + # Run the decoder.
453 + # Use persistent buffers for CUDA graphs.
454 with set_forward_context(attn_metadata):
455 hidden_states = self.model(
348 - input_ids=self.input_ids[:num_input_tokens],
456 + input_ids=None,
457 positions=self.positions[:num_input_tokens],
458 kv_caches=self.kv_caches,
459 attn_metadata=None,
460 + inputs_embeds=self.inputs_embeds[:num_input_tokens],
461 )
462 hidden_states = hidden_states[:num_scheduled_tokens]
463 hidden_states = hidden_states[logits_indices]
@@ -440,13 +549,16 @@ class GPUModelRunner:
549 with set_forward_context(None): # noqa: SIM117
550 with set_compile_context(self.cudagraph_batch_sizes):
551 # Trigger compilation for general shape.
443 - model(self.input_ids,
444 - self.positions,
445 - dummy_kv_caches,
446 - attn_metadata=None)
552 + model(input_ids=None,
553 + positions=self.positions,
554 + kv_caches=dummy_kv_caches,
555 + attn_metadata=None,
556 + inputs_embeds=self.inputs_embeds)
557
558 @torch.inference_mode()
559 def profile_run(self) -> None:
560 + # TODO(woosuk): Profile the max memory usage of the encoder and
561 + # the encoder cache.
562 self._dummy_run(self.model, self.max_num_tokens)
563 torch.cuda.synchronize()
564
@@ -468,10 +580,11 @@ class GPUModelRunner:
580 # can reuse the memory pool allocated for the large shapes.
581 for num_tokens in reversed(self.cudagraph_batch_sizes):
582 self.model(
471 - self.input_ids[:num_tokens],
472 - self.positions[:num_tokens],
583 + input_ids=None,
584 + positions=self.positions[:num_tokens],
585 kv_caches=self.kv_caches,
586 attn_metadata=None,
587 + inputs_embeds=self.inputs_embeds[:num_tokens],
588 )
589
590 end_time = time.perf_counter()
@@ -506,7 +619,8 @@ class CachedRequestState:
619 req_id: str
620 prompt_token_ids: List[int]
621 prompt: Optional[str]
509 - multi_modal_data: Optional["MultiModalDataDict"]
622 + mm_inputs: List[MultiModalKwargs]
623 + mm_positions: List["PlaceholderRange"]
624 sampling_params: SamplingParams
625 generator: Optional[torch.Generator]
626