A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Optimize SPMD architecture with delta + serialization optimization (#7109)
SangBin Cho committed
Aug 18, 2024 at 17:57 UTC
ff7ec82c4dd6170ea8fedbd4d974c0a670e84c97
36 files changed
+727
-351
requirements-common.txt
+1
@@ -21,6 +21,7 @@ outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
21
typing_extensions >= 4.10
22
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
23
pyzmq
24
+msgspec
25
librosa # Required for audio processing
26
soundfile # Required for audio processing
27
gguf == 0.9.1
tests/basic_correctness/test_preemption.py
+18
@@ -8,6 +8,7 @@ pytest tests/basic_correctness/test_preemption.py`.
8
import pytest
9
from prometheus_client import REGISTRY
10
11
+import vllm.envs as envs
12
from vllm import SamplingParams
13
from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
14
ENABLE_ARTIFICIAL_PREEMPT)
@@ -24,6 +25,13 @@ assert ENABLE_ARTIFICIAL_PREEMPT is True, (
25
"tests/basic_correctness/test_preemption.py`")
26
27
28
+@pytest.fixture
29
+def worker_use_ray() -> bool:
30
+ # When SPMD worker is used, use ray_use_worker=True
31
+ # to test delta input optimization works with preemption.
32
+ return envs.VLLM_USE_RAY_SPMD_WORKER
33
+
34
+
35
@pytest.mark.parametrize("model", MODELS)
36
@pytest.mark.parametrize("dtype", ["half"])
37
@pytest.mark.parametrize("max_tokens", [96])
@@ -36,6 +44,7 @@ def test_chunked_prefill_recompute(
44
dtype: str,
45
max_tokens: int,
46
chunked_prefill_token_size: int,
47
+ worker_use_ray: bool,
48
) -> None:
49
"""Ensure that chunked prefill works with preemption."""
50
max_num_seqs = min(chunked_prefill_token_size, 256)
@@ -54,6 +63,7 @@ def test_chunked_prefill_recompute(
63
max_num_batched_tokens=max_num_batched_tokens,
64
enable_chunked_prefill=enable_chunked_prefill,
65
max_num_seqs=max_num_seqs,
66
+ worker_use_ray=worker_use_ray,
67
) as vllm_model:
68
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
69
assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
@@ -79,6 +89,7 @@ def test_preemption(
89
model: str,
90
dtype: str,
91
max_tokens: int,
92
+ worker_use_ray: bool,
93
) -> None:
94
"""By default, recompute preemption is enabled"""
95
@@ -89,6 +100,7 @@ def test_preemption(
100
model,
101
dtype=dtype,
102
disable_log_stats=False,
103
+ worker_use_ray=worker_use_ray,
104
) as vllm_model:
105
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
106
assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
@@ -132,6 +144,7 @@ def test_swap(
144
dtype: str,
145
max_tokens: int,
146
beam_width: int,
147
+ worker_use_ray: bool,
148
) -> None:
149
"""Use beam search enables swapping."""
150
example_prompts = example_prompts[:1]
@@ -144,6 +157,7 @@ def test_swap(
157
dtype=dtype,
158
swap_space=10,
159
disable_log_stats=False,
160
+ worker_use_ray=worker_use_ray,
161
) as vllm_model:
162
vllm_outputs = vllm_model.generate_beam_search(example_prompts,
163
beam_width, max_tokens)
@@ -188,6 +202,7 @@ def test_swap_infeasible(
202
dtype: str,
203
max_tokens: int,
204
beam_width: int,
205
+ worker_use_ray: bool,
206
) -> None:
207
"""Verify infeasible swap request will be ignored."""
208
BLOCK_SIZE = 16
@@ -204,6 +219,7 @@ def test_swap_infeasible(
219
# decode blocks are not enough to finish.
220
num_gpu_blocks_override=prefill_blocks + decode_blocks,
221
max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
222
+ worker_use_ray=worker_use_ray,
223
) as vllm_model:
224
sampling_params = SamplingParams(n=beam_width,
225
use_beam_search=True,
@@ -230,6 +246,7 @@ def test_preemption_infeasible(
246
model: str,
247
dtype: str,
248
max_tokens: int,
249
+ worker_use_ray: bool,
250
) -> None:
251
"""Verify infeasible preemption request will be ignored."""
252
BLOCK_SIZE = 16
@@ -244,6 +261,7 @@ def test_preemption_infeasible(
261
# ignored instead of hanging forever.
262
num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
263
max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
264
+ worker_use_ray=worker_use_ray,
265
) as vllm_model:
266
sampling_params = SamplingParams(max_tokens=max_tokens,
267
ignore_eos=True)
tests/core/test_serialization.py
new
+33
@@ -0,0 +1,33 @@
1
+import msgspec
2
+
3
+from vllm.executor.msgspec_utils import decode_hook, encode_hook
4
+from vllm.sequence import ExecuteModelRequest
5
+
6
+from ..spec_decode.utils import create_batch
7
+
8
+
9
+def test_msgspec_serialization():
10
+ num_lookahead_slots = 4
11
+ seq_group_metadata_list, _, _ = create_batch(16, num_lookahead_slots)
12
+ execute_model_req = ExecuteModelRequest(
13
+ seq_group_metadata_list=seq_group_metadata_list,
14
+ num_lookahead_slots=num_lookahead_slots,
15
+ running_queue_size=4)
16
+
17
+ encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
18
+ decoder = msgspec.msgpack.Decoder(ExecuteModelRequest,
19
+ dec_hook=decode_hook)
20
+ req = decoder.decode(encoder.encode(execute_model_req))
21
+ expected = execute_model_req.seq_group_metadata_list
22
+ actual = req.seq_group_metadata_list
23
+ assert (len(expected) == len(actual))
24
+ expected = expected[0]
25
+ actual = actual[0]
26
+
27
+ assert expected.block_tables == actual.block_tables
28
+ assert expected.is_prompt == actual.is_prompt
29
+ assert expected.request_id == actual.request_id
30
+ assert (expected.seq_data[0].prompt_token_ids ==
31
+ actual.seq_data[0].prompt_token_ids)
32
+ assert (expected.seq_data[0].output_token_ids ==
33
+ actual.seq_data[0].output_token_ids)
tests/distributed/test_basic_distributed_correctness.py
+2
-1
@@ -22,7 +22,8 @@ TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
22
@pytest.mark.skipif(cuda_device_count_stateless() < 2,
23
reason="Need at least 2 GPUs to run the test.")
24
@pytest.mark.parametrize(
25
- "model, distributed_executor_backend, attention_backend, test_suite", [
25
+ "model, distributed_executor_backend, attention_backend, "
26
+ "test_suite", [
27
("facebook/opt-125m", "ray", "", "L4"),
28
("facebook/opt-125m", "mp", "", "L4"),
29
("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
tests/distributed/test_chunked_prefill_distributed.py
+7
@@ -6,6 +6,8 @@ pytest test_chunked_prefill_distributed.py
6
```
7
"""
8
9
+import os
10
+
11
import pytest
12
13
from vllm.utils import cuda_device_count_stateless
@@ -30,6 +32,11 @@ def test_models(
32
model: str,
33
distributed_executor_backend: str,
34
) -> None:
35
+ if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray": # noqa
36
+ assert distributed_executor_backend == "ray"
37
+ # test ray adag
38
+ os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
39
+ os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
40
41
dtype = "half"
42
max_tokens = 5
tests/samplers/test_sampler.py
+19
-6
@@ -1,5 +1,6 @@
1
import itertools
2
import random
3
+from array import array
4
from typing import Dict, List, Optional, Tuple
5
from unittest.mock import Mock, patch
6
@@ -10,7 +11,8 @@ from transformers import GenerationConfig, GenerationMixin
11
from vllm.model_executor.layers.sampler import Sampler
12
from vllm.model_executor.sampling_metadata import SamplingMetadata
13
from vllm.model_executor.utils import set_random_seed
13
-from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
14
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
15
+ SequenceData, SequenceGroupMetadata)
16
from vllm.utils import Counter, is_pin_memory_available
17
18
@@ -56,7 +58,9 @@ def _do_sample(
58
SequenceGroupMetadata(
59
request_id=f"test_{i}",
60
is_prompt=True,
59
- seq_data={0: SequenceData([1, 2, 3])},
61
+ seq_data={
62
+ 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
63
+ },
64
sampling_params=sampling_params,
65
block_tables={0: [1]},
66
))
@@ -201,7 +205,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
205
206
def create_sequence_data(num_input=3, num_generated=0):
207
seq_data = SequenceData(
204
- random.choices(range(0, VOCAB_SIZE), k=num_input))
208
+ array(VLLM_TOKEN_ID_ARRAY_TYPE,
209
+ random.choices(range(0, VOCAB_SIZE), k=num_input)))
210
if num_generated > 0:
211
seq_data.output_token_ids = random.choices(range(0, VOCAB_SIZE),
212
k=num_generated)
@@ -504,7 +509,9 @@ def test_sampler_mixed(seed: int, device: str):
509
SequenceGroupMetadata(
510
request_id=f"test_{i}",
511
is_prompt=True,
507
- seq_data={0: SequenceData([1, 2, 3])},
512
+ seq_data={
513
+ 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
514
+ },
515
sampling_params=sampling_params,
516
block_tables={0: [1]},
517
))
@@ -600,7 +607,9 @@ def test_sampler_top_k_top_p(seed: int, device: str):
607
SequenceGroupMetadata(
608
request_id=f"test_{i}",
609
is_prompt=True,
603
- seq_data={0: SequenceData([1, 2, 3])},
610
+ seq_data={
611
+ 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
612
+ },
613
sampling_params=SamplingParams(
614
temperature=1,
615
top_k=top_k,
@@ -650,7 +659,11 @@ def test_sampler_repetition_penalty_mixed(device: str):
659
SequenceGroupMetadata(
660
request_id=f"test_{i}",
661
is_prompt=True,
653
- seq_data={0: SequenceData([1, 2, 3])},
662
+ seq_data={
663
+ 0:
664
+ SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
665
+ [1, 2, 3]))
666
+ },
667
sampling_params=sampling_params[i],
668
block_tables={0: [1]},
669
))
tests/spec_decode/utils.py
+6
-3
@@ -1,3 +1,4 @@
1
+from array import array
2
from itertools import count
3
from typing import Callable, Dict, List, Optional
4
from typing import Sequence as GenericSequence
@@ -9,7 +10,8 @@ import torch
10
from vllm.engine.arg_utils import EngineArgs
11
from vllm.model_executor.utils import set_random_seed
12
from vllm.sampling_params import SamplingParams
12
-from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
13
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
14
+ CompletionSequenceGroupOutput, Logprob,
15
SamplerOutput, SequenceData, SequenceGroupMetadata,
16
SequenceOutput)
17
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
@@ -138,8 +140,9 @@ def create_seq_group_metadata_from_prompts(
140
seq_data={
141
i:
142
SequenceData(
141
- prompt_token_ids=prompt_token_ids[:],
142
- output_token_ids=cont_token_ids[:],
143
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, prompt_token_ids[:]),
144
+ _output_token_ids=array(VLLM_TOKEN_ID_ARRAY_TYPE,
145
+ cont_token_ids[:]),
146
),
147
},
148
sampling_params=SamplingParams(temperature=0.0, ),
tests/test_logits_processor.py
+6
-2
@@ -1,4 +1,5 @@
1
import random
2
+from array import array
3
from typing import Tuple
4
from unittest.mock import patch
5
@@ -8,7 +9,8 @@ import torch
9
from vllm.model_executor.layers.logits_processor import LogitsProcessor
10
from vllm.model_executor.sampling_metadata import SamplingMetadata
11
from vllm.model_executor.utils import set_random_seed
11
-from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
13
+ SequenceData, SequenceGroupMetadata)
14
from vllm.utils import is_pin_memory_available
15
16
@@ -69,7 +71,9 @@ def test_logits_processors(seed: int, device: str):
71
SequenceGroupMetadata(
72
request_id=f"test_{i}",
73
is_prompt=True,
72
- seq_data={0: SequenceData([1, 2, 3])},
74
+ seq_data={
75
+ 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
76
+ },
77
sampling_params=SamplingParams(temperature=0,
78
logits_processors=[pick_ith]),
79
block_tables={0: [1]},
tests/test_sequence.py
+5
-2
@@ -1,6 +1,9 @@
1
+from array import array
2
+
3
import pytest
4
3
-from vllm.sequence import (CompletionSequenceGroupOutput, SamplerOutput,
5
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
6
+ CompletionSequenceGroupOutput, SamplerOutput,
7
SequenceData, SequenceOutput)
8
9
from .core.utils import create_dummy_prompt
@@ -54,7 +57,7 @@ def test_sampler_output_eq(sample_outputs):
57
58
59
def test_sequence_data_prefill():
57
- seq_data = SequenceData(prompt_token_ids=[1, 2, 3, 4])
60
+ seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3, 4]))
61
assert seq_data.get_num_uncomputed_tokens() == 4
62
assert seq_data.get_num_computed_tokens() == 0
63
# advance by 2
tests/worker/test_encoder_decoder_model_runner.py
+11
-5
@@ -1,10 +1,12 @@
1
+from array import array
2
from typing import List
3
4
import pytest
5
import torch
6
7
from vllm.engine.arg_utils import EngineArgs
7
-from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
8
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
9
+ SequenceData, SequenceGroupMetadata)
10
from vllm.utils import is_cpu
11
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
12
@@ -125,10 +127,12 @@ def test_prepare_prompt(
127
# make sure all tokens fit into one block
128
seq_len = i % (model_runner.block_size - 1) + 1
129
seq_lens.append(seq_len)
128
- seq_data = SequenceData(list(range(seq_len)))
130
+ seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
131
+ range(seq_len)))
132
encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
133
encoder_seq_lens.append(encoder_seq_len)
131
- encoder_seq_data = SequenceData(list(range(encoder_seq_len)))
134
+ encoder_seq_data = SequenceData(
135
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, range(encoder_seq_len)))
136
seq_group_metadata = SequenceGroupMetadata(
137
request_id=f"test_{i}",
138
is_prompt=True,
@@ -319,10 +323,12 @@ def test_prepare_decode(
323
# make sure all tokens fit into one block
324
seq_len = i % (model_runner.block_size - 1) + 1
325
seq_lens.append(seq_len)
322
- seq_data = SequenceData(list(range(seq_len)))
326
+ seq_data = SequenceData(
327
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
328
encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
329
encoder_seq_lens.append(encoder_seq_len)
325
- encoder_seq_data = SequenceData(list(range(encoder_seq_len)))
330
+ encoder_seq_data = SequenceData(
331
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
332
seq_group_metadata = SequenceGroupMetadata(
333
request_id=f"test_{i}",
334
is_prompt=False,
tests/worker/test_model_runner.py
+10
-5
@@ -1,3 +1,4 @@
1
+from array import array
2
from typing import List
3
4
import pytest
@@ -7,7 +8,8 @@ from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
8
init_distributed_environment)
9
from vllm.engine.arg_utils import EngineArgs
10
from vllm.model_executor.sampling_metadata import SamplingMetadata
10
-from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
12
+ SequenceData, SequenceGroupMetadata)
13
from vllm.utils import get_open_port
14
from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
15
@@ -46,7 +48,8 @@ def test_prepare_prompt(batch_size):
48
# make sure all tokens fit into one block
49
seq_len = i % (model_runner.block_size - 1) + 1
50
seq_lens.append(seq_len)
49
- seq_data = SequenceData(list(range(seq_len)))
51
+ seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
52
+ range(seq_len)))
53
seq_group_metadata = SequenceGroupMetadata(
54
request_id=f"test_{i}",
55
is_prompt=True,
@@ -163,7 +166,8 @@ def test_prepare_decode_cuda_graph(batch_size):
166
# make sure all tokens fit into one block
167
context_len = i % (model_runner.block_size - 1) + 1
168
context_lens.append(context_len)
166
- seq_data = SequenceData(list(range(context_len)))
169
+ seq_data = SequenceData(
170
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, range(context_len)))
171
seq_data.update_num_computed_tokens(context_len)
172
# Append one token ID since prefill is finished.
173
seq_data.append_token_id(1, 0)
@@ -324,7 +328,8 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
328
# make sure all tokens fit into one block
329
seq_len = i % (model_runner.block_size - 1) + 1
330
seq_lens.append(seq_len)
327
- seq_data = SequenceData(list(range(seq_len)))
331
+ seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
332
+ range(seq_len)))
333
seq_group_metadata = SequenceGroupMetadata(
334
request_id=f"test_{i}",
335
is_prompt=True,
@@ -340,7 +345,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
345
for i in range(prefill_batch_size, batch_size):
346
# make sure all tokens fit into one block
347
context_len = i % (model_runner.block_size - 1) + 1
343
- prompt_toks = list(range(context_len))
348
+ prompt_toks = array(VLLM_TOKEN_ID_ARRAY_TYPE, range(context_len))
349
seq_data = SequenceData(prompt_toks)
350
seq_data.append_token_id(1, 0)
351
seq_data.update_num_computed_tokens(context_len)
vllm/adapter_commons/request.py
-2
@@ -1,8 +1,6 @@
1
from abc import ABC, abstractmethod
2
-from dataclasses import dataclass
2
3
5
-@dataclass
4
class AdapterRequest(ABC):
5
"""
6
Base class for adapter requests.
vllm/config.py
+9
-2
@@ -770,8 +770,8 @@ class ParallelConfig:
770
self.tokenizer_pool_config = tokenizer_pool_config
771
self.ray_workers_use_nsight = ray_workers_use_nsight
772
self.placement_group = placement_group
773
-
773
self.world_size = pipeline_parallel_size * self.tensor_parallel_size
774
+
775
if worker_use_ray:
776
if self.distributed_executor_backend is None:
777
self.distributed_executor_backend = "ray"
@@ -867,6 +867,11 @@ class SchedulerConfig:
867
swapping. However, when the sequence group has multiple sequences
868
(e.g., beam search), recomputation is not currently supported. In
869
such a case, we use swapping instead.
870
+ send_delta_data: Private API. If used, scheduler sends delta data to
871
+ workers instead of an entire data. It should be enabled only
872
+ when SPMD worker architecture is enabled. I.e.,
873
+ VLLM_USE_RAY_SPMD_WORKER=1
874
+
875
"""
876
877
def __init__(self,
@@ -879,7 +884,8 @@ class SchedulerConfig:
884
enable_chunked_prefill: bool = False,
885
embedding_mode: Optional[bool] = False,
886
preemption_mode: Optional[str] = None,
882
- num_scheduler_steps: int = 1) -> None:
887
+ num_scheduler_steps: int = 1,
888
+ send_delta_data: bool = False) -> None:
889
if max_num_batched_tokens is not None:
890
self.max_num_batched_tokens = max_num_batched_tokens
891
else:
@@ -909,6 +915,7 @@ class SchedulerConfig:
915
self.embedding_mode = embedding_mode
916
self.preemption_mode = preemption_mode
917
self.num_scheduler_steps = num_scheduler_steps
918
+ self.send_delta_data = send_delta_data
919
self._verify_args()
920
921
def _verify_args(self) -> None:
vllm/core/scheduler.py
+50
-38
@@ -12,7 +12,8 @@ from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
13
from vllm.prompt_adapter.request import PromptAdapterRequest
14
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
15
- SequenceGroupMetadata, SequenceStatus)
15
+ SequenceGroupMetadata, SequenceGroupMetadataDelta,
16
+ SequenceStatus)
17
from vllm.utils import PyObjectCache
18
19
logger = init_logger(__name__)
@@ -363,8 +364,6 @@ class Scheduler:
364
self.num_cumulative_preemption: int = 0
365
366
# Used to cache python objects
366
- self._seq_group_metadata_cache: PyObjectCache = PyObjectCache(
367
- seq_group_metadata_builder)
367
self._scheduler_running_outputs_cache: PyObjectCache = PyObjectCache(
368
scheduler_running_outputs_builder)
369
self._scheduled_seq_group_cache: PyObjectCache = PyObjectCache(
@@ -1048,15 +1047,10 @@ class Scheduler:
1047
token_chunk_size = scheduled_seq_group.token_chunk_size
1048
seq_group.maybe_set_first_scheduled_time(now)
1049
1051
- seq_group_metadata = self._seq_group_metadata_cache.get_object()
1052
- seq_group_metadata.seq_data.clear()
1053
- seq_group_metadata.block_tables.clear()
1054
-
1050
# seq_id -> SequenceData
1056
- seq_data: Dict[int, SequenceData] = seq_group_metadata.seq_data
1051
+ seq_data: Dict[int, SequenceData] = {}
1052
# seq_id -> physical block numbers
1058
- block_tables: Dict[int,
1059
- List[int]] = seq_group_metadata.block_tables
1053
+ block_tables: Dict[int, List[int]] = {}
1054
1055
if seq_group.is_encoder_decoder():
1056
# Encoder associated with SequenceGroup
@@ -1081,45 +1075,65 @@ class Scheduler:
1075
seq_group.get_seqs(status=SequenceStatus.RUNNING)))
1076
1077
do_sample = True
1084
- if seq_group.is_prefill():
1078
+ is_prompt = seq_group.is_prefill()
1079
+ # We should send the metadata to workers when the first prefill
1080
+ # is sent. Subsequent requests could be chunked prefill or decode.
1081
+ is_first_prefill = False
1082
+ if is_prompt:
1083
seqs = seq_group.get_seqs()
1084
# Prefill has only 1 sequence.
1085
assert len(seqs) == 1
1086
+ num_computed_tokens = seqs[0].data.get_num_computed_tokens()
1087
+ is_first_prefill = num_computed_tokens == 0
1088
# In the next iteration, all prompt tokens are not computed.
1089
# It means the prefill is chunked, and we don't need sampling.
1090
# NOTE: We use get_len instead of get_prompt_len because when
1091
# a sequence is preempted, prefill includes previous generated
1092
# output tokens.
1093
- if (token_chunk_size + seqs[0].data.get_num_computed_tokens() <
1093
+ if (token_chunk_size + num_computed_tokens <
1094
seqs[0].data.get_len()):
1095
do_sample = False
1096
1097
# It assumes the scheduled_seq_groups is ordered by
1098
# prefill < decoding.
1099
- is_prompt = seq_group.is_prefill()
1100
-
1101
- seq_group_metadata.__init__(
1102
- request_id=seq_group.request_id,
1103
- is_prompt=is_prompt,
1104
- seq_data=seq_data,
1105
- sampling_params=seq_group.sampling_params,
1106
- block_tables=block_tables,
1107
- do_sample=do_sample,
1108
- pooling_params=seq_group.pooling_params,
1109
- token_chunk_size=token_chunk_size,
1110
- lora_request=seq_group.lora_request,
1111
- computed_block_nums=common_computed_block_nums,
1112
- encoder_seq_data=encoder_seq_data,
1113
- cross_block_table=cross_block_table,
1114
- state=seq_group.state,
1115
- # `multi_modal_data` will only be present for the 1st comm
1116
- # between engine and worker.
1117
- # the subsequent comms can still use delta, but
1118
- # `multi_modal_data` will be None.
1119
- multi_modal_data=seq_group.multi_modal_data
1120
- if scheduler_outputs.num_prefill_groups > 0 else None,
1121
- prompt_adapter_request=seq_group.prompt_adapter_request,
1122
- )
1099
+ if is_first_prefill or not self.scheduler_config.send_delta_data:
1100
+ seq_group_metadata = SequenceGroupMetadata(
1101
+ request_id=seq_group.request_id,
1102
+ is_prompt=is_prompt,
1103
+ seq_data=seq_data,
1104
+ sampling_params=seq_group.sampling_params,
1105
+ block_tables=block_tables,
1106
+ do_sample=do_sample,
1107
+ pooling_params=seq_group.pooling_params,
1108
+ token_chunk_size=token_chunk_size,
1109
+ lora_request=seq_group.lora_request,
1110
+ computed_block_nums=common_computed_block_nums,
1111
+ encoder_seq_data=encoder_seq_data,
1112
+ cross_block_table=cross_block_table,
1113
+ state=seq_group.state,
1114
+ # `multi_modal_data` will only be present for the 1st comm
1115
+ # between engine and worker.
1116
+ # the subsequent comms can still use delta, but
1117
+ # `multi_modal_data` will be None.
1118
+ multi_modal_data=seq_group.multi_modal_data
1119
+ if scheduler_outputs.num_prefill_groups > 0 else None,
1120
+ prompt_adapter_request=seq_group.prompt_adapter_request,
1121
+ )
1122
+ else:
1123
+ # When SPMD mode is enabled, we only send delta data except for
1124
+ # the first request to reduce serialization cost.
1125
+ seq_data_delta = {}
1126
+ for id, data in seq_data.items():
1127
+ seq_data_delta[id] = data.get_delta_and_reset()
1128
+ seq_group_metadata = SequenceGroupMetadataDelta(
1129
+ seq_data_delta,
1130
+ seq_group.request_id,
1131
+ block_tables,
1132
+ is_prompt,
1133
+ do_sample=do_sample,
1134
+ token_chunk_size=token_chunk_size,
1135
+ computed_block_nums=common_computed_block_nums,
1136
+ )
1137
seq_group_metadata_list.append(seq_group_metadata)
1138
1139
# Now that the batch has been created, we can assume all blocks in the
@@ -1130,8 +1144,6 @@ class Scheduler:
1144
self.block_manager.mark_blocks_as_computed(
1145
scheduled_seq_group.seq_group)
1146
1133
- self._seq_group_metadata_cache.reset()
1134
-
1147
scheduler_time = time.perf_counter() - scheduler_start_time
1148
# Add this to scheduler time to all the sequences that are currently
1149
# running. This will help estimate if the scheduler is a significant
vllm/engine/arg_utils.py
+3
@@ -5,6 +5,7 @@ from dataclasses import dataclass
5
from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Type,
6
Union)
7
8
+import vllm.envs as envs
9
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
10
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
11
ObservabilityConfig, ParallelConfig,
@@ -905,6 +906,8 @@ class EngineArgs:
906
embedding_mode=model_config.embedding_mode,
907
preemption_mode=self.preemption_mode,
908
num_scheduler_steps=self.num_scheduler_steps,
909
+ send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
910
+ and parallel_config.use_ray),
911
)
912
lora_config = LoRAConfig(
913
max_lora_rank=self.max_lora_rank,
vllm/engine/llm_engine.py
-1
@@ -224,7 +224,6 @@ class LLMEngine:
224
cache_config.enable_prefix_caching,
225
)
226
# TODO(woosuk): Print more configs in debug mode.
227
-
227
from vllm.plugins import load_general_plugins
228
load_general_plugins()
229
vllm/executor/msgspec_utils.py
new
+27
@@ -0,0 +1,27 @@
1
+from array import array
2
+from typing import Any, Type
3
+
4
+from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE
5
+
6
+
7
+def encode_hook(obj: Any) -> Any:
8
+ """Custom msgspec enc hook that supports array types.
9
+
10
+ See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder
11
+ """
12
+ if isinstance(obj, array):
13
+ assert obj.typecode == VLLM_TOKEN_ID_ARRAY_TYPE, (
14
+ f"vLLM array type should use '{VLLM_TOKEN_ID_ARRAY_TYPE}' type. "
15
+ f"Given array has a type code of {obj.typecode}.")
16
+ return obj.tobytes()
17
+
18
+
19
+def decode_hook(type: Type, obj: Any) -> Any:
20
+ """Custom msgspec dec hook that supports array types.
21
+
22
+ See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder
23
+ """
24
+ if type is array:
25
+ deserialized = array(VLLM_TOKEN_ID_ARRAY_TYPE)
26
+ deserialized.frombytes(obj)
27
+ return deserialized
vllm/executor/ray_gpu_executor.py
+15
-4
@@ -4,9 +4,12 @@ from collections import defaultdict
4
from itertools import islice, repeat
5
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
6
7
+import msgspec
8
+
9
import vllm.envs as envs
10
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
11
DistributedGPUExecutor, DistributedGPUExecutorAsync)
12
+from vllm.executor.msgspec_utils import encode_hook
13
from vllm.executor.ray_utils import RayWorkerWrapper, ray
14
from vllm.logger import init_logger
15
from vllm.sequence import ExecuteModelRequest, SamplerOutput
@@ -60,6 +63,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
63
# Create the parallel GPU workers.
64
self._init_workers_ray(placement_group)
65
66
+ self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
67
+ self.output_decoder = msgspec.msgpack.Decoder(
68
+ Optional[List[SamplerOutput]])
69
+
70
def shutdown(self) -> None:
71
if hasattr(self, "forward_dag") and self.forward_dag is not None:
72
self.forward_dag.teardown()
@@ -123,6 +130,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
130
ray_remote_kwargs)
131
132
logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
133
+
134
# Create the workers.
135
driver_ip = get_ip()
136
worker_wrapper_kwargs = self._get_worker_wrapper_args()
@@ -304,8 +312,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
312
if self.forward_dag is None:
313
self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
314
307
- outputs = ray.get(self.forward_dag.execute(execute_model_req))
308
- return outputs[0]
315
+ serialized_data = self.input_encoder.encode(execute_model_req)
316
+ outputs = ray.get(self.forward_dag.execute(serialized_data))
317
+ output = self.output_decoder.decode(outputs[0])
318
+ return output
319
320
def _run_workers(
321
self,
@@ -475,9 +485,10 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
485
if self.forward_dag is None:
486
self.forward_dag = self._compiled_ray_dag(enable_asyncio=True)
487
478
- dag_future = await self.forward_dag.execute_async(execute_model_req)
488
+ serialized_data = self.input_encoder.encode(execute_model_req)
489
+ dag_future = await self.forward_dag.execute_async(serialized_data)
490
outputs = await dag_future
480
- return outputs[0]
491
+ return self.output_decoder.decode(outputs[0])
492
493
async def _driver_execute_model_async(
494
self,
vllm/executor/ray_utils.py
+27
-12
@@ -1,6 +1,9 @@
1
from typing import List, Optional, Tuple, Union
2
3
+import msgspec
4
+
5
from vllm.config import ParallelConfig
6
+from vllm.executor.msgspec_utils import decode_hook, encode_hook
7
from vllm.logger import init_logger
8
from vllm.platforms import current_platform
9
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
@@ -24,6 +27,10 @@ try:
27
# that thread.
28
self.compiled_dag_cuda_device_set = False
29
30
+ self.input_decoder = msgspec.msgpack.Decoder(ExecuteModelRequest,
31
+ dec_hook=decode_hook)
32
+ self.output_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
33
+
34
def get_node_ip(self) -> str:
35
return get_ip()
36
@@ -33,16 +40,26 @@ try:
40
return node_id, gpu_ids
41
42
def execute_model_spmd(
36
- self, req_or_tuple: Union[ExecuteModelRequest,
37
- Tuple[ExecuteModelRequest,
38
- IntermediateTensors]]):
43
+ self, req_or_tuple: Union[bytes,
44
+ Tuple[bytes,
45
+ Optional[IntermediateTensors]]]
46
+ ) -> bytes:
47
"""Execute model in SPMD fashion: used only when SPMD worker and
48
compiled DAG are both enabled.
49
50
Args:
43
- req_or_tuple: The request to execute the model, or a tuple
44
- containing the request and intermediate tensors.
51
+ req_or_tuple: A request or a tuple containing the
52
+ request and intermediate tensors. Intermediate tensors are
53
+ None unless if it is provided because it is > 0 pipeline
54
+ stage. The request is serialized by msgspec.
55
"""
56
+ if isinstance(req_or_tuple, bytes):
57
+ serialized_req, intermediate_tensors = req_or_tuple, None
58
+ else:
59
+ serialized_req, intermediate_tensors = req_or_tuple
60
+
61
+ execute_model_req = self.input_decoder.decode(serialized_req)
62
+
63
# TODO(swang): This is needed right now because Ray aDAG executes
64
# on a background thread, so we need to reset torch's current
65
# device.
@@ -51,16 +68,14 @@ try:
68
torch.cuda.set_device(self.worker.device)
69
self.compiled_dag_cuda_device_set = True
70
54
- if isinstance(req_or_tuple, tuple):
55
- execute_model_req, intermediate_tensors = req_or_tuple
56
- else:
57
- execute_model_req = req_or_tuple
58
- intermediate_tensors = None
59
-
71
output = self.worker._execute_model_spmd(execute_model_req,
72
intermediate_tensors)
73
+ # Pipeline model request and output to the next pipeline stage.
74
if isinstance(output, IntermediateTensors):
63
- return execute_model_req, output
75
+ output = serialized_req, output
76
+ else:
77
+ output = self.output_encoder.encode(output)
78
+
79
return output
80
81
ray_import_err = None
vllm/inputs/registry.py
+7
-1
@@ -1,4 +1,5 @@
1
import functools
2
+from array import array
3
from collections import UserDict
4
from dataclasses import dataclass
5
from typing import (TYPE_CHECKING, Callable, Dict, Mapping, Optional, Protocol,
@@ -21,6 +22,10 @@ logger = init_logger(__name__)
22
23
C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
24
25
+# NOTE: This has to match with sequence.py's VLLM_TOKEN_ID_ARRAY_TYPE.
26
+# We cannot import it here because of circular dependencies.
27
+VLLM_TOKEN_ID_ARRAY_TYPE = "l"
28
+
29
30
@dataclass(frozen=True)
31
class InputContext:
@@ -118,7 +123,8 @@ class InputRegistry:
123
# Avoid circular import
124
from vllm.sequence import SequenceData
125
121
- dummy_seq_data = SequenceData([0] * seq_len)
126
+ dummy_seq_data = SequenceData(
127
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * seq_len)
128
dummy_multi_modal_data = None
129
130
return dummy_seq_data, dummy_multi_modal_data
vllm/lora/request.py
+9
-5
@@ -1,12 +1,15 @@
1
import warnings
2
-from dataclasses import dataclass, field
2
from typing import Optional
3
4
+import msgspec
5
+
6
from vllm.adapter_commons.request import AdapterRequest
7
8
8
-@dataclass
9
-class LoRARequest(AdapterRequest):
9
+class LoRARequest(
10
+ msgspec.Struct,
11
+ omit_defaults=True, # type: ignore[call-arg]
12
+ array_like=True): # type: ignore[call-arg]
13
"""
14
Request for a LoRA adapter.
15
@@ -18,16 +21,17 @@ class LoRARequest(AdapterRequest):
21
lora_int_id must be globally unique for a given adapter.
22
This is currently not enforced in vLLM.
23
"""
24
+ __metaclass__ = AdapterRequest
25
26
lora_name: str
27
lora_int_id: int
28
lora_path: str = ""
25
- lora_local_path: Optional[str] = field(default=None, repr=False)
29
+ lora_local_path: Optional[str] = msgspec.field(default=None)
30
long_lora_max_len: Optional[int] = None
31
__hash__ = AdapterRequest.__hash__
32
33
def __post_init__(self):
30
- if 'lora_local_path' in self.__dict__:
34
+ if 'lora_local_path' in self.__struct_fields__:
35
warnings.warn(
36
"The 'lora_local_path' attribute is deprecated "
37
"and will be removed in a future version. "
vllm/model_executor/models/blip.py
+6
-3
@@ -1,5 +1,6 @@
1
"""Minimal implementation of BlipVisionModel intended to be only used
2
within a vision language model."""
3
+from array import array
4
from typing import Optional, Union
5
6
import torch
@@ -16,7 +17,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.multimodal.image import (cached_get_tokenizer,
19
repeat_and_pad_image_tokens)
19
-from vllm.sequence import SequenceData
20
+from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
21
22
23
def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -53,8 +54,10 @@ def dummy_seq_data_for_blip(
54
else:
55
image_feature_size = image_feature_size_override
56
56
- token_ids = [image_token_id] * image_feature_size
57
- token_ids += [0] * (seq_len - image_feature_size)
57
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
58
+ [image_token_id]) * image_feature_size
59
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
60
+ [0]) * (seq_len - image_feature_size)
61
return SequenceData(token_ids)
62
63
vllm/model_executor/models/blip2.py
+7
-3
@@ -1,3 +1,4 @@
1
+from array import array
2
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
TypedDict, Union)
4
@@ -17,7 +18,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
from vllm.model_executor.models.opt import OPTModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
20
-from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
21
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
22
+ SamplerOutput, SequenceData)
23
24
from .blip import (BlipVisionModel, dummy_image_for_blip,
25
get_max_blip_image_tokens)
@@ -427,8 +429,10 @@ def dummy_seq_data_for_blip2(
429
else:
430
image_feature_size = image_feature_size_override
431
430
- token_ids = [image_token_id] * image_feature_size * num_images
431
- token_ids += [0] * (seq_len - image_feature_size * num_images)
432
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
433
+ [image_token_id]) * image_feature_size * num_images
434
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
435
+ [0]) * (seq_len - image_feature_size * num_images)
436
return SequenceData(token_ids)
437
438
vllm/model_executor/models/chameleon.py
+7
-3
@@ -1,3 +1,4 @@
1
+from array import array
2
from functools import cached_property
3
from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
4
Tuple, TypedDict)
@@ -31,7 +32,8 @@ from vllm.model_executor.utils import set_weight_attrs
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.image import (cached_get_tokenizer,
34
repeat_and_pad_image_tokens)
34
-from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
35
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
36
+ SamplerOutput, SequenceData)
37
from vllm.utils import print_warning_once
38
39
from .interfaces import SupportsMultiModal
@@ -70,8 +72,10 @@ def dummy_seq_data_for_chameleon(
72
else:
73
image_feature_size = image_feature_size_override
74
73
- token_ids = [image_token_id] * image_feature_size * num_images
74
- token_ids += [0] * (seq_len - image_feature_size * num_images)
75
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
76
+ [image_token_id]) * image_feature_size * num_images
77
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
78
+ [0]) * (seq_len - image_feature_size * num_images)
79
return SequenceData(token_ids)
80
81
vllm/model_executor/models/clip.py
+6
-3
@@ -1,5 +1,6 @@
1
"""Minimal implementation of CLIPVisionModel intended to be only used
2
within a vision language model."""
3
+from array import array
4
from typing import Iterable, Optional, Tuple
5
6
import torch
@@ -17,7 +18,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19
from vllm.multimodal.image import (cached_get_tokenizer,
20
repeat_and_pad_image_tokens)
20
-from vllm.sequence import SequenceData
21
+from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
22
23
24
def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -53,8 +54,10 @@ def dummy_seq_data_for_clip(
54
else:
55
image_feature_size = image_feature_size_override
56
56
- token_ids = [image_token_id] * image_feature_size * num_images
57
- token_ids += [0] * (seq_len - image_feature_size * num_images)
57
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
58
+ [image_token_id]) * image_feature_size * num_images
59
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
60
+ [0]) * (seq_len - image_feature_size * num_images)
61
return SequenceData(token_ids)
62
63
vllm/model_executor/models/fuyu.py
+9
-4
@@ -16,6 +16,7 @@
16
# limitations under the License.
17
""" PyTorch Fuyu model."""
18
import math
19
+from array import array
20
from typing import Iterable, List, Literal, Mapping, Optional, Tuple, TypedDict
21
22
import torch
@@ -37,7 +38,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
38
from vllm.multimodal.base import MultiModalInputs
39
from vllm.multimodal.image import (cached_get_image_processor,
40
cached_get_tokenizer)
40
-from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
41
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42
+ SamplerOutput, SequenceData)
43
44
from .interfaces import SupportsMultiModal
45
from .utils import merge_multimodal_embeddings
@@ -97,9 +99,12 @@ def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int):
99
ncol, nrow = get_max_fuyu_image_feature_size()
100
image_feature_size = get_max_fuyu_image_tokens(ctx)
101
100
- image_token_ids = ([_IMAGE_TOKEN_ID] * ncol + [_NEWLINE_TOKEN_ID]) * nrow
101
- token_ids = image_token_ids * num_images
102
- token_ids += [0] * (seq_len - image_feature_size * num_images)
102
+ image_token_ids = (
103
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, [_IMAGE_TOKEN_ID]) * ncol +
104
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, [_NEWLINE_TOKEN_ID])) * nrow
105
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, image_token_ids) * num_images
106
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
107
+ [0]) * (seq_len - image_feature_size * num_images)
108
return SequenceData(token_ids)
109
110
vllm/model_executor/models/minicpmv.py
+4
-2
@@ -23,6 +23,7 @@
23
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
24
import math
25
import re
26
+from array import array
27
from functools import partial
28
from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
29
TypedDict, Union)
@@ -55,7 +56,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
56
from vllm.multimodal import MULTIMODAL_REGISTRY
57
from vllm.multimodal.image import (cached_get_image_processor,
58
cached_get_tokenizer)
58
-from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
59
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
60
+ SamplerOutput, SequenceData)
61
62
from .idefics2_vision_model import Idefics2VisionTransformer
63
@@ -408,7 +410,7 @@ def get_max_minicpmv_image_tokens(ctx: InputContext):
410
411
412
def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
411
- token_ids = [0] * seq_len
413
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * seq_len
414
return SequenceData(token_ids)
415
416
vllm/model_executor/models/siglip.py
+6
-3
@@ -2,6 +2,7 @@
2
within a vision language model."""
3
4
import math
5
+from array import array
6
from typing import Iterable, Optional, Tuple
7
8
import torch
@@ -25,7 +26,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
from vllm.multimodal.image import (cached_get_tokenizer,
28
repeat_and_pad_image_tokens)
28
-from vllm.sequence import SequenceData
29
+from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
30
31
32
def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -62,8 +63,10 @@ def dummy_seq_data_for_siglip(
63
else:
64
image_feature_size = image_feature_size_override
65
65
- token_ids = [image_token_id] * image_feature_size * num_images
66
- token_ids += [0] * (seq_len - image_feature_size * num_images)
66
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
67
+ [image_token_id]) * image_feature_size
68
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
69
+ [0]) * (seq_len - image_feature_size)
70
return SequenceData(token_ids)
71
72
vllm/model_executor/sampling_metadata.py
+6
-3
@@ -6,7 +6,8 @@ from typing import Dict, List, Optional, Tuple
6
import torch
7
8
from vllm.sampling_params import SamplingParams, SamplingType
9
-from vllm.sequence import SequenceData, SequenceGroupMetadata
9
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData,
10
+ SequenceGroupMetadata)
11
from vllm.triton_utils.sample import get_num_triton_sampler_splits
12
from vllm.utils import (PyObjectCache, async_tensor_h2d,
13
is_pin_memory_available, make_tensor_with_pad,
@@ -505,9 +506,11 @@ class SamplingTensors:
506
and sampling_params.prompt_logprobs is not None):
507
prefill_len = len(seq_group.prompt_logprob_indices)
508
prompt_tokens.extend(
508
- array('l') for _ in range(prefill_len))
509
+ array(VLLM_TOKEN_ID_ARRAY_TYPE)
510
+ for _ in range(prefill_len))
511
output_tokens.extend(
510
- array('l') for _ in range(prefill_len))
512
+ array(VLLM_TOKEN_ID_ARRAY_TYPE)
513
+ for _ in range(prefill_len))
514
if seq_group.do_sample:
515
for seq_id in seq_ids:
516
seq_data = seq_group.seq_data[seq_id]
vllm/pooling_params.py
+7
-4
@@ -1,15 +1,18 @@
1
from typing import Any, Optional
2
3
+import msgspec
4
4
-class PoolingParams:
5
+
6
+class PoolingParams(
7
+ msgspec.Struct,
8
+ omit_defaults=True, # type: ignore[call-arg]
9
+ array_like=True): # type: ignore[call-arg]
10
"""Pooling parameters for pooling.
11
12
Attributes:
13
additional_data: Any additional data needed for pooling.
14
"""
10
-
11
- def __init__(self, additional_data: Optional[Any] = None):
12
- self.additional_data = additional_data
15
+ additional_data: Optional[Any] = None
16
17
def clone(self) -> "PoolingParams":
18
"""Returns a deep copy of the PoolingParams instance."""
vllm/prompt_adapter/request.py
+7
-3
@@ -1,13 +1,17 @@
1
-from dataclasses import dataclass
1
+import msgspec
2
3
from vllm.adapter_commons.request import AdapterRequest
4
5
6
-@dataclass
7
-class PromptAdapterRequest(AdapterRequest):
6
+class PromptAdapterRequest(
7
+ msgspec.Struct,
8
+ array_like=True, # type: ignore[call-arg]
9
+ omit_defaults=True, # type: ignore[call-arg]
10
+ frozen=True): # type: ignore[call-arg]
11
"""
12
Request for a Prompt adapter.
13
"""
14
+ __metaclass__ = AdapterRequest
15
16
prompt_adapter_name: str
17
prompt_adapter_id: int
vllm/sampling_params.py
+72
-75
@@ -2,10 +2,10 @@
2
import copy
3
from enum import IntEnum
4
from functools import cached_property
5
-from typing import Any, Callable, Dict, List, Optional, Union
5
+from typing import Any, Callable, Dict, List, Optional, Set, Union
6
7
+import msgspec
8
import torch
8
-from pydantic import Field
9
from typing_extensions import Annotated
10
11
from vllm.logger import init_logger
@@ -33,7 +33,11 @@ first argument, and returns a modified tensor of logits
33
to sample from."""
34
35
36
-class SamplingParams:
36
+class SamplingParams(
37
+ msgspec.Struct,
38
+ omit_defaults=True, # type: ignore[call-arg]
39
+ # required for @cached_property.
40
+ dict=True): # type: ignore[call-arg]
41
"""Sampling parameters for text generation.
42
43
Overall, we follow the sampling parameters from the OpenAI text completion
@@ -112,87 +116,73 @@ class SamplingParams:
116
(i.e., no truncation).
117
"""
118
115
- def __init__(
116
- self,
117
- n: int = 1,
118
- best_of: Optional[int] = None,
119
- presence_penalty: float = 0.0,
120
- frequency_penalty: float = 0.0,
121
- repetition_penalty: float = 1.0,
122
- temperature: float = 1.0,
123
- top_p: float = 1.0,
124
- top_k: int = -1,
125
- min_p: float = 0.0,
126
- seed: Optional[int] = None,
127
- use_beam_search: bool = False,
128
- length_penalty: float = 1.0,
129
- early_stopping: Union[bool, str] = False,
130
- stop: Optional[Union[str, List[str]]] = None,
131
- stop_token_ids: Optional[List[int]] = None,
132
- include_stop_str_in_output: bool = False,
133
- ignore_eos: bool = False,
134
- max_tokens: Optional[int] = 16,
135
- min_tokens: int = 0,
136
- logprobs: Optional[int] = None,
137
- prompt_logprobs: Optional[int] = None,
138
- detokenize: bool = True,
139
- skip_special_tokens: bool = True,
140
- spaces_between_special_tokens: bool = True,
141
- logits_processors: Optional[List[LogitsProcessor]] = None,
142
- truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
143
- ) -> None:
144
- self.n = n
145
- self.best_of = best_of if best_of is not None else n
146
- self.presence_penalty = presence_penalty
147
- self.frequency_penalty = frequency_penalty
148
- self.repetition_penalty = repetition_penalty
149
- if 0 < temperature < _MAX_TEMP:
119
+ n: int = 1
120
+ best_of: Optional[int] = None
121
+ presence_penalty: float = 0.0
122
+ frequency_penalty: float = 0.0
123
+ repetition_penalty: float = 1.0
124
+ temperature: float = 1.0
125
+ top_p: float = 1.0
126
+ top_k: int = -1
127
+ min_p: float = 0.0
128
+ seed: Optional[int] = None
129
+ use_beam_search: bool = False
130
+ length_penalty: float = 1.0
131
+ early_stopping: Union[bool, str] = False
132
+ stop: Optional[Union[str, List[str]]] = None
133
+ stop_token_ids: Optional[List[int]] = None
134
+ ignore_eos: bool = False
135
+ max_tokens: Optional[int] = 16
136
+ min_tokens: int = 0
137
+ logprobs: Optional[int] = None
138
+ prompt_logprobs: Optional[int] = None
139
+ # NOTE: This parameter is only exposed at the engine level for now.
140
+ # It is not exposed in the OpenAI API server, as the OpenAI API does
141
+ # not support returning only a list of token IDs.
142
+ detokenize: bool = True
143
+ skip_special_tokens: bool = True
144
+ spaces_between_special_tokens: bool = True
145
+ # Optional[List[LogitsProcessor]] type. We use Any here because
146
+ # Optional[List[LogitsProcessor]] type is not supported by msgspec.
147
+ logits_processors: Optional[Any] = None
148
+ include_stop_str_in_output: bool = False
149
+ truncate_prompt_tokens: Optional[Annotated[int, msgspec.Meta(ge=1)]] = None
150
+
151
+ # The below fields are not supposed to be used as an input.
152
+ # They are set in post_init.
153
+ output_text_buffer_length: int = 0
154
+ _all_stop_token_ids: Set[int] = msgspec.field(default_factory=set)
155
+
156
+ def __post_init__(self) -> None:
157
+ self.best_of = self.best_of or self.n
158
+ if 0 < self.temperature < _MAX_TEMP:
159
logger.warning(
160
"temperature %s is less than %s, which may cause numerical "
161
"errors nan or inf in tensors. We have maxed it out to %s.",
153
- temperature, _MAX_TEMP, _MAX_TEMP)
154
- temperature = max(temperature, _MAX_TEMP)
155
- self.temperature = temperature
156
- self.top_p = top_p
157
- self.top_k = top_k
158
- self.min_p = min_p
159
- if seed == -1:
162
+ self.temperature, _MAX_TEMP, _MAX_TEMP)
163
+ self.temperature = max(self.temperature, _MAX_TEMP)
164
+ if self.seed == -1:
165
self.seed = None
166
else:
162
- self.seed = seed
163
- self.use_beam_search = use_beam_search
164
- self.length_penalty = length_penalty
165
- self.early_stopping = early_stopping
166
- if stop is None:
167
+ self.seed = self.seed
168
+ if self.stop is None:
169
self.stop = []
168
- elif isinstance(stop, str):
169
- self.stop = [stop]
170
+ elif isinstance(self.stop, str):
171
+ self.stop = [self.stop]
172
else:
171
- self.stop = list(stop)
172
- if stop_token_ids is None:
173
+ self.stop = list(self.stop)
174
+ if self.stop_token_ids is None:
175
self.stop_token_ids = []
176
else:
175
- self.stop_token_ids = list(stop_token_ids)
176
- self.ignore_eos = ignore_eos
177
- self.max_tokens = max_tokens
178
- self.min_tokens = min_tokens
179
- self.logprobs = 1 if logprobs is True else logprobs
180
- self.prompt_logprobs = 1 if prompt_logprobs is True else prompt_logprobs
181
- # NOTE: This parameter is only exposed at the engine level for now.
182
- # It is not exposed in the OpenAI API server, as the OpenAI API does
183
- # not support returning only a list of token IDs.
184
- self.detokenize = detokenize
185
- self.skip_special_tokens = skip_special_tokens
186
- self.spaces_between_special_tokens = spaces_between_special_tokens
187
- self.logits_processors = logits_processors
188
- self.include_stop_str_in_output = include_stop_str_in_output
189
- self.truncate_prompt_tokens = truncate_prompt_tokens
177
+ self.stop_token_ids = list(self.stop_token_ids)
178
+ self.logprobs = 1 if self.logprobs is True else self.logprobs
179
+ self.prompt_logprobs = (1 if self.prompt_logprobs is True else
180
+ self.prompt_logprobs)
181
+
182
# Number of characters to hold back for stop string evaluation
183
# until sequence is finished.
192
- if self.stop and not include_stop_str_in_output:
184
+ if self.stop and not self.include_stop_str_in_output:
185
self.output_text_buffer_length = max(len(s) for s in self.stop) - 1
194
- else:
195
- self.output_text_buffer_length = 0
186
187
self._verify_args()
188
if self.use_beam_search:
@@ -206,11 +196,12 @@ class SamplingParams:
196
self.min_p = 0.0
197
self._verify_greedy_sampling()
198
# eos_token_id is added to this by the engine
209
- self.all_stop_token_ids = set(self.stop_token_ids)
199
+ self._all_stop_token_ids = set(self.stop_token_ids)
200
201
def _verify_args(self) -> None:
202
if self.n < 1:
203
raise ValueError(f"n must be at least 1, got {self.n}.")
204
+ assert isinstance(self.best_of, int)
205
if self.best_of < self.n:
206
raise ValueError(f"best_of must be greater than or equal to n, "
207
f"got n={self.n} and best_of={self.best_of}.")
@@ -257,6 +248,7 @@ class SamplingParams:
248
and self.truncate_prompt_tokens < 1):
249
raise ValueError(f"truncate_prompt_tokens must be >= 1, "
250
f"got {self.truncate_prompt_tokens}")
251
+ assert isinstance(self.stop, list)
252
if any(not stop_str for stop_str in self.stop):
253
raise ValueError("stop cannot contain an empty string.")
254
if self.stop and not self.detokenize:
@@ -290,6 +282,7 @@ class SamplingParams:
282
"default value of 1.0 when not using beam search.")
283
284
def _verify_greedy_sampling(self) -> None:
285
+ assert isinstance(self.best_of, int)
286
if self.best_of > 1:
287
raise ValueError("best_of must be 1 when using greedy sampling."
288
f"Got {self.best_of}.")
@@ -303,7 +296,7 @@ class SamplingParams:
296
if model_eos_token_id is not None:
297
# Add the eos token id into the sampling_params to support
298
# min_tokens processing.
306
- self.all_stop_token_ids.add(model_eos_token_id)
299
+ self._all_stop_token_ids.add(model_eos_token_id)
300
301
# Update eos_token_id for generation
302
if (eos_ids := generation_config.get("eos_token_id")) is not None:
@@ -315,7 +308,7 @@ class SamplingParams:
308
# purposes.
309
eos_ids.discard(model_eos_token_id)
310
if eos_ids:
318
- self.all_stop_token_ids.update(eos_ids)
311
+ self._all_stop_token_ids.update(eos_ids)
312
if not self.ignore_eos:
313
eos_ids.update(self.stop_token_ids)
314
self.stop_token_ids = list(eos_ids)
@@ -330,6 +323,10 @@ class SamplingParams:
323
return SamplingType.RANDOM_SEED
324
return SamplingType.RANDOM
325
326
+ @property
327
+ def all_stop_token_ids(self) -> Set[int]:
328
+ return self._all_stop_token_ids
329
+
330
def clone(self) -> "SamplingParams":
331
"""Deep copy excluding LogitsProcessor objects.
332
vllm/sequence.py
+250
-146
@@ -4,10 +4,11 @@ import enum
4
from abc import ABC, abstractmethod
5
from array import array
6
from collections import defaultdict
7
-from dataclasses import dataclass, field
8
-from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
9
- Union, cast)
7
+from dataclasses import dataclass
8
+from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
9
+ Tuple, Union, cast)
10
11
+import msgspec
12
import numpy
13
import torch
14
@@ -16,13 +17,18 @@ from vllm.lora.request import LoRARequest
17
from vllm.pooling_params import PoolingParams
18
from vllm.prompt_adapter.request import PromptAdapterRequest
19
from vllm.sampling_params import SamplingParams
20
+from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22
if TYPE_CHECKING:
23
from vllm.inputs import LLMInputs
22
- from vllm.multimodal import MultiModalDataDict
23
- from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
24
+ from vllm.multimodal.base import MultiModalDataDict
25
26
+VLLM_TOKEN_ID_ARRAY_TYPE = "l"
27
28
+
29
+# We use dataclass for now because it is used for
30
+# openai server output, and msgspec is not serializable.
31
+# TODO(sang): Fix it.
32
@dataclass
33
class Logprob:
34
"""Infos for supporting OpenAI compatible logprobs and token ranks.
@@ -112,7 +118,23 @@ class RequestMetrics:
118
model_execute_time: Optional[float] = None
119
120
115
-class SequenceData:
121
+class SequenceDataDelta(
122
+ msgspec.Struct,
123
+ array_like=True, # type: ignore[call-arg]
124
+ omit_defaults=True): # type: ignore[call-arg]
125
+ """Delta SequenceData to send to workers per step."""
126
+ # A new token to be appended to existing SequenceData.
127
+ new_output_token_ids: List[int]
128
+ # Overwriting existing `cumulative_logprob`
129
+ new_cumulative_logprob: float
130
+ # Overwriting existing `num_computed_tokens`.
131
+ new_num_computed_tokens: int
132
+ # Overwriting existing `stage`.
133
+ new_stage: SequenceStage
134
+
135
+
136
+class SequenceData(msgspec.Struct,
137
+ omit_defaults=True): # type: ignore[call-arg]
138
"""Data associated with a sequence.
139
140
Args:
@@ -125,40 +147,57 @@ class SequenceData:
147
output_token_ids: The token IDs of the output.
148
cumulative_logprob: The cumulative log probability of the output.
149
"""
128
-
129
- def __init__(
130
- self,
131
- prompt_token_ids: List[int],
132
- output_token_ids: Optional[List[int]] = None,
133
- ) -> None:
134
- self._prompt_token_ids = array('l', prompt_token_ids)
135
- self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
136
- self._output_token_ids = array(
137
- 'l', output_token_ids if output_token_ids is not None else [])
138
-
139
- self.cumulative_logprob = 0.0
140
- # The number of tokens that are computed (that run against the model).
141
- self._num_computed_tokens = 0
142
- self._stage: SequenceStage = SequenceStage.PREFILL
143
-
150
+ # NOTE: we cannot use Union[List, array] because msgspec cannot support
151
+ # union of 2 list types.
152
+ _prompt_token_ids: array
153
+ _output_token_ids: array = msgspec.field(
154
+ default_factory=lambda: array(VLLM_TOKEN_ID_ARRAY_TYPE, []))
155
+
156
+ ### The below fields should not be passed as an argument ###
157
+ _cumulative_logprob: float = 0.0
158
+ _prompt_token_ids_tuple: Tuple[int,
159
+ ...] = msgspec.field(default_factory=tuple)
160
+ # The number of tokens that are computed (that run against the model).
161
+ _num_computed_tokens: int = 0
162
+ _stage: SequenceStage = SequenceStage.PREFILL
163
+ _cached_all_token_ids: List[int] = msgspec.field(default_factory=list)
164
+
165
+ # It is used to get delta input. It is reset when `get_delta_and_reset`
166
+ # is called.
167
+ _new_appended_tokens: List[int] = msgspec.field(default_factory=list)
168
+
169
+ def __post_init__(self) -> None:
170
+ assert self._prompt_token_ids.typecode == "l"
171
+ assert self._output_token_ids.typecode == "l"
172
+ self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(
173
+ self._prompt_token_ids)
174
self._update_cached_all_tokens()
175
176
def _update_cached_all_tokens(self):
177
+ assert isinstance(self._prompt_token_ids, array)
178
+ assert isinstance(self._output_token_ids, array)
179
self._cached_all_token_ids: List[int] = list(self._prompt_token_ids +
180
self._output_token_ids)
181
182
+ @property
183
+ def cumulative_logprob(self) -> float:
184
+ return self._cumulative_logprob
185
+
186
@property
187
def prompt_token_ids(self) -> Tuple[int, ...]:
188
return self._prompt_token_ids_tuple
189
190
@prompt_token_ids.setter
191
def prompt_token_ids(self, new_prompt_token_ids) -> None:
156
- self._prompt_token_ids = array('l', new_prompt_token_ids)
157
- self._prompt_token_ids_tuple = tuple(new_prompt_token_ids)
158
- self._update_cached_all_tokens()
192
+ raise NotImplementedError
193
194
@property
195
def prompt_token_ids_array(self) -> array:
196
+ """Return the prompt token ids in array type.
197
+
198
+ Note that the array is in "I" type, and it is not compatible
199
+ with torch.long (2 bytes vs 4 bytes). So beware of the usage.
200
+ """
201
return self._prompt_token_ids
202
203
@property
@@ -166,18 +205,26 @@ class SequenceData:
205
return tuple(self._output_token_ids)
206
207
@output_token_ids.setter
169
- def output_token_ids(self, new_output_token_ids) -> None:
170
- self._output_token_ids = array('l', new_output_token_ids)
208
+ def output_token_ids(self, new_output_token_ids: List[int]) -> None:
209
+ self._output_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
210
+ new_output_token_ids)
211
self._update_cached_all_tokens()
212
213
@property
214
def output_token_ids_array(self) -> array:
215
+ """Return the prompt token ids in array type.
216
+
217
+ Note that the array is in "I" type, and it is not compatible
218
+ with torch.long (2 bytes vs 4 bytes). So beware of the usage.
219
+ """
220
+ assert isinstance(self._output_token_ids, array)
221
return self._output_token_ids
222
223
def append_token_id(self, token_id: int, logprob: float) -> None:
224
self._output_token_ids.append(token_id)
225
+ self._new_appended_tokens.append(token_id)
226
self._cached_all_token_ids.append(token_id)
180
- self.cumulative_logprob += logprob
227
+ self._cumulative_logprob += logprob
228
229
def get_len(self) -> int:
230
return len(self._output_token_ids) + len(self._prompt_token_ids)
@@ -222,6 +269,7 @@ class SequenceData:
269
"""
270
self._num_computed_tokens = 0
271
self._stage = SequenceStage.PREFILL
272
+ self._new_appended_tokens = []
273
274
def get_num_uncomputed_tokens(self) -> int:
275
"""Return the number of prefill tokens that are not computed."""
@@ -241,6 +289,21 @@ class SequenceData:
289
def get_output_token_ids(self) -> Tuple[int, ...]:
290
return self.output_token_ids
291
292
+ def get_delta_and_reset(self) -> SequenceDataDelta:
293
+ delta = SequenceDataDelta(self._new_appended_tokens,
294
+ self._cumulative_logprob,
295
+ self.get_num_computed_tokens(), self.stage)
296
+ # Reset delta state.
297
+ self._new_appended_tokens = []
298
+ return delta
299
+
300
+ def apply_delta(self, delta: SequenceDataDelta):
301
+ self._num_computed_tokens = delta.new_num_computed_tokens
302
+ self._cumulative_logprob = delta.new_cumulative_logprob
303
+ self._stage = delta.new_stage
304
+ self._output_token_ids.extend(delta.new_output_token_ids)
305
+ self._cached_all_token_ids.extend(delta.new_output_token_ids)
306
+
307
@property
308
def stage(self) -> SequenceStage:
309
return self._stage
@@ -248,8 +311,9 @@ class SequenceData:
311
def __repr__(self) -> str:
312
return (f"SequenceData("
313
f"prompt_token_ids={self._prompt_token_ids}, "
251
- f"output_token_ids={self._output_token_ids}, "
252
- f"cumulative_logprob={self.cumulative_logprob})")
314
+ f"output_token_ids={self.output_token_ids}, "
315
+ f"cumulative_logprob={self.cumulative_logprob}, "
316
+ f"get_num_computed_tokens={self.get_num_computed_tokens()}")
317
318
319
class Sequence:
@@ -325,7 +389,8 @@ class Sequence:
389
f"invalid input {inputs}; did you forget the "
390
"encoder input prompt fields?")
391
328
- self.data = SequenceData(self.prompt_token_ids)
392
+ self.data = SequenceData(
393
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, self.prompt_token_ids))
394
self.output_logprobs: SampleLogprobs = []
395
self.output_text = ""
396
@@ -490,8 +555,8 @@ class Sequence:
555
f"num_blocks={self.n_blocks}, ")
556
557
493
-@dataclass
494
-class SequenceGroupState:
558
+class SequenceGroupState(msgspec.Struct,
559
+ omit_defaults=True): # type: ignore[call-arg]
560
"""Mutable state tied to a specific sequence group"""
561
562
# for multi-step decoding
@@ -647,14 +712,19 @@ class SequenceGroup:
712
if self.sampling_params and self.sampling_params.use_beam_search:
713
# For beam search, maximally there will always be `best_of` beam
714
# candidates running in the future.
650
- return self.sampling_params.best_of
715
+ best_of = self.sampling_params.best_of
716
+ assert isinstance(best_of, int)
717
+ return best_of
718
else:
652
- if (self.sampling_params
653
- and self.sampling_params.best_of > self.num_seqs()):
654
- # At prompt stage, the sequence group is not yet filled up
655
- # and only have one sequence running. However, in the
656
- # generation stage, we will have `best_of` sequences running.
657
- return self.sampling_params.best_of
719
+ if self.sampling_params:
720
+ best_of = self.sampling_params.best_of
721
+ assert isinstance(best_of, int)
722
+ if best_of > self.num_seqs():
723
+ # At prompt stage, the sequence group is not yet filled up
724
+ # and only have one sequence running. However, in the
725
+ # generation stage, we will have `best_of` sequences
726
+ # running.
727
+ return best_of
728
# At sampling stages, return the number of actual sequences
729
# that are not finished yet.
730
return self.num_unfinished_seqs()
@@ -757,7 +827,32 @@ class SequenceGroup:
827
f"num_seqs={len(self.seqs)})")
828
829
760
-class SequenceGroupMetadata:
830
+class SequenceGroupMetadataDelta(
831
+ msgspec.Struct,
832
+ tag=True, # type: ignore[call-arg]
833
+ array_like=True, # type: ignore[call-arg]
834
+ omit_defaults=True): # type: ignore[call-arg]
835
+ """Delta of SequenceGroupMetadata.
836
+
837
+ After sending the first SequenceGroupMetadata, vLLM scheduler
838
+ only sends delta to reduce the data payload size.
839
+ """
840
+ seq_data_delta: Dict[int, SequenceDataDelta]
841
+ request_id: str
842
+ block_tables: Dict[int, List[int]]
843
+ is_prompt: bool
844
+ do_sample: bool = True
845
+ token_chunk_size: Optional[int] = None
846
+ computed_block_nums: Optional[List[int]] = None
847
+ state: Optional[SequenceGroupState] = msgspec.field(
848
+ default_factory=lambda: SequenceGroupState())
849
+
850
+
851
+class SequenceGroupMetadata(
852
+ msgspec.Struct,
853
+ tag=True, # type: ignore[call-arg]
854
+ array_like=True, # type: ignore[call-arg]
855
+ omit_defaults=True): # type: ignore[call-arg]
856
"""Metadata for a sequence group. Used to create `AttentionMetadata`.
857
858
Args:
@@ -789,52 +884,39 @@ class SequenceGroupMetadata:
884
prompt_adapter_request: Prompt Adapter request.
885
"""
886
792
- def __init__(
793
- self,
794
- request_id: str,
795
- is_prompt: bool,
796
- seq_data: Dict[int, SequenceData],
797
- sampling_params: SamplingParams,
798
- block_tables: Dict[int, List[int]],
799
- do_sample: bool = True,
800
- pooling_params: Optional[PoolingParams] = None,
801
- token_chunk_size: Optional[int] = None,
802
- lora_request: Optional[LoRARequest] = None,
803
- computed_block_nums: Optional[List[int]] = None,
804
- state: Optional[SequenceGroupState] = None,
805
- multi_modal_data: Optional["MultiModalDataDict"] = None,
806
- encoder_seq_data: Optional[SequenceData] = None,
807
- cross_block_table: Optional[List[int]] = None,
808
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
809
- ) -> None:
810
- self.request_id = request_id
811
- self.is_prompt = is_prompt
812
- self.seq_data = seq_data
813
- self.sampling_params = sampling_params
814
- self.block_tables = block_tables
815
- self.pooling_params = pooling_params
816
- self.lora_request = lora_request
817
- self.prompt_adapter_request = prompt_adapter_request
818
- self.computed_block_nums = computed_block_nums
819
- self.multi_modal_data = multi_modal_data
820
- self.state = SequenceGroupState() if state is None else state
821
- self.encoder_seq_data = encoder_seq_data
822
- self.cross_block_table = cross_block_table
823
- self._token_chunk_size = token_chunk_size
824
- self.do_sample = do_sample
825
-
826
- # The number of speculative tokens adopted in this request.
827
- # None means specuative decoding is not used.
828
- # Zero means speculative decoding is disabled for some reasons.
829
- # TODO: We should maintain this states out of the sequence group.
830
- self.num_speculative_tokens = None
831
-
832
- if seq_data is not None and self._token_chunk_size is None:
833
- if is_prompt:
834
- self._token_chunk_size = next(iter(
835
- seq_data.values())).get_len()
887
+ request_id: str
888
+ is_prompt: bool
889
+ seq_data: Dict[int, SequenceData]
890
+ sampling_params: SamplingParams
891
+ block_tables: Dict[int, List[int]]
892
+ do_sample: bool = True
893
+ pooling_params: Optional[PoolingParams] = None
894
+ lora_request: Optional[LoRARequest] = None
895
+ computed_block_nums: Optional[List[int]] = None
896
+ state: Optional[SequenceGroupState] = msgspec.field(
897
+ default_factory=lambda: SequenceGroupState())
898
+ # "MultiModalDataDict" types. We have to use Any due to msgspec
899
+ # doesn't allow to have union of 2 different dicts.
900
+ multi_modal_data: Optional[Any] = None
901
+ encoder_seq_data: Optional[SequenceData] = None
902
+ cross_block_table: Optional[List[int]] = None
903
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
904
+ token_chunk_size: Optional[int] = None
905
+
906
+ ### Stateful fields that are lazily defined. ###
907
+ # The number of speculative tokens adopted in this request.
908
+ # None means specuative decoding is not used.
909
+ # Zero means speculative decoding is disabled for some reasons.
910
+ # TODO: We should maintain this states out of the sequence group.
911
+ num_speculative_tokens: Optional[int] = None
912
+
913
+ def __post_init__(self):
914
+ if self.seq_data is not None and self.token_chunk_size is None:
915
+ if self.is_prompt:
916
+ self.token_chunk_size = next(iter(
917
+ self.seq_data.values())).get_len()
918
else:
837
- self._token_chunk_size = 1
919
+ self.token_chunk_size = 1
920
921
@property
922
def lora_int_id(self) -> int:
@@ -850,18 +932,26 @@ class SequenceGroupMetadata:
932
return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens \
933
if self.prompt_adapter_request else 0
934
853
- @property
854
- def token_chunk_size(self) -> int:
855
- """Return the number of tokens to be processed (chunk size)."""
856
- assert self._token_chunk_size is not None
857
- return self._token_chunk_size
935
+ def apply_delta(self,
936
+ sequence_group_metadata_delta: SequenceGroupMetadataDelta):
937
+ for id, delta in sequence_group_metadata_delta.seq_data_delta.items():
938
+ self.seq_data[id].apply_delta(delta)
939
+ assert self.request_id == sequence_group_metadata_delta.request_id
940
+ self.block_tables = sequence_group_metadata_delta.block_tables
941
+ self.token_chunk_size = sequence_group_metadata_delta.token_chunk_size
942
+ self.do_sample = sequence_group_metadata_delta.do_sample
943
+ self.is_prompt = sequence_group_metadata_delta.is_prompt
944
945
def finish_step(self) -> None:
946
+ assert self.state is not None
947
assert self.state.current_step < self.state.num_steps
948
self.state.current_step += 1
949
950
864
-class SequenceOutput:
951
+class SequenceOutput(
952
+ msgspec.Struct,
953
+ omit_defaults=True, # type: ignore[call-arg]
954
+ array_like=True): # type: ignore[call-arg]
955
"""The model output associated with a sequence.
956
957
Args:
@@ -871,16 +961,9 @@ class SequenceOutput:
961
logprobs: The logprobs of the output token.
962
(Token id -> logP(x_i+1 | x_0, ..., x_i))
963
"""
874
-
875
- def __init__(
876
- self,
877
- parent_seq_id: int,
878
- output_token: int,
879
- logprobs: Dict[int, Logprob],
880
- ) -> None:
881
- self.parent_seq_id = parent_seq_id
882
- self.output_token = output_token
883
- self.logprobs = logprobs
964
+ parent_seq_id: int
965
+ output_token: int
966
+ logprobs: Dict[int, Logprob]
967
968
def __repr__(self) -> str:
969
return (f"SequenceOutput(parent_seq_id={self.parent_seq_id}, "
@@ -908,17 +991,15 @@ class SequenceGroupOutput(ABC):
991
pass
992
993
911
-class CompletionSequenceGroupOutput(SequenceGroupOutput):
994
+class CompletionSequenceGroupOutput(
995
+ msgspec.Struct,
996
+ omit_defaults=True, # type: ignore[call-arg]
997
+ array_like=True): # type: ignore[call-arg]
998
+ __metaclass__ = SequenceGroupOutput
999
"""The model output associated with a completion sequence group."""
913
-
914
- def __init__(
915
- self,
916
- samples: List[SequenceOutput],
917
- prompt_logprobs: Optional[PromptLogprobs],
918
- ) -> None:
919
- self.samples = samples
920
- # Prompt logprob for each prompt query token.
921
- self.prompt_logprobs = prompt_logprobs
1000
+ samples: List[SequenceOutput]
1001
+ # Prompt logprob for each prompt query token.
1002
+ prompt_logprobs: Optional[PromptLogprobs]
1003
1004
def __repr__(self) -> str:
1005
return (f"CompletionSequenceGroupOutput(samples={self.samples}, "
@@ -931,14 +1012,14 @@ class CompletionSequenceGroupOutput(SequenceGroupOutput):
1012
and self.prompt_logprobs == other.prompt_logprobs)
1013
1014
934
-class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
1015
+class EmbeddingSequenceGroupOutput(
1016
+ msgspec.Struct,
1017
+ omit_defaults=True, # type: ignore[call-arg]
1018
+ array_like=True, # type: ignore[call-arg]
1019
+):
1020
"""The model output associated with an embedding sequence group."""
936
-
937
- def __init__(
938
- self,
939
- embeddings: List[float],
940
- ) -> None:
941
- self.embeddings = embeddings
1021
+ __metaclass__ = SequenceGroupOutput
1022
+ embeddings: List[int]
1023
1024
def __repr__(self) -> str:
1025
return (f"EmbeddingSequenceGroupOutput("
@@ -950,8 +1031,10 @@ class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
1031
return self.embeddings == other.embeddings
1032
1033
953
-@dataclass
954
-class IntermediateTensors:
1034
+class IntermediateTensors(
1035
+ msgspec.Struct,
1036
+ omit_defaults=True, # type: ignore[call-arg]
1037
+ array_like=True): # type: ignore[call-arg]
1038
"""For all pipeline stages except the last, we need to return the hidden
1039
states and residuals to be sent to the next stage. This data structure
1040
contains the hidden states and residuals for a request.
@@ -978,8 +1061,10 @@ class IntermediateTensors:
1061
return f"IntermediateTensors(tensors={self.tensors})"
1062
1063
981
-@dataclass
982
-class SamplerOutput:
1064
+class SamplerOutput(
1065
+ msgspec.Struct,
1066
+ omit_defaults=True, # type: ignore[call-arg]
1067
+ array_like=True): # type: ignore[call-arg]
1068
"""For each sequence group, we generate a list of SequenceOutput object,
1069
each of which contains one possible candidate for the next token.
1070
@@ -1000,7 +1085,7 @@ class SamplerOutput:
1085
sampled_token_ids_numpy: Optional[numpy.ndarray] = None
1086
1087
# Spec decode metrics populated by workers.
1003
- spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
1088
+ spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1089
1090
# Optional last hidden states from the model.
1091
hidden_states: Optional[torch.Tensor] = None
@@ -1039,12 +1124,14 @@ class SamplerOutput:
1124
f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
1125
1126
1042
-@dataclass
1043
-class PoolerOutput:
1127
+class PoolerOutput(
1128
+ msgspec.Struct,
1129
+ omit_defaults=True, # type: ignore[call-arg]
1130
+ array_like=True): # type: ignore[call-arg]
1131
"""The output from a pooling operation in the embedding model."""
1132
outputs: List[EmbeddingSequenceGroupOutput]
1133
1047
- spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
1134
+ spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1135
1136
def __getitem__(self, idx: int):
1137
return self.outputs[idx]
@@ -1083,7 +1170,8 @@ def get_all_seq_ids_and_request_ids(
1170
return seq_ids, request_id_seq_ids_mapping
1171
1172
1086
-class HiddenStates:
1173
+class HiddenStates(msgspec.Struct, array_like=True,
1174
+ omit_defaults=True): # type: ignore[call-arg]
1175
"""Hidden states corresponding to in-progress sequences.
1176
Used in speculative decoding to pass hidden states from
1177
the target model to the proposer model in the subsequent step.
@@ -1091,42 +1179,53 @@ class HiddenStates:
1179
seq_ids are the sequence ids of each entry of the batch
1180
dimension of the hidden_states tensor"""
1181
1094
- def __init__(self, seq_group_metadata_list: List[SequenceGroupMetadata],
1095
- hidden_states: torch.Tensor):
1096
- assert len(seq_group_metadata_list) == len(hidden_states)
1097
- self.seq_ids: List[int] = get_all_seq_ids(seq_group_metadata_list)
1098
- self.hidden_states: torch.Tensor = hidden_states
1182
+ seq_group_metadata_list: List[SequenceGroupMetadata]
1183
+ hidden_states: torch.Tensor
1184
+ _seq_ids: List[int] = msgspec.field(default_factory=list)
1185
+
1186
+ def __post_init__(self):
1187
+ self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list)
1188
+ assert len(self.seq_group_metadata_list) == len(self.hidden_states)
1189
+
1190
+ @property
1191
+ def seq_ids(self) -> List[int]:
1192
+ return self._seq_ids
1193
1194
def update(self, seq_group_metadata_list: List[SequenceGroupMetadata],
1195
hidden_states: torch.Tensor) -> None:
1196
"""Update hidden states from target model invocation."""
1197
assert len(seq_group_metadata_list) == len(hidden_states)
1104
- self.seq_ids.extend(get_all_seq_ids(seq_group_metadata_list))
1198
+ self._seq_ids.extend(get_all_seq_ids(seq_group_metadata_list))
1199
self.hidden_states = torch.cat([self.hidden_states, hidden_states])
1200
1201
def prune(self,
1202
seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
1203
"""Prune to provided list of sequence ids."""
1204
seq_ids = get_all_seq_ids(seq_group_metadata_list)
1111
- if seq_ids != self.seq_ids:
1205
+ if seq_ids != self._seq_ids:
1206
# Batch contents changed - prune removed sequences.
1113
- index = [self.seq_ids.index(seq_id) for seq_id in seq_ids]
1207
+ index = [self._seq_ids.index(seq_id) for seq_id in seq_ids]
1208
self.hidden_states = self.hidden_states[index]
1115
- self.seq_ids = seq_ids
1209
+ self._seq_ids = seq_ids
1210
1211
1118
-@dataclass
1119
-class ExecuteModelRequest:
1212
+class ExecuteModelRequest(
1213
+ msgspec.Struct,
1214
+ array_like=True, # type: ignore[call-arg]
1215
+ omit_defaults=True): # type: ignore[call-arg]
1216
"""The model execution request, containing CPU metadata only. The LLM
1217
engine should create an instance of this class for each request batch."""
1218
# The sequence group metadata list.
1123
- seq_group_metadata_list: List[SequenceGroupMetadata]
1219
+ seq_group_metadata_list: List[Union[SequenceGroupMetadata,
1220
+ SequenceGroupMetadataDelta]]
1221
# Blocks to swap in. List of CPU -> GPU block number.
1125
- blocks_to_swap_in: List[Tuple[int, int]] = field(default_factory=list)
1222
+ blocks_to_swap_in: List[Tuple[int,
1223
+ int]] = msgspec.field(default_factory=list)
1224
# Blocks to swap out. List of GPU -> CPU block number.
1127
- blocks_to_swap_out: List[Tuple[int, int]] = field(default_factory=list)
1225
+ blocks_to_swap_out: List[Tuple[int,
1226
+ int]] = msgspec.field(default_factory=list)
1227
# Blocks to copy. Source to dest block.
1129
- blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
1228
+ blocks_to_copy: List[Tuple[int, int]] = msgspec.field(default_factory=list)
1229
# Virtual engine ID for pipeline parallel.
1230
virtual_engine: int = 0
1231
# The number of slots for lookahead decoding.
@@ -1138,7 +1237,7 @@ class ExecuteModelRequest:
1237
# The number of forward steps to run.
1238
num_steps: int = 1
1239
# Finished request ids since last step.
1141
- finished_requests_ids: List[str] = field(default_factory=list)
1240
+ finished_requests_ids: List[str] = msgspec.field(default_factory=list)
1241
# The last sampled token ids for multi step decoding.
1242
last_sampled_token_ids: Optional[torch.Tensor] = None
1243
@@ -1148,6 +1247,7 @@ class ExecuteModelRequest:
1247
# steps
1248
assert len(self.seq_group_metadata_list) > 0
1249
first_seq_group = self.seq_group_metadata_list[0]
1250
+ assert first_seq_group.state is not None
1251
return first_seq_group.state.current_step == 0
1252
1253
@property
@@ -1156,6 +1256,7 @@ class ExecuteModelRequest:
1256
# steps
1257
assert len(self.seq_group_metadata_list) > 0
1258
first_seq_group = self.seq_group_metadata_list[0]
1259
+ assert first_seq_group.state is not None
1260
num_steps = first_seq_group.state.num_steps
1261
current_step = first_seq_group.state.current_step
1262
return num_steps - current_step == 1
@@ -1165,10 +1266,13 @@ class ExecuteModelRequest:
1266
# TODO(will) make this be able to handle batches with variable number of
1267
# steps
1268
assert len(self.seq_group_metadata_list) > 0
1168
- return self.seq_group_metadata_list[0].state.current_step
1269
+ state = self.seq_group_metadata_list[0].state
1270
+ assert state is not None
1271
+ return state.current_step
1272
1273
def clone(
1171
- self, seq_group_metadata_list: List[SequenceGroupMetadata]
1274
+ self, seq_group_metadata_list: List[Union[SequenceGroupMetadata,
1275
+ SequenceGroupMetadataDelta]]
1276
) -> "ExecuteModelRequest":
1277
"""Clone the request with a new sequence group metadata list."""
1278
return ExecuteModelRequest(
vllm/spec_decode/batch_expansion.py
+8
-5
@@ -1,11 +1,13 @@
1
+from array import array
2
from itertools import chain, count
3
from typing import Iterator, List, Tuple
4
5
import torch
6
7
from vllm import SamplingParams
7
-from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8
- SequenceGroupMetadata, get_all_seq_ids)
8
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, ExecuteModelRequest,
9
+ SamplerOutput, SequenceData, SequenceGroupMetadata,
10
+ get_all_seq_ids)
11
from vllm.spec_decode.interfaces import (SpeculativeProposals,
12
SpeculativeScorer, SpeculativeScores)
13
from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
@@ -293,14 +295,15 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
295
input sequence.
296
"""
297
seq_data = seq_group_metadata.seq_data[seq_id]
296
- prompt_token_ids = seq_data.get_prompt_token_ids()
298
+ prompt_token_ids = seq_data.prompt_token_ids_array
299
new_output_token_ids = [*seq_data.get_output_token_ids(), *token_ids]
300
301
new_seq_data_dict = {
302
target_seq_id:
303
SequenceData(
302
- prompt_token_ids=prompt_token_ids,
303
- output_token_ids=new_output_token_ids,
304
+ prompt_token_ids,
305
+ _output_token_ids=array(VLLM_TOKEN_ID_ARRAY_TYPE,
306
+ new_output_token_ids),
307
),
308
}
309
# This is a hack. Technically, spec decoding should compute
vllm/spec_decode/metrics.py
+5
-3
@@ -1,7 +1,7 @@
1
import time
2
-from dataclasses import dataclass
2
from typing import Callable, Optional
3
4
+import msgspec
5
import torch
6
7
from vllm.model_executor.layers.spec_decode_base_sampler import (
@@ -9,8 +9,10 @@ from vllm.model_executor.layers.spec_decode_base_sampler import (
9
from vllm.utils import is_pin_memory_available
10
11
12
-@dataclass
13
-class SpecDecodeWorkerMetrics:
12
+class SpecDecodeWorkerMetrics(
13
+ msgspec.Struct,
14
+ omit_defaults=True, # type: ignore[call-arg]
15
+ array_like=True): # type: ignore[call-arg]
16
"""Dataclass holding metrics emitted from the spec decode worker.
17
"""
18
vllm/worker/worker.py
+62
-2
@@ -1,7 +1,7 @@
1
"""A GPU worker class."""
2
import gc
3
import os
4
-from typing import List, Optional, Set, Tuple, Type
4
+from typing import Dict, List, Optional, Set, Tuple, Type, Union
5
6
import torch
7
import torch.distributed
@@ -18,7 +18,9 @@ from vllm.model_executor import set_random_seed
18
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19
from vllm.platforms import current_platform
20
from vllm.prompt_adapter.request import PromptAdapterRequest
21
-from vllm.sequence import ExecuteModelRequest
21
+from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
22
+ SamplerOutput, SequenceGroupMetadata,
23
+ SequenceGroupMetadataDelta)
24
from vllm.worker.cache_engine import CacheEngine
25
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
26
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
@@ -109,6 +111,7 @@ class Worker(LocalOrDistributedWorkerBase):
111
self.cache_engine: List[CacheEngine]
112
# Initialize gpu_cache as embedding models don't initialize kv_caches
113
self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
114
+ self._seq_group_metadata_cache: Dict[str, SequenceGroupMetadata] = {}
115
116
def _is_encoder_decoder_model(self):
117
return self.model_config.is_encoder_decoder_model
@@ -303,6 +306,63 @@ class Worker(LocalOrDistributedWorkerBase):
306
and worker_input.blocks_to_copy.numel() > 0):
307
self.cache_engine[virtual_engine].copy(worker_input.blocks_to_copy)
308
309
+ def _get_cached_seq_group_metadata(
310
+ self,
311
+ seq_group_metadata_list: List[Union[SequenceGroupMetadata,
312
+ SequenceGroupMetadataDelta]],
313
+ finished_request_ids: List[str]) -> List[SequenceGroupMetadata]:
314
+ """Return a list of cached Sequence Group Metadata after updating its
315
+ state.
316
+
317
+ It is used because scheduler only sends delta to workers to reduce
318
+ the data payload size. The function also cleans up cache based on
319
+ a given `finished_request_ids`.
320
+ """
321
+ new_seq_group_metadata_list = []
322
+ for metadata_or_delta in seq_group_metadata_list:
323
+ request_id = metadata_or_delta.request_id
324
+ if request_id not in self._seq_group_metadata_cache:
325
+ # The first prefill.
326
+ assert isinstance(metadata_or_delta, SequenceGroupMetadata)
327
+ self._seq_group_metadata_cache[request_id] = metadata_or_delta
328
+ else:
329
+ # The first prefill is already cached.
330
+ if isinstance(metadata_or_delta, SequenceGroupMetadataDelta):
331
+ self._seq_group_metadata_cache[request_id].apply_delta(
332
+ metadata_or_delta)
333
+ else:
334
+ # If metadata snapshot is sent again, it is
335
+ # preempted. Reset the cache because we need to start
336
+ # from scratch.
337
+ assert isinstance(metadata_or_delta, SequenceGroupMetadata)
338
+ self._seq_group_metadata_cache[
339
+ request_id] = metadata_or_delta
340
+
341
+ new_seq_group_metadata_list.append(
342
+ self._seq_group_metadata_cache[request_id])
343
+
344
+ # Clean up finished ids
345
+ for finished_id in finished_request_ids:
346
+ del self._seq_group_metadata_cache[finished_id]
347
+
348
+ return new_seq_group_metadata_list
349
+
350
+ def _execute_model_spmd(
351
+ self,
352
+ execute_model_req: ExecuteModelRequest,
353
+ intermediate_tensors: Optional[IntermediateTensors] = None,
354
+ ) -> Optional[List[SamplerOutput]]:
355
+ if execute_model_req is not None:
356
+ new_seq_group_metadata_list = self._get_cached_seq_group_metadata(
357
+ execute_model_req.seq_group_metadata_list,
358
+ execute_model_req.finished_requests_ids)
359
+
360
+ execute_model_req.seq_group_metadata_list = (
361
+ new_seq_group_metadata_list)
362
+ output = super()._execute_model_spmd(execute_model_req,
363
+ intermediate_tensors)
364
+ return output
365
+
366
def add_lora(self, lora_request: LoRARequest) -> bool:
367
return self.model_runner.add_lora(lora_request)
368