@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ff7ec82c4d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Optimize SPMD architecture with delta + serialization optimization (#7109)

SangBin Cho committed Aug 18, 2024 at 17:57 UTC ff7ec82c4dd6170ea8fedbd4d974c0a670e84c97
36 files changed +727 -351
requirements-common.txt
+1
@@ -21,6 +21,7 @@ outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
21 typing_extensions >= 4.10
22 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
23 pyzmq
24 +msgspec
25 librosa # Required for audio processing
26 soundfile # Required for audio processing
27 gguf == 0.9.1
tests/basic_correctness/test_preemption.py
+18
@@ -8,6 +8,7 @@ pytest tests/basic_correctness/test_preemption.py`.
8 import pytest
9 from prometheus_client import REGISTRY
10
11 +import vllm.envs as envs
12 from vllm import SamplingParams
13 from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
14 ENABLE_ARTIFICIAL_PREEMPT)
@@ -24,6 +25,13 @@ assert ENABLE_ARTIFICIAL_PREEMPT is True, (
25 "tests/basic_correctness/test_preemption.py`")
26
27
28 +@pytest.fixture
29 +def worker_use_ray() -> bool:
30 + # When SPMD worker is used, use ray_use_worker=True
31 + # to test delta input optimization works with preemption.
32 + return envs.VLLM_USE_RAY_SPMD_WORKER
33 +
34 +
35 @pytest.mark.parametrize("model", MODELS)
36 @pytest.mark.parametrize("dtype", ["half"])
37 @pytest.mark.parametrize("max_tokens", [96])
@@ -36,6 +44,7 @@ def test_chunked_prefill_recompute(
44 dtype: str,
45 max_tokens: int,
46 chunked_prefill_token_size: int,
47 + worker_use_ray: bool,
48 ) -> None:
49 """Ensure that chunked prefill works with preemption."""
50 max_num_seqs = min(chunked_prefill_token_size, 256)
@@ -54,6 +63,7 @@ def test_chunked_prefill_recompute(
63 max_num_batched_tokens=max_num_batched_tokens,
64 enable_chunked_prefill=enable_chunked_prefill,
65 max_num_seqs=max_num_seqs,
66 + worker_use_ray=worker_use_ray,
67 ) as vllm_model:
68 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
69 assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
@@ -79,6 +89,7 @@ def test_preemption(
89 model: str,
90 dtype: str,
91 max_tokens: int,
92 + worker_use_ray: bool,
93 ) -> None:
94 """By default, recompute preemption is enabled"""
95
@@ -89,6 +100,7 @@ def test_preemption(
100 model,
101 dtype=dtype,
102 disable_log_stats=False,
103 + worker_use_ray=worker_use_ray,
104 ) as vllm_model:
105 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
106 assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
@@ -132,6 +144,7 @@ def test_swap(
144 dtype: str,
145 max_tokens: int,
146 beam_width: int,
147 + worker_use_ray: bool,
148 ) -> None:
149 """Use beam search enables swapping."""
150 example_prompts = example_prompts[:1]
@@ -144,6 +157,7 @@ def test_swap(
157 dtype=dtype,
158 swap_space=10,
159 disable_log_stats=False,
160 + worker_use_ray=worker_use_ray,
161 ) as vllm_model:
162 vllm_outputs = vllm_model.generate_beam_search(example_prompts,
163 beam_width, max_tokens)
@@ -188,6 +202,7 @@ def test_swap_infeasible(
202 dtype: str,
203 max_tokens: int,
204 beam_width: int,
205 + worker_use_ray: bool,
206 ) -> None:
207 """Verify infeasible swap request will be ignored."""
208 BLOCK_SIZE = 16
@@ -204,6 +219,7 @@ def test_swap_infeasible(
219 # decode blocks are not enough to finish.
220 num_gpu_blocks_override=prefill_blocks + decode_blocks,
221 max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
222 + worker_use_ray=worker_use_ray,
223 ) as vllm_model:
224 sampling_params = SamplingParams(n=beam_width,
225 use_beam_search=True,
@@ -230,6 +246,7 @@ def test_preemption_infeasible(
246 model: str,
247 dtype: str,
248 max_tokens: int,
249 + worker_use_ray: bool,
250 ) -> None:
251 """Verify infeasible preemption request will be ignored."""
252 BLOCK_SIZE = 16
@@ -244,6 +261,7 @@ def test_preemption_infeasible(
261 # ignored instead of hanging forever.
262 num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
263 max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
264 + worker_use_ray=worker_use_ray,
265 ) as vllm_model:
266 sampling_params = SamplingParams(max_tokens=max_tokens,
267 ignore_eos=True)
tests/core/test_serialization.py new
+33
@@ -0,0 +1,33 @@
1 +import msgspec
2 +
3 +from vllm.executor.msgspec_utils import decode_hook, encode_hook
4 +from vllm.sequence import ExecuteModelRequest
5 +
6 +from ..spec_decode.utils import create_batch
7 +
8 +
9 +def test_msgspec_serialization():
10 + num_lookahead_slots = 4
11 + seq_group_metadata_list, _, _ = create_batch(16, num_lookahead_slots)
12 + execute_model_req = ExecuteModelRequest(
13 + seq_group_metadata_list=seq_group_metadata_list,
14 + num_lookahead_slots=num_lookahead_slots,
15 + running_queue_size=4)
16 +
17 + encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
18 + decoder = msgspec.msgpack.Decoder(ExecuteModelRequest,
19 + dec_hook=decode_hook)
20 + req = decoder.decode(encoder.encode(execute_model_req))
21 + expected = execute_model_req.seq_group_metadata_list
22 + actual = req.seq_group_metadata_list
23 + assert (len(expected) == len(actual))
24 + expected = expected[0]
25 + actual = actual[0]
26 +
27 + assert expected.block_tables == actual.block_tables
28 + assert expected.is_prompt == actual.is_prompt
29 + assert expected.request_id == actual.request_id
30 + assert (expected.seq_data[0].prompt_token_ids ==
31 + actual.seq_data[0].prompt_token_ids)
32 + assert (expected.seq_data[0].output_token_ids ==
33 + actual.seq_data[0].output_token_ids)
tests/distributed/test_basic_distributed_correctness.py
+2 -1
@@ -22,7 +22,8 @@ TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
22 @pytest.mark.skipif(cuda_device_count_stateless() < 2,
23 reason="Need at least 2 GPUs to run the test.")
24 @pytest.mark.parametrize(
25 - "model, distributed_executor_backend, attention_backend, test_suite", [
25 + "model, distributed_executor_backend, attention_backend, "
26 + "test_suite", [
27 ("facebook/opt-125m", "ray", "", "L4"),
28 ("facebook/opt-125m", "mp", "", "L4"),
29 ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
tests/distributed/test_chunked_prefill_distributed.py
+7
@@ -6,6 +6,8 @@ pytest test_chunked_prefill_distributed.py
6 ```
7 """
8
9 +import os
10 +
11 import pytest
12
13 from vllm.utils import cuda_device_count_stateless
@@ -30,6 +32,11 @@ def test_models(
32 model: str,
33 distributed_executor_backend: str,
34 ) -> None:
35 + if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray": # noqa
36 + assert distributed_executor_backend == "ray"
37 + # test ray adag
38 + os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
39 + os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
40
41 dtype = "half"
42 max_tokens = 5
tests/samplers/test_sampler.py
+19 -6
@@ -1,5 +1,6 @@
1 import itertools
2 import random
3 +from array import array
4 from typing import Dict, List, Optional, Tuple
5 from unittest.mock import Mock, patch
6
@@ -10,7 +11,8 @@ from transformers import GenerationConfig, GenerationMixin
11 from vllm.model_executor.layers.sampler import Sampler
12 from vllm.model_executor.sampling_metadata import SamplingMetadata
13 from vllm.model_executor.utils import set_random_seed
13 -from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
14 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
15 + SequenceData, SequenceGroupMetadata)
16 from vllm.utils import Counter, is_pin_memory_available
17
18
@@ -56,7 +58,9 @@ def _do_sample(
58 SequenceGroupMetadata(
59 request_id=f"test_{i}",
60 is_prompt=True,
59 - seq_data={0: SequenceData([1, 2, 3])},
61 + seq_data={
62 + 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
63 + },
64 sampling_params=sampling_params,
65 block_tables={0: [1]},
66 ))
@@ -201,7 +205,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
205
206 def create_sequence_data(num_input=3, num_generated=0):
207 seq_data = SequenceData(
204 - random.choices(range(0, VOCAB_SIZE), k=num_input))
208 + array(VLLM_TOKEN_ID_ARRAY_TYPE,
209 + random.choices(range(0, VOCAB_SIZE), k=num_input)))
210 if num_generated > 0:
211 seq_data.output_token_ids = random.choices(range(0, VOCAB_SIZE),
212 k=num_generated)
@@ -504,7 +509,9 @@ def test_sampler_mixed(seed: int, device: str):
509 SequenceGroupMetadata(
510 request_id=f"test_{i}",
511 is_prompt=True,
507 - seq_data={0: SequenceData([1, 2, 3])},
512 + seq_data={
513 + 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
514 + },
515 sampling_params=sampling_params,
516 block_tables={0: [1]},
517 ))
@@ -600,7 +607,9 @@ def test_sampler_top_k_top_p(seed: int, device: str):
607 SequenceGroupMetadata(
608 request_id=f"test_{i}",
609 is_prompt=True,
603 - seq_data={0: SequenceData([1, 2, 3])},
610 + seq_data={
611 + 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
612 + },
613 sampling_params=SamplingParams(
614 temperature=1,
615 top_k=top_k,
@@ -650,7 +659,11 @@ def test_sampler_repetition_penalty_mixed(device: str):
659 SequenceGroupMetadata(
660 request_id=f"test_{i}",
661 is_prompt=True,
653 - seq_data={0: SequenceData([1, 2, 3])},
662 + seq_data={
663 + 0:
664 + SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
665 + [1, 2, 3]))
666 + },
667 sampling_params=sampling_params[i],
668 block_tables={0: [1]},
669 ))
tests/spec_decode/utils.py
+6 -3
@@ -1,3 +1,4 @@
1 +from array import array
2 from itertools import count
3 from typing import Callable, Dict, List, Optional
4 from typing import Sequence as GenericSequence
@@ -9,7 +10,8 @@ import torch
10 from vllm.engine.arg_utils import EngineArgs
11 from vllm.model_executor.utils import set_random_seed
12 from vllm.sampling_params import SamplingParams
12 -from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
13 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
14 + CompletionSequenceGroupOutput, Logprob,
15 SamplerOutput, SequenceData, SequenceGroupMetadata,
16 SequenceOutput)
17 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
@@ -138,8 +140,9 @@ def create_seq_group_metadata_from_prompts(
140 seq_data={
141 i:
142 SequenceData(
141 - prompt_token_ids=prompt_token_ids[:],
142 - output_token_ids=cont_token_ids[:],
143 + array(VLLM_TOKEN_ID_ARRAY_TYPE, prompt_token_ids[:]),
144 + _output_token_ids=array(VLLM_TOKEN_ID_ARRAY_TYPE,
145 + cont_token_ids[:]),
146 ),
147 },
148 sampling_params=SamplingParams(temperature=0.0, ),
tests/test_logits_processor.py
+6 -2
@@ -1,4 +1,5 @@
1 import random
2 +from array import array
3 from typing import Tuple
4 from unittest.mock import patch
5
@@ -8,7 +9,8 @@ import torch
9 from vllm.model_executor.layers.logits_processor import LogitsProcessor
10 from vllm.model_executor.sampling_metadata import SamplingMetadata
11 from vllm.model_executor.utils import set_random_seed
11 -from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
13 + SequenceData, SequenceGroupMetadata)
14 from vllm.utils import is_pin_memory_available
15
16
@@ -69,7 +71,9 @@ def test_logits_processors(seed: int, device: str):
71 SequenceGroupMetadata(
72 request_id=f"test_{i}",
73 is_prompt=True,
72 - seq_data={0: SequenceData([1, 2, 3])},
74 + seq_data={
75 + 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
76 + },
77 sampling_params=SamplingParams(temperature=0,
78 logits_processors=[pick_ith]),
79 block_tables={0: [1]},
tests/test_sequence.py
+5 -2
@@ -1,6 +1,9 @@
1 +from array import array
2 +
3 import pytest
4
3 -from vllm.sequence import (CompletionSequenceGroupOutput, SamplerOutput,
5 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
6 + CompletionSequenceGroupOutput, SamplerOutput,
7 SequenceData, SequenceOutput)
8
9 from .core.utils import create_dummy_prompt
@@ -54,7 +57,7 @@ def test_sampler_output_eq(sample_outputs):
57
58
59 def test_sequence_data_prefill():
57 - seq_data = SequenceData(prompt_token_ids=[1, 2, 3, 4])
60 + seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3, 4]))
61 assert seq_data.get_num_uncomputed_tokens() == 4
62 assert seq_data.get_num_computed_tokens() == 0
63 # advance by 2
tests/worker/test_encoder_decoder_model_runner.py
+11 -5
@@ -1,10 +1,12 @@
1 +from array import array
2 from typing import List
3
4 import pytest
5 import torch
6
7 from vllm.engine.arg_utils import EngineArgs
7 -from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
8 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
9 + SequenceData, SequenceGroupMetadata)
10 from vllm.utils import is_cpu
11 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
12
@@ -125,10 +127,12 @@ def test_prepare_prompt(
127 # make sure all tokens fit into one block
128 seq_len = i % (model_runner.block_size - 1) + 1
129 seq_lens.append(seq_len)
128 - seq_data = SequenceData(list(range(seq_len)))
130 + seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
131 + range(seq_len)))
132 encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
133 encoder_seq_lens.append(encoder_seq_len)
131 - encoder_seq_data = SequenceData(list(range(encoder_seq_len)))
134 + encoder_seq_data = SequenceData(
135 + array(VLLM_TOKEN_ID_ARRAY_TYPE, range(encoder_seq_len)))
136 seq_group_metadata = SequenceGroupMetadata(
137 request_id=f"test_{i}",
138 is_prompt=True,
@@ -319,10 +323,12 @@ def test_prepare_decode(
323 # make sure all tokens fit into one block
324 seq_len = i % (model_runner.block_size - 1) + 1
325 seq_lens.append(seq_len)
322 - seq_data = SequenceData(list(range(seq_len)))
326 + seq_data = SequenceData(
327 + array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
328 encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
329 encoder_seq_lens.append(encoder_seq_len)
325 - encoder_seq_data = SequenceData(list(range(encoder_seq_len)))
330 + encoder_seq_data = SequenceData(
331 + array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
332 seq_group_metadata = SequenceGroupMetadata(
333 request_id=f"test_{i}",
334 is_prompt=False,
tests/worker/test_model_runner.py
+10 -5
@@ -1,3 +1,4 @@
1 +from array import array
2 from typing import List
3
4 import pytest
@@ -7,7 +8,8 @@ from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
8 init_distributed_environment)
9 from vllm.engine.arg_utils import EngineArgs
10 from vllm.model_executor.sampling_metadata import SamplingMetadata
10 -from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
12 + SequenceData, SequenceGroupMetadata)
13 from vllm.utils import get_open_port
14 from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
15
@@ -46,7 +48,8 @@ def test_prepare_prompt(batch_size):
48 # make sure all tokens fit into one block
49 seq_len = i % (model_runner.block_size - 1) + 1
50 seq_lens.append(seq_len)
49 - seq_data = SequenceData(list(range(seq_len)))
51 + seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
52 + range(seq_len)))
53 seq_group_metadata = SequenceGroupMetadata(
54 request_id=f"test_{i}",
55 is_prompt=True,
@@ -163,7 +166,8 @@ def test_prepare_decode_cuda_graph(batch_size):
166 # make sure all tokens fit into one block
167 context_len = i % (model_runner.block_size - 1) + 1
168 context_lens.append(context_len)
166 - seq_data = SequenceData(list(range(context_len)))
169 + seq_data = SequenceData(
170 + array(VLLM_TOKEN_ID_ARRAY_TYPE, range(context_len)))
171 seq_data.update_num_computed_tokens(context_len)
172 # Append one token ID since prefill is finished.
173 seq_data.append_token_id(1, 0)
@@ -324,7 +328,8 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
328 # make sure all tokens fit into one block
329 seq_len = i % (model_runner.block_size - 1) + 1
330 seq_lens.append(seq_len)
327 - seq_data = SequenceData(list(range(seq_len)))
331 + seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
332 + range(seq_len)))
333 seq_group_metadata = SequenceGroupMetadata(
334 request_id=f"test_{i}",
335 is_prompt=True,
@@ -340,7 +345,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
345 for i in range(prefill_batch_size, batch_size):
346 # make sure all tokens fit into one block
347 context_len = i % (model_runner.block_size - 1) + 1
343 - prompt_toks = list(range(context_len))
348 + prompt_toks = array(VLLM_TOKEN_ID_ARRAY_TYPE, range(context_len))
349 seq_data = SequenceData(prompt_toks)
350 seq_data.append_token_id(1, 0)
351 seq_data.update_num_computed_tokens(context_len)
vllm/adapter_commons/request.py
-2
@@ -1,8 +1,6 @@
1 from abc import ABC, abstractmethod
2 -from dataclasses import dataclass
2
3
5 -@dataclass
4 class AdapterRequest(ABC):
5 """
6 Base class for adapter requests.
vllm/config.py
+9 -2
@@ -770,8 +770,8 @@ class ParallelConfig:
770 self.tokenizer_pool_config = tokenizer_pool_config
771 self.ray_workers_use_nsight = ray_workers_use_nsight
772 self.placement_group = placement_group
773 -
773 self.world_size = pipeline_parallel_size * self.tensor_parallel_size
774 +
775 if worker_use_ray:
776 if self.distributed_executor_backend is None:
777 self.distributed_executor_backend = "ray"
@@ -867,6 +867,11 @@ class SchedulerConfig:
867 swapping. However, when the sequence group has multiple sequences
868 (e.g., beam search), recomputation is not currently supported. In
869 such a case, we use swapping instead.
870 + send_delta_data: Private API. If used, scheduler sends delta data to
871 + workers instead of an entire data. It should be enabled only
872 + when SPMD worker architecture is enabled. I.e.,
873 + VLLM_USE_RAY_SPMD_WORKER=1
874 +
875 """
876
877 def __init__(self,
@@ -879,7 +884,8 @@ class SchedulerConfig:
884 enable_chunked_prefill: bool = False,
885 embedding_mode: Optional[bool] = False,
886 preemption_mode: Optional[str] = None,
882 - num_scheduler_steps: int = 1) -> None:
887 + num_scheduler_steps: int = 1,
888 + send_delta_data: bool = False) -> None:
889 if max_num_batched_tokens is not None:
890 self.max_num_batched_tokens = max_num_batched_tokens
891 else:
@@ -909,6 +915,7 @@ class SchedulerConfig:
915 self.embedding_mode = embedding_mode
916 self.preemption_mode = preemption_mode
917 self.num_scheduler_steps = num_scheduler_steps
918 + self.send_delta_data = send_delta_data
919 self._verify_args()
920
921 def _verify_args(self) -> None:
vllm/core/scheduler.py
+50 -38
@@ -12,7 +12,8 @@ from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 from vllm.prompt_adapter.request import PromptAdapterRequest
14 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
15 - SequenceGroupMetadata, SequenceStatus)
15 + SequenceGroupMetadata, SequenceGroupMetadataDelta,
16 + SequenceStatus)
17 from vllm.utils import PyObjectCache
18
19 logger = init_logger(__name__)
@@ -363,8 +364,6 @@ class Scheduler:
364 self.num_cumulative_preemption: int = 0
365
366 # Used to cache python objects
366 - self._seq_group_metadata_cache: PyObjectCache = PyObjectCache(
367 - seq_group_metadata_builder)
367 self._scheduler_running_outputs_cache: PyObjectCache = PyObjectCache(
368 scheduler_running_outputs_builder)
369 self._scheduled_seq_group_cache: PyObjectCache = PyObjectCache(
@@ -1048,15 +1047,10 @@ class Scheduler:
1047 token_chunk_size = scheduled_seq_group.token_chunk_size
1048 seq_group.maybe_set_first_scheduled_time(now)
1049
1051 - seq_group_metadata = self._seq_group_metadata_cache.get_object()
1052 - seq_group_metadata.seq_data.clear()
1053 - seq_group_metadata.block_tables.clear()
1054 -
1050 # seq_id -> SequenceData
1056 - seq_data: Dict[int, SequenceData] = seq_group_metadata.seq_data
1051 + seq_data: Dict[int, SequenceData] = {}
1052 # seq_id -> physical block numbers
1058 - block_tables: Dict[int,
1059 - List[int]] = seq_group_metadata.block_tables
1053 + block_tables: Dict[int, List[int]] = {}
1054
1055 if seq_group.is_encoder_decoder():
1056 # Encoder associated with SequenceGroup
@@ -1081,45 +1075,65 @@ class Scheduler:
1075 seq_group.get_seqs(status=SequenceStatus.RUNNING)))
1076
1077 do_sample = True
1084 - if seq_group.is_prefill():
1078 + is_prompt = seq_group.is_prefill()
1079 + # We should send the metadata to workers when the first prefill
1080 + # is sent. Subsequent requests could be chunked prefill or decode.
1081 + is_first_prefill = False
1082 + if is_prompt:
1083 seqs = seq_group.get_seqs()
1084 # Prefill has only 1 sequence.
1085 assert len(seqs) == 1
1086 + num_computed_tokens = seqs[0].data.get_num_computed_tokens()
1087 + is_first_prefill = num_computed_tokens == 0
1088 # In the next iteration, all prompt tokens are not computed.
1089 # It means the prefill is chunked, and we don't need sampling.
1090 # NOTE: We use get_len instead of get_prompt_len because when
1091 # a sequence is preempted, prefill includes previous generated
1092 # output tokens.
1093 - if (token_chunk_size + seqs[0].data.get_num_computed_tokens() <
1093 + if (token_chunk_size + num_computed_tokens <
1094 seqs[0].data.get_len()):
1095 do_sample = False
1096
1097 # It assumes the scheduled_seq_groups is ordered by
1098 # prefill < decoding.
1099 - is_prompt = seq_group.is_prefill()
1100 -
1101 - seq_group_metadata.__init__(
1102 - request_id=seq_group.request_id,
1103 - is_prompt=is_prompt,
1104 - seq_data=seq_data,
1105 - sampling_params=seq_group.sampling_params,
1106 - block_tables=block_tables,
1107 - do_sample=do_sample,
1108 - pooling_params=seq_group.pooling_params,
1109 - token_chunk_size=token_chunk_size,
1110 - lora_request=seq_group.lora_request,
1111 - computed_block_nums=common_computed_block_nums,
1112 - encoder_seq_data=encoder_seq_data,
1113 - cross_block_table=cross_block_table,
1114 - state=seq_group.state,
1115 - # `multi_modal_data` will only be present for the 1st comm
1116 - # between engine and worker.
1117 - # the subsequent comms can still use delta, but
1118 - # `multi_modal_data` will be None.
1119 - multi_modal_data=seq_group.multi_modal_data
1120 - if scheduler_outputs.num_prefill_groups > 0 else None,
1121 - prompt_adapter_request=seq_group.prompt_adapter_request,
1122 - )
1099 + if is_first_prefill or not self.scheduler_config.send_delta_data:
1100 + seq_group_metadata = SequenceGroupMetadata(
1101 + request_id=seq_group.request_id,
1102 + is_prompt=is_prompt,
1103 + seq_data=seq_data,
1104 + sampling_params=seq_group.sampling_params,
1105 + block_tables=block_tables,
1106 + do_sample=do_sample,
1107 + pooling_params=seq_group.pooling_params,
1108 + token_chunk_size=token_chunk_size,
1109 + lora_request=seq_group.lora_request,
1110 + computed_block_nums=common_computed_block_nums,
1111 + encoder_seq_data=encoder_seq_data,
1112 + cross_block_table=cross_block_table,
1113 + state=seq_group.state,
1114 + # `multi_modal_data` will only be present for the 1st comm
1115 + # between engine and worker.
1116 + # the subsequent comms can still use delta, but
1117 + # `multi_modal_data` will be None.
1118 + multi_modal_data=seq_group.multi_modal_data
1119 + if scheduler_outputs.num_prefill_groups > 0 else None,
1120 + prompt_adapter_request=seq_group.prompt_adapter_request,
1121 + )
1122 + else:
1123 + # When SPMD mode is enabled, we only send delta data except for
1124 + # the first request to reduce serialization cost.
1125 + seq_data_delta = {}
1126 + for id, data in seq_data.items():
1127 + seq_data_delta[id] = data.get_delta_and_reset()
1128 + seq_group_metadata = SequenceGroupMetadataDelta(
1129 + seq_data_delta,
1130 + seq_group.request_id,
1131 + block_tables,
1132 + is_prompt,
1133 + do_sample=do_sample,
1134 + token_chunk_size=token_chunk_size,
1135 + computed_block_nums=common_computed_block_nums,
1136 + )
1137 seq_group_metadata_list.append(seq_group_metadata)
1138
1139 # Now that the batch has been created, we can assume all blocks in the
@@ -1130,8 +1144,6 @@ class Scheduler:
1144 self.block_manager.mark_blocks_as_computed(
1145 scheduled_seq_group.seq_group)
1146
1133 - self._seq_group_metadata_cache.reset()
1134 -
1147 scheduler_time = time.perf_counter() - scheduler_start_time
1148 # Add this to scheduler time to all the sequences that are currently
1149 # running. This will help estimate if the scheduler is a significant
vllm/engine/arg_utils.py
+3
@@ -5,6 +5,7 @@ from dataclasses import dataclass
5 from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Type,
6 Union)
7
8 +import vllm.envs as envs
9 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
10 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
11 ObservabilityConfig, ParallelConfig,
@@ -905,6 +906,8 @@ class EngineArgs:
906 embedding_mode=model_config.embedding_mode,
907 preemption_mode=self.preemption_mode,
908 num_scheduler_steps=self.num_scheduler_steps,
909 + send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
910 + and parallel_config.use_ray),
911 )
912 lora_config = LoRAConfig(
913 max_lora_rank=self.max_lora_rank,
vllm/engine/llm_engine.py
-1
@@ -224,7 +224,6 @@ class LLMEngine:
224 cache_config.enable_prefix_caching,
225 )
226 # TODO(woosuk): Print more configs in debug mode.
227 -
227 from vllm.plugins import load_general_plugins
228 load_general_plugins()
229
vllm/executor/msgspec_utils.py new
+27
@@ -0,0 +1,27 @@
1 +from array import array
2 +from typing import Any, Type
3 +
4 +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE
5 +
6 +
7 +def encode_hook(obj: Any) -> Any:
8 + """Custom msgspec enc hook that supports array types.
9 +
10 + See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder
11 + """
12 + if isinstance(obj, array):
13 + assert obj.typecode == VLLM_TOKEN_ID_ARRAY_TYPE, (
14 + f"vLLM array type should use '{VLLM_TOKEN_ID_ARRAY_TYPE}' type. "
15 + f"Given array has a type code of {obj.typecode}.")
16 + return obj.tobytes()
17 +
18 +
19 +def decode_hook(type: Type, obj: Any) -> Any:
20 + """Custom msgspec dec hook that supports array types.
21 +
22 + See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder
23 + """
24 + if type is array:
25 + deserialized = array(VLLM_TOKEN_ID_ARRAY_TYPE)
26 + deserialized.frombytes(obj)
27 + return deserialized
vllm/executor/ray_gpu_executor.py
+15 -4
@@ -4,9 +4,12 @@ from collections import defaultdict
4 from itertools import islice, repeat
5 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
6
7 +import msgspec
8 +
9 import vllm.envs as envs
10 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
11 DistributedGPUExecutor, DistributedGPUExecutorAsync)
12 +from vllm.executor.msgspec_utils import encode_hook
13 from vllm.executor.ray_utils import RayWorkerWrapper, ray
14 from vllm.logger import init_logger
15 from vllm.sequence import ExecuteModelRequest, SamplerOutput
@@ -60,6 +63,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
63 # Create the parallel GPU workers.
64 self._init_workers_ray(placement_group)
65
66 + self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
67 + self.output_decoder = msgspec.msgpack.Decoder(
68 + Optional[List[SamplerOutput]])
69 +
70 def shutdown(self) -> None:
71 if hasattr(self, "forward_dag") and self.forward_dag is not None:
72 self.forward_dag.teardown()
@@ -123,6 +130,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
130 ray_remote_kwargs)
131
132 logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
133 +
134 # Create the workers.
135 driver_ip = get_ip()
136 worker_wrapper_kwargs = self._get_worker_wrapper_args()
@@ -304,8 +312,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
312 if self.forward_dag is None:
313 self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
314
307 - outputs = ray.get(self.forward_dag.execute(execute_model_req))
308 - return outputs[0]
315 + serialized_data = self.input_encoder.encode(execute_model_req)
316 + outputs = ray.get(self.forward_dag.execute(serialized_data))
317 + output = self.output_decoder.decode(outputs[0])
318 + return output
319
320 def _run_workers(
321 self,
@@ -475,9 +485,10 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
485 if self.forward_dag is None:
486 self.forward_dag = self._compiled_ray_dag(enable_asyncio=True)
487
478 - dag_future = await self.forward_dag.execute_async(execute_model_req)
488 + serialized_data = self.input_encoder.encode(execute_model_req)
489 + dag_future = await self.forward_dag.execute_async(serialized_data)
490 outputs = await dag_future
480 - return outputs[0]
491 + return self.output_decoder.decode(outputs[0])
492
493 async def _driver_execute_model_async(
494 self,
vllm/executor/ray_utils.py
+27 -12
@@ -1,6 +1,9 @@
1 from typing import List, Optional, Tuple, Union
2
3 +import msgspec
4 +
5 from vllm.config import ParallelConfig
6 +from vllm.executor.msgspec_utils import decode_hook, encode_hook
7 from vllm.logger import init_logger
8 from vllm.platforms import current_platform
9 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
@@ -24,6 +27,10 @@ try:
27 # that thread.
28 self.compiled_dag_cuda_device_set = False
29
30 + self.input_decoder = msgspec.msgpack.Decoder(ExecuteModelRequest,
31 + dec_hook=decode_hook)
32 + self.output_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
33 +
34 def get_node_ip(self) -> str:
35 return get_ip()
36
@@ -33,16 +40,26 @@ try:
40 return node_id, gpu_ids
41
42 def execute_model_spmd(
36 - self, req_or_tuple: Union[ExecuteModelRequest,
37 - Tuple[ExecuteModelRequest,
38 - IntermediateTensors]]):
43 + self, req_or_tuple: Union[bytes,
44 + Tuple[bytes,
45 + Optional[IntermediateTensors]]]
46 + ) -> bytes:
47 """Execute model in SPMD fashion: used only when SPMD worker and
48 compiled DAG are both enabled.
49
50 Args:
43 - req_or_tuple: The request to execute the model, or a tuple
44 - containing the request and intermediate tensors.
51 + req_or_tuple: A request or a tuple containing the
52 + request and intermediate tensors. Intermediate tensors are
53 + None unless if it is provided because it is > 0 pipeline
54 + stage. The request is serialized by msgspec.
55 """
56 + if isinstance(req_or_tuple, bytes):
57 + serialized_req, intermediate_tensors = req_or_tuple, None
58 + else:
59 + serialized_req, intermediate_tensors = req_or_tuple
60 +
61 + execute_model_req = self.input_decoder.decode(serialized_req)
62 +
63 # TODO(swang): This is needed right now because Ray aDAG executes
64 # on a background thread, so we need to reset torch's current
65 # device.
@@ -51,16 +68,14 @@ try:
68 torch.cuda.set_device(self.worker.device)
69 self.compiled_dag_cuda_device_set = True
70
54 - if isinstance(req_or_tuple, tuple):
55 - execute_model_req, intermediate_tensors = req_or_tuple
56 - else:
57 - execute_model_req = req_or_tuple
58 - intermediate_tensors = None
59 -
71 output = self.worker._execute_model_spmd(execute_model_req,
72 intermediate_tensors)
73 + # Pipeline model request and output to the next pipeline stage.
74 if isinstance(output, IntermediateTensors):
63 - return execute_model_req, output
75 + output = serialized_req, output
76 + else:
77 + output = self.output_encoder.encode(output)
78 +
79 return output
80
81 ray_import_err = None
vllm/inputs/registry.py
+7 -1
@@ -1,4 +1,5 @@
1 import functools
2 +from array import array
3 from collections import UserDict
4 from dataclasses import dataclass
5 from typing import (TYPE_CHECKING, Callable, Dict, Mapping, Optional, Protocol,
@@ -21,6 +22,10 @@ logger = init_logger(__name__)
22
23 C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
24
25 +# NOTE: This has to match with sequence.py's VLLM_TOKEN_ID_ARRAY_TYPE.
26 +# We cannot import it here because of circular dependencies.
27 +VLLM_TOKEN_ID_ARRAY_TYPE = "l"
28 +
29
30 @dataclass(frozen=True)
31 class InputContext:
@@ -118,7 +123,8 @@ class InputRegistry:
123 # Avoid circular import
124 from vllm.sequence import SequenceData
125
121 - dummy_seq_data = SequenceData([0] * seq_len)
126 + dummy_seq_data = SequenceData(
127 + array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * seq_len)
128 dummy_multi_modal_data = None
129
130 return dummy_seq_data, dummy_multi_modal_data
vllm/lora/request.py
+9 -5
@@ -1,12 +1,15 @@
1 import warnings
2 -from dataclasses import dataclass, field
2 from typing import Optional
3
4 +import msgspec
5 +
6 from vllm.adapter_commons.request import AdapterRequest
7
8
8 -@dataclass
9 -class LoRARequest(AdapterRequest):
9 +class LoRARequest(
10 + msgspec.Struct,
11 + omit_defaults=True, # type: ignore[call-arg]
12 + array_like=True): # type: ignore[call-arg]
13 """
14 Request for a LoRA adapter.
15
@@ -18,16 +21,17 @@ class LoRARequest(AdapterRequest):
21 lora_int_id must be globally unique for a given adapter.
22 This is currently not enforced in vLLM.
23 """
24 + __metaclass__ = AdapterRequest
25
26 lora_name: str
27 lora_int_id: int
28 lora_path: str = ""
25 - lora_local_path: Optional[str] = field(default=None, repr=False)
29 + lora_local_path: Optional[str] = msgspec.field(default=None)
30 long_lora_max_len: Optional[int] = None
31 __hash__ = AdapterRequest.__hash__
32
33 def __post_init__(self):
30 - if 'lora_local_path' in self.__dict__:
34 + if 'lora_local_path' in self.__struct_fields__:
35 warnings.warn(
36 "The 'lora_local_path' attribute is deprecated "
37 "and will be removed in a future version. "
vllm/model_executor/models/blip.py
+6 -3
@@ -1,5 +1,6 @@
1 """Minimal implementation of BlipVisionModel intended to be only used
2 within a vision language model."""
3 +from array import array
4 from typing import Optional, Union
5
6 import torch
@@ -16,7 +17,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.multimodal.image import (cached_get_tokenizer,
19 repeat_and_pad_image_tokens)
19 -from vllm.sequence import SequenceData
20 +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
21
22
23 def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -53,8 +54,10 @@ def dummy_seq_data_for_blip(
54 else:
55 image_feature_size = image_feature_size_override
56
56 - token_ids = [image_token_id] * image_feature_size
57 - token_ids += [0] * (seq_len - image_feature_size)
57 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
58 + [image_token_id]) * image_feature_size
59 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
60 + [0]) * (seq_len - image_feature_size)
61 return SequenceData(token_ids)
62
63
vllm/model_executor/models/blip2.py
+7 -3
@@ -1,3 +1,4 @@
1 +from array import array
2 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 TypedDict, Union)
4
@@ -17,7 +18,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18 from vllm.model_executor.models.opt import OPTModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
20 -from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
21 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
22 + SamplerOutput, SequenceData)
23
24 from .blip import (BlipVisionModel, dummy_image_for_blip,
25 get_max_blip_image_tokens)
@@ -427,8 +429,10 @@ def dummy_seq_data_for_blip2(
429 else:
430 image_feature_size = image_feature_size_override
431
430 - token_ids = [image_token_id] * image_feature_size * num_images
431 - token_ids += [0] * (seq_len - image_feature_size * num_images)
432 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
433 + [image_token_id]) * image_feature_size * num_images
434 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
435 + [0]) * (seq_len - image_feature_size * num_images)
436 return SequenceData(token_ids)
437
438
vllm/model_executor/models/chameleon.py
+7 -3
@@ -1,3 +1,4 @@
1 +from array import array
2 from functools import cached_property
3 from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
4 Tuple, TypedDict)
@@ -31,7 +32,8 @@ from vllm.model_executor.utils import set_weight_attrs
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.image import (cached_get_tokenizer,
34 repeat_and_pad_image_tokens)
34 -from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
35 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
36 + SamplerOutput, SequenceData)
37 from vllm.utils import print_warning_once
38
39 from .interfaces import SupportsMultiModal
@@ -70,8 +72,10 @@ def dummy_seq_data_for_chameleon(
72 else:
73 image_feature_size = image_feature_size_override
74
73 - token_ids = [image_token_id] * image_feature_size * num_images
74 - token_ids += [0] * (seq_len - image_feature_size * num_images)
75 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
76 + [image_token_id]) * image_feature_size * num_images
77 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
78 + [0]) * (seq_len - image_feature_size * num_images)
79 return SequenceData(token_ids)
80
81
vllm/model_executor/models/clip.py
+6 -3
@@ -1,5 +1,6 @@
1 """Minimal implementation of CLIPVisionModel intended to be only used
2 within a vision language model."""
3 +from array import array
4 from typing import Iterable, Optional, Tuple
5
6 import torch
@@ -17,7 +18,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19 from vllm.multimodal.image import (cached_get_tokenizer,
20 repeat_and_pad_image_tokens)
20 -from vllm.sequence import SequenceData
21 +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
22
23
24 def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -53,8 +54,10 @@ def dummy_seq_data_for_clip(
54 else:
55 image_feature_size = image_feature_size_override
56
56 - token_ids = [image_token_id] * image_feature_size * num_images
57 - token_ids += [0] * (seq_len - image_feature_size * num_images)
57 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
58 + [image_token_id]) * image_feature_size * num_images
59 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
60 + [0]) * (seq_len - image_feature_size * num_images)
61 return SequenceData(token_ids)
62
63
vllm/model_executor/models/fuyu.py
+9 -4
@@ -16,6 +16,7 @@
16 # limitations under the License.
17 """ PyTorch Fuyu model."""
18 import math
19 +from array import array
20 from typing import Iterable, List, Literal, Mapping, Optional, Tuple, TypedDict
21
22 import torch
@@ -37,7 +38,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
38 from vllm.multimodal.base import MultiModalInputs
39 from vllm.multimodal.image import (cached_get_image_processor,
40 cached_get_tokenizer)
40 -from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
41 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42 + SamplerOutput, SequenceData)
43
44 from .interfaces import SupportsMultiModal
45 from .utils import merge_multimodal_embeddings
@@ -97,9 +99,12 @@ def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int):
99 ncol, nrow = get_max_fuyu_image_feature_size()
100 image_feature_size = get_max_fuyu_image_tokens(ctx)
101
100 - image_token_ids = ([_IMAGE_TOKEN_ID] * ncol + [_NEWLINE_TOKEN_ID]) * nrow
101 - token_ids = image_token_ids * num_images
102 - token_ids += [0] * (seq_len - image_feature_size * num_images)
102 + image_token_ids = (
103 + array(VLLM_TOKEN_ID_ARRAY_TYPE, [_IMAGE_TOKEN_ID]) * ncol +
104 + array(VLLM_TOKEN_ID_ARRAY_TYPE, [_NEWLINE_TOKEN_ID])) * nrow
105 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, image_token_ids) * num_images
106 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
107 + [0]) * (seq_len - image_feature_size * num_images)
108 return SequenceData(token_ids)
109
110
vllm/model_executor/models/minicpmv.py
+4 -2
@@ -23,6 +23,7 @@
23 """Inference-only MiniCPM-V model compatible with HuggingFace weights."""
24 import math
25 import re
26 +from array import array
27 from functools import partial
28 from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
29 TypedDict, Union)
@@ -55,7 +56,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
56 from vllm.multimodal import MULTIMODAL_REGISTRY
57 from vllm.multimodal.image import (cached_get_image_processor,
58 cached_get_tokenizer)
58 -from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
59 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
60 + SamplerOutput, SequenceData)
61
62 from .idefics2_vision_model import Idefics2VisionTransformer
63
@@ -408,7 +410,7 @@ def get_max_minicpmv_image_tokens(ctx: InputContext):
410
411
412 def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
411 - token_ids = [0] * seq_len
413 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * seq_len
414 return SequenceData(token_ids)
415
416
vllm/model_executor/models/siglip.py
+6 -3
@@ -2,6 +2,7 @@
2 within a vision language model."""
3
4 import math
5 +from array import array
6 from typing import Iterable, Optional, Tuple
7
8 import torch
@@ -25,7 +26,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 from vllm.multimodal.image import (cached_get_tokenizer,
28 repeat_and_pad_image_tokens)
28 -from vllm.sequence import SequenceData
29 +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
30
31
32 def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -62,8 +63,10 @@ def dummy_seq_data_for_siglip(
63 else:
64 image_feature_size = image_feature_size_override
65
65 - token_ids = [image_token_id] * image_feature_size * num_images
66 - token_ids += [0] * (seq_len - image_feature_size * num_images)
66 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
67 + [image_token_id]) * image_feature_size
68 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
69 + [0]) * (seq_len - image_feature_size)
70 return SequenceData(token_ids)
71
72
vllm/model_executor/sampling_metadata.py
+6 -3
@@ -6,7 +6,8 @@ from typing import Dict, List, Optional, Tuple
6 import torch
7
8 from vllm.sampling_params import SamplingParams, SamplingType
9 -from vllm.sequence import SequenceData, SequenceGroupMetadata
9 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData,
10 + SequenceGroupMetadata)
11 from vllm.triton_utils.sample import get_num_triton_sampler_splits
12 from vllm.utils import (PyObjectCache, async_tensor_h2d,
13 is_pin_memory_available, make_tensor_with_pad,
@@ -505,9 +506,11 @@ class SamplingTensors:
506 and sampling_params.prompt_logprobs is not None):
507 prefill_len = len(seq_group.prompt_logprob_indices)
508 prompt_tokens.extend(
508 - array('l') for _ in range(prefill_len))
509 + array(VLLM_TOKEN_ID_ARRAY_TYPE)
510 + for _ in range(prefill_len))
511 output_tokens.extend(
510 - array('l') for _ in range(prefill_len))
512 + array(VLLM_TOKEN_ID_ARRAY_TYPE)
513 + for _ in range(prefill_len))
514 if seq_group.do_sample:
515 for seq_id in seq_ids:
516 seq_data = seq_group.seq_data[seq_id]
vllm/pooling_params.py
+7 -4
@@ -1,15 +1,18 @@
1 from typing import Any, Optional
2
3 +import msgspec
4
4 -class PoolingParams:
5 +
6 +class PoolingParams(
7 + msgspec.Struct,
8 + omit_defaults=True, # type: ignore[call-arg]
9 + array_like=True): # type: ignore[call-arg]
10 """Pooling parameters for pooling.
11
12 Attributes:
13 additional_data: Any additional data needed for pooling.
14 """
10 -
11 - def __init__(self, additional_data: Optional[Any] = None):
12 - self.additional_data = additional_data
15 + additional_data: Optional[Any] = None
16
17 def clone(self) -> "PoolingParams":
18 """Returns a deep copy of the PoolingParams instance."""
vllm/prompt_adapter/request.py
+7 -3
@@ -1,13 +1,17 @@
1 -from dataclasses import dataclass
1 +import msgspec
2
3 from vllm.adapter_commons.request import AdapterRequest
4
5
6 -@dataclass
7 -class PromptAdapterRequest(AdapterRequest):
6 +class PromptAdapterRequest(
7 + msgspec.Struct,
8 + array_like=True, # type: ignore[call-arg]
9 + omit_defaults=True, # type: ignore[call-arg]
10 + frozen=True): # type: ignore[call-arg]
11 """
12 Request for a Prompt adapter.
13 """
14 + __metaclass__ = AdapterRequest
15
16 prompt_adapter_name: str
17 prompt_adapter_id: int
vllm/sampling_params.py
+72 -75
@@ -2,10 +2,10 @@
2 import copy
3 from enum import IntEnum
4 from functools import cached_property
5 -from typing import Any, Callable, Dict, List, Optional, Union
5 +from typing import Any, Callable, Dict, List, Optional, Set, Union
6
7 +import msgspec
8 import torch
8 -from pydantic import Field
9 from typing_extensions import Annotated
10
11 from vllm.logger import init_logger
@@ -33,7 +33,11 @@ first argument, and returns a modified tensor of logits
33 to sample from."""
34
35
36 -class SamplingParams:
36 +class SamplingParams(
37 + msgspec.Struct,
38 + omit_defaults=True, # type: ignore[call-arg]
39 + # required for @cached_property.
40 + dict=True): # type: ignore[call-arg]
41 """Sampling parameters for text generation.
42
43 Overall, we follow the sampling parameters from the OpenAI text completion
@@ -112,87 +116,73 @@ class SamplingParams:
116 (i.e., no truncation).
117 """
118
115 - def __init__(
116 - self,
117 - n: int = 1,
118 - best_of: Optional[int] = None,
119 - presence_penalty: float = 0.0,
120 - frequency_penalty: float = 0.0,
121 - repetition_penalty: float = 1.0,
122 - temperature: float = 1.0,
123 - top_p: float = 1.0,
124 - top_k: int = -1,
125 - min_p: float = 0.0,
126 - seed: Optional[int] = None,
127 - use_beam_search: bool = False,
128 - length_penalty: float = 1.0,
129 - early_stopping: Union[bool, str] = False,
130 - stop: Optional[Union[str, List[str]]] = None,
131 - stop_token_ids: Optional[List[int]] = None,
132 - include_stop_str_in_output: bool = False,
133 - ignore_eos: bool = False,
134 - max_tokens: Optional[int] = 16,
135 - min_tokens: int = 0,
136 - logprobs: Optional[int] = None,
137 - prompt_logprobs: Optional[int] = None,
138 - detokenize: bool = True,
139 - skip_special_tokens: bool = True,
140 - spaces_between_special_tokens: bool = True,
141 - logits_processors: Optional[List[LogitsProcessor]] = None,
142 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
143 - ) -> None:
144 - self.n = n
145 - self.best_of = best_of if best_of is not None else n
146 - self.presence_penalty = presence_penalty
147 - self.frequency_penalty = frequency_penalty
148 - self.repetition_penalty = repetition_penalty
149 - if 0 < temperature < _MAX_TEMP:
119 + n: int = 1
120 + best_of: Optional[int] = None
121 + presence_penalty: float = 0.0
122 + frequency_penalty: float = 0.0
123 + repetition_penalty: float = 1.0
124 + temperature: float = 1.0
125 + top_p: float = 1.0
126 + top_k: int = -1
127 + min_p: float = 0.0
128 + seed: Optional[int] = None
129 + use_beam_search: bool = False
130 + length_penalty: float = 1.0
131 + early_stopping: Union[bool, str] = False
132 + stop: Optional[Union[str, List[str]]] = None
133 + stop_token_ids: Optional[List[int]] = None
134 + ignore_eos: bool = False
135 + max_tokens: Optional[int] = 16
136 + min_tokens: int = 0
137 + logprobs: Optional[int] = None
138 + prompt_logprobs: Optional[int] = None
139 + # NOTE: This parameter is only exposed at the engine level for now.
140 + # It is not exposed in the OpenAI API server, as the OpenAI API does
141 + # not support returning only a list of token IDs.
142 + detokenize: bool = True
143 + skip_special_tokens: bool = True
144 + spaces_between_special_tokens: bool = True
145 + # Optional[List[LogitsProcessor]] type. We use Any here because
146 + # Optional[List[LogitsProcessor]] type is not supported by msgspec.
147 + logits_processors: Optional[Any] = None
148 + include_stop_str_in_output: bool = False
149 + truncate_prompt_tokens: Optional[Annotated[int, msgspec.Meta(ge=1)]] = None
150 +
151 + # The below fields are not supposed to be used as an input.
152 + # They are set in post_init.
153 + output_text_buffer_length: int = 0
154 + _all_stop_token_ids: Set[int] = msgspec.field(default_factory=set)
155 +
156 + def __post_init__(self) -> None:
157 + self.best_of = self.best_of or self.n
158 + if 0 < self.temperature < _MAX_TEMP:
159 logger.warning(
160 "temperature %s is less than %s, which may cause numerical "
161 "errors nan or inf in tensors. We have maxed it out to %s.",
153 - temperature, _MAX_TEMP, _MAX_TEMP)
154 - temperature = max(temperature, _MAX_TEMP)
155 - self.temperature = temperature
156 - self.top_p = top_p
157 - self.top_k = top_k
158 - self.min_p = min_p
159 - if seed == -1:
162 + self.temperature, _MAX_TEMP, _MAX_TEMP)
163 + self.temperature = max(self.temperature, _MAX_TEMP)
164 + if self.seed == -1:
165 self.seed = None
166 else:
162 - self.seed = seed
163 - self.use_beam_search = use_beam_search
164 - self.length_penalty = length_penalty
165 - self.early_stopping = early_stopping
166 - if stop is None:
167 + self.seed = self.seed
168 + if self.stop is None:
169 self.stop = []
168 - elif isinstance(stop, str):
169 - self.stop = [stop]
170 + elif isinstance(self.stop, str):
171 + self.stop = [self.stop]
172 else:
171 - self.stop = list(stop)
172 - if stop_token_ids is None:
173 + self.stop = list(self.stop)
174 + if self.stop_token_ids is None:
175 self.stop_token_ids = []
176 else:
175 - self.stop_token_ids = list(stop_token_ids)
176 - self.ignore_eos = ignore_eos
177 - self.max_tokens = max_tokens
178 - self.min_tokens = min_tokens
179 - self.logprobs = 1 if logprobs is True else logprobs
180 - self.prompt_logprobs = 1 if prompt_logprobs is True else prompt_logprobs
181 - # NOTE: This parameter is only exposed at the engine level for now.
182 - # It is not exposed in the OpenAI API server, as the OpenAI API does
183 - # not support returning only a list of token IDs.
184 - self.detokenize = detokenize
185 - self.skip_special_tokens = skip_special_tokens
186 - self.spaces_between_special_tokens = spaces_between_special_tokens
187 - self.logits_processors = logits_processors
188 - self.include_stop_str_in_output = include_stop_str_in_output
189 - self.truncate_prompt_tokens = truncate_prompt_tokens
177 + self.stop_token_ids = list(self.stop_token_ids)
178 + self.logprobs = 1 if self.logprobs is True else self.logprobs
179 + self.prompt_logprobs = (1 if self.prompt_logprobs is True else
180 + self.prompt_logprobs)
181 +
182 # Number of characters to hold back for stop string evaluation
183 # until sequence is finished.
192 - if self.stop and not include_stop_str_in_output:
184 + if self.stop and not self.include_stop_str_in_output:
185 self.output_text_buffer_length = max(len(s) for s in self.stop) - 1
194 - else:
195 - self.output_text_buffer_length = 0
186
187 self._verify_args()
188 if self.use_beam_search:
@@ -206,11 +196,12 @@ class SamplingParams:
196 self.min_p = 0.0
197 self._verify_greedy_sampling()
198 # eos_token_id is added to this by the engine
209 - self.all_stop_token_ids = set(self.stop_token_ids)
199 + self._all_stop_token_ids = set(self.stop_token_ids)
200
201 def _verify_args(self) -> None:
202 if self.n < 1:
203 raise ValueError(f"n must be at least 1, got {self.n}.")
204 + assert isinstance(self.best_of, int)
205 if self.best_of < self.n:
206 raise ValueError(f"best_of must be greater than or equal to n, "
207 f"got n={self.n} and best_of={self.best_of}.")
@@ -257,6 +248,7 @@ class SamplingParams:
248 and self.truncate_prompt_tokens < 1):
249 raise ValueError(f"truncate_prompt_tokens must be >= 1, "
250 f"got {self.truncate_prompt_tokens}")
251 + assert isinstance(self.stop, list)
252 if any(not stop_str for stop_str in self.stop):
253 raise ValueError("stop cannot contain an empty string.")
254 if self.stop and not self.detokenize:
@@ -290,6 +282,7 @@ class SamplingParams:
282 "default value of 1.0 when not using beam search.")
283
284 def _verify_greedy_sampling(self) -> None:
285 + assert isinstance(self.best_of, int)
286 if self.best_of > 1:
287 raise ValueError("best_of must be 1 when using greedy sampling."
288 f"Got {self.best_of}.")
@@ -303,7 +296,7 @@ class SamplingParams:
296 if model_eos_token_id is not None:
297 # Add the eos token id into the sampling_params to support
298 # min_tokens processing.
306 - self.all_stop_token_ids.add(model_eos_token_id)
299 + self._all_stop_token_ids.add(model_eos_token_id)
300
301 # Update eos_token_id for generation
302 if (eos_ids := generation_config.get("eos_token_id")) is not None:
@@ -315,7 +308,7 @@ class SamplingParams:
308 # purposes.
309 eos_ids.discard(model_eos_token_id)
310 if eos_ids:
318 - self.all_stop_token_ids.update(eos_ids)
311 + self._all_stop_token_ids.update(eos_ids)
312 if not self.ignore_eos:
313 eos_ids.update(self.stop_token_ids)
314 self.stop_token_ids = list(eos_ids)
@@ -330,6 +323,10 @@ class SamplingParams:
323 return SamplingType.RANDOM_SEED
324 return SamplingType.RANDOM
325
326 + @property
327 + def all_stop_token_ids(self) -> Set[int]:
328 + return self._all_stop_token_ids
329 +
330 def clone(self) -> "SamplingParams":
331 """Deep copy excluding LogitsProcessor objects.
332
vllm/sequence.py
+250 -146
@@ -4,10 +4,11 @@ import enum
4 from abc import ABC, abstractmethod
5 from array import array
6 from collections import defaultdict
7 -from dataclasses import dataclass, field
8 -from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
9 - Union, cast)
7 +from dataclasses import dataclass
8 +from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
9 + Tuple, Union, cast)
10
11 +import msgspec
12 import numpy
13 import torch
14
@@ -16,13 +17,18 @@ from vllm.lora.request import LoRARequest
17 from vllm.pooling_params import PoolingParams
18 from vllm.prompt_adapter.request import PromptAdapterRequest
19 from vllm.sampling_params import SamplingParams
20 +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22 if TYPE_CHECKING:
23 from vllm.inputs import LLMInputs
22 - from vllm.multimodal import MultiModalDataDict
23 - from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
24 + from vllm.multimodal.base import MultiModalDataDict
25
26 +VLLM_TOKEN_ID_ARRAY_TYPE = "l"
27
28 +
29 +# We use dataclass for now because it is used for
30 +# openai server output, and msgspec is not serializable.
31 +# TODO(sang): Fix it.
32 @dataclass
33 class Logprob:
34 """Infos for supporting OpenAI compatible logprobs and token ranks.
@@ -112,7 +118,23 @@ class RequestMetrics:
118 model_execute_time: Optional[float] = None
119
120
115 -class SequenceData:
121 +class SequenceDataDelta(
122 + msgspec.Struct,
123 + array_like=True, # type: ignore[call-arg]
124 + omit_defaults=True): # type: ignore[call-arg]
125 + """Delta SequenceData to send to workers per step."""
126 + # A new token to be appended to existing SequenceData.
127 + new_output_token_ids: List[int]
128 + # Overwriting existing `cumulative_logprob`
129 + new_cumulative_logprob: float
130 + # Overwriting existing `num_computed_tokens`.
131 + new_num_computed_tokens: int
132 + # Overwriting existing `stage`.
133 + new_stage: SequenceStage
134 +
135 +
136 +class SequenceData(msgspec.Struct,
137 + omit_defaults=True): # type: ignore[call-arg]
138 """Data associated with a sequence.
139
140 Args:
@@ -125,40 +147,57 @@ class SequenceData:
147 output_token_ids: The token IDs of the output.
148 cumulative_logprob: The cumulative log probability of the output.
149 """
128 -
129 - def __init__(
130 - self,
131 - prompt_token_ids: List[int],
132 - output_token_ids: Optional[List[int]] = None,
133 - ) -> None:
134 - self._prompt_token_ids = array('l', prompt_token_ids)
135 - self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
136 - self._output_token_ids = array(
137 - 'l', output_token_ids if output_token_ids is not None else [])
138 -
139 - self.cumulative_logprob = 0.0
140 - # The number of tokens that are computed (that run against the model).
141 - self._num_computed_tokens = 0
142 - self._stage: SequenceStage = SequenceStage.PREFILL
143 -
150 + # NOTE: we cannot use Union[List, array] because msgspec cannot support
151 + # union of 2 list types.
152 + _prompt_token_ids: array
153 + _output_token_ids: array = msgspec.field(
154 + default_factory=lambda: array(VLLM_TOKEN_ID_ARRAY_TYPE, []))
155 +
156 + ### The below fields should not be passed as an argument ###
157 + _cumulative_logprob: float = 0.0
158 + _prompt_token_ids_tuple: Tuple[int,
159 + ...] = msgspec.field(default_factory=tuple)
160 + # The number of tokens that are computed (that run against the model).
161 + _num_computed_tokens: int = 0
162 + _stage: SequenceStage = SequenceStage.PREFILL
163 + _cached_all_token_ids: List[int] = msgspec.field(default_factory=list)
164 +
165 + # It is used to get delta input. It is reset when `get_delta_and_reset`
166 + # is called.
167 + _new_appended_tokens: List[int] = msgspec.field(default_factory=list)
168 +
169 + def __post_init__(self) -> None:
170 + assert self._prompt_token_ids.typecode == "l"
171 + assert self._output_token_ids.typecode == "l"
172 + self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(
173 + self._prompt_token_ids)
174 self._update_cached_all_tokens()
175
176 def _update_cached_all_tokens(self):
177 + assert isinstance(self._prompt_token_ids, array)
178 + assert isinstance(self._output_token_ids, array)
179 self._cached_all_token_ids: List[int] = list(self._prompt_token_ids +
180 self._output_token_ids)
181
182 + @property
183 + def cumulative_logprob(self) -> float:
184 + return self._cumulative_logprob
185 +
186 @property
187 def prompt_token_ids(self) -> Tuple[int, ...]:
188 return self._prompt_token_ids_tuple
189
190 @prompt_token_ids.setter
191 def prompt_token_ids(self, new_prompt_token_ids) -> None:
156 - self._prompt_token_ids = array('l', new_prompt_token_ids)
157 - self._prompt_token_ids_tuple = tuple(new_prompt_token_ids)
158 - self._update_cached_all_tokens()
192 + raise NotImplementedError
193
194 @property
195 def prompt_token_ids_array(self) -> array:
196 + """Return the prompt token ids in array type.
197 +
198 + Note that the array is in "I" type, and it is not compatible
199 + with torch.long (2 bytes vs 4 bytes). So beware of the usage.
200 + """
201 return self._prompt_token_ids
202
203 @property
@@ -166,18 +205,26 @@ class SequenceData:
205 return tuple(self._output_token_ids)
206
207 @output_token_ids.setter
169 - def output_token_ids(self, new_output_token_ids) -> None:
170 - self._output_token_ids = array('l', new_output_token_ids)
208 + def output_token_ids(self, new_output_token_ids: List[int]) -> None:
209 + self._output_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
210 + new_output_token_ids)
211 self._update_cached_all_tokens()
212
213 @property
214 def output_token_ids_array(self) -> array:
215 + """Return the prompt token ids in array type.
216 +
217 + Note that the array is in "I" type, and it is not compatible
218 + with torch.long (2 bytes vs 4 bytes). So beware of the usage.
219 + """
220 + assert isinstance(self._output_token_ids, array)
221 return self._output_token_ids
222
223 def append_token_id(self, token_id: int, logprob: float) -> None:
224 self._output_token_ids.append(token_id)
225 + self._new_appended_tokens.append(token_id)
226 self._cached_all_token_ids.append(token_id)
180 - self.cumulative_logprob += logprob
227 + self._cumulative_logprob += logprob
228
229 def get_len(self) -> int:
230 return len(self._output_token_ids) + len(self._prompt_token_ids)
@@ -222,6 +269,7 @@ class SequenceData:
269 """
270 self._num_computed_tokens = 0
271 self._stage = SequenceStage.PREFILL
272 + self._new_appended_tokens = []
273
274 def get_num_uncomputed_tokens(self) -> int:
275 """Return the number of prefill tokens that are not computed."""
@@ -241,6 +289,21 @@ class SequenceData:
289 def get_output_token_ids(self) -> Tuple[int, ...]:
290 return self.output_token_ids
291
292 + def get_delta_and_reset(self) -> SequenceDataDelta:
293 + delta = SequenceDataDelta(self._new_appended_tokens,
294 + self._cumulative_logprob,
295 + self.get_num_computed_tokens(), self.stage)
296 + # Reset delta state.
297 + self._new_appended_tokens = []
298 + return delta
299 +
300 + def apply_delta(self, delta: SequenceDataDelta):
301 + self._num_computed_tokens = delta.new_num_computed_tokens
302 + self._cumulative_logprob = delta.new_cumulative_logprob
303 + self._stage = delta.new_stage
304 + self._output_token_ids.extend(delta.new_output_token_ids)
305 + self._cached_all_token_ids.extend(delta.new_output_token_ids)
306 +
307 @property
308 def stage(self) -> SequenceStage:
309 return self._stage
@@ -248,8 +311,9 @@ class SequenceData:
311 def __repr__(self) -> str:
312 return (f"SequenceData("
313 f"prompt_token_ids={self._prompt_token_ids}, "
251 - f"output_token_ids={self._output_token_ids}, "
252 - f"cumulative_logprob={self.cumulative_logprob})")
314 + f"output_token_ids={self.output_token_ids}, "
315 + f"cumulative_logprob={self.cumulative_logprob}, "
316 + f"get_num_computed_tokens={self.get_num_computed_tokens()}")
317
318
319 class Sequence:
@@ -325,7 +389,8 @@ class Sequence:
389 f"invalid input {inputs}; did you forget the "
390 "encoder input prompt fields?")
391
328 - self.data = SequenceData(self.prompt_token_ids)
392 + self.data = SequenceData(
393 + array(VLLM_TOKEN_ID_ARRAY_TYPE, self.prompt_token_ids))
394 self.output_logprobs: SampleLogprobs = []
395 self.output_text = ""
396
@@ -490,8 +555,8 @@ class Sequence:
555 f"num_blocks={self.n_blocks}, ")
556
557
493 -@dataclass
494 -class SequenceGroupState:
558 +class SequenceGroupState(msgspec.Struct,
559 + omit_defaults=True): # type: ignore[call-arg]
560 """Mutable state tied to a specific sequence group"""
561
562 # for multi-step decoding
@@ -647,14 +712,19 @@ class SequenceGroup:
712 if self.sampling_params and self.sampling_params.use_beam_search:
713 # For beam search, maximally there will always be `best_of` beam
714 # candidates running in the future.
650 - return self.sampling_params.best_of
715 + best_of = self.sampling_params.best_of
716 + assert isinstance(best_of, int)
717 + return best_of
718 else:
652 - if (self.sampling_params
653 - and self.sampling_params.best_of > self.num_seqs()):
654 - # At prompt stage, the sequence group is not yet filled up
655 - # and only have one sequence running. However, in the
656 - # generation stage, we will have `best_of` sequences running.
657 - return self.sampling_params.best_of
719 + if self.sampling_params:
720 + best_of = self.sampling_params.best_of
721 + assert isinstance(best_of, int)
722 + if best_of > self.num_seqs():
723 + # At prompt stage, the sequence group is not yet filled up
724 + # and only have one sequence running. However, in the
725 + # generation stage, we will have `best_of` sequences
726 + # running.
727 + return best_of
728 # At sampling stages, return the number of actual sequences
729 # that are not finished yet.
730 return self.num_unfinished_seqs()
@@ -757,7 +827,32 @@ class SequenceGroup:
827 f"num_seqs={len(self.seqs)})")
828
829
760 -class SequenceGroupMetadata:
830 +class SequenceGroupMetadataDelta(
831 + msgspec.Struct,
832 + tag=True, # type: ignore[call-arg]
833 + array_like=True, # type: ignore[call-arg]
834 + omit_defaults=True): # type: ignore[call-arg]
835 + """Delta of SequenceGroupMetadata.
836 +
837 + After sending the first SequenceGroupMetadata, vLLM scheduler
838 + only sends delta to reduce the data payload size.
839 + """
840 + seq_data_delta: Dict[int, SequenceDataDelta]
841 + request_id: str
842 + block_tables: Dict[int, List[int]]
843 + is_prompt: bool
844 + do_sample: bool = True
845 + token_chunk_size: Optional[int] = None
846 + computed_block_nums: Optional[List[int]] = None
847 + state: Optional[SequenceGroupState] = msgspec.field(
848 + default_factory=lambda: SequenceGroupState())
849 +
850 +
851 +class SequenceGroupMetadata(
852 + msgspec.Struct,
853 + tag=True, # type: ignore[call-arg]
854 + array_like=True, # type: ignore[call-arg]
855 + omit_defaults=True): # type: ignore[call-arg]
856 """Metadata for a sequence group. Used to create `AttentionMetadata`.
857
858 Args:
@@ -789,52 +884,39 @@ class SequenceGroupMetadata:
884 prompt_adapter_request: Prompt Adapter request.
885 """
886
792 - def __init__(
793 - self,
794 - request_id: str,
795 - is_prompt: bool,
796 - seq_data: Dict[int, SequenceData],
797 - sampling_params: SamplingParams,
798 - block_tables: Dict[int, List[int]],
799 - do_sample: bool = True,
800 - pooling_params: Optional[PoolingParams] = None,
801 - token_chunk_size: Optional[int] = None,
802 - lora_request: Optional[LoRARequest] = None,
803 - computed_block_nums: Optional[List[int]] = None,
804 - state: Optional[SequenceGroupState] = None,
805 - multi_modal_data: Optional["MultiModalDataDict"] = None,
806 - encoder_seq_data: Optional[SequenceData] = None,
807 - cross_block_table: Optional[List[int]] = None,
808 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
809 - ) -> None:
810 - self.request_id = request_id
811 - self.is_prompt = is_prompt
812 - self.seq_data = seq_data
813 - self.sampling_params = sampling_params
814 - self.block_tables = block_tables
815 - self.pooling_params = pooling_params
816 - self.lora_request = lora_request
817 - self.prompt_adapter_request = prompt_adapter_request
818 - self.computed_block_nums = computed_block_nums
819 - self.multi_modal_data = multi_modal_data
820 - self.state = SequenceGroupState() if state is None else state
821 - self.encoder_seq_data = encoder_seq_data
822 - self.cross_block_table = cross_block_table
823 - self._token_chunk_size = token_chunk_size
824 - self.do_sample = do_sample
825 -
826 - # The number of speculative tokens adopted in this request.
827 - # None means specuative decoding is not used.
828 - # Zero means speculative decoding is disabled for some reasons.
829 - # TODO: We should maintain this states out of the sequence group.
830 - self.num_speculative_tokens = None
831 -
832 - if seq_data is not None and self._token_chunk_size is None:
833 - if is_prompt:
834 - self._token_chunk_size = next(iter(
835 - seq_data.values())).get_len()
887 + request_id: str
888 + is_prompt: bool
889 + seq_data: Dict[int, SequenceData]
890 + sampling_params: SamplingParams
891 + block_tables: Dict[int, List[int]]
892 + do_sample: bool = True
893 + pooling_params: Optional[PoolingParams] = None
894 + lora_request: Optional[LoRARequest] = None
895 + computed_block_nums: Optional[List[int]] = None
896 + state: Optional[SequenceGroupState] = msgspec.field(
897 + default_factory=lambda: SequenceGroupState())
898 + # "MultiModalDataDict" types. We have to use Any due to msgspec
899 + # doesn't allow to have union of 2 different dicts.
900 + multi_modal_data: Optional[Any] = None
901 + encoder_seq_data: Optional[SequenceData] = None
902 + cross_block_table: Optional[List[int]] = None
903 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
904 + token_chunk_size: Optional[int] = None
905 +
906 + ### Stateful fields that are lazily defined. ###
907 + # The number of speculative tokens adopted in this request.
908 + # None means specuative decoding is not used.
909 + # Zero means speculative decoding is disabled for some reasons.
910 + # TODO: We should maintain this states out of the sequence group.
911 + num_speculative_tokens: Optional[int] = None
912 +
913 + def __post_init__(self):
914 + if self.seq_data is not None and self.token_chunk_size is None:
915 + if self.is_prompt:
916 + self.token_chunk_size = next(iter(
917 + self.seq_data.values())).get_len()
918 else:
837 - self._token_chunk_size = 1
919 + self.token_chunk_size = 1
920
921 @property
922 def lora_int_id(self) -> int:
@@ -850,18 +932,26 @@ class SequenceGroupMetadata:
932 return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens \
933 if self.prompt_adapter_request else 0
934
853 - @property
854 - def token_chunk_size(self) -> int:
855 - """Return the number of tokens to be processed (chunk size)."""
856 - assert self._token_chunk_size is not None
857 - return self._token_chunk_size
935 + def apply_delta(self,
936 + sequence_group_metadata_delta: SequenceGroupMetadataDelta):
937 + for id, delta in sequence_group_metadata_delta.seq_data_delta.items():
938 + self.seq_data[id].apply_delta(delta)
939 + assert self.request_id == sequence_group_metadata_delta.request_id
940 + self.block_tables = sequence_group_metadata_delta.block_tables
941 + self.token_chunk_size = sequence_group_metadata_delta.token_chunk_size
942 + self.do_sample = sequence_group_metadata_delta.do_sample
943 + self.is_prompt = sequence_group_metadata_delta.is_prompt
944
945 def finish_step(self) -> None:
946 + assert self.state is not None
947 assert self.state.current_step < self.state.num_steps
948 self.state.current_step += 1
949
950
864 -class SequenceOutput:
951 +class SequenceOutput(
952 + msgspec.Struct,
953 + omit_defaults=True, # type: ignore[call-arg]
954 + array_like=True): # type: ignore[call-arg]
955 """The model output associated with a sequence.
956
957 Args:
@@ -871,16 +961,9 @@ class SequenceOutput:
961 logprobs: The logprobs of the output token.
962 (Token id -> logP(x_i+1 | x_0, ..., x_i))
963 """
874 -
875 - def __init__(
876 - self,
877 - parent_seq_id: int,
878 - output_token: int,
879 - logprobs: Dict[int, Logprob],
880 - ) -> None:
881 - self.parent_seq_id = parent_seq_id
882 - self.output_token = output_token
883 - self.logprobs = logprobs
964 + parent_seq_id: int
965 + output_token: int
966 + logprobs: Dict[int, Logprob]
967
968 def __repr__(self) -> str:
969 return (f"SequenceOutput(parent_seq_id={self.parent_seq_id}, "
@@ -908,17 +991,15 @@ class SequenceGroupOutput(ABC):
991 pass
992
993
911 -class CompletionSequenceGroupOutput(SequenceGroupOutput):
994 +class CompletionSequenceGroupOutput(
995 + msgspec.Struct,
996 + omit_defaults=True, # type: ignore[call-arg]
997 + array_like=True): # type: ignore[call-arg]
998 + __metaclass__ = SequenceGroupOutput
999 """The model output associated with a completion sequence group."""
913 -
914 - def __init__(
915 - self,
916 - samples: List[SequenceOutput],
917 - prompt_logprobs: Optional[PromptLogprobs],
918 - ) -> None:
919 - self.samples = samples
920 - # Prompt logprob for each prompt query token.
921 - self.prompt_logprobs = prompt_logprobs
1000 + samples: List[SequenceOutput]
1001 + # Prompt logprob for each prompt query token.
1002 + prompt_logprobs: Optional[PromptLogprobs]
1003
1004 def __repr__(self) -> str:
1005 return (f"CompletionSequenceGroupOutput(samples={self.samples}, "
@@ -931,14 +1012,14 @@ class CompletionSequenceGroupOutput(SequenceGroupOutput):
1012 and self.prompt_logprobs == other.prompt_logprobs)
1013
1014
934 -class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
1015 +class EmbeddingSequenceGroupOutput(
1016 + msgspec.Struct,
1017 + omit_defaults=True, # type: ignore[call-arg]
1018 + array_like=True, # type: ignore[call-arg]
1019 +):
1020 """The model output associated with an embedding sequence group."""
936 -
937 - def __init__(
938 - self,
939 - embeddings: List[float],
940 - ) -> None:
941 - self.embeddings = embeddings
1021 + __metaclass__ = SequenceGroupOutput
1022 + embeddings: List[int]
1023
1024 def __repr__(self) -> str:
1025 return (f"EmbeddingSequenceGroupOutput("
@@ -950,8 +1031,10 @@ class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
1031 return self.embeddings == other.embeddings
1032
1033
953 -@dataclass
954 -class IntermediateTensors:
1034 +class IntermediateTensors(
1035 + msgspec.Struct,
1036 + omit_defaults=True, # type: ignore[call-arg]
1037 + array_like=True): # type: ignore[call-arg]
1038 """For all pipeline stages except the last, we need to return the hidden
1039 states and residuals to be sent to the next stage. This data structure
1040 contains the hidden states and residuals for a request.
@@ -978,8 +1061,10 @@ class IntermediateTensors:
1061 return f"IntermediateTensors(tensors={self.tensors})"
1062
1063
981 -@dataclass
982 -class SamplerOutput:
1064 +class SamplerOutput(
1065 + msgspec.Struct,
1066 + omit_defaults=True, # type: ignore[call-arg]
1067 + array_like=True): # type: ignore[call-arg]
1068 """For each sequence group, we generate a list of SequenceOutput object,
1069 each of which contains one possible candidate for the next token.
1070
@@ -1000,7 +1085,7 @@ class SamplerOutput:
1085 sampled_token_ids_numpy: Optional[numpy.ndarray] = None
1086
1087 # Spec decode metrics populated by workers.
1003 - spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
1088 + spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1089
1090 # Optional last hidden states from the model.
1091 hidden_states: Optional[torch.Tensor] = None
@@ -1039,12 +1124,14 @@ class SamplerOutput:
1124 f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
1125
1126
1042 -@dataclass
1043 -class PoolerOutput:
1127 +class PoolerOutput(
1128 + msgspec.Struct,
1129 + omit_defaults=True, # type: ignore[call-arg]
1130 + array_like=True): # type: ignore[call-arg]
1131 """The output from a pooling operation in the embedding model."""
1132 outputs: List[EmbeddingSequenceGroupOutput]
1133
1047 - spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
1134 + spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1135
1136 def __getitem__(self, idx: int):
1137 return self.outputs[idx]
@@ -1083,7 +1170,8 @@ def get_all_seq_ids_and_request_ids(
1170 return seq_ids, request_id_seq_ids_mapping
1171
1172
1086 -class HiddenStates:
1173 +class HiddenStates(msgspec.Struct, array_like=True,
1174 + omit_defaults=True): # type: ignore[call-arg]
1175 """Hidden states corresponding to in-progress sequences.
1176 Used in speculative decoding to pass hidden states from
1177 the target model to the proposer model in the subsequent step.
@@ -1091,42 +1179,53 @@ class HiddenStates:
1179 seq_ids are the sequence ids of each entry of the batch
1180 dimension of the hidden_states tensor"""
1181
1094 - def __init__(self, seq_group_metadata_list: List[SequenceGroupMetadata],
1095 - hidden_states: torch.Tensor):
1096 - assert len(seq_group_metadata_list) == len(hidden_states)
1097 - self.seq_ids: List[int] = get_all_seq_ids(seq_group_metadata_list)
1098 - self.hidden_states: torch.Tensor = hidden_states
1182 + seq_group_metadata_list: List[SequenceGroupMetadata]
1183 + hidden_states: torch.Tensor
1184 + _seq_ids: List[int] = msgspec.field(default_factory=list)
1185 +
1186 + def __post_init__(self):
1187 + self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list)
1188 + assert len(self.seq_group_metadata_list) == len(self.hidden_states)
1189 +
1190 + @property
1191 + def seq_ids(self) -> List[int]:
1192 + return self._seq_ids
1193
1194 def update(self, seq_group_metadata_list: List[SequenceGroupMetadata],
1195 hidden_states: torch.Tensor) -> None:
1196 """Update hidden states from target model invocation."""
1197 assert len(seq_group_metadata_list) == len(hidden_states)
1104 - self.seq_ids.extend(get_all_seq_ids(seq_group_metadata_list))
1198 + self._seq_ids.extend(get_all_seq_ids(seq_group_metadata_list))
1199 self.hidden_states = torch.cat([self.hidden_states, hidden_states])
1200
1201 def prune(self,
1202 seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
1203 """Prune to provided list of sequence ids."""
1204 seq_ids = get_all_seq_ids(seq_group_metadata_list)
1111 - if seq_ids != self.seq_ids:
1205 + if seq_ids != self._seq_ids:
1206 # Batch contents changed - prune removed sequences.
1113 - index = [self.seq_ids.index(seq_id) for seq_id in seq_ids]
1207 + index = [self._seq_ids.index(seq_id) for seq_id in seq_ids]
1208 self.hidden_states = self.hidden_states[index]
1115 - self.seq_ids = seq_ids
1209 + self._seq_ids = seq_ids
1210
1211
1118 -@dataclass
1119 -class ExecuteModelRequest:
1212 +class ExecuteModelRequest(
1213 + msgspec.Struct,
1214 + array_like=True, # type: ignore[call-arg]
1215 + omit_defaults=True): # type: ignore[call-arg]
1216 """The model execution request, containing CPU metadata only. The LLM
1217 engine should create an instance of this class for each request batch."""
1218 # The sequence group metadata list.
1123 - seq_group_metadata_list: List[SequenceGroupMetadata]
1219 + seq_group_metadata_list: List[Union[SequenceGroupMetadata,
1220 + SequenceGroupMetadataDelta]]
1221 # Blocks to swap in. List of CPU -> GPU block number.
1125 - blocks_to_swap_in: List[Tuple[int, int]] = field(default_factory=list)
1222 + blocks_to_swap_in: List[Tuple[int,
1223 + int]] = msgspec.field(default_factory=list)
1224 # Blocks to swap out. List of GPU -> CPU block number.
1127 - blocks_to_swap_out: List[Tuple[int, int]] = field(default_factory=list)
1225 + blocks_to_swap_out: List[Tuple[int,
1226 + int]] = msgspec.field(default_factory=list)
1227 # Blocks to copy. Source to dest block.
1129 - blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
1228 + blocks_to_copy: List[Tuple[int, int]] = msgspec.field(default_factory=list)
1229 # Virtual engine ID for pipeline parallel.
1230 virtual_engine: int = 0
1231 # The number of slots for lookahead decoding.
@@ -1138,7 +1237,7 @@ class ExecuteModelRequest:
1237 # The number of forward steps to run.
1238 num_steps: int = 1
1239 # Finished request ids since last step.
1141 - finished_requests_ids: List[str] = field(default_factory=list)
1240 + finished_requests_ids: List[str] = msgspec.field(default_factory=list)
1241 # The last sampled token ids for multi step decoding.
1242 last_sampled_token_ids: Optional[torch.Tensor] = None
1243
@@ -1148,6 +1247,7 @@ class ExecuteModelRequest:
1247 # steps
1248 assert len(self.seq_group_metadata_list) > 0
1249 first_seq_group = self.seq_group_metadata_list[0]
1250 + assert first_seq_group.state is not None
1251 return first_seq_group.state.current_step == 0
1252
1253 @property
@@ -1156,6 +1256,7 @@ class ExecuteModelRequest:
1256 # steps
1257 assert len(self.seq_group_metadata_list) > 0
1258 first_seq_group = self.seq_group_metadata_list[0]
1259 + assert first_seq_group.state is not None
1260 num_steps = first_seq_group.state.num_steps
1261 current_step = first_seq_group.state.current_step
1262 return num_steps - current_step == 1
@@ -1165,10 +1266,13 @@ class ExecuteModelRequest:
1266 # TODO(will) make this be able to handle batches with variable number of
1267 # steps
1268 assert len(self.seq_group_metadata_list) > 0
1168 - return self.seq_group_metadata_list[0].state.current_step
1269 + state = self.seq_group_metadata_list[0].state
1270 + assert state is not None
1271 + return state.current_step
1272
1273 def clone(
1171 - self, seq_group_metadata_list: List[SequenceGroupMetadata]
1274 + self, seq_group_metadata_list: List[Union[SequenceGroupMetadata,
1275 + SequenceGroupMetadataDelta]]
1276 ) -> "ExecuteModelRequest":
1277 """Clone the request with a new sequence group metadata list."""
1278 return ExecuteModelRequest(
vllm/spec_decode/batch_expansion.py
+8 -5
@@ -1,11 +1,13 @@
1 +from array import array
2 from itertools import chain, count
3 from typing import Iterator, List, Tuple
4
5 import torch
6
7 from vllm import SamplingParams
7 -from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8 - SequenceGroupMetadata, get_all_seq_ids)
8 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, ExecuteModelRequest,
9 + SamplerOutput, SequenceData, SequenceGroupMetadata,
10 + get_all_seq_ids)
11 from vllm.spec_decode.interfaces import (SpeculativeProposals,
12 SpeculativeScorer, SpeculativeScores)
13 from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
@@ -293,14 +295,15 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
295 input sequence.
296 """
297 seq_data = seq_group_metadata.seq_data[seq_id]
296 - prompt_token_ids = seq_data.get_prompt_token_ids()
298 + prompt_token_ids = seq_data.prompt_token_ids_array
299 new_output_token_ids = [*seq_data.get_output_token_ids(), *token_ids]
300
301 new_seq_data_dict = {
302 target_seq_id:
303 SequenceData(
302 - prompt_token_ids=prompt_token_ids,
303 - output_token_ids=new_output_token_ids,
304 + prompt_token_ids,
305 + _output_token_ids=array(VLLM_TOKEN_ID_ARRAY_TYPE,
306 + new_output_token_ids),
307 ),
308 }
309 # This is a hack. Technically, spec decoding should compute
vllm/spec_decode/metrics.py
+5 -3
@@ -1,7 +1,7 @@
1 import time
2 -from dataclasses import dataclass
2 from typing import Callable, Optional
3
4 +import msgspec
5 import torch
6
7 from vllm.model_executor.layers.spec_decode_base_sampler import (
@@ -9,8 +9,10 @@ from vllm.model_executor.layers.spec_decode_base_sampler import (
9 from vllm.utils import is_pin_memory_available
10
11
12 -@dataclass
13 -class SpecDecodeWorkerMetrics:
12 +class SpecDecodeWorkerMetrics(
13 + msgspec.Struct,
14 + omit_defaults=True, # type: ignore[call-arg]
15 + array_like=True): # type: ignore[call-arg]
16 """Dataclass holding metrics emitted from the spec decode worker.
17 """
18
vllm/worker/worker.py
+62 -2
@@ -1,7 +1,7 @@
1 """A GPU worker class."""
2 import gc
3 import os
4 -from typing import List, Optional, Set, Tuple, Type
4 +from typing import Dict, List, Optional, Set, Tuple, Type, Union
5
6 import torch
7 import torch.distributed
@@ -18,7 +18,9 @@ from vllm.model_executor import set_random_seed
18 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19 from vllm.platforms import current_platform
20 from vllm.prompt_adapter.request import PromptAdapterRequest
21 -from vllm.sequence import ExecuteModelRequest
21 +from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
22 + SamplerOutput, SequenceGroupMetadata,
23 + SequenceGroupMetadataDelta)
24 from vllm.worker.cache_engine import CacheEngine
25 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
26 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
@@ -109,6 +111,7 @@ class Worker(LocalOrDistributedWorkerBase):
111 self.cache_engine: List[CacheEngine]
112 # Initialize gpu_cache as embedding models don't initialize kv_caches
113 self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
114 + self._seq_group_metadata_cache: Dict[str, SequenceGroupMetadata] = {}
115
116 def _is_encoder_decoder_model(self):
117 return self.model_config.is_encoder_decoder_model
@@ -303,6 +306,63 @@ class Worker(LocalOrDistributedWorkerBase):
306 and worker_input.blocks_to_copy.numel() > 0):
307 self.cache_engine[virtual_engine].copy(worker_input.blocks_to_copy)
308
309 + def _get_cached_seq_group_metadata(
310 + self,
311 + seq_group_metadata_list: List[Union[SequenceGroupMetadata,
312 + SequenceGroupMetadataDelta]],
313 + finished_request_ids: List[str]) -> List[SequenceGroupMetadata]:
314 + """Return a list of cached Sequence Group Metadata after updating its
315 + state.
316 +
317 + It is used because scheduler only sends delta to workers to reduce
318 + the data payload size. The function also cleans up cache based on
319 + a given `finished_request_ids`.
320 + """
321 + new_seq_group_metadata_list = []
322 + for metadata_or_delta in seq_group_metadata_list:
323 + request_id = metadata_or_delta.request_id
324 + if request_id not in self._seq_group_metadata_cache:
325 + # The first prefill.
326 + assert isinstance(metadata_or_delta, SequenceGroupMetadata)
327 + self._seq_group_metadata_cache[request_id] = metadata_or_delta
328 + else:
329 + # The first prefill is already cached.
330 + if isinstance(metadata_or_delta, SequenceGroupMetadataDelta):
331 + self._seq_group_metadata_cache[request_id].apply_delta(
332 + metadata_or_delta)
333 + else:
334 + # If metadata snapshot is sent again, it is
335 + # preempted. Reset the cache because we need to start
336 + # from scratch.
337 + assert isinstance(metadata_or_delta, SequenceGroupMetadata)
338 + self._seq_group_metadata_cache[
339 + request_id] = metadata_or_delta
340 +
341 + new_seq_group_metadata_list.append(
342 + self._seq_group_metadata_cache[request_id])
343 +
344 + # Clean up finished ids
345 + for finished_id in finished_request_ids:
346 + del self._seq_group_metadata_cache[finished_id]
347 +
348 + return new_seq_group_metadata_list
349 +
350 + def _execute_model_spmd(
351 + self,
352 + execute_model_req: ExecuteModelRequest,
353 + intermediate_tensors: Optional[IntermediateTensors] = None,
354 + ) -> Optional[List[SamplerOutput]]:
355 + if execute_model_req is not None:
356 + new_seq_group_metadata_list = self._get_cached_seq_group_metadata(
357 + execute_model_req.seq_group_metadata_list,
358 + execute_model_req.finished_requests_ids)
359 +
360 + execute_model_req.seq_group_metadata_list = (
361 + new_seq_group_metadata_list)
362 + output = super()._execute_model_spmd(execute_model_req,
363 + intermediate_tensors)
364 + return output
365 +
366 def add_lora(self, lora_request: LoRARequest) -> bool:
367 return self.model_runner.add_lora(lora_request)
368