A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
Remove hard-dependencies of Speculative decode to CUDA workers (#10587)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
Chendi.Xue committed
Nov 26, 2024 at 19:57 UTC
0a71900bc92b4a18d5545e9d5dc0ca750add3c69
19 files changed
+219
-77
tests/spec_decode/test_spec_decode_worker.py
+2
-2
@@ -595,8 +595,8 @@ def test_init_device(acceptance_sampler_method: str):
595
596
target_worker.init_device.assert_called_once()
597
598
- metrics_collector.init_gpu_tensors.assert_called_once()
599
- spec_decode_sampler.init_gpu_tensors.assert_called_once()
598
+ metrics_collector.init_tensors.assert_called_once()
599
+ spec_decode_sampler.init_tensors.assert_called_once()
600
601
602
@pytest.mark.parametrize("acceptance_sampler_method",
vllm/config.py
+1
@@ -990,6 +990,7 @@ class ParallelConfig:
990
# the full name of the worker class to use. If "auto", the worker class
991
# will be determined based on the platform.
992
worker_cls: str = "auto"
993
+ sd_worker_cls: str = "auto"
994
995
world_size: int = field(init=False)
996
vllm/model_executor/layers/spec_decode_base_sampler.py
+16
-1
@@ -43,6 +43,21 @@ class SpecDecodeBaseSampler(nn.Module):
43
dtype=torch.long,
44
device=device)
45
46
+ def init_tensors(self,
47
+ device: Union[int, str],
48
+ device_type: Union[torch.device, str] = 'cuda') -> None:
49
+ assert self.num_accepted_tokens is None
50
+ if isinstance(device_type, torch.device):
51
+ device_type = device_type.type
52
+ if isinstance(device, int):
53
+ device = f"{device_type}:{device}"
54
+ self.num_accepted_tokens = torch.tensor(0,
55
+ dtype=torch.long,
56
+ device=device)
57
+ self.num_emitted_tokens = torch.tensor(0,
58
+ dtype=torch.long,
59
+ device=device)
60
+
61
@property
62
def probs_dtype(self):
63
return torch.float32
@@ -77,7 +92,7 @@ class SpecDecodeBaseSampler(nn.Module):
92
tensor is [batch_size, k + num_bonus_tokens]
93
"""
94
batch_size, k = substitute_token_ids.shape
80
- bonus_token_ids = bonus_token_ids.squeeze()
95
+ bonus_token_ids = bonus_token_ids.squeeze(-1)
96
# Determine the index of the first False value for each row.
97
limits = (accepted == 0).max(1).indices
98
limits[~(accepted == 0).any(1)] = k
vllm/platforms/cpu.py
+7
-1
@@ -86,4 +86,10 @@ class CpuPlatform(Platform):
86
parallel_config.distributed_executor_backend)
87
parallel_config.distributed_executor_backend = "mp"
88
if parallel_config.worker_cls == "auto":
89
- parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
89
+ if vllm_config.speculative_config:
90
+ parallel_config.worker_cls = \
91
+ "vllm.spec_decode.spec_decode_worker.create_spec_worker"
92
+ parallel_config.sd_worker_cls = \
93
+ "vllm.worker.cpu_worker.CPUWorker"
94
+ else:
95
+ parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
vllm/platforms/cuda.py
+3
-1
@@ -106,6 +106,8 @@ class CudaPlatformBase(Platform):
106
elif vllm_config.speculative_config:
107
parallel_config.worker_cls = \
108
"vllm.spec_decode.spec_decode_worker.create_spec_worker"
109
+ parallel_config.sd_worker_cls = \
110
+ "vllm.worker.worker.Worker"
111
else:
112
parallel_config.worker_cls = "vllm.worker.worker.Worker"
113
@@ -236,4 +238,4 @@ try:
238
if not isinstance(pynvml, _MockModule):
239
CudaPlatform.log_warnings()
240
except ModuleNotFoundError:
239
- CudaPlatform.log_warnings()
241
+ CudaPlatform.log_warnings()
\ No newline at end of file
vllm/spec_decode/draft_model_runner.py
+12
-12
@@ -20,8 +20,9 @@ except (ModuleNotFoundError, ImportError) as err:
20
from vllm.logger import init_logger
21
from vllm.multimodal import MultiModalKwargs
22
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
23
-from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
24
- ModelRunner)
23
+from vllm.worker.model_runner_base import (ModelRunnerBase,
24
+ ModelRunnerInputBase,
25
+ ModelRunnerWrapperBase)
26
27
logger = init_logger(__name__)
28
@@ -33,7 +34,7 @@ debug_advance_input = False
34
allow_gpu_advance_step = True
35
36
36
-class TP1DraftModelRunner(ModelRunner):
37
+class TP1DraftModelRunner(ModelRunnerWrapperBase):
38
"""Specialized model runner for speculative decoding draft model.
39
Since the draft model always execute k forward passes consecutively to
40
generate k speculative tokens in a single speculative decoding step,
@@ -46,13 +47,14 @@ class TP1DraftModelRunner(ModelRunner):
47
any broadcasting inside execute_model).
48
"""
49
49
- def __init__(self, *args, **kwargs):
50
- if kwargs.get("return_hidden_states"):
50
+ def __init__(self, model_runner: ModelRunnerBase):
51
+ if hasattr(
52
+ model_runner,
53
+ "return_hidden_states") and model_runner.return_hidden_states:
54
raise ValueError(
55
"return_hidden_states is not supported for TP1DraftModelRunner."
56
)
54
-
55
- super().__init__(*args, **kwargs)
57
+ super().__init__(model_runner)
58
59
self.indices_of_seq_with_bonus_tokens = None
60
@@ -73,10 +75,8 @@ class TP1DraftModelRunner(ModelRunner):
75
assert seq_group.prompt_logprob_indices == [] # No prompt
76
assert seq_group.sample_indices == [i] # Simple
77
76
- def _gpu_advance_step(
77
- self, model_input: ModelInputForGPUWithSamplingMetadata,
78
- last_output: SamplerOutput
79
- ) -> ModelInputForGPUWithSamplingMetadata:
78
+ def _gpu_advance_step(self, model_input: ModelRunnerInputBase,
79
+ last_output: SamplerOutput) -> ModelRunnerInputBase:
80
# Currently, we expect "decode mode" only
81
assert not model_input.is_prompt
82
@@ -168,7 +168,7 @@ class TP1DraftModelRunner(ModelRunner):
168
@torch.inference_mode()
169
def execute_model(
170
self,
171
- model_input: ModelInputForGPUWithSamplingMetadata,
171
+ model_input: ModelRunnerInputBase,
172
kv_caches: List[torch.Tensor],
173
previous_hidden_states: Optional[torch.Tensor] = None,
174
intermediate_tensors: Optional[IntermediateTensors] = None,
vllm/spec_decode/interfaces.py
+5
-3
@@ -1,6 +1,6 @@
1
from abc import ABC, abstractmethod
2
from dataclasses import dataclass
3
-from typing import Optional, Set
3
+from typing import Optional, Set, Union
4
5
import torch
6
@@ -75,9 +75,11 @@ class SpeculativeProposer(ABC):
75
76
class SpeculativeScorer(ABC):
77
78
- def __init__(self, scorer_worker: WorkerBase, device: str,
79
- vocab_size: int):
78
+ def __init__(self, scorer_worker: WorkerBase,
79
+ device: Union[torch.device, str], vocab_size: int):
80
self._scorer_worker = scorer_worker
81
+ if isinstance(device, torch.device):
82
+ device = device.type
83
self._device = device
84
self._vocab_size = vocab_size
85
vllm/spec_decode/medusa_worker.py
+5
-4
@@ -9,21 +9,22 @@ from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
9
from vllm.spec_decode.interfaces import SpeculativeProposals
10
from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11
from vllm.spec_decode.top1_proposer import Top1Proposer
12
-from vllm.worker.worker import Worker
12
+from vllm.worker.worker_base import WorkerWrapperBase
13
14
15
-class MedusaWorker(NonLLMProposerWorkerBase, Worker):
15
+class MedusaWorker(NonLLMProposerWorkerBase, WorkerWrapperBase):
16
"""Worker for Medusa.
17
"""
18
19
def __init__(self, *args, **kwargs):
20
- super().__init__(*args, **kwargs)
20
+ super().__init__(kwargs.get("vllm_config"))
21
+ self.init_worker(*args, **kwargs)
22
23
# Lazy initialization list.
24
self._proposer: Top1Proposer
25
26
def init_device(self):
26
- super().init_device()
27
+ self.worker.init_device()
28
29
self._proposer = Top1Proposer(
30
weakref.proxy(self), # type: ignore[arg-type]
vllm/spec_decode/metrics.py
+14
-1
@@ -1,11 +1,12 @@
1
import time
2
-from typing import Callable, Optional
2
+from typing import Callable, Optional, Union
3
4
import msgspec
5
import torch
6
7
from vllm.model_executor.layers.spec_decode_base_sampler import (
8
SpecDecodeBaseSampler)
9
+from vllm.platforms import current_platform
10
from vllm.utils import is_pin_memory_available
11
12
@@ -81,8 +82,20 @@ class AsyncMetricsCollector:
82
self._rank = rank
83
self._copy_stream = torch.cuda.Stream()
84
85
+ def init_tensors(self,
86
+ rank: int,
87
+ device_type: Union[torch.device, str] = 'cuda') -> None:
88
+ self._rank = rank
89
+ if isinstance(device_type, torch.device):
90
+ device_type = device_type.type
91
+ if device_type == 'cuda':
92
+ self._copy_stream = torch.cuda.Stream()
93
+
94
def maybe_collect_rejsample_metrics(
95
self, k: int) -> Optional[SpecDecodeWorkerMetrics]:
96
+ # currently using cuda.Event, skip for any non_cuda_alike platform
97
+ if not current_platform.is_cuda_alike():
98
+ return None
99
100
# If a copy was initiated in the previous call, collect and return.
101
if self._in_flight_copy is not None:
vllm/spec_decode/multi_step_worker.py
+24
-7
@@ -5,17 +5,21 @@ from typing import Dict, List, Set, Tuple
5
import torch
6
7
from vllm.model_executor.layers.sampler import SamplerOutput
8
+from vllm.platforms import current_platform
9
from vllm.sequence import (ExecuteModelRequest, HiddenStates, SequenceData,
10
SequenceGroupMetadata)
10
-from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
11
+
12
+if current_platform.is_cuda_alike():
13
+ from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
14
+
15
from vllm.spec_decode.interfaces import (SpeculativeProposals,
16
SpeculativeProposer)
17
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
18
from vllm.spec_decode.top1_proposer import Top1Proposer
15
-from vllm.worker.worker import Worker
19
+from vllm.worker.worker_base import WorkerWrapperBase
20
21
18
-class MultiStepWorker(Worker, ProposerWorkerBase):
22
+class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
23
"""The MultiStepWorker is equivalent to a Worker except that it allows
24
multiple forward passes in a single call, assuming the scheduler has
25
allocated enough space to store the additional KV. This reduces overhead
@@ -28,13 +32,14 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
32
"""
33
34
def __init__(self, *args, **kwargs):
31
- super().__init__(*args, **kwargs)
35
+ super().__init__(kwargs.get("vllm_config"))
36
+ self.init_worker(*args, **kwargs)
37
38
# Lazy initialization list.
39
self._proposer: SpeculativeProposer
40
41
def init_device(self) -> None:
37
- super().init_device()
42
+ self.worker.init_device()
43
44
self._proposer = Top1Proposer(
45
weakref.proxy(self), # type: ignore[arg-type]
@@ -51,6 +56,18 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
56
self.model_runner.model.sampler.should_modify_greedy_probs_inplace = (
57
True)
58
59
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
60
+ return self.worker.determine_num_available_blocks()
61
+
62
+ def get_cache_block_size_bytes(self) -> int:
63
+ return self.worker.get_cache_block_size_bytes()
64
+
65
+ def initialize_cache(self, *args, **kwargs) -> None:
66
+ self.worker.initialize_cache(*args, **kwargs)
67
+
68
+ def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
69
+ return self.worker.execute_model(*args, **kwargs)
70
+
71
@torch.inference_mode()
72
def sampler_output(
73
self,
@@ -75,7 +92,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
92
93
# Run model sample_len times.
94
model_outputs: List[SamplerOutput] = []
78
- if isinstance(
95
+ if current_platform.is_cuda_alike() and isinstance(
96
self.model_runner, TP1DraftModelRunner
97
) and self.model_runner.supports_gpu_multi_step(expanded_request):
98
# Here we run the draft_model_runner with multi-step prepare
@@ -92,7 +109,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
109
# and other restrictions that are part of DraftModelRunner's
110
# supports_gpu_multi_step(..)
111
for _ in range(sample_len):
95
- model_output: List[SamplerOutput] = super().execute_model(
112
+ model_output: List[SamplerOutput] = self.worker.execute_model(
113
execute_model_req=expanded_request)
114
assert (len(model_output) == 1
115
), "composing multistep workers not supported"
vllm/spec_decode/ngram_worker.py
+2
-1
@@ -22,6 +22,7 @@ class NGramWorker(NonLLMProposerWorkerBase):
22
# Get local_rank/vocab_size from kwargs attribute
23
self.local_rank = kwargs["local_rank"]
24
self.vocab_size = kwargs["vllm_config"].model_config.get_vocab_size()
25
+ self.device_type = kwargs.get("device_type", "cuda")
26
27
# Lazy initialization list.
28
self._proposer: Top1Proposer
@@ -34,7 +35,7 @@ class NGramWorker(NonLLMProposerWorkerBase):
35
self.ngram_prompt_lookup_min = ngram_prompt_lookup_min
36
37
def init_device(self):
37
- self.device = torch.device(f"cuda:{self.local_rank}")
38
+ self.device = torch.device(f"{self.device_type}:{self.local_rank}")
39
self.load_model = lambda *args, **kwargs: None
40
41
# Current NGramWorker only supports Top1Proposer
vllm/spec_decode/spec_decode_worker.py
+25
-11
@@ -14,12 +14,16 @@ from vllm.model_executor.layers.spec_decode_base_sampler import (
14
SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
15
from vllm.model_executor.layers.typical_acceptance_sampler import (
16
TypicalAcceptanceSampler)
17
+from vllm.platforms import current_platform
18
from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
19
CompletionSequenceGroupOutput, ExecuteModelRequest,
20
HiddenStates, SequenceGroupMetadata,
21
get_all_seq_ids_and_request_ids)
22
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
22
-from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
23
+
24
+if current_platform.is_cuda_alike():
25
+ from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
26
+
27
from vllm.spec_decode.interfaces import (SpeculativeProposals,
28
SpeculativeScorer, SpeculativeScores)
29
from vllm.spec_decode.medusa_worker import MedusaWorker
@@ -36,8 +40,8 @@ from vllm.spec_decode.util import (Timer, create_logprobs_output,
40
get_all_num_logprobs,
41
get_sampled_token_logprobs, nvtx_range,
42
split_batch_by_proposal_len)
39
-from vllm.worker.worker import Worker
40
-from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
43
+from vllm.worker.worker_base import (LoraNotSupportedWorkerBase, WorkerBase,
44
+ WorkerWrapperBase)
45
46
logger = init_logger(__name__)
47
@@ -53,7 +57,11 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
57
draft_worker_kwargs = kwargs.copy()
58
59
kwargs["model_runner_cls"] = TargetModelRunner
56
- target_worker = Worker(*args, **kwargs)
60
+ target_worker_config = copy.deepcopy(vllm_config)
61
+ target_worker_config.parallel_config.worker_cls =\
62
+ target_worker_config.parallel_config.sd_worker_cls
63
+ target_worker = WorkerWrapperBase(vllm_config=target_worker_config)
64
+ target_worker.init_worker(*args, **kwargs)
65
# Set the disable_logprobs variable in the TargetModelRunner instance
66
# as per its value specified in the SpeculativeConfig.
67
target_worker.model_runner.disable_logprobs =\
@@ -65,6 +73,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
73
draft_worker_config.model_config,
74
vllm_config.load_config,
75
)
76
+ speculative_config.draft_parallel_config.worker_cls =\
77
+ draft_worker_config.parallel_config.sd_worker_cls
78
draft_worker_config.parallel_config = speculative_config.draft_parallel_config # noqa
79
# TODO allow draft-model specific load config.
80
@@ -125,7 +135,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
135
@classmethod
136
def create_worker(
137
cls,
128
- scorer_worker: Worker,
138
+ scorer_worker: WorkerBase,
139
draft_worker_kwargs: Dict[str, Any],
140
disable_mqa_scorer: bool,
141
disable_by_batch_size: Optional[int],
@@ -145,6 +155,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
155
draft_parallel_config: ParallelConfig = draft_worker_kwargs[
156
'vllm_config'].parallel_config
157
if ngram_prompt_lookup_max > 0:
158
+ draft_worker_kwargs[
159
+ "device_type"] = scorer_worker.device_config.device.type
160
proposer_worker = NGramWorker(**draft_worker_kwargs)
161
proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
162
ngram_prompt_lookup_max)
@@ -158,8 +170,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
170
proposer_worker = MedusaWorker(**draft_worker_kwargs)
171
else:
172
if draft_tp == 1:
161
- draft_worker_kwargs[
162
- "model_runner_cls"] = TP1DraftModelRunner
173
+ if current_platform.is_cuda_alike():
174
+ draft_worker_kwargs[
175
+ "model_runner_cls"] = TP1DraftModelRunner
176
else:
177
if draft_model_config.hf_config.model_type == "eagle":
178
raise NotImplementedError(
@@ -306,8 +319,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
319
self.scorer_worker.load_model()
320
self.proposer_worker.load_model()
321
309
- self._metrics.init_gpu_tensors(self.rank)
310
- self.spec_decode_sampler.init_gpu_tensors(self.rank)
322
+ self._metrics.init_tensors(self.rank, device_type=self.device)
323
+ self.spec_decode_sampler.init_tensors(self.rank,
324
+ device_type=self.device)
325
326
scorer_cls: Type[SpeculativeScorer]
327
if self.disable_mqa_scorer:
@@ -1111,11 +1125,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
1125
raise NotImplementedError
1126
1127
def start_profile(self):
1114
- if isinstance(self.scorer_worker, Worker):
1128
+ if isinstance(self.scorer_worker, WorkerBase):
1129
self.scorer_worker.start_profile()
1130
1131
def stop_profile(self):
1118
- if isinstance(self.scorer_worker, Worker):
1132
+ if isinstance(self.scorer_worker, WorkerBase):
1133
self.scorer_worker.stop_profile()
1134
1135
vllm/spec_decode/target_model_runner.py
+11
-22
@@ -1,12 +1,12 @@
1
from typing import List, Optional
2
3
-from vllm.config import VllmConfig
3
from vllm.sequence import SequenceGroupMetadata
5
-from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
6
- ModelRunner)
4
+from vllm.worker.model_runner_base import (ModelRunnerBase,
5
+ ModelRunnerInputBase,
6
+ ModelRunnerWrapperBase)
7
8
9
-class TargetModelRunner(ModelRunner):
9
+class TargetModelRunner(ModelRunnerWrapperBase):
10
"""Specialized model runner for speculative decoding target model.
11
In speculative decoding, the log probabilities selected finally may not
12
be the same ones as selected by the target model sampling. This means
@@ -18,32 +18,21 @@ class TargetModelRunner(ModelRunner):
18
requested or not.
19
"""
20
21
- def __init__(
22
- self,
23
- vllm_config: VllmConfig,
24
- kv_cache_dtype: Optional[str] = "auto",
25
- is_driver_worker: bool = False,
26
- return_hidden_states: bool = False,
27
- ):
21
+ def __init__(self, model_runner: ModelRunnerBase):
22
# An internal boolean member variable to indicate if token log
23
# probabilities are needed or not.
24
+ super().__init__(model_runner)
25
self.disable_logprobs = True
31
- super().__init__(
32
- vllm_config=vllm_config,
33
- kv_cache_dtype=kv_cache_dtype,
34
- is_driver_worker=is_driver_worker,
35
- return_hidden_states=return_hidden_states,
36
- )
26
27
def prepare_model_input(
28
self,
29
seq_group_metadata_list: List[SequenceGroupMetadata],
30
virtual_engine: int = 0,
42
- finished_requests_ids: Optional[List[str]] = None
43
- ) -> ModelInputForGPUWithSamplingMetadata:
44
- model_input: ModelInputForGPUWithSamplingMetadata = super(
45
- ).prepare_model_input(seq_group_metadata_list, virtual_engine,
46
- finished_requests_ids)
31
+ finished_requests_ids: Optional[List[str]] = None,
32
+ ) -> ModelRunnerInputBase:
33
+ model_input: ModelRunnerInputBase =\
34
+ self.model_runner.prepare_model_input(
35
+ seq_group_metadata_list, virtual_engine, finished_requests_ids)
36
# If token log probabilities is disabled then skip generating sampler
37
# CPU output. We directly serialize the GPU sampled_token_id tensors
38
# as needed. If log probabilities is enabled then synchronize all the
vllm/spec_decode/util.py
+8
-4
@@ -5,6 +5,7 @@ from typing import Dict, List, Optional, Sequence, Tuple
5
import torch
6
7
from vllm.model_executor.layers.sampler import SamplerOutput
8
+from vllm.platforms import current_platform
9
from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
10
PromptLogprobs, SequenceGroupMetadata,
11
SequenceOutput)
@@ -247,11 +248,14 @@ def nvtx_range(msg, *args, **kwargs):
248
Arguments:
249
msg (string): message to associate with the range
250
"""
250
- torch.cuda.nvtx.range_push(msg.format(*args, **kwargs))
251
- try:
251
+ if current_platform.is_cuda_alike():
252
+ torch.cuda.nvtx.range_push(msg.format(*args, **kwargs))
253
+ try:
254
+ yield
255
+ finally:
256
+ torch.cuda.nvtx.range_pop()
257
+ else:
258
yield
253
- finally:
254
- torch.cuda.nvtx.range_pop()
259
260
261
class Timer:
vllm/worker/cpu_model_runner.py
+37
-2
@@ -80,6 +80,7 @@ class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
80
Used by the ModelRunner.
81
"""
82
sampling_metadata: Optional["SamplingMetadata"] = None
83
+ is_prompt: Optional[bool] = None
84
85
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
86
tensor_dict = {
@@ -395,6 +396,7 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
396
vllm_config: VllmConfig,
397
kv_cache_dtype: Optional[str] = "auto",
398
is_driver_worker: bool = False,
399
+ return_hidden_states: bool = False,
400
*args,
401
**kwargs,
402
):
@@ -403,19 +405,25 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
405
cache_config = self.cache_config
406
407
self.is_driver_worker = is_driver_worker
408
+ self.return_hidden_states = return_hidden_states
409
410
self.device = self.device_config.device
411
+ self.pin_memory = False
412
413
self.kv_cache_dtype = kv_cache_dtype
414
self.sliding_window = model_config.get_sliding_window()
415
self.block_size = cache_config.block_size
416
+ num_attn_heads = self.model_config.get_num_attention_heads(
417
+ self.parallel_config)
418
+ needs_attn_backend = (num_attn_heads != 0
419
+ or self.model_config.is_attention_free)
420
self.attn_backend = get_attn_backend(
421
self.model_config.get_head_size(),
422
self.model_config.dtype,
423
self.kv_cache_dtype,
424
self.block_size,
425
self.model_config.is_attention_free,
418
- )
426
+ ) if needs_attn_backend else None
427
428
# Multi-modal data support
429
self.mm_registry = MULTIMODAL_REGISTRY
@@ -444,6 +452,15 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
452
453
return builder.build() # type: ignore
454
455
+ # sampler property will be used by spec_decode_worker
456
+ @property
457
+ def sampler(self):
458
+ return self.model.sampler
459
+
460
+ @property
461
+ def vocab_size(self) -> int:
462
+ return self.model_config.get_vocab_size()
463
+
464
465
class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
466
_model_input_cls: Type[ModelInputForCPUWithSamplingMetadata] = (
@@ -480,9 +497,12 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
497
pin_memory=False,
498
generators=generators)
499
500
+ is_prompt = (seq_group_metadata_list[0].is_prompt
501
+ if seq_group_metadata_list else None)
502
return dataclasses.replace(model_input,
503
sampling_metadata=sampling_metadata,
485
- virtual_engine=virtual_engine)
504
+ virtual_engine=virtual_engine,
505
+ is_prompt=is_prompt)
506
507
@torch.no_grad()
508
def execute_model(
@@ -491,16 +511,22 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
511
kv_caches: List[torch.Tensor],
512
intermediate_tensors: Optional[IntermediateTensors] = None,
513
num_steps: int = 1,
514
+ previous_hidden_states: Optional[torch.Tensor] = None,
515
) -> Optional[List[SamplerOutput]]:
516
if num_steps > 1:
517
raise ValueError(
518
"CPU worker does not support multi-step execution.")
519
520
model_executable = self.model
521
+
522
multimodal_kwargs = {}
523
if model_input.multi_modal_kwargs is not None:
524
multimodal_kwargs = MultiModalKwargs.as_kwargs(
525
model_input.multi_modal_kwargs, device=self.device)
526
+ execute_model_kwargs = {}
527
+ if previous_hidden_states is not None:
528
+ execute_model_kwargs.update(
529
+ {"previous_hidden_states": previous_hidden_states})
530
531
with set_forward_context(model_input.attn_metadata, self.vllm_config):
532
hidden_states = model_executable(
@@ -509,6 +535,7 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
535
kv_caches=kv_caches,
536
attn_metadata=model_input.attn_metadata,
537
intermediate_tensors=intermediate_tensors,
538
+ **execute_model_kwargs,
539
**multimodal_kwargs,
540
)
541
@@ -525,4 +552,12 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
552
logits=logits,
553
sampling_metadata=model_input.sampling_metadata,
554
)
555
+ if self.return_hidden_states:
556
+ # we only need to pass hidden states of most recent token
557
+ if model_input.is_prompt:
558
+ output.prefill_hidden_states = hidden_states
559
+ output.hidden_states = hidden_states
560
return [output]
561
+
562
+ def generate_proposals(self, *args, **kwargs):
563
+ return self.model.generate_proposals(*args, **kwargs)
vllm/worker/cpu_worker.py
+25
-2
@@ -128,6 +128,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
128
distributed_init_method: str,
129
kv_cache_dtype: Optional[str] = "auto",
130
is_driver_worker: bool = False,
131
+ model_runner_cls: Optional[Type[CPUModelRunner]] = None,
132
) -> None:
133
WorkerBase.__init__(self, vllm_config=vllm_config)
134
@@ -151,6 +152,16 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
152
else:
153
self.local_omp_cpuid = omp_cpuids.split("|")[rank]
154
155
+ # Return hidden states from target model if the draft model is an
156
+ # mlp_speculator
157
+ speculative_config = self.speculative_config
158
+ model_config = self.model_config
159
+ speculative_args = {} if speculative_config is None \
160
+ or (speculative_config.draft_model_config.model ==
161
+ model_config.model) \
162
+ or (speculative_config.draft_model_config.hf_config.model_type
163
+ not in ["medusa", "mlp_speculator", "eagle"]) \
164
+ else {"return_hidden_states": True}
165
ModelRunnerClass: Type[CPUModelRunnerBase] = CPUModelRunner
166
if self.model_config.task == "embedding":
167
ModelRunnerClass = CPUEmbeddingModelRunner
@@ -159,7 +170,11 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
170
self.model_runner: CPUModelRunnerBase = ModelRunnerClass(
171
vllm_config=vllm_config,
172
kv_cache_dtype=kv_cache_dtype,
162
- is_driver_worker=is_driver_worker)
173
+ is_driver_worker=is_driver_worker,
174
+ **speculative_args,
175
+ )
176
+ if model_runner_cls is not None:
177
+ self.model_runner = model_runner_cls(self.model_runner)
178
# Uninitialized cache engine. Will be initialized by
179
# initialize_cache.
180
self.cache_engine: List[CPUCacheEngine]
@@ -197,7 +212,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
212
ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
213
if ret:
214
logger.info(ret)
200
-
215
+ self.device = torch.device("cpu")
216
self.init_distributed_environment()
217
# Set random seed.
218
set_random_seed(self.model_config.seed)
@@ -297,6 +312,14 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
312
def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
313
return self.cpu_cache
314
315
+ @property
316
+ def vocab_size(self) -> int:
317
+ return self.model_runner.vocab_size
318
+
319
+ @property
320
+ def max_model_len(self) -> int:
321
+ return self.model_config.max_model_len
322
+
323
def execute_worker(
324
self,
325
worker_input: WorkerInput,
vllm/worker/model_runner_base.py
+15
@@ -289,3 +289,18 @@ class ModelRunnerBase(ABC, Generic[T]):
289
self.generators.pop(request_id, None)
290
291
return self.generators
292
+
293
+
294
+class ModelRunnerWrapperBase:
295
+ """
296
+ The whole point of this class is to lazily initialize the model_runner.
297
+ """
298
+
299
+ def __init__(
300
+ self,
301
+ moderl_runner: ModelRunnerBase,
302
+ ) -> None:
303
+ self.model_runner: ModelRunnerBase = moderl_runner
304
+
305
+ def __getattr__(self, attr):
306
+ return getattr(self.model_runner, attr)
vllm/worker/worker.py
+4
-3
@@ -74,9 +74,7 @@ class Worker(LocalOrDistributedWorkerBase):
74
else {"return_hidden_states": True}
75
76
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
77
- if model_runner_cls is not None:
78
- ModelRunnerClass = model_runner_cls
79
- elif model_config.task == "embedding":
77
+ if model_config.task == "embedding":
78
ModelRunnerClass = EmbeddingModelRunner
79
elif self.model_config.is_encoder_decoder:
80
ModelRunnerClass = EncoderDecoderModelRunner
@@ -86,6 +84,9 @@ class Worker(LocalOrDistributedWorkerBase):
84
is_driver_worker=is_driver_worker,
85
**speculative_args,
86
)
87
+ if model_runner_cls is not None:
88
+ self.model_runner = model_runner_cls(self.model_runner)
89
+
90
# Uninitialized cache engine. Will be initialized by
91
# initialize_cache.
92
self.cache_engine: List[CacheEngine]
vllm/worker/worker_base.py
+3
@@ -466,6 +466,9 @@ class WorkerWrapperBase:
466
logger.exception(msg)
467
raise e
468
469
+ def __getattr__(self, attr):
470
+ return getattr(self.worker, attr)
471
+
472
473
def extract_previous_hidden_states(
474
data: Union[ExecuteModelRequest, Dict[str, torch.Tensor]]) -> \