@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0a71900bc9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

Remove hard-dependencies of Speculative decode to CUDA workers (#10587)

Signed-off-by: Chendi Xue <chendi.xue@intel.com>

Chendi.Xue committed Nov 26, 2024 at 19:57 UTC 0a71900bc92b4a18d5545e9d5dc0ca750add3c69
19 files changed +219 -77
tests/spec_decode/test_spec_decode_worker.py
+2 -2
@@ -595,8 +595,8 @@ def test_init_device(acceptance_sampler_method: str):
595
596 target_worker.init_device.assert_called_once()
597
598 - metrics_collector.init_gpu_tensors.assert_called_once()
599 - spec_decode_sampler.init_gpu_tensors.assert_called_once()
598 + metrics_collector.init_tensors.assert_called_once()
599 + spec_decode_sampler.init_tensors.assert_called_once()
600
601
602 @pytest.mark.parametrize("acceptance_sampler_method",
vllm/config.py
+1
@@ -990,6 +990,7 @@ class ParallelConfig:
990 # the full name of the worker class to use. If "auto", the worker class
991 # will be determined based on the platform.
992 worker_cls: str = "auto"
993 + sd_worker_cls: str = "auto"
994
995 world_size: int = field(init=False)
996
vllm/model_executor/layers/spec_decode_base_sampler.py
+16 -1
@@ -43,6 +43,21 @@ class SpecDecodeBaseSampler(nn.Module):
43 dtype=torch.long,
44 device=device)
45
46 + def init_tensors(self,
47 + device: Union[int, str],
48 + device_type: Union[torch.device, str] = 'cuda') -> None:
49 + assert self.num_accepted_tokens is None
50 + if isinstance(device_type, torch.device):
51 + device_type = device_type.type
52 + if isinstance(device, int):
53 + device = f"{device_type}:{device}"
54 + self.num_accepted_tokens = torch.tensor(0,
55 + dtype=torch.long,
56 + device=device)
57 + self.num_emitted_tokens = torch.tensor(0,
58 + dtype=torch.long,
59 + device=device)
60 +
61 @property
62 def probs_dtype(self):
63 return torch.float32
@@ -77,7 +92,7 @@ class SpecDecodeBaseSampler(nn.Module):
92 tensor is [batch_size, k + num_bonus_tokens]
93 """
94 batch_size, k = substitute_token_ids.shape
80 - bonus_token_ids = bonus_token_ids.squeeze()
95 + bonus_token_ids = bonus_token_ids.squeeze(-1)
96 # Determine the index of the first False value for each row.
97 limits = (accepted == 0).max(1).indices
98 limits[~(accepted == 0).any(1)] = k
vllm/platforms/cpu.py
+7 -1
@@ -86,4 +86,10 @@ class CpuPlatform(Platform):
86 parallel_config.distributed_executor_backend)
87 parallel_config.distributed_executor_backend = "mp"
88 if parallel_config.worker_cls == "auto":
89 - parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
89 + if vllm_config.speculative_config:
90 + parallel_config.worker_cls = \
91 + "vllm.spec_decode.spec_decode_worker.create_spec_worker"
92 + parallel_config.sd_worker_cls = \
93 + "vllm.worker.cpu_worker.CPUWorker"
94 + else:
95 + parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
vllm/platforms/cuda.py
+3 -1
@@ -106,6 +106,8 @@ class CudaPlatformBase(Platform):
106 elif vllm_config.speculative_config:
107 parallel_config.worker_cls = \
108 "vllm.spec_decode.spec_decode_worker.create_spec_worker"
109 + parallel_config.sd_worker_cls = \
110 + "vllm.worker.worker.Worker"
111 else:
112 parallel_config.worker_cls = "vllm.worker.worker.Worker"
113
@@ -236,4 +238,4 @@ try:
238 if not isinstance(pynvml, _MockModule):
239 CudaPlatform.log_warnings()
240 except ModuleNotFoundError:
239 - CudaPlatform.log_warnings()
241 + CudaPlatform.log_warnings()
\ No newline at end of file
vllm/spec_decode/draft_model_runner.py
+12 -12
@@ -20,8 +20,9 @@ except (ModuleNotFoundError, ImportError) as err:
20 from vllm.logger import init_logger
21 from vllm.multimodal import MultiModalKwargs
22 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
23 -from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
24 - ModelRunner)
23 +from vllm.worker.model_runner_base import (ModelRunnerBase,
24 + ModelRunnerInputBase,
25 + ModelRunnerWrapperBase)
26
27 logger = init_logger(__name__)
28
@@ -33,7 +34,7 @@ debug_advance_input = False
34 allow_gpu_advance_step = True
35
36
36 -class TP1DraftModelRunner(ModelRunner):
37 +class TP1DraftModelRunner(ModelRunnerWrapperBase):
38 """Specialized model runner for speculative decoding draft model.
39 Since the draft model always execute k forward passes consecutively to
40 generate k speculative tokens in a single speculative decoding step,
@@ -46,13 +47,14 @@ class TP1DraftModelRunner(ModelRunner):
47 any broadcasting inside execute_model).
48 """
49
49 - def __init__(self, *args, **kwargs):
50 - if kwargs.get("return_hidden_states"):
50 + def __init__(self, model_runner: ModelRunnerBase):
51 + if hasattr(
52 + model_runner,
53 + "return_hidden_states") and model_runner.return_hidden_states:
54 raise ValueError(
55 "return_hidden_states is not supported for TP1DraftModelRunner."
56 )
54 -
55 - super().__init__(*args, **kwargs)
57 + super().__init__(model_runner)
58
59 self.indices_of_seq_with_bonus_tokens = None
60
@@ -73,10 +75,8 @@ class TP1DraftModelRunner(ModelRunner):
75 assert seq_group.prompt_logprob_indices == [] # No prompt
76 assert seq_group.sample_indices == [i] # Simple
77
76 - def _gpu_advance_step(
77 - self, model_input: ModelInputForGPUWithSamplingMetadata,
78 - last_output: SamplerOutput
79 - ) -> ModelInputForGPUWithSamplingMetadata:
78 + def _gpu_advance_step(self, model_input: ModelRunnerInputBase,
79 + last_output: SamplerOutput) -> ModelRunnerInputBase:
80 # Currently, we expect "decode mode" only
81 assert not model_input.is_prompt
82
@@ -168,7 +168,7 @@ class TP1DraftModelRunner(ModelRunner):
168 @torch.inference_mode()
169 def execute_model(
170 self,
171 - model_input: ModelInputForGPUWithSamplingMetadata,
171 + model_input: ModelRunnerInputBase,
172 kv_caches: List[torch.Tensor],
173 previous_hidden_states: Optional[torch.Tensor] = None,
174 intermediate_tensors: Optional[IntermediateTensors] = None,
vllm/spec_decode/interfaces.py
+5 -3
@@ -1,6 +1,6 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass
3 -from typing import Optional, Set
3 +from typing import Optional, Set, Union
4
5 import torch
6
@@ -75,9 +75,11 @@ class SpeculativeProposer(ABC):
75
76 class SpeculativeScorer(ABC):
77
78 - def __init__(self, scorer_worker: WorkerBase, device: str,
79 - vocab_size: int):
78 + def __init__(self, scorer_worker: WorkerBase,
79 + device: Union[torch.device, str], vocab_size: int):
80 self._scorer_worker = scorer_worker
81 + if isinstance(device, torch.device):
82 + device = device.type
83 self._device = device
84 self._vocab_size = vocab_size
85
vllm/spec_decode/medusa_worker.py
+5 -4
@@ -9,21 +9,22 @@ from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
9 from vllm.spec_decode.interfaces import SpeculativeProposals
10 from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11 from vllm.spec_decode.top1_proposer import Top1Proposer
12 -from vllm.worker.worker import Worker
12 +from vllm.worker.worker_base import WorkerWrapperBase
13
14
15 -class MedusaWorker(NonLLMProposerWorkerBase, Worker):
15 +class MedusaWorker(NonLLMProposerWorkerBase, WorkerWrapperBase):
16 """Worker for Medusa.
17 """
18
19 def __init__(self, *args, **kwargs):
20 - super().__init__(*args, **kwargs)
20 + super().__init__(kwargs.get("vllm_config"))
21 + self.init_worker(*args, **kwargs)
22
23 # Lazy initialization list.
24 self._proposer: Top1Proposer
25
26 def init_device(self):
26 - super().init_device()
27 + self.worker.init_device()
28
29 self._proposer = Top1Proposer(
30 weakref.proxy(self), # type: ignore[arg-type]
vllm/spec_decode/metrics.py
+14 -1
@@ -1,11 +1,12 @@
1 import time
2 -from typing import Callable, Optional
2 +from typing import Callable, Optional, Union
3
4 import msgspec
5 import torch
6
7 from vllm.model_executor.layers.spec_decode_base_sampler import (
8 SpecDecodeBaseSampler)
9 +from vllm.platforms import current_platform
10 from vllm.utils import is_pin_memory_available
11
12
@@ -81,8 +82,20 @@ class AsyncMetricsCollector:
82 self._rank = rank
83 self._copy_stream = torch.cuda.Stream()
84
85 + def init_tensors(self,
86 + rank: int,
87 + device_type: Union[torch.device, str] = 'cuda') -> None:
88 + self._rank = rank
89 + if isinstance(device_type, torch.device):
90 + device_type = device_type.type
91 + if device_type == 'cuda':
92 + self._copy_stream = torch.cuda.Stream()
93 +
94 def maybe_collect_rejsample_metrics(
95 self, k: int) -> Optional[SpecDecodeWorkerMetrics]:
96 + # currently using cuda.Event, skip for any non_cuda_alike platform
97 + if not current_platform.is_cuda_alike():
98 + return None
99
100 # If a copy was initiated in the previous call, collect and return.
101 if self._in_flight_copy is not None:
vllm/spec_decode/multi_step_worker.py
+24 -7
@@ -5,17 +5,21 @@ from typing import Dict, List, Set, Tuple
5 import torch
6
7 from vllm.model_executor.layers.sampler import SamplerOutput
8 +from vllm.platforms import current_platform
9 from vllm.sequence import (ExecuteModelRequest, HiddenStates, SequenceData,
10 SequenceGroupMetadata)
10 -from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
11 +
12 +if current_platform.is_cuda_alike():
13 + from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
14 +
15 from vllm.spec_decode.interfaces import (SpeculativeProposals,
16 SpeculativeProposer)
17 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
18 from vllm.spec_decode.top1_proposer import Top1Proposer
15 -from vllm.worker.worker import Worker
19 +from vllm.worker.worker_base import WorkerWrapperBase
20
21
18 -class MultiStepWorker(Worker, ProposerWorkerBase):
22 +class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
23 """The MultiStepWorker is equivalent to a Worker except that it allows
24 multiple forward passes in a single call, assuming the scheduler has
25 allocated enough space to store the additional KV. This reduces overhead
@@ -28,13 +32,14 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
32 """
33
34 def __init__(self, *args, **kwargs):
31 - super().__init__(*args, **kwargs)
35 + super().__init__(kwargs.get("vllm_config"))
36 + self.init_worker(*args, **kwargs)
37
38 # Lazy initialization list.
39 self._proposer: SpeculativeProposer
40
41 def init_device(self) -> None:
37 - super().init_device()
42 + self.worker.init_device()
43
44 self._proposer = Top1Proposer(
45 weakref.proxy(self), # type: ignore[arg-type]
@@ -51,6 +56,18 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
56 self.model_runner.model.sampler.should_modify_greedy_probs_inplace = (
57 True)
58
59 + def determine_num_available_blocks(self) -> Tuple[int, int]:
60 + return self.worker.determine_num_available_blocks()
61 +
62 + def get_cache_block_size_bytes(self) -> int:
63 + return self.worker.get_cache_block_size_bytes()
64 +
65 + def initialize_cache(self, *args, **kwargs) -> None:
66 + self.worker.initialize_cache(*args, **kwargs)
67 +
68 + def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
69 + return self.worker.execute_model(*args, **kwargs)
70 +
71 @torch.inference_mode()
72 def sampler_output(
73 self,
@@ -75,7 +92,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
92
93 # Run model sample_len times.
94 model_outputs: List[SamplerOutput] = []
78 - if isinstance(
95 + if current_platform.is_cuda_alike() and isinstance(
96 self.model_runner, TP1DraftModelRunner
97 ) and self.model_runner.supports_gpu_multi_step(expanded_request):
98 # Here we run the draft_model_runner with multi-step prepare
@@ -92,7 +109,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
109 # and other restrictions that are part of DraftModelRunner's
110 # supports_gpu_multi_step(..)
111 for _ in range(sample_len):
95 - model_output: List[SamplerOutput] = super().execute_model(
112 + model_output: List[SamplerOutput] = self.worker.execute_model(
113 execute_model_req=expanded_request)
114 assert (len(model_output) == 1
115 ), "composing multistep workers not supported"
vllm/spec_decode/ngram_worker.py
+2 -1
@@ -22,6 +22,7 @@ class NGramWorker(NonLLMProposerWorkerBase):
22 # Get local_rank/vocab_size from kwargs attribute
23 self.local_rank = kwargs["local_rank"]
24 self.vocab_size = kwargs["vllm_config"].model_config.get_vocab_size()
25 + self.device_type = kwargs.get("device_type", "cuda")
26
27 # Lazy initialization list.
28 self._proposer: Top1Proposer
@@ -34,7 +35,7 @@ class NGramWorker(NonLLMProposerWorkerBase):
35 self.ngram_prompt_lookup_min = ngram_prompt_lookup_min
36
37 def init_device(self):
37 - self.device = torch.device(f"cuda:{self.local_rank}")
38 + self.device = torch.device(f"{self.device_type}:{self.local_rank}")
39 self.load_model = lambda *args, **kwargs: None
40
41 # Current NGramWorker only supports Top1Proposer
vllm/spec_decode/spec_decode_worker.py
+25 -11
@@ -14,12 +14,16 @@ from vllm.model_executor.layers.spec_decode_base_sampler import (
14 SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
15 from vllm.model_executor.layers.typical_acceptance_sampler import (
16 TypicalAcceptanceSampler)
17 +from vllm.platforms import current_platform
18 from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
19 CompletionSequenceGroupOutput, ExecuteModelRequest,
20 HiddenStates, SequenceGroupMetadata,
21 get_all_seq_ids_and_request_ids)
22 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
22 -from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
23 +
24 +if current_platform.is_cuda_alike():
25 + from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
26 +
27 from vllm.spec_decode.interfaces import (SpeculativeProposals,
28 SpeculativeScorer, SpeculativeScores)
29 from vllm.spec_decode.medusa_worker import MedusaWorker
@@ -36,8 +40,8 @@ from vllm.spec_decode.util import (Timer, create_logprobs_output,
40 get_all_num_logprobs,
41 get_sampled_token_logprobs, nvtx_range,
42 split_batch_by_proposal_len)
39 -from vllm.worker.worker import Worker
40 -from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
43 +from vllm.worker.worker_base import (LoraNotSupportedWorkerBase, WorkerBase,
44 + WorkerWrapperBase)
45
46 logger = init_logger(__name__)
47
@@ -53,7 +57,11 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
57 draft_worker_kwargs = kwargs.copy()
58
59 kwargs["model_runner_cls"] = TargetModelRunner
56 - target_worker = Worker(*args, **kwargs)
60 + target_worker_config = copy.deepcopy(vllm_config)
61 + target_worker_config.parallel_config.worker_cls =\
62 + target_worker_config.parallel_config.sd_worker_cls
63 + target_worker = WorkerWrapperBase(vllm_config=target_worker_config)
64 + target_worker.init_worker(*args, **kwargs)
65 # Set the disable_logprobs variable in the TargetModelRunner instance
66 # as per its value specified in the SpeculativeConfig.
67 target_worker.model_runner.disable_logprobs =\
@@ -65,6 +73,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
73 draft_worker_config.model_config,
74 vllm_config.load_config,
75 )
76 + speculative_config.draft_parallel_config.worker_cls =\
77 + draft_worker_config.parallel_config.sd_worker_cls
78 draft_worker_config.parallel_config = speculative_config.draft_parallel_config # noqa
79 # TODO allow draft-model specific load config.
80
@@ -125,7 +135,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
135 @classmethod
136 def create_worker(
137 cls,
128 - scorer_worker: Worker,
138 + scorer_worker: WorkerBase,
139 draft_worker_kwargs: Dict[str, Any],
140 disable_mqa_scorer: bool,
141 disable_by_batch_size: Optional[int],
@@ -145,6 +155,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
155 draft_parallel_config: ParallelConfig = draft_worker_kwargs[
156 'vllm_config'].parallel_config
157 if ngram_prompt_lookup_max > 0:
158 + draft_worker_kwargs[
159 + "device_type"] = scorer_worker.device_config.device.type
160 proposer_worker = NGramWorker(**draft_worker_kwargs)
161 proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
162 ngram_prompt_lookup_max)
@@ -158,8 +170,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
170 proposer_worker = MedusaWorker(**draft_worker_kwargs)
171 else:
172 if draft_tp == 1:
161 - draft_worker_kwargs[
162 - "model_runner_cls"] = TP1DraftModelRunner
173 + if current_platform.is_cuda_alike():
174 + draft_worker_kwargs[
175 + "model_runner_cls"] = TP1DraftModelRunner
176 else:
177 if draft_model_config.hf_config.model_type == "eagle":
178 raise NotImplementedError(
@@ -306,8 +319,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
319 self.scorer_worker.load_model()
320 self.proposer_worker.load_model()
321
309 - self._metrics.init_gpu_tensors(self.rank)
310 - self.spec_decode_sampler.init_gpu_tensors(self.rank)
322 + self._metrics.init_tensors(self.rank, device_type=self.device)
323 + self.spec_decode_sampler.init_tensors(self.rank,
324 + device_type=self.device)
325
326 scorer_cls: Type[SpeculativeScorer]
327 if self.disable_mqa_scorer:
@@ -1111,11 +1125,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
1125 raise NotImplementedError
1126
1127 def start_profile(self):
1114 - if isinstance(self.scorer_worker, Worker):
1128 + if isinstance(self.scorer_worker, WorkerBase):
1129 self.scorer_worker.start_profile()
1130
1131 def stop_profile(self):
1118 - if isinstance(self.scorer_worker, Worker):
1132 + if isinstance(self.scorer_worker, WorkerBase):
1133 self.scorer_worker.stop_profile()
1134
1135
vllm/spec_decode/target_model_runner.py
+11 -22
@@ -1,12 +1,12 @@
1 from typing import List, Optional
2
3 -from vllm.config import VllmConfig
3 from vllm.sequence import SequenceGroupMetadata
5 -from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
6 - ModelRunner)
4 +from vllm.worker.model_runner_base import (ModelRunnerBase,
5 + ModelRunnerInputBase,
6 + ModelRunnerWrapperBase)
7
8
9 -class TargetModelRunner(ModelRunner):
9 +class TargetModelRunner(ModelRunnerWrapperBase):
10 """Specialized model runner for speculative decoding target model.
11 In speculative decoding, the log probabilities selected finally may not
12 be the same ones as selected by the target model sampling. This means
@@ -18,32 +18,21 @@ class TargetModelRunner(ModelRunner):
18 requested or not.
19 """
20
21 - def __init__(
22 - self,
23 - vllm_config: VllmConfig,
24 - kv_cache_dtype: Optional[str] = "auto",
25 - is_driver_worker: bool = False,
26 - return_hidden_states: bool = False,
27 - ):
21 + def __init__(self, model_runner: ModelRunnerBase):
22 # An internal boolean member variable to indicate if token log
23 # probabilities are needed or not.
24 + super().__init__(model_runner)
25 self.disable_logprobs = True
31 - super().__init__(
32 - vllm_config=vllm_config,
33 - kv_cache_dtype=kv_cache_dtype,
34 - is_driver_worker=is_driver_worker,
35 - return_hidden_states=return_hidden_states,
36 - )
26
27 def prepare_model_input(
28 self,
29 seq_group_metadata_list: List[SequenceGroupMetadata],
30 virtual_engine: int = 0,
42 - finished_requests_ids: Optional[List[str]] = None
43 - ) -> ModelInputForGPUWithSamplingMetadata:
44 - model_input: ModelInputForGPUWithSamplingMetadata = super(
45 - ).prepare_model_input(seq_group_metadata_list, virtual_engine,
46 - finished_requests_ids)
31 + finished_requests_ids: Optional[List[str]] = None,
32 + ) -> ModelRunnerInputBase:
33 + model_input: ModelRunnerInputBase =\
34 + self.model_runner.prepare_model_input(
35 + seq_group_metadata_list, virtual_engine, finished_requests_ids)
36 # If token log probabilities is disabled then skip generating sampler
37 # CPU output. We directly serialize the GPU sampled_token_id tensors
38 # as needed. If log probabilities is enabled then synchronize all the
vllm/spec_decode/util.py
+8 -4
@@ -5,6 +5,7 @@ from typing import Dict, List, Optional, Sequence, Tuple
5 import torch
6
7 from vllm.model_executor.layers.sampler import SamplerOutput
8 +from vllm.platforms import current_platform
9 from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
10 PromptLogprobs, SequenceGroupMetadata,
11 SequenceOutput)
@@ -247,11 +248,14 @@ def nvtx_range(msg, *args, **kwargs):
248 Arguments:
249 msg (string): message to associate with the range
250 """
250 - torch.cuda.nvtx.range_push(msg.format(*args, **kwargs))
251 - try:
251 + if current_platform.is_cuda_alike():
252 + torch.cuda.nvtx.range_push(msg.format(*args, **kwargs))
253 + try:
254 + yield
255 + finally:
256 + torch.cuda.nvtx.range_pop()
257 + else:
258 yield
253 - finally:
254 - torch.cuda.nvtx.range_pop()
259
260
261 class Timer:
vllm/worker/cpu_model_runner.py
+37 -2
@@ -80,6 +80,7 @@ class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
80 Used by the ModelRunner.
81 """
82 sampling_metadata: Optional["SamplingMetadata"] = None
83 + is_prompt: Optional[bool] = None
84
85 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
86 tensor_dict = {
@@ -395,6 +396,7 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
396 vllm_config: VllmConfig,
397 kv_cache_dtype: Optional[str] = "auto",
398 is_driver_worker: bool = False,
399 + return_hidden_states: bool = False,
400 *args,
401 **kwargs,
402 ):
@@ -403,19 +405,25 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
405 cache_config = self.cache_config
406
407 self.is_driver_worker = is_driver_worker
408 + self.return_hidden_states = return_hidden_states
409
410 self.device = self.device_config.device
411 + self.pin_memory = False
412
413 self.kv_cache_dtype = kv_cache_dtype
414 self.sliding_window = model_config.get_sliding_window()
415 self.block_size = cache_config.block_size
416 + num_attn_heads = self.model_config.get_num_attention_heads(
417 + self.parallel_config)
418 + needs_attn_backend = (num_attn_heads != 0
419 + or self.model_config.is_attention_free)
420 self.attn_backend = get_attn_backend(
421 self.model_config.get_head_size(),
422 self.model_config.dtype,
423 self.kv_cache_dtype,
424 self.block_size,
425 self.model_config.is_attention_free,
418 - )
426 + ) if needs_attn_backend else None
427
428 # Multi-modal data support
429 self.mm_registry = MULTIMODAL_REGISTRY
@@ -444,6 +452,15 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
452
453 return builder.build() # type: ignore
454
455 + # sampler property will be used by spec_decode_worker
456 + @property
457 + def sampler(self):
458 + return self.model.sampler
459 +
460 + @property
461 + def vocab_size(self) -> int:
462 + return self.model_config.get_vocab_size()
463 +
464
465 class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
466 _model_input_cls: Type[ModelInputForCPUWithSamplingMetadata] = (
@@ -480,9 +497,12 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
497 pin_memory=False,
498 generators=generators)
499
500 + is_prompt = (seq_group_metadata_list[0].is_prompt
501 + if seq_group_metadata_list else None)
502 return dataclasses.replace(model_input,
503 sampling_metadata=sampling_metadata,
485 - virtual_engine=virtual_engine)
504 + virtual_engine=virtual_engine,
505 + is_prompt=is_prompt)
506
507 @torch.no_grad()
508 def execute_model(
@@ -491,16 +511,22 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
511 kv_caches: List[torch.Tensor],
512 intermediate_tensors: Optional[IntermediateTensors] = None,
513 num_steps: int = 1,
514 + previous_hidden_states: Optional[torch.Tensor] = None,
515 ) -> Optional[List[SamplerOutput]]:
516 if num_steps > 1:
517 raise ValueError(
518 "CPU worker does not support multi-step execution.")
519
520 model_executable = self.model
521 +
522 multimodal_kwargs = {}
523 if model_input.multi_modal_kwargs is not None:
524 multimodal_kwargs = MultiModalKwargs.as_kwargs(
525 model_input.multi_modal_kwargs, device=self.device)
526 + execute_model_kwargs = {}
527 + if previous_hidden_states is not None:
528 + execute_model_kwargs.update(
529 + {"previous_hidden_states": previous_hidden_states})
530
531 with set_forward_context(model_input.attn_metadata, self.vllm_config):
532 hidden_states = model_executable(
@@ -509,6 +535,7 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
535 kv_caches=kv_caches,
536 attn_metadata=model_input.attn_metadata,
537 intermediate_tensors=intermediate_tensors,
538 + **execute_model_kwargs,
539 **multimodal_kwargs,
540 )
541
@@ -525,4 +552,12 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
552 logits=logits,
553 sampling_metadata=model_input.sampling_metadata,
554 )
555 + if self.return_hidden_states:
556 + # we only need to pass hidden states of most recent token
557 + if model_input.is_prompt:
558 + output.prefill_hidden_states = hidden_states
559 + output.hidden_states = hidden_states
560 return [output]
561 +
562 + def generate_proposals(self, *args, **kwargs):
563 + return self.model.generate_proposals(*args, **kwargs)
vllm/worker/cpu_worker.py
+25 -2
@@ -128,6 +128,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
128 distributed_init_method: str,
129 kv_cache_dtype: Optional[str] = "auto",
130 is_driver_worker: bool = False,
131 + model_runner_cls: Optional[Type[CPUModelRunner]] = None,
132 ) -> None:
133 WorkerBase.__init__(self, vllm_config=vllm_config)
134
@@ -151,6 +152,16 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
152 else:
153 self.local_omp_cpuid = omp_cpuids.split("|")[rank]
154
155 + # Return hidden states from target model if the draft model is an
156 + # mlp_speculator
157 + speculative_config = self.speculative_config
158 + model_config = self.model_config
159 + speculative_args = {} if speculative_config is None \
160 + or (speculative_config.draft_model_config.model ==
161 + model_config.model) \
162 + or (speculative_config.draft_model_config.hf_config.model_type
163 + not in ["medusa", "mlp_speculator", "eagle"]) \
164 + else {"return_hidden_states": True}
165 ModelRunnerClass: Type[CPUModelRunnerBase] = CPUModelRunner
166 if self.model_config.task == "embedding":
167 ModelRunnerClass = CPUEmbeddingModelRunner
@@ -159,7 +170,11 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
170 self.model_runner: CPUModelRunnerBase = ModelRunnerClass(
171 vllm_config=vllm_config,
172 kv_cache_dtype=kv_cache_dtype,
162 - is_driver_worker=is_driver_worker)
173 + is_driver_worker=is_driver_worker,
174 + **speculative_args,
175 + )
176 + if model_runner_cls is not None:
177 + self.model_runner = model_runner_cls(self.model_runner)
178 # Uninitialized cache engine. Will be initialized by
179 # initialize_cache.
180 self.cache_engine: List[CPUCacheEngine]
@@ -197,7 +212,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
212 ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
213 if ret:
214 logger.info(ret)
200 -
215 + self.device = torch.device("cpu")
216 self.init_distributed_environment()
217 # Set random seed.
218 set_random_seed(self.model_config.seed)
@@ -297,6 +312,14 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
312 def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
313 return self.cpu_cache
314
315 + @property
316 + def vocab_size(self) -> int:
317 + return self.model_runner.vocab_size
318 +
319 + @property
320 + def max_model_len(self) -> int:
321 + return self.model_config.max_model_len
322 +
323 def execute_worker(
324 self,
325 worker_input: WorkerInput,
vllm/worker/model_runner_base.py
+15
@@ -289,3 +289,18 @@ class ModelRunnerBase(ABC, Generic[T]):
289 self.generators.pop(request_id, None)
290
291 return self.generators
292 +
293 +
294 +class ModelRunnerWrapperBase:
295 + """
296 + The whole point of this class is to lazily initialize the model_runner.
297 + """
298 +
299 + def __init__(
300 + self,
301 + moderl_runner: ModelRunnerBase,
302 + ) -> None:
303 + self.model_runner: ModelRunnerBase = moderl_runner
304 +
305 + def __getattr__(self, attr):
306 + return getattr(self.model_runner, attr)
vllm/worker/worker.py
+4 -3
@@ -74,9 +74,7 @@ class Worker(LocalOrDistributedWorkerBase):
74 else {"return_hidden_states": True}
75
76 ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
77 - if model_runner_cls is not None:
78 - ModelRunnerClass = model_runner_cls
79 - elif model_config.task == "embedding":
77 + if model_config.task == "embedding":
78 ModelRunnerClass = EmbeddingModelRunner
79 elif self.model_config.is_encoder_decoder:
80 ModelRunnerClass = EncoderDecoderModelRunner
@@ -86,6 +84,9 @@ class Worker(LocalOrDistributedWorkerBase):
84 is_driver_worker=is_driver_worker,
85 **speculative_args,
86 )
87 + if model_runner_cls is not None:
88 + self.model_runner = model_runner_cls(self.model_runner)
89 +
90 # Uninitialized cache engine. Will be initialized by
91 # initialize_cache.
92 self.cache_engine: List[CacheEngine]
vllm/worker/worker_base.py
+3
@@ -466,6 +466,9 @@ class WorkerWrapperBase:
466 logger.exception(msg)
467 raise e
468
469 + def __getattr__(self, attr):
470 + return getattr(self.worker, attr)
471 +
472
473 def extract_previous_hidden_states(
474 data: Union[ExecuteModelRequest, Dict[str, torch.Tensor]]) -> \