A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Speculative decoding][Re-take] Enable TP>1 speculative decoding (#4840)
Co-authored-by: Cade Daniel <edacih@gmail.com> Co-authored-by: Cade Daniel <cade@anyscale.com>
Cody Yu committed
May 16, 2024 at 00:53 UTC
973617ae02a4e8e6190674cf1cdb0c0803b65ae6
12 files changed
+297
-182
.buildkite/test-pipeline.yaml
+1
@@ -42,6 +42,7 @@ steps:
42
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
43
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
44
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
45
+ - pytest -v -s spec_decode/e2e/test_integration_dist.py
46
47
- label: Distributed Tests (Multiple Groups)
48
working_dir: "/vllm-workspace/tests"
benchmarks/benchmark_latency.py
+6
@@ -18,6 +18,8 @@ def main(args: argparse.Namespace):
18
# NOTE(woosuk): If the request cannot be processed in a single batch,
19
# the engine will automatically process the request in multiple batches.
20
llm = LLM(model=args.model,
21
+ speculative_model=args.speculative_model,
22
+ num_speculative_tokens=args.num_speculative_tokens,
23
tokenizer=args.tokenizer,
24
quantization=args.quantization,
25
tensor_parallel_size=args.tensor_parallel_size,
@@ -28,6 +30,7 @@ def main(args: argparse.Namespace):
30
quantization_param_path=args.quantization_param_path,
31
device=args.device,
32
ray_workers_use_nsight=args.ray_workers_use_nsight,
33
+ use_v2_block_manager=args.use_v2_block_manager,
34
enable_chunked_prefill=args.enable_chunked_prefill,
35
download_dir=args.download_dir,
36
block_size=args.block_size)
@@ -99,6 +102,8 @@ if __name__ == '__main__':
102
description='Benchmark the latency of processing a single batch of '
103
'requests till completion.')
104
parser.add_argument('--model', type=str, default='facebook/opt-125m')
105
+ parser.add_argument('--speculative-model', type=str, default=None)
106
+ parser.add_argument('--num-speculative-tokens', type=int, default=None)
107
parser.add_argument('--tokenizer', type=str, default=None)
108
parser.add_argument('--quantization',
109
'-q',
@@ -181,6 +186,7 @@ if __name__ == '__main__':
186
action='store_true',
187
help='If True, the prefill requests can be chunked based on the '
188
'max_num_batched_tokens')
189
+ parser.add_argument('--use-v2-block-manager', action='store_true')
190
parser.add_argument(
191
"--ray-workers-use-nsight",
192
action='store_true',
tests/spec_decode/e2e/test_compatibility.py
-50
@@ -5,56 +5,6 @@ from vllm import SamplingParams
5
from .conftest import get_output_from_llm_generator
6
7
8
-@pytest.mark.parametrize(
9
- "common_llm_kwargs",
10
- [{
11
- "model": "JackFram/llama-68m",
12
- "speculative_model": "JackFram/llama-68m",
13
- "num_speculative_tokens": 5,
14
-
15
- # Required for spec decode.
16
- "use_v2_block_manager": True
17
- }])
18
-@pytest.mark.parametrize(
19
- "per_test_common_llm_kwargs",
20
- [
21
- {
22
- # Expect failure as spec decode not supported by
23
- # Ray backend.
24
- "worker_use_ray": True,
25
- },
26
- ])
27
-@pytest.mark.parametrize("test_llm_kwargs", [{}])
28
-@pytest.mark.parametrize("seed", [1])
29
-def test_spec_decode_xfail_ray(test_llm_generator):
30
- """Verify that speculative decoding with Ray fails.
31
- """
32
- output_len = 128
33
- temperature = 0.0
34
-
35
- prompts = [
36
- "Hello, my name is",
37
- ]
38
-
39
- sampling_params = SamplingParams(
40
- max_tokens=output_len,
41
- ignore_eos=True,
42
- temperature=temperature,
43
- )
44
-
45
- try:
46
- with pytest.raises(
47
- AssertionError,
48
- match="Speculative decoding not yet supported for "):
49
- get_output_from_llm_generator(test_llm_generator, prompts,
50
- sampling_params)
51
- finally:
52
- # we need to free up ray resource,
53
- # so that latter test could use the gpu we allocated here
54
- import ray
55
- ray.shutdown()
56
-
57
-
8
@pytest.mark.parametrize(
9
"common_llm_kwargs",
10
[{
tests/spec_decode/e2e/test_integration.py
new
+44
@@ -0,0 +1,44 @@
1
+"""Tests which cover integration of the speculative decoding framework with
2
+other features, e.g. cuda graphs.
3
+"""
4
+
5
+import pytest
6
+
7
+from .conftest import run_greedy_equality_correctness_test
8
+
9
+
10
+@pytest.mark.parametrize(
11
+ "common_llm_kwargs",
12
+ [{
13
+ # Required for spec decode.
14
+ "use_v2_block_manager": True,
15
+
16
+ # Verify equality when cuda graphs allowed.
17
+ "enforce_eager": False,
18
+ "model": "JackFram/llama-68m",
19
+ }])
20
+@pytest.mark.parametrize(
21
+ "per_test_common_llm_kwargs",
22
+ [
23
+ {
24
+ # Identical models.
25
+ "speculative_model": "JackFram/llama-68m",
26
+ "num_speculative_tokens": 5,
27
+ },
28
+ ])
29
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
30
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
31
+@pytest.mark.parametrize("batch_size", [8])
32
+@pytest.mark.parametrize("output_len", [32])
33
+@pytest.mark.parametrize("seed", [1])
34
+def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
35
+ batch_size, output_len):
36
+ """Verify spec decode equality when cuda graphs are enabled.
37
+ """
38
+ run_greedy_equality_correctness_test(
39
+ baseline_llm_generator,
40
+ test_llm_generator,
41
+ batch_size,
42
+ max_output_len=output_len,
43
+ force_output_len=True,
44
+ )
tests/spec_decode/e2e/test_integration_dist.py
new
+65
@@ -0,0 +1,65 @@
1
+"""Tests which cover integration of the speculative decoding framework with
2
+tensor parallelism.
3
+"""
4
+
5
+import pytest
6
+import torch
7
+
8
+from vllm.utils import is_hip
9
+
10
+from .conftest import run_greedy_equality_correctness_test
11
+
12
+
13
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
14
+ reason="Need at least 2 GPUs to run the test.")
15
+@pytest.mark.parametrize(
16
+ "common_llm_kwargs",
17
+ [{
18
+ "model": "JackFram/llama-68m",
19
+
20
+ # Skip cuda graph recording for fast test.
21
+ "enforce_eager": True,
22
+
23
+ # Required for spec decode.
24
+ "use_v2_block_manager": True,
25
+ "tensor_parallel_size": 2,
26
+
27
+ # Use AsyncLLM engine, so that the engine runs in its own process.
28
+ # Otherwise, since vLLM does not follow true SPMD, the test runner
29
+ # process will have both the engine and the rank0 worker. NCCL is not
30
+ # cleaned up properly, and its server host thread leaks, causing the
31
+ # second run of the test to fail with internal NCCL error.
32
+ "use_async": True,
33
+ }])
34
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
35
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
36
+@pytest.mark.parametrize("test_llm_kwargs", [
37
+ {
38
+ "speculative_model": "JackFram/llama-68m",
39
+ "num_speculative_tokens": 3,
40
+ },
41
+ {
42
+ "speculative_model": "[ngram]",
43
+ "num_speculative_tokens": 5,
44
+ "ngram_prompt_lookup_max": 3,
45
+ },
46
+])
47
+@pytest.mark.parametrize("batch_size", [2])
48
+@pytest.mark.parametrize(
49
+ "output_len",
50
+ [
51
+ # Use smaller output len for fast test.
52
+ 32,
53
+ ])
54
+@pytest.mark.parametrize("seed", [1])
55
+def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
56
+ batch_size: int, output_len: int):
57
+ """Verify greedy equality when tensor parallelism is used.
58
+ """
59
+ if is_hip():
60
+ pytest.skip("hip is not well-supported yet")
61
+ run_greedy_equality_correctness_test(baseline_llm_generator,
62
+ test_llm_generator,
63
+ batch_size,
64
+ max_output_len=output_len,
65
+ force_output_len=True)
tests/spec_decode/e2e/test_multistep_correctness.py
-37
@@ -611,40 +611,3 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
611
batch_size,
612
max_output_len=output_len,
613
force_output_len=True)
614
-
615
-
616
-@pytest.mark.parametrize(
617
- "common_llm_kwargs",
618
- [{
619
- # Required for spec decode.
620
- "use_v2_block_manager": True,
621
-
622
- # Verify equality when cuda graphs allowed.
623
- "enforce_eager": False,
624
- "model": "JackFram/llama-68m",
625
- }])
626
-@pytest.mark.parametrize(
627
- "per_test_common_llm_kwargs",
628
- [
629
- {
630
- # Identical models.
631
- "speculative_model": "JackFram/llama-68m",
632
- "num_speculative_tokens": 5,
633
- },
634
- ])
635
-@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
636
-@pytest.mark.parametrize("test_llm_kwargs", [{}])
637
-@pytest.mark.parametrize("batch_size", [8])
638
-@pytest.mark.parametrize("output_len", [32])
639
-@pytest.mark.parametrize("seed", [1])
640
-def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
641
- batch_size, output_len):
642
- """Verify spec decode equality when cuda graphs are enabled.
643
- """
644
- run_greedy_equality_correctness_test(
645
- baseline_llm_generator,
646
- test_llm_generator,
647
- batch_size,
648
- max_output_len=output_len,
649
- force_output_len=True,
650
- )
vllm/distributed/communication_op.py
+5
-5
@@ -219,16 +219,16 @@ def broadcast_tensor_dict(
219
to broadcast the metadata of the dict (e.g. dict structure, tensor sizes,
220
dtypes).
221
"""
222
+ # Bypass the function if we are using only 1 GPU.
223
+ if (not torch.distributed.is_initialized()
224
+ or torch.distributed.get_world_size(group=group) == 1):
225
+ return tensor_dict
226
+
227
group = group or torch.distributed.group.WORLD
228
metadata_group = metadata_group or get_cpu_world_group()
229
ranks = torch.distributed.get_process_group_ranks(group)
230
assert src in ranks, f"Invalid src rank ({src})"
231
227
- # Bypass the function if we are using only 1 GPU.
228
- world_size = torch.distributed.get_world_size(group=group)
229
- if world_size == 1:
230
- return tensor_dict
231
-
232
rank = torch.distributed.get_rank()
233
if rank == src:
234
metadata_list: List[Tuple[Any, Any]] = []
vllm/executor/gpu_executor.py
+17
-54
@@ -15,14 +15,13 @@ class GPUExecutor(ExecutorBase):
15
16
def _init_executor(self) -> None:
17
"""Initialize the worker and load the model.
18
-
19
- If speculative decoding is enabled, we instead create the speculative
20
- worker.
18
"""
22
- if self.speculative_config is None:
23
- self._init_non_spec_worker()
24
- else:
25
- self._init_spec_worker()
19
+ assert self.parallel_config.world_size == 1, (
20
+ "GPUExecutor only supports single GPU.")
21
+
22
+ self.driver_worker = self._create_worker()
23
+ self.driver_worker.init_device()
24
+ self.driver_worker.load_model()
25
26
def _get_worker_kwargs(
27
self,
@@ -45,6 +44,7 @@ class GPUExecutor(ExecutorBase):
44
distributed_init_method=distributed_init_method,
45
lora_config=self.lora_config,
46
vision_language_config=self.vision_language_config,
47
+ speculative_config=self.speculative_config,
48
is_driver_worker=rank == 0,
49
)
50
@@ -52,59 +52,22 @@ class GPUExecutor(ExecutorBase):
52
local_rank: int = 0,
53
rank: int = 0,
54
distributed_init_method: Optional[str] = None):
55
+
56
+ if self.speculative_config is None:
57
+ worker_module_name = "vllm.worker.worker"
58
+ worker_class_name = "Worker"
59
+ else:
60
+ worker_module_name = "vllm.spec_decode.spec_decode_worker"
61
+ worker_class_name = "create_spec_worker"
62
+
63
wrapper = WorkerWrapperBase(
56
- worker_module_name="vllm.worker.worker",
57
- worker_class_name="Worker",
64
+ worker_module_name=worker_module_name,
65
+ worker_class_name=worker_class_name,
66
)
67
wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
68
distributed_init_method))
69
return wrapper.worker
70
63
- def _init_non_spec_worker(self):
64
- assert self.parallel_config.world_size == 1, (
65
- "GPUExecutor only supports single GPU.")
66
-
67
- self.driver_worker = self._create_worker()
68
- self.driver_worker.init_device()
69
- self.driver_worker.load_model()
70
-
71
- def _init_spec_worker(self):
72
- """Initialize a SpecDecodeWorker, using a draft model for proposals.
73
- """
74
- assert self.speculative_config is not None
75
-
76
- from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
77
-
78
- target_worker = self._create_worker()
79
-
80
- draft_worker_kwargs = self._get_worker_kwargs()
81
- # Override draft-model specific worker args.
82
- draft_worker_kwargs.update(
83
- model_config=self.speculative_config.draft_model_config,
84
- parallel_config=self.speculative_config.draft_parallel_config,
85
- ngram_prompt_lookup_max=self.speculative_config.
86
- ngram_prompt_lookup_max,
87
- ngram_prompt_lookup_min=self.speculative_config.
88
- ngram_prompt_lookup_min,
89
- # TODO allow draft-model specific load config.
90
- #load_config=self.load_config,
91
- )
92
-
93
- spec_decode_worker = SpecDecodeWorker.create_worker(
94
- scorer_worker=target_worker,
95
- draft_worker_kwargs=draft_worker_kwargs,
96
- disable_by_batch_size=self.speculative_config.
97
- speculative_disable_by_batch_size,
98
- )
99
-
100
- assert self.parallel_config.world_size == 1, (
101
- "GPUExecutor only supports single GPU.")
102
-
103
- self.driver_worker = spec_decode_worker
104
-
105
- # Load model handled in spec decode worker.
106
- self.driver_worker.init_device()
107
-
71
def determine_num_available_blocks(self) -> Tuple[int, int]:
72
"""Determine the number of available KV blocks by invoking the
73
underlying worker.
vllm/executor/ray_gpu_executor.py
+12
-7
@@ -28,9 +28,6 @@ USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
28
class RayGPUExecutor(DistributedGPUExecutor):
29
30
def _init_executor(self) -> None:
31
- assert (not self.speculative_config
32
- ), "Speculative decoding not yet supported for RayGPU backend."
33
-
31
assert self.parallel_config.distributed_executor_backend == "ray"
32
placement_group = self.parallel_config.placement_group
33
@@ -90,14 +87,22 @@ class RayGPUExecutor(DistributedGPUExecutor):
87
placement_group_capture_child_tasks=True,
88
placement_group_bundle_index=bundle_id,
89
)
90
+
91
+ if self.speculative_config is not None:
92
+ worker_module_name = "vllm.spec_decode.spec_decode_worker"
93
+ worker_class_name = "create_spec_worker"
94
+ else:
95
+ worker_module_name = "vllm.worker.worker"
96
+ worker_class_name = "Worker"
97
+
98
worker = ray.remote(
99
num_cpus=0,
100
num_gpus=num_gpus,
101
scheduling_strategy=scheduling_strategy,
102
**ray_remote_kwargs,
103
)(RayWorkerWrapper).remote(
99
- worker_module_name="vllm.worker.worker",
100
- worker_class_name="Worker",
104
+ worker_module_name=worker_module_name,
105
+ worker_class_name=worker_class_name,
106
trust_remote_code=self.model_config.trust_remote_code,
107
)
108
@@ -107,8 +112,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
112
# as the resource holder for the driver process.
113
self.driver_dummy_worker = worker
114
self.driver_worker = RayWorkerWrapper(
110
- worker_module_name="vllm.worker.worker",
111
- worker_class_name="Worker",
115
+ worker_module_name=worker_module_name,
116
+ worker_class_name=worker_class_name,
117
trust_remote_code=self.model_config.trust_remote_code,
118
)
119
else:
vllm/spec_decode/spec_decode_worker.py
+144
-27
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple
3
4
import torch
5
6
+from vllm.distributed.communication_op import broadcast_tensor_dict
7
from vllm.logger import init_logger
8
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
9
from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
@@ -17,11 +18,43 @@ from vllm.spec_decode.util import (create_sequence_group_output,
18
get_all_num_logprobs, get_all_seq_ids,
19
get_sampled_token_logprobs, nvtx_range,
20
split_batch_by_proposal_len)
21
+from vllm.worker.worker import Worker
22
from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
23
24
logger = init_logger(__name__)
25
26
27
+def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
28
+ """Helper method that is the entrypoint for Executors which use
29
+ WorkerWrapper. It constructs a SpecDecodeWorker from the speculative config.
30
+ """
31
+ assert "speculative_config" in kwargs
32
+ speculative_config = kwargs.get("speculative_config")
33
+ assert speculative_config is not None
34
+
35
+ target_worker = Worker(*args, **kwargs)
36
+
37
+ draft_worker_kwargs = kwargs.copy()
38
+ # Override draft-model specific worker args.
39
+ draft_worker_kwargs.update(
40
+ model_config=speculative_config.draft_model_config,
41
+ parallel_config=speculative_config.draft_parallel_config,
42
+ ngram_prompt_lookup_max=speculative_config.ngram_prompt_lookup_max,
43
+ ngram_prompt_lookup_min=speculative_config.ngram_prompt_lookup_min,
44
+ # TODO allow draft-model specific load config.
45
+ #load_config=load_config,
46
+ )
47
+
48
+ spec_decode_worker = SpecDecodeWorker.create_worker(
49
+ scorer_worker=target_worker,
50
+ draft_worker_kwargs=draft_worker_kwargs,
51
+ disable_by_batch_size=speculative_config.
52
+ speculative_disable_by_batch_size,
53
+ )
54
+
55
+ return spec_decode_worker
56
+
57
+
58
class SpecDecodeWorker(LoraNotSupportedWorkerBase):
59
"""Worker which implements speculative decoding.
60
@@ -142,6 +175,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
175
176
self._configure_model_sampler_for_spec_decode()
177
178
+ def load_model(self, *args, **kwargs):
179
+ pass
180
+
181
def _configure_model_sampler_for_spec_decode(self):
182
"""Configure model sampler to emit GPU tensors. This allows spec decode
183
to keep data on device without transferring to CPU and serializing,
@@ -195,39 +231,97 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
231
self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
232
num_cpu_blocks=num_cpu_blocks)
233
234
+ def _broadcast_control_flow_decision(
235
+ self,
236
+ execute_model_req: Optional[ExecuteModelRequest] = None,
237
+ disable_all_speculation: bool = False) -> Tuple[int, bool]:
238
+ """Broadcast how many lookahead slots are scheduled for this step, and
239
+ whether all speculation is disabled, to all non-driver workers.
240
+
241
+ This is required as if the number of draft model runs changes
242
+ dynamically, the non-driver workers won't know unless we perform a
243
+ communication to inform then.
244
+
245
+ Returns the broadcasted num_lookahead_slots and disable_all_speculation.
246
+ """
247
+
248
+ if self.rank == self._driver_rank:
249
+ assert execute_model_req is not None
250
+
251
+ broadcast_dict = dict(
252
+ num_lookahead_slots=execute_model_req.num_lookahead_slots,
253
+ disable_all_speculation=disable_all_speculation,
254
+ )
255
+ broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
256
+ else:
257
+ assert execute_model_req is None
258
+ broadcast_dict = broadcast_tensor_dict(src=self._driver_rank)
259
+
260
+ return (broadcast_dict["num_lookahead_slots"],
261
+ broadcast_dict["disable_all_speculation"])
262
+
263
@torch.inference_mode()
264
def execute_model(
200
- self,
201
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
265
+ self,
266
+ execute_model_req: Optional[ExecuteModelRequest] = None
267
+ ) -> List[SamplerOutput]:
268
"""Perform speculative decoding on the input batch.
269
"""
270
205
- assert execute_model_req.seq_group_metadata_list is not None, (
206
- "speculative decoding "
207
- "requires non-None seq_group_metadata_list")
271
+ disable_all_speculation = False
272
+ if self.rank == self._driver_rank:
273
+ disable_all_speculation = self._should_disable_all_speculation(
274
+ execute_model_req)
275
+
276
+ (num_lookahead_slots,
277
+ disable_all_speculation) = self._broadcast_control_flow_decision(
278
+ execute_model_req, disable_all_speculation)
279
+
280
+ if self.rank == self._driver_rank:
281
+ assert execute_model_req is not None
282
+ assert execute_model_req.seq_group_metadata_list is not None, (
283
+ "speculative decoding requires non-None seq_group_metadata_list"
284
+ )
285
+
286
+ self._maybe_disable_speculative_tokens(
287
+ disable_all_speculation,
288
+ execute_model_req.seq_group_metadata_list)
289
+
290
+ # If no spec tokens, call the proposer and scorer workers normally.
291
+ # Used for prefill.
292
+ if num_lookahead_slots == 0 or len(
293
+ execute_model_req.seq_group_metadata_list) == 0:
294
+ return self._run_no_spec(execute_model_req,
295
+ skip_proposer=disable_all_speculation)
296
+
297
+ return self._run_speculative_decoding_step(execute_model_req,
298
+ num_lookahead_slots)
299
+ else:
300
+ self._run_non_driver_rank(num_lookahead_slots)
301
+ return []
302
303
+ def _should_disable_all_speculation(
304
+ self, execute_model_req: ExecuteModelRequest) -> bool:
305
# When the batch size is too large, disable speculative decoding
306
# to stop trading off throughput for latency.
211
- disable_all = (execute_model_req.running_queue_size >=
212
- self.disable_by_batch_size)
213
- if disable_all:
214
- for seq_group_metadata in execute_model_req.seq_group_metadata_list:
215
- # Once num_speculative_tokens is set to 0, the spec decode
216
- # of this request will be disabled forever.
217
- # TODO(comaniac): We currently store spec decoding specific
218
- # state in the global data structure, but we should maintain
219
- # this state within spec decode worker.
220
- seq_group_metadata.num_speculative_tokens = 0
221
-
222
- # If no spec tokens, call the proposer and scorer workers normally.
223
- # This happens for prefill, or when the spec decode is disabled
224
- # for this batch.
225
- if execute_model_req.num_lookahead_slots == 0 or len(
226
- execute_model_req.seq_group_metadata_list) == 0:
227
- return self._run_no_spec(execute_model_req,
228
- skip_proposer=disable_all)
229
-
230
- return self._run_speculative_decoding_step(execute_model_req)
307
+ disable_all_speculation = (execute_model_req.running_queue_size >=
308
+ self.disable_by_batch_size)
309
+
310
+ return disable_all_speculation
311
+
312
+ def _maybe_disable_speculative_tokens(
313
+ self, disable_all_speculation: bool,
314
+ seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
315
+ if not disable_all_speculation:
316
+ return
317
+
318
+ for seq_group_metadata in seq_group_metadata_list:
319
+ # Once num_speculative_tokens is set to 0, the spec decode
320
+ # of this request will be disabled forever.
321
+ # TODO(comaniac): We currently store spec decoding specific
322
+ # state in the global data structure, but we should maintain
323
+ # this state within spec decode worker.
324
+ seq_group_metadata.num_speculative_tokens = 0
325
326
@nvtx_range("spec_decode_worker._run_no_spec")
327
def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
@@ -252,10 +346,28 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
346
sampler_output.logprobs = None
347
return [sampler_output]
348
349
+ def _run_non_driver_rank(self, num_lookahead_slots: int) -> None:
350
+ """Run proposer and verifier model in non-driver workers. This is used
351
+ for both speculation cases (num_lookahead_slots>0) and non-speculation
352
+ cases (e.g. prefill).
353
+ """
354
+ # In non-driver workers the input is None
355
+ execute_model_req = None
356
+
357
+ # Even if num_lookahead_slots is zero, we want to run the proposer model
358
+ # as it may have KV.
359
+ #
360
+ # We run the proposer once per lookahead slot. In the future we should
361
+ # delegate how many times it runs to the proposer.
362
+ for _ in range(max(num_lookahead_slots, 1)):
363
+ self.proposer_worker.execute_model(execute_model_req)
364
+
365
+ self.scorer_worker.execute_model(execute_model_req)
366
+
367
@nvtx_range("spec_decode_worker._run_speculative_decoding_step")
368
def _run_speculative_decoding_step(
257
- self,
258
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
369
+ self, execute_model_req: ExecuteModelRequest,
370
+ num_lookahead_slots: int) -> List[SamplerOutput]:
371
"""Execute a single step of speculative decoding.
372
373
This invokes the proposer worker to get k speculative tokens for each
@@ -264,6 +376,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
376
Returns a list of SamplerOutput, each containing a single token per
377
sequence.
378
"""
379
+ assert num_lookahead_slots == execute_model_req.num_lookahead_slots
380
381
# Generate proposals using draft worker.
382
proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
@@ -455,6 +568,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
568
def device(self):
569
return self.scorer_worker.device
570
571
+ @property
572
+ def _driver_rank(self) -> int:
573
+ return 0
574
+
575
def get_cache_block_size_bytes(self):
576
"""Return the size of a cache block in bytes.
577
vllm/worker/worker.py
+2
-1
@@ -8,7 +8,7 @@ import torch.distributed
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, ParallelConfig, SchedulerConfig,
11
- VisionLanguageConfig)
11
+ SpeculativeConfig, VisionLanguageConfig)
12
from vllm.distributed import (broadcast_tensor_dict,
13
ensure_model_parallel_initialized,
14
init_distributed_environment,
@@ -43,6 +43,7 @@ class Worker(WorkerBase):
43
distributed_init_method: str,
44
lora_config: Optional[LoRAConfig] = None,
45
vision_language_config: Optional[VisionLanguageConfig] = None,
46
+ speculative_config: Optional[SpeculativeConfig] = None,
47
is_driver_worker: bool = False,
48
) -> None:
49
self.model_config = model_config
vllm/worker/worker_base.py
+1
-1
@@ -121,7 +121,7 @@ class WorkerWrapperBase:
121
def init_worker(self, *args, **kwargs):
122
"""
123
Actual initialization of the worker class, and set up
124
- function tracing if required.
124
+ function tracing if required.
125
Arguments are passed to the worker class constructor.
126
"""
127
enable_trace_function_call_for_thread()