@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 973617ae02
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Speculative decoding][Re-take] Enable TP>1 speculative decoding (#4840)

Co-authored-by: Cade Daniel <edacih@gmail.com> Co-authored-by: Cade Daniel <cade@anyscale.com>

Cody Yu committed May 16, 2024 at 00:53 UTC 973617ae02a4e8e6190674cf1cdb0c0803b65ae6
12 files changed +297 -182
.buildkite/test-pipeline.yaml
+1
@@ -42,6 +42,7 @@ steps:
42 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
43 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
44 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
45 + - pytest -v -s spec_decode/e2e/test_integration_dist.py
46
47 - label: Distributed Tests (Multiple Groups)
48 working_dir: "/vllm-workspace/tests"
benchmarks/benchmark_latency.py
+6
@@ -18,6 +18,8 @@ def main(args: argparse.Namespace):
18 # NOTE(woosuk): If the request cannot be processed in a single batch,
19 # the engine will automatically process the request in multiple batches.
20 llm = LLM(model=args.model,
21 + speculative_model=args.speculative_model,
22 + num_speculative_tokens=args.num_speculative_tokens,
23 tokenizer=args.tokenizer,
24 quantization=args.quantization,
25 tensor_parallel_size=args.tensor_parallel_size,
@@ -28,6 +30,7 @@ def main(args: argparse.Namespace):
30 quantization_param_path=args.quantization_param_path,
31 device=args.device,
32 ray_workers_use_nsight=args.ray_workers_use_nsight,
33 + use_v2_block_manager=args.use_v2_block_manager,
34 enable_chunked_prefill=args.enable_chunked_prefill,
35 download_dir=args.download_dir,
36 block_size=args.block_size)
@@ -99,6 +102,8 @@ if __name__ == '__main__':
102 description='Benchmark the latency of processing a single batch of '
103 'requests till completion.')
104 parser.add_argument('--model', type=str, default='facebook/opt-125m')
105 + parser.add_argument('--speculative-model', type=str, default=None)
106 + parser.add_argument('--num-speculative-tokens', type=int, default=None)
107 parser.add_argument('--tokenizer', type=str, default=None)
108 parser.add_argument('--quantization',
109 '-q',
@@ -181,6 +186,7 @@ if __name__ == '__main__':
186 action='store_true',
187 help='If True, the prefill requests can be chunked based on the '
188 'max_num_batched_tokens')
189 + parser.add_argument('--use-v2-block-manager', action='store_true')
190 parser.add_argument(
191 "--ray-workers-use-nsight",
192 action='store_true',
tests/spec_decode/e2e/test_compatibility.py
-50
@@ -5,56 +5,6 @@ from vllm import SamplingParams
5 from .conftest import get_output_from_llm_generator
6
7
8 -@pytest.mark.parametrize(
9 - "common_llm_kwargs",
10 - [{
11 - "model": "JackFram/llama-68m",
12 - "speculative_model": "JackFram/llama-68m",
13 - "num_speculative_tokens": 5,
14 -
15 - # Required for spec decode.
16 - "use_v2_block_manager": True
17 - }])
18 -@pytest.mark.parametrize(
19 - "per_test_common_llm_kwargs",
20 - [
21 - {
22 - # Expect failure as spec decode not supported by
23 - # Ray backend.
24 - "worker_use_ray": True,
25 - },
26 - ])
27 -@pytest.mark.parametrize("test_llm_kwargs", [{}])
28 -@pytest.mark.parametrize("seed", [1])
29 -def test_spec_decode_xfail_ray(test_llm_generator):
30 - """Verify that speculative decoding with Ray fails.
31 - """
32 - output_len = 128
33 - temperature = 0.0
34 -
35 - prompts = [
36 - "Hello, my name is",
37 - ]
38 -
39 - sampling_params = SamplingParams(
40 - max_tokens=output_len,
41 - ignore_eos=True,
42 - temperature=temperature,
43 - )
44 -
45 - try:
46 - with pytest.raises(
47 - AssertionError,
48 - match="Speculative decoding not yet supported for "):
49 - get_output_from_llm_generator(test_llm_generator, prompts,
50 - sampling_params)
51 - finally:
52 - # we need to free up ray resource,
53 - # so that latter test could use the gpu we allocated here
54 - import ray
55 - ray.shutdown()
56 -
57 -
8 @pytest.mark.parametrize(
9 "common_llm_kwargs",
10 [{
tests/spec_decode/e2e/test_integration.py new
+44
@@ -0,0 +1,44 @@
1 +"""Tests which cover integration of the speculative decoding framework with
2 +other features, e.g. cuda graphs.
3 +"""
4 +
5 +import pytest
6 +
7 +from .conftest import run_greedy_equality_correctness_test
8 +
9 +
10 +@pytest.mark.parametrize(
11 + "common_llm_kwargs",
12 + [{
13 + # Required for spec decode.
14 + "use_v2_block_manager": True,
15 +
16 + # Verify equality when cuda graphs allowed.
17 + "enforce_eager": False,
18 + "model": "JackFram/llama-68m",
19 + }])
20 +@pytest.mark.parametrize(
21 + "per_test_common_llm_kwargs",
22 + [
23 + {
24 + # Identical models.
25 + "speculative_model": "JackFram/llama-68m",
26 + "num_speculative_tokens": 5,
27 + },
28 + ])
29 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
30 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
31 +@pytest.mark.parametrize("batch_size", [8])
32 +@pytest.mark.parametrize("output_len", [32])
33 +@pytest.mark.parametrize("seed", [1])
34 +def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
35 + batch_size, output_len):
36 + """Verify spec decode equality when cuda graphs are enabled.
37 + """
38 + run_greedy_equality_correctness_test(
39 + baseline_llm_generator,
40 + test_llm_generator,
41 + batch_size,
42 + max_output_len=output_len,
43 + force_output_len=True,
44 + )
tests/spec_decode/e2e/test_integration_dist.py new
+65
@@ -0,0 +1,65 @@
1 +"""Tests which cover integration of the speculative decoding framework with
2 +tensor parallelism.
3 +"""
4 +
5 +import pytest
6 +import torch
7 +
8 +from vllm.utils import is_hip
9 +
10 +from .conftest import run_greedy_equality_correctness_test
11 +
12 +
13 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
14 + reason="Need at least 2 GPUs to run the test.")
15 +@pytest.mark.parametrize(
16 + "common_llm_kwargs",
17 + [{
18 + "model": "JackFram/llama-68m",
19 +
20 + # Skip cuda graph recording for fast test.
21 + "enforce_eager": True,
22 +
23 + # Required for spec decode.
24 + "use_v2_block_manager": True,
25 + "tensor_parallel_size": 2,
26 +
27 + # Use AsyncLLM engine, so that the engine runs in its own process.
28 + # Otherwise, since vLLM does not follow true SPMD, the test runner
29 + # process will have both the engine and the rank0 worker. NCCL is not
30 + # cleaned up properly, and its server host thread leaks, causing the
31 + # second run of the test to fail with internal NCCL error.
32 + "use_async": True,
33 + }])
34 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
35 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
36 +@pytest.mark.parametrize("test_llm_kwargs", [
37 + {
38 + "speculative_model": "JackFram/llama-68m",
39 + "num_speculative_tokens": 3,
40 + },
41 + {
42 + "speculative_model": "[ngram]",
43 + "num_speculative_tokens": 5,
44 + "ngram_prompt_lookup_max": 3,
45 + },
46 +])
47 +@pytest.mark.parametrize("batch_size", [2])
48 +@pytest.mark.parametrize(
49 + "output_len",
50 + [
51 + # Use smaller output len for fast test.
52 + 32,
53 + ])
54 +@pytest.mark.parametrize("seed", [1])
55 +def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
56 + batch_size: int, output_len: int):
57 + """Verify greedy equality when tensor parallelism is used.
58 + """
59 + if is_hip():
60 + pytest.skip("hip is not well-supported yet")
61 + run_greedy_equality_correctness_test(baseline_llm_generator,
62 + test_llm_generator,
63 + batch_size,
64 + max_output_len=output_len,
65 + force_output_len=True)
tests/spec_decode/e2e/test_multistep_correctness.py
-37
@@ -611,40 +611,3 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
611 batch_size,
612 max_output_len=output_len,
613 force_output_len=True)
614 -
615 -
616 -@pytest.mark.parametrize(
617 - "common_llm_kwargs",
618 - [{
619 - # Required for spec decode.
620 - "use_v2_block_manager": True,
621 -
622 - # Verify equality when cuda graphs allowed.
623 - "enforce_eager": False,
624 - "model": "JackFram/llama-68m",
625 - }])
626 -@pytest.mark.parametrize(
627 - "per_test_common_llm_kwargs",
628 - [
629 - {
630 - # Identical models.
631 - "speculative_model": "JackFram/llama-68m",
632 - "num_speculative_tokens": 5,
633 - },
634 - ])
635 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
636 -@pytest.mark.parametrize("test_llm_kwargs", [{}])
637 -@pytest.mark.parametrize("batch_size", [8])
638 -@pytest.mark.parametrize("output_len", [32])
639 -@pytest.mark.parametrize("seed", [1])
640 -def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
641 - batch_size, output_len):
642 - """Verify spec decode equality when cuda graphs are enabled.
643 - """
644 - run_greedy_equality_correctness_test(
645 - baseline_llm_generator,
646 - test_llm_generator,
647 - batch_size,
648 - max_output_len=output_len,
649 - force_output_len=True,
650 - )
vllm/distributed/communication_op.py
+5 -5
@@ -219,16 +219,16 @@ def broadcast_tensor_dict(
219 to broadcast the metadata of the dict (e.g. dict structure, tensor sizes,
220 dtypes).
221 """
222 + # Bypass the function if we are using only 1 GPU.
223 + if (not torch.distributed.is_initialized()
224 + or torch.distributed.get_world_size(group=group) == 1):
225 + return tensor_dict
226 +
227 group = group or torch.distributed.group.WORLD
228 metadata_group = metadata_group or get_cpu_world_group()
229 ranks = torch.distributed.get_process_group_ranks(group)
230 assert src in ranks, f"Invalid src rank ({src})"
231
227 - # Bypass the function if we are using only 1 GPU.
228 - world_size = torch.distributed.get_world_size(group=group)
229 - if world_size == 1:
230 - return tensor_dict
231 -
232 rank = torch.distributed.get_rank()
233 if rank == src:
234 metadata_list: List[Tuple[Any, Any]] = []
vllm/executor/gpu_executor.py
+17 -54
@@ -15,14 +15,13 @@ class GPUExecutor(ExecutorBase):
15
16 def _init_executor(self) -> None:
17 """Initialize the worker and load the model.
18 -
19 - If speculative decoding is enabled, we instead create the speculative
20 - worker.
18 """
22 - if self.speculative_config is None:
23 - self._init_non_spec_worker()
24 - else:
25 - self._init_spec_worker()
19 + assert self.parallel_config.world_size == 1, (
20 + "GPUExecutor only supports single GPU.")
21 +
22 + self.driver_worker = self._create_worker()
23 + self.driver_worker.init_device()
24 + self.driver_worker.load_model()
25
26 def _get_worker_kwargs(
27 self,
@@ -45,6 +44,7 @@ class GPUExecutor(ExecutorBase):
44 distributed_init_method=distributed_init_method,
45 lora_config=self.lora_config,
46 vision_language_config=self.vision_language_config,
47 + speculative_config=self.speculative_config,
48 is_driver_worker=rank == 0,
49 )
50
@@ -52,59 +52,22 @@ class GPUExecutor(ExecutorBase):
52 local_rank: int = 0,
53 rank: int = 0,
54 distributed_init_method: Optional[str] = None):
55 +
56 + if self.speculative_config is None:
57 + worker_module_name = "vllm.worker.worker"
58 + worker_class_name = "Worker"
59 + else:
60 + worker_module_name = "vllm.spec_decode.spec_decode_worker"
61 + worker_class_name = "create_spec_worker"
62 +
63 wrapper = WorkerWrapperBase(
56 - worker_module_name="vllm.worker.worker",
57 - worker_class_name="Worker",
64 + worker_module_name=worker_module_name,
65 + worker_class_name=worker_class_name,
66 )
67 wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
68 distributed_init_method))
69 return wrapper.worker
70
63 - def _init_non_spec_worker(self):
64 - assert self.parallel_config.world_size == 1, (
65 - "GPUExecutor only supports single GPU.")
66 -
67 - self.driver_worker = self._create_worker()
68 - self.driver_worker.init_device()
69 - self.driver_worker.load_model()
70 -
71 - def _init_spec_worker(self):
72 - """Initialize a SpecDecodeWorker, using a draft model for proposals.
73 - """
74 - assert self.speculative_config is not None
75 -
76 - from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
77 -
78 - target_worker = self._create_worker()
79 -
80 - draft_worker_kwargs = self._get_worker_kwargs()
81 - # Override draft-model specific worker args.
82 - draft_worker_kwargs.update(
83 - model_config=self.speculative_config.draft_model_config,
84 - parallel_config=self.speculative_config.draft_parallel_config,
85 - ngram_prompt_lookup_max=self.speculative_config.
86 - ngram_prompt_lookup_max,
87 - ngram_prompt_lookup_min=self.speculative_config.
88 - ngram_prompt_lookup_min,
89 - # TODO allow draft-model specific load config.
90 - #load_config=self.load_config,
91 - )
92 -
93 - spec_decode_worker = SpecDecodeWorker.create_worker(
94 - scorer_worker=target_worker,
95 - draft_worker_kwargs=draft_worker_kwargs,
96 - disable_by_batch_size=self.speculative_config.
97 - speculative_disable_by_batch_size,
98 - )
99 -
100 - assert self.parallel_config.world_size == 1, (
101 - "GPUExecutor only supports single GPU.")
102 -
103 - self.driver_worker = spec_decode_worker
104 -
105 - # Load model handled in spec decode worker.
106 - self.driver_worker.init_device()
107 -
71 def determine_num_available_blocks(self) -> Tuple[int, int]:
72 """Determine the number of available KV blocks by invoking the
73 underlying worker.
vllm/executor/ray_gpu_executor.py
+12 -7
@@ -28,9 +28,6 @@ USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
28 class RayGPUExecutor(DistributedGPUExecutor):
29
30 def _init_executor(self) -> None:
31 - assert (not self.speculative_config
32 - ), "Speculative decoding not yet supported for RayGPU backend."
33 -
31 assert self.parallel_config.distributed_executor_backend == "ray"
32 placement_group = self.parallel_config.placement_group
33
@@ -90,14 +87,22 @@ class RayGPUExecutor(DistributedGPUExecutor):
87 placement_group_capture_child_tasks=True,
88 placement_group_bundle_index=bundle_id,
89 )
90 +
91 + if self.speculative_config is not None:
92 + worker_module_name = "vllm.spec_decode.spec_decode_worker"
93 + worker_class_name = "create_spec_worker"
94 + else:
95 + worker_module_name = "vllm.worker.worker"
96 + worker_class_name = "Worker"
97 +
98 worker = ray.remote(
99 num_cpus=0,
100 num_gpus=num_gpus,
101 scheduling_strategy=scheduling_strategy,
102 **ray_remote_kwargs,
103 )(RayWorkerWrapper).remote(
99 - worker_module_name="vllm.worker.worker",
100 - worker_class_name="Worker",
104 + worker_module_name=worker_module_name,
105 + worker_class_name=worker_class_name,
106 trust_remote_code=self.model_config.trust_remote_code,
107 )
108
@@ -107,8 +112,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
112 # as the resource holder for the driver process.
113 self.driver_dummy_worker = worker
114 self.driver_worker = RayWorkerWrapper(
110 - worker_module_name="vllm.worker.worker",
111 - worker_class_name="Worker",
115 + worker_module_name=worker_module_name,
116 + worker_class_name=worker_class_name,
117 trust_remote_code=self.model_config.trust_remote_code,
118 )
119 else:
vllm/spec_decode/spec_decode_worker.py
+144 -27
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple
3
4 import torch
5
6 +from vllm.distributed.communication_op import broadcast_tensor_dict
7 from vllm.logger import init_logger
8 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
9 from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
@@ -17,11 +18,43 @@ from vllm.spec_decode.util import (create_sequence_group_output,
18 get_all_num_logprobs, get_all_seq_ids,
19 get_sampled_token_logprobs, nvtx_range,
20 split_batch_by_proposal_len)
21 +from vllm.worker.worker import Worker
22 from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
23
24 logger = init_logger(__name__)
25
26
27 +def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
28 + """Helper method that is the entrypoint for Executors which use
29 + WorkerWrapper. It constructs a SpecDecodeWorker from the speculative config.
30 + """
31 + assert "speculative_config" in kwargs
32 + speculative_config = kwargs.get("speculative_config")
33 + assert speculative_config is not None
34 +
35 + target_worker = Worker(*args, **kwargs)
36 +
37 + draft_worker_kwargs = kwargs.copy()
38 + # Override draft-model specific worker args.
39 + draft_worker_kwargs.update(
40 + model_config=speculative_config.draft_model_config,
41 + parallel_config=speculative_config.draft_parallel_config,
42 + ngram_prompt_lookup_max=speculative_config.ngram_prompt_lookup_max,
43 + ngram_prompt_lookup_min=speculative_config.ngram_prompt_lookup_min,
44 + # TODO allow draft-model specific load config.
45 + #load_config=load_config,
46 + )
47 +
48 + spec_decode_worker = SpecDecodeWorker.create_worker(
49 + scorer_worker=target_worker,
50 + draft_worker_kwargs=draft_worker_kwargs,
51 + disable_by_batch_size=speculative_config.
52 + speculative_disable_by_batch_size,
53 + )
54 +
55 + return spec_decode_worker
56 +
57 +
58 class SpecDecodeWorker(LoraNotSupportedWorkerBase):
59 """Worker which implements speculative decoding.
60
@@ -142,6 +175,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
175
176 self._configure_model_sampler_for_spec_decode()
177
178 + def load_model(self, *args, **kwargs):
179 + pass
180 +
181 def _configure_model_sampler_for_spec_decode(self):
182 """Configure model sampler to emit GPU tensors. This allows spec decode
183 to keep data on device without transferring to CPU and serializing,
@@ -195,39 +231,97 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
231 self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
232 num_cpu_blocks=num_cpu_blocks)
233
234 + def _broadcast_control_flow_decision(
235 + self,
236 + execute_model_req: Optional[ExecuteModelRequest] = None,
237 + disable_all_speculation: bool = False) -> Tuple[int, bool]:
238 + """Broadcast how many lookahead slots are scheduled for this step, and
239 + whether all speculation is disabled, to all non-driver workers.
240 +
241 + This is required as if the number of draft model runs changes
242 + dynamically, the non-driver workers won't know unless we perform a
243 + communication to inform then.
244 +
245 + Returns the broadcasted num_lookahead_slots and disable_all_speculation.
246 + """
247 +
248 + if self.rank == self._driver_rank:
249 + assert execute_model_req is not None
250 +
251 + broadcast_dict = dict(
252 + num_lookahead_slots=execute_model_req.num_lookahead_slots,
253 + disable_all_speculation=disable_all_speculation,
254 + )
255 + broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
256 + else:
257 + assert execute_model_req is None
258 + broadcast_dict = broadcast_tensor_dict(src=self._driver_rank)
259 +
260 + return (broadcast_dict["num_lookahead_slots"],
261 + broadcast_dict["disable_all_speculation"])
262 +
263 @torch.inference_mode()
264 def execute_model(
200 - self,
201 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
265 + self,
266 + execute_model_req: Optional[ExecuteModelRequest] = None
267 + ) -> List[SamplerOutput]:
268 """Perform speculative decoding on the input batch.
269 """
270
205 - assert execute_model_req.seq_group_metadata_list is not None, (
206 - "speculative decoding "
207 - "requires non-None seq_group_metadata_list")
271 + disable_all_speculation = False
272 + if self.rank == self._driver_rank:
273 + disable_all_speculation = self._should_disable_all_speculation(
274 + execute_model_req)
275 +
276 + (num_lookahead_slots,
277 + disable_all_speculation) = self._broadcast_control_flow_decision(
278 + execute_model_req, disable_all_speculation)
279 +
280 + if self.rank == self._driver_rank:
281 + assert execute_model_req is not None
282 + assert execute_model_req.seq_group_metadata_list is not None, (
283 + "speculative decoding requires non-None seq_group_metadata_list"
284 + )
285 +
286 + self._maybe_disable_speculative_tokens(
287 + disable_all_speculation,
288 + execute_model_req.seq_group_metadata_list)
289 +
290 + # If no spec tokens, call the proposer and scorer workers normally.
291 + # Used for prefill.
292 + if num_lookahead_slots == 0 or len(
293 + execute_model_req.seq_group_metadata_list) == 0:
294 + return self._run_no_spec(execute_model_req,
295 + skip_proposer=disable_all_speculation)
296 +
297 + return self._run_speculative_decoding_step(execute_model_req,
298 + num_lookahead_slots)
299 + else:
300 + self._run_non_driver_rank(num_lookahead_slots)
301 + return []
302
303 + def _should_disable_all_speculation(
304 + self, execute_model_req: ExecuteModelRequest) -> bool:
305 # When the batch size is too large, disable speculative decoding
306 # to stop trading off throughput for latency.
211 - disable_all = (execute_model_req.running_queue_size >=
212 - self.disable_by_batch_size)
213 - if disable_all:
214 - for seq_group_metadata in execute_model_req.seq_group_metadata_list:
215 - # Once num_speculative_tokens is set to 0, the spec decode
216 - # of this request will be disabled forever.
217 - # TODO(comaniac): We currently store spec decoding specific
218 - # state in the global data structure, but we should maintain
219 - # this state within spec decode worker.
220 - seq_group_metadata.num_speculative_tokens = 0
221 -
222 - # If no spec tokens, call the proposer and scorer workers normally.
223 - # This happens for prefill, or when the spec decode is disabled
224 - # for this batch.
225 - if execute_model_req.num_lookahead_slots == 0 or len(
226 - execute_model_req.seq_group_metadata_list) == 0:
227 - return self._run_no_spec(execute_model_req,
228 - skip_proposer=disable_all)
229 -
230 - return self._run_speculative_decoding_step(execute_model_req)
307 + disable_all_speculation = (execute_model_req.running_queue_size >=
308 + self.disable_by_batch_size)
309 +
310 + return disable_all_speculation
311 +
312 + def _maybe_disable_speculative_tokens(
313 + self, disable_all_speculation: bool,
314 + seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
315 + if not disable_all_speculation:
316 + return
317 +
318 + for seq_group_metadata in seq_group_metadata_list:
319 + # Once num_speculative_tokens is set to 0, the spec decode
320 + # of this request will be disabled forever.
321 + # TODO(comaniac): We currently store spec decoding specific
322 + # state in the global data structure, but we should maintain
323 + # this state within spec decode worker.
324 + seq_group_metadata.num_speculative_tokens = 0
325
326 @nvtx_range("spec_decode_worker._run_no_spec")
327 def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
@@ -252,10 +346,28 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
346 sampler_output.logprobs = None
347 return [sampler_output]
348
349 + def _run_non_driver_rank(self, num_lookahead_slots: int) -> None:
350 + """Run proposer and verifier model in non-driver workers. This is used
351 + for both speculation cases (num_lookahead_slots>0) and non-speculation
352 + cases (e.g. prefill).
353 + """
354 + # In non-driver workers the input is None
355 + execute_model_req = None
356 +
357 + # Even if num_lookahead_slots is zero, we want to run the proposer model
358 + # as it may have KV.
359 + #
360 + # We run the proposer once per lookahead slot. In the future we should
361 + # delegate how many times it runs to the proposer.
362 + for _ in range(max(num_lookahead_slots, 1)):
363 + self.proposer_worker.execute_model(execute_model_req)
364 +
365 + self.scorer_worker.execute_model(execute_model_req)
366 +
367 @nvtx_range("spec_decode_worker._run_speculative_decoding_step")
368 def _run_speculative_decoding_step(
257 - self,
258 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
369 + self, execute_model_req: ExecuteModelRequest,
370 + num_lookahead_slots: int) -> List[SamplerOutput]:
371 """Execute a single step of speculative decoding.
372
373 This invokes the proposer worker to get k speculative tokens for each
@@ -264,6 +376,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
376 Returns a list of SamplerOutput, each containing a single token per
377 sequence.
378 """
379 + assert num_lookahead_slots == execute_model_req.num_lookahead_slots
380
381 # Generate proposals using draft worker.
382 proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
@@ -455,6 +568,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
568 def device(self):
569 return self.scorer_worker.device
570
571 + @property
572 + def _driver_rank(self) -> int:
573 + return 0
574 +
575 def get_cache_block_size_bytes(self):
576 """Return the size of a cache block in bytes.
577
vllm/worker/worker.py
+2 -1
@@ -8,7 +8,7 @@ import torch.distributed
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, ParallelConfig, SchedulerConfig,
11 - VisionLanguageConfig)
11 + SpeculativeConfig, VisionLanguageConfig)
12 from vllm.distributed import (broadcast_tensor_dict,
13 ensure_model_parallel_initialized,
14 init_distributed_environment,
@@ -43,6 +43,7 @@ class Worker(WorkerBase):
43 distributed_init_method: str,
44 lora_config: Optional[LoRAConfig] = None,
45 vision_language_config: Optional[VisionLanguageConfig] = None,
46 + speculative_config: Optional[SpeculativeConfig] = None,
47 is_driver_worker: bool = False,
48 ) -> None:
49 self.model_config = model_config
vllm/worker/worker_base.py
+1 -1
@@ -121,7 +121,7 @@ class WorkerWrapperBase:
121 def init_worker(self, *args, **kwargs):
122 """
123 Actual initialization of the worker class, and set up
124 - function tracing if required.
124 + function tracing if required.
125 Arguments are passed to the worker class constructor.
126 """
127 enable_trace_function_call_for_thread()