A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Speculative Decoding] Support draft model on different tensor-parallel size than target model (#5414)
[Speculative Decoding] Support draft model on different tensor-parallel size than target model (#5414)
Woo-Yeon Lee committed
Jun 25, 2024 at 18:56 UTC
2ce5d6688bae64e467640b05e73af2888e93afcf
11 files changed
+388
-59
.buildkite/test-pipeline.yaml
+2
-1
@@ -54,7 +54,7 @@ steps:
54
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
55
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
56
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
57
- - pytest -v -s spec_decode/e2e/test_integration_dist.py
57
+ - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
58
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
59
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
60
@@ -71,6 +71,7 @@ steps:
71
# See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
72
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
73
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
74
+ - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
75
76
- label: Engine Test
77
mirror_hardwares: [amd]
benchmarks/benchmark_latency.py
+6
@@ -25,6 +25,8 @@ def main(args: argparse.Namespace):
25
model=args.model,
26
speculative_model=args.speculative_model,
27
num_speculative_tokens=args.num_speculative_tokens,
28
+ speculative_draft_tensor_parallel_size=\
29
+ args.speculative_draft_tensor_parallel_size,
30
tokenizer=args.tokenizer,
31
quantization=args.quantization,
32
tensor_parallel_size=args.tensor_parallel_size,
@@ -127,6 +129,10 @@ if __name__ == '__main__':
129
parser.add_argument('--model', type=str, default='facebook/opt-125m')
130
parser.add_argument('--speculative-model', type=str, default=None)
131
parser.add_argument('--num-speculative-tokens', type=int, default=None)
132
+ parser.add_argument('--speculative-draft-tensor-parallel-size',
133
+ '-spec-draft-tp',
134
+ type=int,
135
+ default=None)
136
parser.add_argument('--tokenizer', type=str, default=None)
137
parser.add_argument('--quantization',
138
'-q',
tests/spec_decode/e2e/test_integration_dist_tp2.py
new
+111
@@ -0,0 +1,111 @@
1
+"""Tests which cover integration of the speculative decoding framework with
2
+tensor parallelism.
3
+"""
4
+
5
+import pytest
6
+import torch
7
+
8
+from vllm.utils import is_hip
9
+
10
+from .conftest import run_greedy_equality_correctness_test
11
+
12
+
13
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
14
+ reason="Need at least 2 GPUs to run the test.")
15
+@pytest.mark.parametrize(
16
+ "common_llm_kwargs",
17
+ [{
18
+ "model": "JackFram/llama-68m",
19
+
20
+ # Skip cuda graph recording for fast test.
21
+ "enforce_eager": True,
22
+
23
+ # Required for spec decode.
24
+ "use_v2_block_manager": True,
25
+ "tensor_parallel_size": 2,
26
+
27
+ # Use AsyncLLM engine, so that the engine runs in its own process.
28
+ # Otherwise, since vLLM does not follow true SPMD, the test runner
29
+ # process will have both the engine and the rank0 worker. NCCL is not
30
+ # cleaned up properly, and its server host thread leaks, causing the
31
+ # second run of the test to fail with internal NCCL error.
32
+ "use_async": True,
33
+ }])
34
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
35
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
36
+@pytest.mark.parametrize("test_llm_kwargs", [
37
+ {
38
+ "speculative_model": "JackFram/llama-68m",
39
+ "num_speculative_tokens": 3,
40
+ },
41
+ {
42
+ "speculative_model": "[ngram]",
43
+ "num_speculative_tokens": 5,
44
+ "ngram_prompt_lookup_max": 3,
45
+ },
46
+])
47
+@pytest.mark.parametrize("batch_size", [2])
48
+@pytest.mark.parametrize(
49
+ "output_len",
50
+ [
51
+ # Use smaller output len for fast test.
52
+ 32,
53
+ ])
54
+@pytest.mark.parametrize("seed", [1])
55
+def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
56
+ batch_size: int, output_len: int):
57
+ """Verify greedy equality when tensor parallelism is used.
58
+ """
59
+ if is_hip():
60
+ pytest.skip("hip is not well-supported yet")
61
+ run_greedy_equality_correctness_test(baseline_llm_generator,
62
+ test_llm_generator,
63
+ batch_size,
64
+ max_output_len=output_len,
65
+ force_output_len=True)
66
+
67
+
68
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
69
+ reason="Need at least 2 GPUs to run the test.")
70
+@pytest.mark.parametrize(
71
+ "common_llm_kwargs",
72
+ [{
73
+ # Use a small model for a fast test.
74
+ # Note this is repeated in the test body; to initialize a tokenizer.
75
+ "model": "JackFram/llama-68m",
76
+
77
+ # Skip cuda graph recording for fast test.
78
+ "enforce_eager": True,
79
+
80
+ # Required for spec decode.
81
+ "use_v2_block_manager": True,
82
+ "tensor_parallel_size": 2,
83
+
84
+ # Use AsyncLLM engine, so that the engine runs in its own process.
85
+ # Otherwise, since vLLM does not follow true SPMD, the test runner
86
+ # process will have both the engine and the rank0 worker. NCCL is not
87
+ # cleaned up properly, and its server host thread leaks, causing the
88
+ # second run of the test to fail with internal NCCL error.
89
+ "use_async": True,
90
+ }])
91
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
92
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
93
+@pytest.mark.parametrize("test_llm_kwargs", [
94
+ {
95
+ "speculative_model": "JackFram/llama-68m",
96
+ "num_speculative_tokens": 5,
97
+ "speculative_draft_tensor_parallel_size": 1,
98
+ },
99
+])
100
+@pytest.mark.parametrize("batch_size", [2])
101
+@pytest.mark.parametrize("seed", [1])
102
+def test_draft_model_tp_lt_target_model_tp2(test_llm_generator,
103
+ baseline_llm_generator,
104
+ batch_size: int):
105
+ """Verify spec decode works well with smaller tp for draft models.
106
+ """
107
+ run_greedy_equality_correctness_test(baseline_llm_generator,
108
+ test_llm_generator,
109
+ batch_size,
110
+ max_output_len=32,
111
+ force_output_len=True)
tests/spec_decode/e2e/test_integration_dist_tp4.py
renamed
+18
-23
@@ -5,16 +5,16 @@ tensor parallelism.
5
import pytest
6
import torch
7
8
-from vllm.utils import is_hip
9
-
8
from .conftest import run_greedy_equality_correctness_test
9
10
13
-@pytest.mark.skipif(torch.cuda.device_count() < 2,
14
- reason="Need at least 2 GPUs to run the test.")
11
+@pytest.mark.skipif(torch.cuda.device_count() < 4,
12
+ reason="Need at least 4 GPUs to run the test.")
13
@pytest.mark.parametrize(
14
"common_llm_kwargs",
15
[{
16
+ # Use a small model for a fast test.
17
+ # Note this is repeated in the test body; to initialize a tokenizer.
18
"model": "JackFram/llama-68m",
19
20
# Skip cuda graph recording for fast test.
@@ -22,7 +22,7 @@ from .conftest import run_greedy_equality_correctness_test
22
23
# Required for spec decode.
24
"use_v2_block_manager": True,
25
- "tensor_parallel_size": 2,
25
+ "tensor_parallel_size": 4,
26
27
# Use AsyncLLM engine, so that the engine runs in its own process.
28
# Otherwise, since vLLM does not follow true SPMD, the test runner
@@ -31,35 +31,30 @@ from .conftest import run_greedy_equality_correctness_test
31
# second run of the test to fail with internal NCCL error.
32
"use_async": True,
33
}])
34
-@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
35
-@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
36
-@pytest.mark.parametrize("test_llm_kwargs", [
34
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [
35
{
36
"speculative_model": "JackFram/llama-68m",
39
- "num_speculative_tokens": 3,
40
- },
41
- {
42
- "speculative_model": "[ngram]",
37
"num_speculative_tokens": 5,
44
- "ngram_prompt_lookup_max": 3,
38
},
39
])
47
-@pytest.mark.parametrize("batch_size", [2])
40
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
41
@pytest.mark.parametrize(
49
- "output_len",
42
+ "test_llm_kwargs",
43
[
51
- # Use smaller output len for fast test.
52
- 32,
44
+ #TODO(wooyeon): add spec_draft_dp=2 case
45
+ {
46
+ "speculative_draft_tensor_parallel_size": 1,
47
+ },
48
])
49
+@pytest.mark.parametrize("batch_size", [2])
50
@pytest.mark.parametrize("seed", [1])
55
-def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
56
- batch_size: int, output_len: int):
57
- """Verify greedy equality when tensor parallelism is used.
51
+def test_draft_model_tp_lt_target_model_tp4(test_llm_generator,
52
+ baseline_llm_generator,
53
+ batch_size: int):
54
+ """Verify spec decode works well with smaller tp for draft models.
55
"""
59
- if is_hip():
60
- pytest.skip("hip is not well-supported yet")
56
run_greedy_equality_correctness_test(baseline_llm_generator,
57
test_llm_generator,
58
batch_size,
64
- max_output_len=output_len,
59
+ max_output_len=32,
60
force_output_len=True)
vllm/config.py
+19
-5
@@ -797,6 +797,7 @@ class SpeculativeConfig:
797
target_parallel_config: ParallelConfig,
798
target_dtype: str,
799
speculative_model: Optional[str],
800
+ speculative_draft_tensor_parallel_size: Optional[int],
801
num_speculative_tokens: Optional[int],
802
speculative_max_model_len: Optional[int],
803
enable_chunked_prefill: bool,
@@ -819,6 +820,8 @@ class SpeculativeConfig:
820
target_dtype (str): The data type used for the target model.
821
speculative_model (Optional[str]): The name of the speculative
822
model, if provided.
823
+ speculative_draft_tensor_parallel_size (Optional[int]): The degree
824
+ of the tensor parallelism for the draft model.
825
num_speculative_tokens (Optional[int]): The number of speculative
826
tokens, if provided. Will default to the number in the draft
827
model config if present, otherwise is required.
@@ -939,7 +942,8 @@ class SpeculativeConfig:
942
943
draft_parallel_config = (
944
SpeculativeConfig.create_draft_parallel_config(
942
- target_parallel_config))
945
+ target_parallel_config,
946
+ speculative_draft_tensor_parallel_size))
947
948
if num_speculative_tokens is None:
949
raise ValueError(
@@ -993,16 +997,26 @@ class SpeculativeConfig:
997
998
@staticmethod
999
def create_draft_parallel_config(
996
- target_parallel_config: ParallelConfig) -> ParallelConfig:
1000
+ target_parallel_config: ParallelConfig,
1001
+ speculative_draft_tensor_parallel_size: Optional[int]
1002
+ ) -> ParallelConfig:
1003
"""Create a parallel config for use by the draft worker.
1004
999
- This is mostly a copy of the target parallel config. In the future the
1000
- draft worker can have a different parallel strategy, e.g. TP=1.
1005
+ This is mostly a copy of the target parallel config, except the tp_size.
1006
"""
1007
+ if speculative_draft_tensor_parallel_size is None:
1008
+ speculative_draft_tensor_parallel_size = \
1009
+ target_parallel_config.tensor_parallel_size
1010
+ elif speculative_draft_tensor_parallel_size != 1:
1011
+ # TODO(wooyeon): allow tp values larger than 1
1012
+ raise ValueError(
1013
+ f"{speculative_draft_tensor_parallel_size=} cannot be"
1014
+ f"other value than 1")
1015
+
1016
draft_parallel_config = ParallelConfig(
1017
pipeline_parallel_size=target_parallel_config.
1018
pipeline_parallel_size,
1005
- tensor_parallel_size=target_parallel_config.tensor_parallel_size,
1019
+ tensor_parallel_size=speculative_draft_tensor_parallel_size,
1020
distributed_executor_backend=target_parallel_config.
1021
distributed_executor_backend,
1022
max_parallel_loading_workers=target_parallel_config.
vllm/distributed/parallel_state.py
+55
-21
@@ -676,6 +676,28 @@ def get_world_group() -> GroupCoordinator:
676
return _WORLD
677
678
679
+def init_world_group(ranks: List[int], local_rank: int,
680
+ backend: str) -> GroupCoordinator:
681
+ return GroupCoordinator(
682
+ group_ranks=[ranks],
683
+ local_rank=local_rank,
684
+ torch_distributed_backend=backend,
685
+ use_pynccl=False,
686
+ use_custom_allreduce=False,
687
+ )
688
+
689
+
690
+def init_model_parallel_group(group_ranks: List[List[int]], local_rank: int,
691
+ backend: str) -> GroupCoordinator:
692
+ return GroupCoordinator(
693
+ group_ranks=group_ranks,
694
+ local_rank=local_rank,
695
+ torch_distributed_backend=backend,
696
+ use_pynccl=True,
697
+ use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
698
+ )
699
+
700
+
701
_TP: Optional[GroupCoordinator] = None
702
703
@@ -764,13 +786,7 @@ def init_distributed_environment(
786
global _WORLD
787
if _WORLD is None:
788
ranks = list(range(torch.distributed.get_world_size()))
767
- _WORLD = GroupCoordinator(
768
- group_ranks=[ranks],
769
- local_rank=local_rank,
770
- torch_distributed_backend=backend,
771
- use_pynccl=False,
772
- use_custom_allreduce=False,
773
- )
789
+ _WORLD = init_world_group(ranks, local_rank, backend)
790
else:
791
assert _WORLD.world_size == torch.distributed.get_world_size(), (
792
"world group already initialized with a different world size")
@@ -827,13 +843,8 @@ def initialize_model_parallel(
843
range(i * tensor_model_parallel_size,
844
(i + 1) * tensor_model_parallel_size))
845
group_ranks.append(ranks)
830
- _TP = GroupCoordinator(
831
- group_ranks=group_ranks,
832
- local_rank=get_world_group().local_rank,
833
- torch_distributed_backend=backend,
834
- use_pynccl=True,
835
- use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
836
- )
846
+ _TP = init_model_parallel_group(group_ranks,
847
+ get_world_group().local_rank, backend)
848
849
# Build the pipeline model-parallel groups.
850
num_pipeline_model_parallel_groups: int = (world_size //
@@ -845,13 +856,8 @@ def initialize_model_parallel(
856
for i in range(num_pipeline_model_parallel_groups):
857
ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
858
group_ranks.append(ranks)
848
- _PP = GroupCoordinator(
849
- group_ranks=group_ranks,
850
- local_rank=get_world_group().local_rank,
851
- torch_distributed_backend=backend,
852
- use_pynccl=True,
853
- use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
854
- )
859
+ _PP = init_model_parallel_group(group_ranks,
860
+ get_world_group().local_rank, backend)
861
862
863
def ensure_model_parallel_initialized(
@@ -887,6 +893,34 @@ def model_parallel_is_initialized():
893
return (_TP is not None and _PP is not None)
894
895
896
+_TP_STATE_PATCHED = False
897
+
898
+
899
+@contextmanager
900
+def patch_tensor_parallel_group(tp_group: GroupCoordinator):
901
+ """Patch the tp group temporarily until this function ends.
902
+
903
+ This method is for draft workers of speculative decoding to run draft model
904
+ with different tp degree from that of target model workers.
905
+
906
+ Args:
907
+ tp_group (GroupCoordinator): the tp group coordinator
908
+ """
909
+ global _TP_STATE_PATCHED
910
+ assert not _TP_STATE_PATCHED, "Should not call when it's already patched"
911
+
912
+ _TP_STATE_PATCHED = True
913
+ old_tp_group = get_tp_group()
914
+ global _TP
915
+ _TP = tp_group
916
+ try:
917
+ yield
918
+ finally:
919
+ # restore the original state
920
+ _TP_STATE_PATCHED = False
921
+ _TP = old_tp_group
922
+
923
+
924
def get_tensor_model_parallel_world_size():
925
"""Return world size for the tensor model parallel group."""
926
return get_tp_group().world_size
vllm/engine/arg_utils.py
+10
@@ -94,6 +94,7 @@ class EngineArgs:
94
guided_decoding_backend: str = 'outlines'
95
# Speculative decoding configuration.
96
speculative_model: Optional[str] = None
97
+ speculative_draft_tensor_parallel_size: Optional[int] = None
98
num_speculative_tokens: Optional[int] = None
99
speculative_max_model_len: Optional[int] = None
100
speculative_disable_by_batch_size: Optional[int] = None
@@ -537,6 +538,13 @@ class EngineArgs:
538
default=EngineArgs.num_speculative_tokens,
539
help='The number of speculative tokens to sample from '
540
'the draft model in speculative decoding.')
541
+ parser.add_argument(
542
+ '--speculative-draft-tensor-parallel-size',
543
+ '-spec-draft-tp',
544
+ type=int,
545
+ default=EngineArgs.speculative_draft_tensor_parallel_size,
546
+ help='Number of tensor parallel replicas for '
547
+ 'the draft model in speculative decoding.')
548
549
parser.add_argument(
550
'--speculative-max-model-len',
@@ -686,6 +694,8 @@ class EngineArgs:
694
target_parallel_config=parallel_config,
695
target_dtype=self.dtype,
696
speculative_model=self.speculative_model,
697
+ speculative_draft_tensor_parallel_size = \
698
+ self.speculative_draft_tensor_parallel_size,
699
num_speculative_tokens=self.num_speculative_tokens,
700
speculative_disable_by_batch_size=self.
701
speculative_disable_by_batch_size,
vllm/spec_decode/multi_step_worker.py
+6
-5
@@ -6,7 +6,8 @@ import torch
6
7
from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8
SequenceGroupMetadata)
9
-from vllm.spec_decode.interfaces import SpeculativeProposals
9
+from vllm.spec_decode.interfaces import (SpeculativeProposals,
10
+ SpeculativeProposer)
11
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
12
from vllm.spec_decode.top1_proposer import Top1Proposer
13
from vllm.worker.worker import Worker
@@ -28,9 +29,9 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
29
super().__init__(*args, **kwargs)
30
31
# Lazy initialization list.
31
- self._proposer: Top1Proposer
32
+ self._proposer: SpeculativeProposer
33
33
- def init_device(self):
34
+ def init_device(self) -> None:
35
super().init_device()
36
37
self._proposer = Top1Proposer(
@@ -40,7 +41,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
41
max_proposal_len=self.max_model_len,
42
)
43
43
- def set_include_gpu_probs_tensor(self):
44
+ def set_include_gpu_probs_tensor(self) -> None:
45
# Need include_gpu_probs_tensor for multi_step_worker
46
self.model_runner.model.sampler.include_gpu_probs_tensor = True
47
@@ -73,7 +74,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
74
# Run model sample_len times.
75
model_outputs: List[SamplerOutput] = []
76
for _ in range(sample_len):
76
- model_output = super().execute_model(
77
+ model_output: List[SamplerOutput] = super().execute_model(
78
execute_model_req=copied_execute_model_req)
79
assert (len(model_output) == 1
80
), "composing multistep workers not supported"
vllm/spec_decode/proposer_worker_base.py
+2
-2
@@ -3,10 +3,10 @@ from typing import List, Optional, Tuple
3
4
from vllm.sequence import ExecuteModelRequest, SamplerOutput
5
from vllm.spec_decode.interfaces import SpeculativeProposer
6
-from vllm.worker.worker_base import WorkerBase
6
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase
7
8
9
-class ProposerWorkerBase(WorkerBase, SpeculativeProposer):
9
+class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer):
10
"""Interface for proposer workers"""
11
12
@abstractmethod
vllm/spec_decode/smaller_tp_proposer_worker.py
new
+149
@@ -0,0 +1,149 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.distributed.parallel_state import (get_tp_group,
6
+ init_model_parallel_group,
7
+ patch_tensor_parallel_group)
8
+from vllm.logger import init_logger
9
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
10
+from vllm.spec_decode.interfaces import SpeculativeProposals
11
+from vllm.spec_decode.multi_step_worker import MultiStepWorker
12
+from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+class SmallerTpProposerWorker(ProposerWorkerBase):
18
+ """Class which allows a speculative draft model to run with smaller tensor
19
+ parallel degree than target model.
20
+ This reduces the communication overhead of small draft models.
21
+
22
+ To implement this feature, this class differs behavior based on is_dummy
23
+ flag, where dummy means worker that does not participate draft generation.
24
+ Participating workers use a smaller tp group by patching vLLM's tensor
25
+ parallel group temporarily during forward passes of draft models.
26
+ """
27
+
28
+ @classmethod
29
+ def maybe_wrap_worker(cls, worker, draft_tensor_parallel_size: int,
30
+ target_tensor_parallel_size: int):
31
+ """Wrap the worker in a SmallerTpProposerWorker if necessary.
32
+ """
33
+ if draft_tensor_parallel_size == target_tensor_parallel_size:
34
+ return worker
35
+
36
+ # gpu ranks that will generate draft tokens together
37
+ draft_ranks = list(range(draft_tensor_parallel_size))
38
+
39
+ logger.info("Wrapping {%s} in {%s}", type(worker), cls)
40
+ return cls(worker, draft_ranks)
41
+
42
+ def __init__(self, worker: MultiStepWorker, draft_ranks: List[int]):
43
+ """Create a SmallerTpProposerWorker.
44
+
45
+ Args:
46
+ worker (MultiStepWorker): an actual worker wrapped with this class
47
+ draft_ranks (List[int]): if this value is given, only the GPU ranks
48
+ written in this value participate in draft generation
49
+ """
50
+ self._worker = worker
51
+ self._draft_ranks = draft_ranks
52
+
53
+ # init during init_device
54
+ self._is_dummy = False
55
+ self._tp_group = None
56
+
57
+ def _patch_tensor_parallel_group(self):
58
+ """Temporarily patch the global tp group state with its own tp group
59
+ state.
60
+ """
61
+ return patch_tensor_parallel_group(self._tp_group)
62
+
63
+ def init_device(self) -> None:
64
+ self._is_dummy = get_tp_group().rank not in self._draft_ranks
65
+
66
+ # dummy workers do nothing
67
+ if self._is_dummy:
68
+ return
69
+
70
+ # creates tp process group containing only a subset of gpu ranks
71
+ local_rank = get_tp_group().local_rank
72
+ tp_backend = torch.distributed.get_backend(get_tp_group().device_group)
73
+ self._tp_group = init_model_parallel_group([self._draft_ranks],
74
+ local_rank, tp_backend)
75
+
76
+ with self._patch_tensor_parallel_group():
77
+ self._worker.init_device()
78
+
79
+ def set_include_gpu_probs_tensor(self) -> None:
80
+ if self._is_dummy:
81
+ return
82
+
83
+ # Need include_gpu_probs_tensor for multi_step_worker
84
+ self._worker.set_include_gpu_probs_tensor()
85
+
86
+ def load_model(self) -> None:
87
+ if self._is_dummy:
88
+ return
89
+
90
+ with self._patch_tensor_parallel_group():
91
+ self._worker.load_model()
92
+
93
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
94
+ if self._is_dummy:
95
+ # this case is not used now
96
+ return -1, -1
97
+
98
+ with self._patch_tensor_parallel_group():
99
+ return self._worker.determine_num_available_blocks()
100
+
101
+ def initialize_cache(self, num_gpu_blocks: int,
102
+ num_cpu_blocks: int) -> None:
103
+ if self._is_dummy:
104
+ return
105
+
106
+ with self._patch_tensor_parallel_group():
107
+ self._worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
108
+
109
+ def sampler_output(
110
+ self,
111
+ execute_model_req: ExecuteModelRequest,
112
+ sample_len: int,
113
+ ) -> Tuple[List[SamplerOutput], bool]:
114
+ # Do not check _is_dummy, as it's always called by get_spec_proposals
115
+ return self._worker.sampler_output(execute_model_req, sample_len)
116
+
117
+ def get_spec_proposals(
118
+ self,
119
+ execute_model_req: ExecuteModelRequest,
120
+ ) -> SpeculativeProposals:
121
+ """Produce speculations given an input batch of sequences. The number of
122
+ speculative tokens per sequence is determined by max_proposal_len.
123
+ """
124
+ if self._is_dummy:
125
+ return SpeculativeProposals(None, None, None)
126
+
127
+ with self._patch_tensor_parallel_group():
128
+ return self._worker.get_spec_proposals(execute_model_req)
129
+
130
+ def execute_model(
131
+ self,
132
+ execute_model_req: Optional[ExecuteModelRequest] = None
133
+ ) -> List[SamplerOutput]:
134
+ if self._is_dummy:
135
+ return []
136
+
137
+ with self._patch_tensor_parallel_group():
138
+ return self._worker.execute_model(execute_model_req)
139
+
140
+ def get_cache_block_size_bytes(self) -> int:
141
+ if self._is_dummy:
142
+ # by returning zero, target worker can use the entire kv cache space
143
+ return 0
144
+
145
+ return self._worker.get_cache_block_size_bytes()
146
+
147
+ @property
148
+ def vocab_size(self) -> int:
149
+ return self._worker.vocab_size
vllm/spec_decode/spec_decode_worker.py
+10
-2
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple
3
4
import torch
5
6
-from vllm.config import SpeculativeConfig
6
+from vllm.config import ParallelConfig, SpeculativeConfig
7
from vllm.distributed.communication_op import broadcast_tensor_dict
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
@@ -18,6 +18,7 @@ from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker
18
from vllm.spec_decode.multi_step_worker import MultiStepWorker
19
from vllm.spec_decode.ngram_worker import NGramWorker
20
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
21
+from vllm.spec_decode.smaller_tp_proposer_worker import SmallerTpProposerWorker
22
from vllm.spec_decode.util import (create_sequence_group_output,
23
get_all_num_logprobs,
24
get_sampled_token_logprobs, nvtx_range,
@@ -90,7 +91,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
91
@classmethod
92
def create_worker(
93
cls,
93
- scorer_worker: WorkerBase,
94
+ scorer_worker: Worker,
95
draft_worker_kwargs: Dict[str, Any],
96
disable_by_batch_size: Optional[int],
97
) -> "SpecDecodeWorker":
@@ -111,7 +112,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
112
proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
113
disable_bonus_tokens = False
114
else:
115
+ draft_parallel_config: ParallelConfig = draft_worker_kwargs[
116
+ 'parallel_config']
117
+ draft_tp = draft_parallel_config.tensor_parallel_size
118
+ target_tp = scorer_worker.parallel_config.tensor_parallel_size
119
+
120
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
121
+ proposer_worker = SmallerTpProposerWorker.maybe_wrap_worker(
122
+ proposer_worker, draft_tp, target_tp)
123
124
logger.info("Configuring SpecDecodeWorker with proposer=%s",
125
type(proposer_worker))