@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ce5d6688b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Speculative Decoding] Support draft model on different tensor-parallel size than target model (#5414)

[Speculative Decoding] Support draft model on different tensor-parallel size than target model (#5414)

Woo-Yeon Lee committed Jun 25, 2024 at 18:56 UTC 2ce5d6688bae64e467640b05e73af2888e93afcf
11 files changed +388 -59
.buildkite/test-pipeline.yaml
+2 -1
@@ -54,7 +54,7 @@ steps:
54 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
55 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
56 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
57 - - pytest -v -s spec_decode/e2e/test_integration_dist.py
57 + - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
58 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
59 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
60
@@ -71,6 +71,7 @@ steps:
71 # See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
72 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
73 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
74 + - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
75
76 - label: Engine Test
77 mirror_hardwares: [amd]
benchmarks/benchmark_latency.py
+6
@@ -25,6 +25,8 @@ def main(args: argparse.Namespace):
25 model=args.model,
26 speculative_model=args.speculative_model,
27 num_speculative_tokens=args.num_speculative_tokens,
28 + speculative_draft_tensor_parallel_size=\
29 + args.speculative_draft_tensor_parallel_size,
30 tokenizer=args.tokenizer,
31 quantization=args.quantization,
32 tensor_parallel_size=args.tensor_parallel_size,
@@ -127,6 +129,10 @@ if __name__ == '__main__':
129 parser.add_argument('--model', type=str, default='facebook/opt-125m')
130 parser.add_argument('--speculative-model', type=str, default=None)
131 parser.add_argument('--num-speculative-tokens', type=int, default=None)
132 + parser.add_argument('--speculative-draft-tensor-parallel-size',
133 + '-spec-draft-tp',
134 + type=int,
135 + default=None)
136 parser.add_argument('--tokenizer', type=str, default=None)
137 parser.add_argument('--quantization',
138 '-q',
tests/spec_decode/e2e/test_integration_dist_tp2.py new
+111
@@ -0,0 +1,111 @@
1 +"""Tests which cover integration of the speculative decoding framework with
2 +tensor parallelism.
3 +"""
4 +
5 +import pytest
6 +import torch
7 +
8 +from vllm.utils import is_hip
9 +
10 +from .conftest import run_greedy_equality_correctness_test
11 +
12 +
13 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
14 + reason="Need at least 2 GPUs to run the test.")
15 +@pytest.mark.parametrize(
16 + "common_llm_kwargs",
17 + [{
18 + "model": "JackFram/llama-68m",
19 +
20 + # Skip cuda graph recording for fast test.
21 + "enforce_eager": True,
22 +
23 + # Required for spec decode.
24 + "use_v2_block_manager": True,
25 + "tensor_parallel_size": 2,
26 +
27 + # Use AsyncLLM engine, so that the engine runs in its own process.
28 + # Otherwise, since vLLM does not follow true SPMD, the test runner
29 + # process will have both the engine and the rank0 worker. NCCL is not
30 + # cleaned up properly, and its server host thread leaks, causing the
31 + # second run of the test to fail with internal NCCL error.
32 + "use_async": True,
33 + }])
34 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
35 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
36 +@pytest.mark.parametrize("test_llm_kwargs", [
37 + {
38 + "speculative_model": "JackFram/llama-68m",
39 + "num_speculative_tokens": 3,
40 + },
41 + {
42 + "speculative_model": "[ngram]",
43 + "num_speculative_tokens": 5,
44 + "ngram_prompt_lookup_max": 3,
45 + },
46 +])
47 +@pytest.mark.parametrize("batch_size", [2])
48 +@pytest.mark.parametrize(
49 + "output_len",
50 + [
51 + # Use smaller output len for fast test.
52 + 32,
53 + ])
54 +@pytest.mark.parametrize("seed", [1])
55 +def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
56 + batch_size: int, output_len: int):
57 + """Verify greedy equality when tensor parallelism is used.
58 + """
59 + if is_hip():
60 + pytest.skip("hip is not well-supported yet")
61 + run_greedy_equality_correctness_test(baseline_llm_generator,
62 + test_llm_generator,
63 + batch_size,
64 + max_output_len=output_len,
65 + force_output_len=True)
66 +
67 +
68 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
69 + reason="Need at least 2 GPUs to run the test.")
70 +@pytest.mark.parametrize(
71 + "common_llm_kwargs",
72 + [{
73 + # Use a small model for a fast test.
74 + # Note this is repeated in the test body; to initialize a tokenizer.
75 + "model": "JackFram/llama-68m",
76 +
77 + # Skip cuda graph recording for fast test.
78 + "enforce_eager": True,
79 +
80 + # Required for spec decode.
81 + "use_v2_block_manager": True,
82 + "tensor_parallel_size": 2,
83 +
84 + # Use AsyncLLM engine, so that the engine runs in its own process.
85 + # Otherwise, since vLLM does not follow true SPMD, the test runner
86 + # process will have both the engine and the rank0 worker. NCCL is not
87 + # cleaned up properly, and its server host thread leaks, causing the
88 + # second run of the test to fail with internal NCCL error.
89 + "use_async": True,
90 + }])
91 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
92 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
93 +@pytest.mark.parametrize("test_llm_kwargs", [
94 + {
95 + "speculative_model": "JackFram/llama-68m",
96 + "num_speculative_tokens": 5,
97 + "speculative_draft_tensor_parallel_size": 1,
98 + },
99 +])
100 +@pytest.mark.parametrize("batch_size", [2])
101 +@pytest.mark.parametrize("seed", [1])
102 +def test_draft_model_tp_lt_target_model_tp2(test_llm_generator,
103 + baseline_llm_generator,
104 + batch_size: int):
105 + """Verify spec decode works well with smaller tp for draft models.
106 + """
107 + run_greedy_equality_correctness_test(baseline_llm_generator,
108 + test_llm_generator,
109 + batch_size,
110 + max_output_len=32,
111 + force_output_len=True)
tests/spec_decode/e2e/test_integration_dist_tp4.py renamed
+18 -23
@@ -5,16 +5,16 @@ tensor parallelism.
5 import pytest
6 import torch
7
8 -from vllm.utils import is_hip
9 -
8 from .conftest import run_greedy_equality_correctness_test
9
10
13 -@pytest.mark.skipif(torch.cuda.device_count() < 2,
14 - reason="Need at least 2 GPUs to run the test.")
11 +@pytest.mark.skipif(torch.cuda.device_count() < 4,
12 + reason="Need at least 4 GPUs to run the test.")
13 @pytest.mark.parametrize(
14 "common_llm_kwargs",
15 [{
16 + # Use a small model for a fast test.
17 + # Note this is repeated in the test body; to initialize a tokenizer.
18 "model": "JackFram/llama-68m",
19
20 # Skip cuda graph recording for fast test.
@@ -22,7 +22,7 @@ from .conftest import run_greedy_equality_correctness_test
22
23 # Required for spec decode.
24 "use_v2_block_manager": True,
25 - "tensor_parallel_size": 2,
25 + "tensor_parallel_size": 4,
26
27 # Use AsyncLLM engine, so that the engine runs in its own process.
28 # Otherwise, since vLLM does not follow true SPMD, the test runner
@@ -31,35 +31,30 @@ from .conftest import run_greedy_equality_correctness_test
31 # second run of the test to fail with internal NCCL error.
32 "use_async": True,
33 }])
34 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
35 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
36 -@pytest.mark.parametrize("test_llm_kwargs", [
34 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
35 {
36 "speculative_model": "JackFram/llama-68m",
39 - "num_speculative_tokens": 3,
40 - },
41 - {
42 - "speculative_model": "[ngram]",
37 "num_speculative_tokens": 5,
44 - "ngram_prompt_lookup_max": 3,
38 },
39 ])
47 -@pytest.mark.parametrize("batch_size", [2])
40 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
41 @pytest.mark.parametrize(
49 - "output_len",
42 + "test_llm_kwargs",
43 [
51 - # Use smaller output len for fast test.
52 - 32,
44 + #TODO(wooyeon): add spec_draft_dp=2 case
45 + {
46 + "speculative_draft_tensor_parallel_size": 1,
47 + },
48 ])
49 +@pytest.mark.parametrize("batch_size", [2])
50 @pytest.mark.parametrize("seed", [1])
55 -def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
56 - batch_size: int, output_len: int):
57 - """Verify greedy equality when tensor parallelism is used.
51 +def test_draft_model_tp_lt_target_model_tp4(test_llm_generator,
52 + baseline_llm_generator,
53 + batch_size: int):
54 + """Verify spec decode works well with smaller tp for draft models.
55 """
59 - if is_hip():
60 - pytest.skip("hip is not well-supported yet")
56 run_greedy_equality_correctness_test(baseline_llm_generator,
57 test_llm_generator,
58 batch_size,
64 - max_output_len=output_len,
59 + max_output_len=32,
60 force_output_len=True)
vllm/config.py
+19 -5
@@ -797,6 +797,7 @@ class SpeculativeConfig:
797 target_parallel_config: ParallelConfig,
798 target_dtype: str,
799 speculative_model: Optional[str],
800 + speculative_draft_tensor_parallel_size: Optional[int],
801 num_speculative_tokens: Optional[int],
802 speculative_max_model_len: Optional[int],
803 enable_chunked_prefill: bool,
@@ -819,6 +820,8 @@ class SpeculativeConfig:
820 target_dtype (str): The data type used for the target model.
821 speculative_model (Optional[str]): The name of the speculative
822 model, if provided.
823 + speculative_draft_tensor_parallel_size (Optional[int]): The degree
824 + of the tensor parallelism for the draft model.
825 num_speculative_tokens (Optional[int]): The number of speculative
826 tokens, if provided. Will default to the number in the draft
827 model config if present, otherwise is required.
@@ -939,7 +942,8 @@ class SpeculativeConfig:
942
943 draft_parallel_config = (
944 SpeculativeConfig.create_draft_parallel_config(
942 - target_parallel_config))
945 + target_parallel_config,
946 + speculative_draft_tensor_parallel_size))
947
948 if num_speculative_tokens is None:
949 raise ValueError(
@@ -993,16 +997,26 @@ class SpeculativeConfig:
997
998 @staticmethod
999 def create_draft_parallel_config(
996 - target_parallel_config: ParallelConfig) -> ParallelConfig:
1000 + target_parallel_config: ParallelConfig,
1001 + speculative_draft_tensor_parallel_size: Optional[int]
1002 + ) -> ParallelConfig:
1003 """Create a parallel config for use by the draft worker.
1004
999 - This is mostly a copy of the target parallel config. In the future the
1000 - draft worker can have a different parallel strategy, e.g. TP=1.
1005 + This is mostly a copy of the target parallel config, except the tp_size.
1006 """
1007 + if speculative_draft_tensor_parallel_size is None:
1008 + speculative_draft_tensor_parallel_size = \
1009 + target_parallel_config.tensor_parallel_size
1010 + elif speculative_draft_tensor_parallel_size != 1:
1011 + # TODO(wooyeon): allow tp values larger than 1
1012 + raise ValueError(
1013 + f"{speculative_draft_tensor_parallel_size=} cannot be"
1014 + f"other value than 1")
1015 +
1016 draft_parallel_config = ParallelConfig(
1017 pipeline_parallel_size=target_parallel_config.
1018 pipeline_parallel_size,
1005 - tensor_parallel_size=target_parallel_config.tensor_parallel_size,
1019 + tensor_parallel_size=speculative_draft_tensor_parallel_size,
1020 distributed_executor_backend=target_parallel_config.
1021 distributed_executor_backend,
1022 max_parallel_loading_workers=target_parallel_config.
vllm/distributed/parallel_state.py
+55 -21
@@ -676,6 +676,28 @@ def get_world_group() -> GroupCoordinator:
676 return _WORLD
677
678
679 +def init_world_group(ranks: List[int], local_rank: int,
680 + backend: str) -> GroupCoordinator:
681 + return GroupCoordinator(
682 + group_ranks=[ranks],
683 + local_rank=local_rank,
684 + torch_distributed_backend=backend,
685 + use_pynccl=False,
686 + use_custom_allreduce=False,
687 + )
688 +
689 +
690 +def init_model_parallel_group(group_ranks: List[List[int]], local_rank: int,
691 + backend: str) -> GroupCoordinator:
692 + return GroupCoordinator(
693 + group_ranks=group_ranks,
694 + local_rank=local_rank,
695 + torch_distributed_backend=backend,
696 + use_pynccl=True,
697 + use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
698 + )
699 +
700 +
701 _TP: Optional[GroupCoordinator] = None
702
703
@@ -764,13 +786,7 @@ def init_distributed_environment(
786 global _WORLD
787 if _WORLD is None:
788 ranks = list(range(torch.distributed.get_world_size()))
767 - _WORLD = GroupCoordinator(
768 - group_ranks=[ranks],
769 - local_rank=local_rank,
770 - torch_distributed_backend=backend,
771 - use_pynccl=False,
772 - use_custom_allreduce=False,
773 - )
789 + _WORLD = init_world_group(ranks, local_rank, backend)
790 else:
791 assert _WORLD.world_size == torch.distributed.get_world_size(), (
792 "world group already initialized with a different world size")
@@ -827,13 +843,8 @@ def initialize_model_parallel(
843 range(i * tensor_model_parallel_size,
844 (i + 1) * tensor_model_parallel_size))
845 group_ranks.append(ranks)
830 - _TP = GroupCoordinator(
831 - group_ranks=group_ranks,
832 - local_rank=get_world_group().local_rank,
833 - torch_distributed_backend=backend,
834 - use_pynccl=True,
835 - use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
836 - )
846 + _TP = init_model_parallel_group(group_ranks,
847 + get_world_group().local_rank, backend)
848
849 # Build the pipeline model-parallel groups.
850 num_pipeline_model_parallel_groups: int = (world_size //
@@ -845,13 +856,8 @@ def initialize_model_parallel(
856 for i in range(num_pipeline_model_parallel_groups):
857 ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
858 group_ranks.append(ranks)
848 - _PP = GroupCoordinator(
849 - group_ranks=group_ranks,
850 - local_rank=get_world_group().local_rank,
851 - torch_distributed_backend=backend,
852 - use_pynccl=True,
853 - use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
854 - )
859 + _PP = init_model_parallel_group(group_ranks,
860 + get_world_group().local_rank, backend)
861
862
863 def ensure_model_parallel_initialized(
@@ -887,6 +893,34 @@ def model_parallel_is_initialized():
893 return (_TP is not None and _PP is not None)
894
895
896 +_TP_STATE_PATCHED = False
897 +
898 +
899 +@contextmanager
900 +def patch_tensor_parallel_group(tp_group: GroupCoordinator):
901 + """Patch the tp group temporarily until this function ends.
902 +
903 + This method is for draft workers of speculative decoding to run draft model
904 + with different tp degree from that of target model workers.
905 +
906 + Args:
907 + tp_group (GroupCoordinator): the tp group coordinator
908 + """
909 + global _TP_STATE_PATCHED
910 + assert not _TP_STATE_PATCHED, "Should not call when it's already patched"
911 +
912 + _TP_STATE_PATCHED = True
913 + old_tp_group = get_tp_group()
914 + global _TP
915 + _TP = tp_group
916 + try:
917 + yield
918 + finally:
919 + # restore the original state
920 + _TP_STATE_PATCHED = False
921 + _TP = old_tp_group
922 +
923 +
924 def get_tensor_model_parallel_world_size():
925 """Return world size for the tensor model parallel group."""
926 return get_tp_group().world_size
vllm/engine/arg_utils.py
+10
@@ -94,6 +94,7 @@ class EngineArgs:
94 guided_decoding_backend: str = 'outlines'
95 # Speculative decoding configuration.
96 speculative_model: Optional[str] = None
97 + speculative_draft_tensor_parallel_size: Optional[int] = None
98 num_speculative_tokens: Optional[int] = None
99 speculative_max_model_len: Optional[int] = None
100 speculative_disable_by_batch_size: Optional[int] = None
@@ -537,6 +538,13 @@ class EngineArgs:
538 default=EngineArgs.num_speculative_tokens,
539 help='The number of speculative tokens to sample from '
540 'the draft model in speculative decoding.')
541 + parser.add_argument(
542 + '--speculative-draft-tensor-parallel-size',
543 + '-spec-draft-tp',
544 + type=int,
545 + default=EngineArgs.speculative_draft_tensor_parallel_size,
546 + help='Number of tensor parallel replicas for '
547 + 'the draft model in speculative decoding.')
548
549 parser.add_argument(
550 '--speculative-max-model-len',
@@ -686,6 +694,8 @@ class EngineArgs:
694 target_parallel_config=parallel_config,
695 target_dtype=self.dtype,
696 speculative_model=self.speculative_model,
697 + speculative_draft_tensor_parallel_size = \
698 + self.speculative_draft_tensor_parallel_size,
699 num_speculative_tokens=self.num_speculative_tokens,
700 speculative_disable_by_batch_size=self.
701 speculative_disable_by_batch_size,
vllm/spec_decode/multi_step_worker.py
+6 -5
@@ -6,7 +6,8 @@ import torch
6
7 from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8 SequenceGroupMetadata)
9 -from vllm.spec_decode.interfaces import SpeculativeProposals
9 +from vllm.spec_decode.interfaces import (SpeculativeProposals,
10 + SpeculativeProposer)
11 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
12 from vllm.spec_decode.top1_proposer import Top1Proposer
13 from vllm.worker.worker import Worker
@@ -28,9 +29,9 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
29 super().__init__(*args, **kwargs)
30
31 # Lazy initialization list.
31 - self._proposer: Top1Proposer
32 + self._proposer: SpeculativeProposer
33
33 - def init_device(self):
34 + def init_device(self) -> None:
35 super().init_device()
36
37 self._proposer = Top1Proposer(
@@ -40,7 +41,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
41 max_proposal_len=self.max_model_len,
42 )
43
43 - def set_include_gpu_probs_tensor(self):
44 + def set_include_gpu_probs_tensor(self) -> None:
45 # Need include_gpu_probs_tensor for multi_step_worker
46 self.model_runner.model.sampler.include_gpu_probs_tensor = True
47
@@ -73,7 +74,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
74 # Run model sample_len times.
75 model_outputs: List[SamplerOutput] = []
76 for _ in range(sample_len):
76 - model_output = super().execute_model(
77 + model_output: List[SamplerOutput] = super().execute_model(
78 execute_model_req=copied_execute_model_req)
79 assert (len(model_output) == 1
80 ), "composing multistep workers not supported"
vllm/spec_decode/proposer_worker_base.py
+2 -2
@@ -3,10 +3,10 @@ from typing import List, Optional, Tuple
3
4 from vllm.sequence import ExecuteModelRequest, SamplerOutput
5 from vllm.spec_decode.interfaces import SpeculativeProposer
6 -from vllm.worker.worker_base import WorkerBase
6 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase
7
8
9 -class ProposerWorkerBase(WorkerBase, SpeculativeProposer):
9 +class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer):
10 """Interface for proposer workers"""
11
12 @abstractmethod
vllm/spec_decode/smaller_tp_proposer_worker.py new
+149
@@ -0,0 +1,149 @@
1 +from typing import List, Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm.distributed.parallel_state import (get_tp_group,
6 + init_model_parallel_group,
7 + patch_tensor_parallel_group)
8 +from vllm.logger import init_logger
9 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
10 +from vllm.spec_decode.interfaces import SpeculativeProposals
11 +from vllm.spec_decode.multi_step_worker import MultiStepWorker
12 +from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +class SmallerTpProposerWorker(ProposerWorkerBase):
18 + """Class which allows a speculative draft model to run with smaller tensor
19 + parallel degree than target model.
20 + This reduces the communication overhead of small draft models.
21 +
22 + To implement this feature, this class differs behavior based on is_dummy
23 + flag, where dummy means worker that does not participate draft generation.
24 + Participating workers use a smaller tp group by patching vLLM's tensor
25 + parallel group temporarily during forward passes of draft models.
26 + """
27 +
28 + @classmethod
29 + def maybe_wrap_worker(cls, worker, draft_tensor_parallel_size: int,
30 + target_tensor_parallel_size: int):
31 + """Wrap the worker in a SmallerTpProposerWorker if necessary.
32 + """
33 + if draft_tensor_parallel_size == target_tensor_parallel_size:
34 + return worker
35 +
36 + # gpu ranks that will generate draft tokens together
37 + draft_ranks = list(range(draft_tensor_parallel_size))
38 +
39 + logger.info("Wrapping {%s} in {%s}", type(worker), cls)
40 + return cls(worker, draft_ranks)
41 +
42 + def __init__(self, worker: MultiStepWorker, draft_ranks: List[int]):
43 + """Create a SmallerTpProposerWorker.
44 +
45 + Args:
46 + worker (MultiStepWorker): an actual worker wrapped with this class
47 + draft_ranks (List[int]): if this value is given, only the GPU ranks
48 + written in this value participate in draft generation
49 + """
50 + self._worker = worker
51 + self._draft_ranks = draft_ranks
52 +
53 + # init during init_device
54 + self._is_dummy = False
55 + self._tp_group = None
56 +
57 + def _patch_tensor_parallel_group(self):
58 + """Temporarily patch the global tp group state with its own tp group
59 + state.
60 + """
61 + return patch_tensor_parallel_group(self._tp_group)
62 +
63 + def init_device(self) -> None:
64 + self._is_dummy = get_tp_group().rank not in self._draft_ranks
65 +
66 + # dummy workers do nothing
67 + if self._is_dummy:
68 + return
69 +
70 + # creates tp process group containing only a subset of gpu ranks
71 + local_rank = get_tp_group().local_rank
72 + tp_backend = torch.distributed.get_backend(get_tp_group().device_group)
73 + self._tp_group = init_model_parallel_group([self._draft_ranks],
74 + local_rank, tp_backend)
75 +
76 + with self._patch_tensor_parallel_group():
77 + self._worker.init_device()
78 +
79 + def set_include_gpu_probs_tensor(self) -> None:
80 + if self._is_dummy:
81 + return
82 +
83 + # Need include_gpu_probs_tensor for multi_step_worker
84 + self._worker.set_include_gpu_probs_tensor()
85 +
86 + def load_model(self) -> None:
87 + if self._is_dummy:
88 + return
89 +
90 + with self._patch_tensor_parallel_group():
91 + self._worker.load_model()
92 +
93 + def determine_num_available_blocks(self) -> Tuple[int, int]:
94 + if self._is_dummy:
95 + # this case is not used now
96 + return -1, -1
97 +
98 + with self._patch_tensor_parallel_group():
99 + return self._worker.determine_num_available_blocks()
100 +
101 + def initialize_cache(self, num_gpu_blocks: int,
102 + num_cpu_blocks: int) -> None:
103 + if self._is_dummy:
104 + return
105 +
106 + with self._patch_tensor_parallel_group():
107 + self._worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
108 +
109 + def sampler_output(
110 + self,
111 + execute_model_req: ExecuteModelRequest,
112 + sample_len: int,
113 + ) -> Tuple[List[SamplerOutput], bool]:
114 + # Do not check _is_dummy, as it's always called by get_spec_proposals
115 + return self._worker.sampler_output(execute_model_req, sample_len)
116 +
117 + def get_spec_proposals(
118 + self,
119 + execute_model_req: ExecuteModelRequest,
120 + ) -> SpeculativeProposals:
121 + """Produce speculations given an input batch of sequences. The number of
122 + speculative tokens per sequence is determined by max_proposal_len.
123 + """
124 + if self._is_dummy:
125 + return SpeculativeProposals(None, None, None)
126 +
127 + with self._patch_tensor_parallel_group():
128 + return self._worker.get_spec_proposals(execute_model_req)
129 +
130 + def execute_model(
131 + self,
132 + execute_model_req: Optional[ExecuteModelRequest] = None
133 + ) -> List[SamplerOutput]:
134 + if self._is_dummy:
135 + return []
136 +
137 + with self._patch_tensor_parallel_group():
138 + return self._worker.execute_model(execute_model_req)
139 +
140 + def get_cache_block_size_bytes(self) -> int:
141 + if self._is_dummy:
142 + # by returning zero, target worker can use the entire kv cache space
143 + return 0
144 +
145 + return self._worker.get_cache_block_size_bytes()
146 +
147 + @property
148 + def vocab_size(self) -> int:
149 + return self._worker.vocab_size
vllm/spec_decode/spec_decode_worker.py
+10 -2
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple
3
4 import torch
5
6 -from vllm.config import SpeculativeConfig
6 +from vllm.config import ParallelConfig, SpeculativeConfig
7 from vllm.distributed.communication_op import broadcast_tensor_dict
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
@@ -18,6 +18,7 @@ from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker
18 from vllm.spec_decode.multi_step_worker import MultiStepWorker
19 from vllm.spec_decode.ngram_worker import NGramWorker
20 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
21 +from vllm.spec_decode.smaller_tp_proposer_worker import SmallerTpProposerWorker
22 from vllm.spec_decode.util import (create_sequence_group_output,
23 get_all_num_logprobs,
24 get_sampled_token_logprobs, nvtx_range,
@@ -90,7 +91,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
91 @classmethod
92 def create_worker(
93 cls,
93 - scorer_worker: WorkerBase,
94 + scorer_worker: Worker,
95 draft_worker_kwargs: Dict[str, Any],
96 disable_by_batch_size: Optional[int],
97 ) -> "SpecDecodeWorker":
@@ -111,7 +112,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
112 proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
113 disable_bonus_tokens = False
114 else:
115 + draft_parallel_config: ParallelConfig = draft_worker_kwargs[
116 + 'parallel_config']
117 + draft_tp = draft_parallel_config.tensor_parallel_size
118 + target_tp = scorer_worker.parallel_config.tensor_parallel_size
119 +
120 proposer_worker = MultiStepWorker(**draft_worker_kwargs)
121 + proposer_worker = SmallerTpProposerWorker.maybe_wrap_worker(
122 + proposer_worker, draft_tp, target_tp)
123
124 logger.info("Configuring SpecDecodeWorker with proposer=%s",
125 type(proposer_worker))