A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Speculative decoding] [Multi-Step] decouple should_modify_greedy_probs_inplace (#6971)
William Lin committed
Aug 8, 2024 at 22:42 UTC
57b7be0e1c4e594c58a78297ab65fbb3ec206958
8 files changed
+52
-3
tests/samplers/test_sampler.py
+26
-1
@@ -1,7 +1,7 @@
1
import itertools
2
import random
3
from typing import Dict, List, Optional, Tuple
4
-from unittest.mock import patch
4
+from unittest.mock import Mock, patch
5
6
import pytest
7
import torch
@@ -703,3 +703,28 @@ def test_sampler_repetition_penalty_mixed(device: str):
703
704
assert tokens1[0] == tokens2[1]
705
assert tokens1[1] == tokens2[0]
706
+
707
+
708
+@pytest.mark.parametrize("device", CUDA_DEVICES)
709
+def test_sampler_include_gpu_probs_tensor(device: str):
710
+ set_random_seed(42)
711
+ torch.set_default_device(device)
712
+ batch_size = random.randint(1, 256)
713
+ _, fake_logits, sampler = _prepare_test(batch_size)
714
+ sampler.include_gpu_probs_tensor = True
715
+ sampler.should_modify_greedy_probs_inplace = False
716
+
717
+ sampling_params = SamplingParams(temperature=0)
718
+
719
+ mock_inplace = Mock()
720
+ with patch(
721
+ "vllm.model_executor.layers.sampler._modify_greedy_probs_inplace",
722
+ mock_inplace):
723
+
724
+ sampler_output = _do_sample(batch_size, fake_logits, sampler,
725
+ sampling_params, device)
726
+ mock_inplace.assert_not_called()
727
+
728
+ assert sampler_output.sampled_token_probs is not None
729
+ assert sampler_output.logprobs is not None
730
+ assert sampler_output.sampled_token_ids is not None
vllm/lora/layers.py
+4
@@ -1067,6 +1067,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1067
def include_gpu_probs_tensor(self):
1068
return self.base_layer.include_gpu_probs_tensor
1069
1070
+ @property
1071
+ def should_modify_greedy_probs_inplace(self):
1072
+ return self.base_layer.should_modify_greedy_probs_inplace
1073
+
1074
def create_lora_weights(
1075
self,
1076
max_loras: int,
vllm/model_executor/layers/sampler.py
+2
-2
@@ -51,6 +51,7 @@ class Sampler(nn.Module):
51
# containing the sampled token ids and probabilities. This is used by
52
# speculative decoding.
53
self.include_gpu_probs_tensor = False
54
+ self.should_modify_greedy_probs_inplace = False
55
56
def _init_sampling_tensors(
57
self,
@@ -177,8 +178,7 @@ class Sampler(nn.Module):
178
This is used by speculative decoding, which requires that the sampling
179
method be encoded into the probability distribution.
180
"""
180
- # Modify greedy probs if include_gpu_probs_tensor is set.
181
- return self.include_gpu_probs_tensor
181
+ return self.should_modify_greedy_probs_inplace
182
183
184
def _get_bin_counts_and_mask(
vllm/spec_decode/medusa_worker.py
+3
@@ -35,6 +35,9 @@ class MedusaWorker(NonLLMProposerWorkerBase, Worker):
35
def set_include_gpu_probs_tensor(self):
36
pass
37
38
+ def set_should_modify_greedy_probs_inplace(self):
39
+ pass
40
+
41
@torch.inference_mode()
42
def sampler_output(
43
self,
vllm/spec_decode/multi_step_worker.py
+4
@@ -46,6 +46,10 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
46
# Need include_gpu_probs_tensor for MultiStepWorker
47
self.model_runner.model.sampler.include_gpu_probs_tensor = True
48
49
+ def set_should_modify_greedy_probs_inplace(self) -> None:
50
+ self.model_runner.model.sampler.should_modify_greedy_probs_inplace = (
51
+ True)
52
+
53
@torch.inference_mode()
54
def sampler_output(
55
self,
vllm/spec_decode/proposer_worker_base.py
+4
@@ -28,6 +28,10 @@ class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer):
28
"""Implementation optional"""
29
pass
30
31
+ def set_should_modify_greedy_probs_inplace(self) -> None:
32
+ """Implementation optional"""
33
+ pass
34
+
35
36
class NonLLMProposerWorkerBase(ProposerWorkerBase, ABC):
37
"""Proposer worker which does not use a model with kvcache"""
vllm/spec_decode/smaller_tp_proposer_worker.py
+6
@@ -83,6 +83,12 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
83
# Need include_gpu_probs_tensor for multi_step_worker
84
self._worker.set_include_gpu_probs_tensor()
85
86
+ def set_should_modify_greedy_probs_inplace(self) -> None:
87
+ if self._is_dummy:
88
+ return
89
+
90
+ self._worker.set_should_modify_greedy_probs_inplace()
91
+
92
def load_model(self) -> None:
93
if self._is_dummy:
94
return
vllm/spec_decode/spec_decode_worker.py
+3
@@ -295,7 +295,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
295
"""
296
(self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor
297
) = True
298
+ (self.scorer_worker.model_runner.model.sampler.
299
+ should_modify_greedy_probs_inplace) = True
300
self.proposer_worker.set_include_gpu_probs_tensor()
301
+ self.proposer_worker.set_should_modify_greedy_probs_inplace()
302
303
def determine_num_available_blocks(self) -> Tuple[int, int]:
304
"""Determine the number of cache blocks to use.