@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 57b7be0e1c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Speculative decoding] [Multi-Step] decouple should_modify_greedy_probs_inplace (#6971)

William Lin committed Aug 8, 2024 at 22:42 UTC 57b7be0e1c4e594c58a78297ab65fbb3ec206958
8 files changed +52 -3
tests/samplers/test_sampler.py
+26 -1
@@ -1,7 +1,7 @@
1 import itertools
2 import random
3 from typing import Dict, List, Optional, Tuple
4 -from unittest.mock import patch
4 +from unittest.mock import Mock, patch
5
6 import pytest
7 import torch
@@ -703,3 +703,28 @@ def test_sampler_repetition_penalty_mixed(device: str):
703
704 assert tokens1[0] == tokens2[1]
705 assert tokens1[1] == tokens2[0]
706 +
707 +
708 +@pytest.mark.parametrize("device", CUDA_DEVICES)
709 +def test_sampler_include_gpu_probs_tensor(device: str):
710 + set_random_seed(42)
711 + torch.set_default_device(device)
712 + batch_size = random.randint(1, 256)
713 + _, fake_logits, sampler = _prepare_test(batch_size)
714 + sampler.include_gpu_probs_tensor = True
715 + sampler.should_modify_greedy_probs_inplace = False
716 +
717 + sampling_params = SamplingParams(temperature=0)
718 +
719 + mock_inplace = Mock()
720 + with patch(
721 + "vllm.model_executor.layers.sampler._modify_greedy_probs_inplace",
722 + mock_inplace):
723 +
724 + sampler_output = _do_sample(batch_size, fake_logits, sampler,
725 + sampling_params, device)
726 + mock_inplace.assert_not_called()
727 +
728 + assert sampler_output.sampled_token_probs is not None
729 + assert sampler_output.logprobs is not None
730 + assert sampler_output.sampled_token_ids is not None
vllm/lora/layers.py
+4
@@ -1067,6 +1067,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1067 def include_gpu_probs_tensor(self):
1068 return self.base_layer.include_gpu_probs_tensor
1069
1070 + @property
1071 + def should_modify_greedy_probs_inplace(self):
1072 + return self.base_layer.should_modify_greedy_probs_inplace
1073 +
1074 def create_lora_weights(
1075 self,
1076 max_loras: int,
vllm/model_executor/layers/sampler.py
+2 -2
@@ -51,6 +51,7 @@ class Sampler(nn.Module):
51 # containing the sampled token ids and probabilities. This is used by
52 # speculative decoding.
53 self.include_gpu_probs_tensor = False
54 + self.should_modify_greedy_probs_inplace = False
55
56 def _init_sampling_tensors(
57 self,
@@ -177,8 +178,7 @@ class Sampler(nn.Module):
178 This is used by speculative decoding, which requires that the sampling
179 method be encoded into the probability distribution.
180 """
180 - # Modify greedy probs if include_gpu_probs_tensor is set.
181 - return self.include_gpu_probs_tensor
181 + return self.should_modify_greedy_probs_inplace
182
183
184 def _get_bin_counts_and_mask(
vllm/spec_decode/medusa_worker.py
+3
@@ -35,6 +35,9 @@ class MedusaWorker(NonLLMProposerWorkerBase, Worker):
35 def set_include_gpu_probs_tensor(self):
36 pass
37
38 + def set_should_modify_greedy_probs_inplace(self):
39 + pass
40 +
41 @torch.inference_mode()
42 def sampler_output(
43 self,
vllm/spec_decode/multi_step_worker.py
+4
@@ -46,6 +46,10 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
46 # Need include_gpu_probs_tensor for MultiStepWorker
47 self.model_runner.model.sampler.include_gpu_probs_tensor = True
48
49 + def set_should_modify_greedy_probs_inplace(self) -> None:
50 + self.model_runner.model.sampler.should_modify_greedy_probs_inplace = (
51 + True)
52 +
53 @torch.inference_mode()
54 def sampler_output(
55 self,
vllm/spec_decode/proposer_worker_base.py
+4
@@ -28,6 +28,10 @@ class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer):
28 """Implementation optional"""
29 pass
30
31 + def set_should_modify_greedy_probs_inplace(self) -> None:
32 + """Implementation optional"""
33 + pass
34 +
35
36 class NonLLMProposerWorkerBase(ProposerWorkerBase, ABC):
37 """Proposer worker which does not use a model with kvcache"""
vllm/spec_decode/smaller_tp_proposer_worker.py
+6
@@ -83,6 +83,12 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
83 # Need include_gpu_probs_tensor for multi_step_worker
84 self._worker.set_include_gpu_probs_tensor()
85
86 + def set_should_modify_greedy_probs_inplace(self) -> None:
87 + if self._is_dummy:
88 + return
89 +
90 + self._worker.set_should_modify_greedy_probs_inplace()
91 +
92 def load_model(self) -> None:
93 if self._is_dummy:
94 return
vllm/spec_decode/spec_decode_worker.py
+3
@@ -295,7 +295,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
295 """
296 (self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor
297 ) = True
298 + (self.scorer_worker.model_runner.model.sampler.
299 + should_modify_greedy_probs_inplace) = True
300 self.proposer_worker.set_include_gpu_probs_tensor()
301 + self.proposer_worker.set_should_modify_greedy_probs_inplace()
302
303 def determine_num_available_blocks(self) -> Tuple[int, int]:
304 """Determine the number of cache blocks to use.