A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Remove Sampler copy stream (#2209)
Antoni Baum committed
Dec 20, 2023 at 00:04 UTC
bd29cf3d3ad3dd06105f1a4bb9023bb23bdfd5ed
1 file changed
+4
-9
vllm/model_executor/layers/sampler.py
+4
-9
@@ -30,7 +30,6 @@ class Sampler(nn.Module):
30
def __init__(self, vocab_size: int) -> None:
31
super().__init__()
32
self.vocab_size = vocab_size
33
- self._copy_stream: torch.cuda.Stream = torch.cuda.Stream()
33
34
def forward(
35
self,
@@ -51,14 +50,10 @@ class Sampler(nn.Module):
50
# Apply logits processors (if any).
51
logits = _apply_logits_processors(logits, sampling_metadata)
52
54
- # Prepare sampling tensors in another stream to overlap
55
- # CPU<->GPU data transfer with GPU computation in forward pass.
56
- with torch.cuda.stream(self._copy_stream):
57
- (sampling_tensors, do_penalties, do_top_p_top_k,
58
- do_min_p) = SamplingTensors.from_sampling_metadata(
59
- sampling_metadata, vocab_size, logits.device, logits.dtype)
60
-
61
- torch.cuda.current_stream().wait_stream(self._copy_stream)
53
+ # Prepare sampling tensors with pinned memory to avoid blocking.
54
+ (sampling_tensors, do_penalties, do_top_p_top_k,
55
+ do_min_p) = SamplingTensors.from_sampling_metadata(
56
+ sampling_metadata, vocab_size, logits.device, logits.dtype)
57
58
# Apply presence and frequency penalties.
59
if do_penalties: