@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bd29cf3d3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Remove Sampler copy stream (#2209)

Antoni Baum committed Dec 20, 2023 at 00:04 UTC bd29cf3d3ad3dd06105f1a4bb9023bb23bdfd5ed
1 file changed +4 -9
vllm/model_executor/layers/sampler.py
+4 -9
@@ -30,7 +30,6 @@ class Sampler(nn.Module):
30 def __init__(self, vocab_size: int) -> None:
31 super().__init__()
32 self.vocab_size = vocab_size
33 - self._copy_stream: torch.cuda.Stream = torch.cuda.Stream()
33
34 def forward(
35 self,
@@ -51,14 +50,10 @@ class Sampler(nn.Module):
50 # Apply logits processors (if any).
51 logits = _apply_logits_processors(logits, sampling_metadata)
52
54 - # Prepare sampling tensors in another stream to overlap
55 - # CPU<->GPU data transfer with GPU computation in forward pass.
56 - with torch.cuda.stream(self._copy_stream):
57 - (sampling_tensors, do_penalties, do_top_p_top_k,
58 - do_min_p) = SamplingTensors.from_sampling_metadata(
59 - sampling_metadata, vocab_size, logits.device, logits.dtype)
60 -
61 - torch.cuda.current_stream().wait_stream(self._copy_stream)
53 + # Prepare sampling tensors with pinned memory to avoid blocking.
54 + (sampling_tensors, do_penalties, do_top_p_top_k,
55 + do_min_p) = SamplingTensors.from_sampling_metadata(
56 + sampling_metadata, vocab_size, logits.device, logits.dtype)
57
58 # Apply presence and frequency penalties.
59 if do_penalties: