@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f710fb5265
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Use flashinfer sampling kernel when available (#7137)

Co-authored-by: Michael Goin <michael@neuralmagic.com>

Peng Guanwen committed Aug 19, 2024 at 11:24 UTC f710fb5265abf4e299c9e5b8a3391e71339da883
5 files changed +130 -28
.buildkite/test-pipeline.yaml
+3 -1
@@ -192,7 +192,9 @@ steps:
192 - vllm/model_executor/layers
193 - vllm/sampling_metadata.py
194 - tests/samplers
195 - command: pytest -v -s samplers
195 + commands:
196 + - pytest -v -s samplers
197 + - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers
198
199 - label: LogitsProcessor Test # 5min
200 mirror_hardwares: [amd]
Dockerfile
+1 -1
@@ -194,7 +194,7 @@ RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamb
194 python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
195
196 RUN --mount=type=cache,target=/root/.cache/pip \
197 - python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.3/flashinfer-0.1.3+cu121torch2.4-cp310-cp310-linux_x86_64.whl
197 + python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.4/flashinfer-0.1.4+cu121torch2.4-cp310-cp310-linux_x86_64.whl
198 #################### vLLM installation IMAGE ####################
199
200
tests/samplers/test_sampler.py
+36 -1
@@ -8,6 +8,7 @@ import pytest
8 import torch
9 from transformers import GenerationConfig, GenerationMixin
10
11 +import vllm.envs as envs
12 from vllm.model_executor.layers.sampler import Sampler
13 from vllm.model_executor.sampling_metadata import SamplingMetadata
14 from vllm.model_executor.utils import set_random_seed
@@ -634,7 +635,10 @@ def test_sampler_top_k_top_p(seed: int, device: str):
635 return ([[prob.topk(1, dim=-1).indices.tolist(), [0]]
636 for prob in probs], None)
637
637 - with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
638 + # top-k and top-p is only calculated when flashinfer kernel is not available
639 + with patch("vllm.model_executor.layers.sampler._sample", mock_sample), \
640 + patch("vllm.model_executor.layers.sampler."
641 + "flashinfer_top_k_top_p_sampling", None):
642 sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
643
644 assert sample_probs is not None
@@ -645,6 +649,37 @@ def test_sampler_top_k_top_p(seed: int, device: str):
649 assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
650
651
652 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
653 +@pytest.mark.parametrize("device", CUDA_DEVICES)
654 +def test_flashinfer_fallback(seed: int, device: str):
655 + if not envs.VLLM_USE_FLASHINFER_SAMPLER:
656 + pytest.skip("Flashinfer sampler is disabled")
657 +
658 + set_random_seed(seed)
659 + torch.set_default_device(device)
660 + batch_size = random.randint(1, 256)
661 + _, fake_logits, sampler = _prepare_test(batch_size)
662 +
663 + def failing_flashinfer_sampling(*_args, **_kwargs):
664 + return None, torch.zeros(batch_size, device=device, dtype=torch.int32)
665 +
666 + sampling_params = SamplingParams(
667 + temperature=1.0,
668 + n=random.randint(1, 10),
669 + seed=random.randint(0, 10000),
670 + )
671 + sampler_output = _do_sample(batch_size, fake_logits, sampler,
672 + sampling_params, device)
673 +
674 + with patch(
675 + "vllm.model_executor.layers.sampler."
676 + "flashinfer_top_k_top_p_sampling", failing_flashinfer_sampling):
677 + fallback_sampler_output = _do_sample(batch_size, fake_logits, sampler,
678 + sampling_params, device)
679 +
680 + assert sampler_output == fallback_sampler_output
681 +
682 +
683 @pytest.mark.parametrize("device", CUDA_DEVICES)
684 def test_sampler_repetition_penalty_mixed(device: str):
685
vllm/envs.py
+5
@@ -30,6 +30,7 @@ if TYPE_CHECKING:
30 VLLM_LOGGING_CONFIG_PATH: Optional[str] = None
31 VLLM_TRACE_FUNCTION: int = 0
32 VLLM_ATTENTION_BACKEND: Optional[str] = None
33 + VLLM_USE_FLASHINFER_SAMPLER: bool = False
34 VLLM_PP_LAYER_PARTITION: Optional[str] = None
35 VLLM_CPU_KVCACHE_SPACE: int = 0
36 VLLM_CPU_OMP_THREADS_BIND: str = ""
@@ -256,6 +257,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
257 "VLLM_ATTENTION_BACKEND":
258 lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
259
260 + # If set, vllm will use flashinfer sampler
261 + "VLLM_USE_FLASHINFER_SAMPLER":
262 + lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_SAMPLER", "0"))),
263 +
264 # Pipeline stage partition strategy
265 "VLLM_PP_LAYER_PARTITION":
266 lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
vllm/model_executor/layers/sampler.py
+85 -25
@@ -1,5 +1,7 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 import itertools
3 +import warnings
4 +from importlib.util import find_spec
5 from math import inf
6 from typing import Dict, List, Optional, Tuple
7
@@ -11,6 +13,7 @@ from vllm.triton_utils import HAS_TRITON
13 if HAS_TRITON:
14 from vllm.model_executor.layers.ops.sample import sample as sample_triton
15
16 +import vllm.envs as envs
17 from vllm.model_executor.sampling_metadata import (SamplingMetadata,
18 SamplingTensors,
19 SequenceGroupToSample)
@@ -19,6 +22,16 @@ from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
22 PromptLogprobs, SampleLogprobs, SamplerOutput,
23 SequenceOutput)
24
25 +if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
26 + import flashinfer.sampling
27 + # yapf: disable
28 + from flashinfer.sampling import (
29 + top_k_top_p_sampling_from_probs as flashinfer_top_k_top_p_sampling)
30 +
31 + # yapf: enable
32 +else:
33 + flashinfer_top_k_top_p_sampling = None
34 +
35 # (num_token_ids, num_parent_ids) per sequence group.
36 SampleResultType = List[Tuple[List[int], List[int]]]
37
@@ -123,7 +136,7 @@ class Sampler(nn.Module):
136 logits = logits.to(torch.float)
137 logits.div_(sampling_tensors.temperatures.unsqueeze(dim=1))
138
126 - if do_top_p_top_k:
139 + if do_top_p_top_k and flashinfer_top_k_top_p_sampling is None:
140 logits = _apply_top_k_top_p(logits, sampling_tensors.top_ps,
141 sampling_tensors.top_ks)
142
@@ -476,14 +489,7 @@ def _multinomial(
489 seq_groups: Optional[List[SequenceGroupToSample]] = None,
490 ) -> torch.Tensor:
491 if num_samples > 1:
479 - # This is equivalent to torch.repeat_interleaved (which also
480 - # forces a GPU<->CPU sync).
481 - # This allows us to do sampling with replacement by creating
482 - # num_samples copies of each row in the tensor, and then
483 - # batch sampling the resulting tensor.
484 - probs = probs[:, None, :].expand(probs.shape[0], num_samples,
485 - probs.shape[1]).contiguous().view(
486 - -1, probs.shape[1])
492 + probs = probs.repeat_interleave(num_samples, dim=0)
493 q = torch.empty_like(probs)
494 if seq_groups is None:
495 q.exponential_()
@@ -491,17 +497,57 @@ def _multinomial(
497 sample_idx = 0
498 for seq_group in seq_groups:
499 seq_ids = seq_group.seq_ids
494 - next_sample_idx = sample_idx + len(seq_ids) * num_samples
495 - q[sample_idx:next_sample_idx].exponential_(
496 - generator=seq_group.generator)
497 - sample_idx = next_sample_idx
500 + stride = len(seq_ids) * num_samples
501 + assert seq_group.generator is not None
502 + q[sample_idx:sample_idx +
503 + stride].exponential_(generator=seq_group.generator)
504 + sample_idx += stride
505 return probs.div_(q).argmax(dim=1).view(-1, num_samples)
506
507
508 +def _top_k_top_p_multinomial_with_flashinfer(
509 + probs: torch.Tensor, top_ks: torch.Tensor, top_ps: torch.Tensor,
510 + num_samples: int, seq_groups: Optional[List[SequenceGroupToSample]]):
511 + max_top_k_round = 32
512 + if num_samples > 1:
513 + probs = probs.repeat_interleave(num_samples, dim=0)
514 + top_ks = top_ks.repeat_interleave(num_samples)
515 + top_ps = top_ps.repeat_interleave(num_samples)
516 + batch_size = probs.shape[0]
517 + uniform_samples = torch.empty((max_top_k_round, batch_size),
518 + device=probs.device)
519 + if seq_groups is None:
520 + uniform_samples.uniform_()
521 + else:
522 + sample_idx = 0
523 + for seq_group in seq_groups:
524 + seq_ids = seq_group.seq_ids
525 + stride = len(seq_ids) * num_samples
526 + assert seq_group.generator is not None
527 + uniform_samples[:, sample_idx:sample_idx +
528 + stride].uniform_(generator=seq_group.generator)
529 + sample_idx += stride
530 + batch_next_token_ids, success = flashinfer_top_k_top_p_sampling(
531 + probs,
532 + uniform_samples,
533 + top_ks,
534 + top_ps,
535 + )
536 + if not success.all():
537 + warnings.warn("FlashInfer rejection sampling failed, fallback.",
538 + stacklevel=1)
539 + probs = flashinfer.sampling.top_k_renorm_prob(probs, top_ks)
540 + probs = flashinfer.sampling.top_p_renorm_prob(probs, top_ps)
541 + batch_next_token_ids = flashinfer.sampling.sampling_from_probs(
542 + probs, uniform_samples[0])
543 + return batch_next_token_ids.view(-1, num_samples)
544 +
545 +
546 def _sample_with_torch(
547 probs: torch.Tensor,
548 logprobs: torch.Tensor,
549 sampling_metadata: SamplingMetadata,
550 + sampling_tensors: SamplingTensors,
551 include_gpu_probs_tensor: bool,
552 modify_greedy_probs: bool,
553 ) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
@@ -564,18 +610,28 @@ def _sample_with_torch(
610 sampling_params = seq_group.sampling_params
611 max_best_of_in_batch = max(max_best_of_in_batch,
612 sampling_params.best_of)
567 - seeded_args = {} if sampling_type == SamplingType.RANDOM else {
568 - "seq_groups": seq_groups,
569 - }
570 -
571 - multinomial_samples[sampling_type] = _multinomial(
572 - probs[long_sample_indices], max_best_of_in_batch,
573 - **seeded_args)
613 + seq_groups_arg = (None if sampling_type == SamplingType.RANDOM else
614 + seq_groups)
615 +
616 + if flashinfer_top_k_top_p_sampling is not None:
617 + multinomial_samples[
618 + sampling_type] = _top_k_top_p_multinomial_with_flashinfer(
619 + probs[long_sample_indices],
620 + sampling_tensors.top_ks[long_sample_indices],
621 + sampling_tensors.top_ps[long_sample_indices],
622 + max_best_of_in_batch,
623 + seq_groups_arg,
624 + )
625 + else:
626 + multinomial_samples[sampling_type] = _multinomial(
627 + probs[long_sample_indices],
628 + max_best_of_in_batch,
629 + seq_groups=seq_groups_arg)
630
631 if sampled_token_ids_tensor is not None:
632 # Store sampled tokens in output tensor.
577 - sampled_token_ids_tensor[
578 - long_sample_indices] = multinomial_samples[sampling_type]
633 + sampled_token_ids_tensor[long_sample_indices] = \
634 + multinomial_samples[sampling_type].to(torch.long)
635
636 elif sampling_type == SamplingType.BEAM:
637 beam_search_logprobs = logprobs[sample_indices]
@@ -693,9 +749,12 @@ def _sample_with_triton_kernel(
749
750
751 def _sample(
696 - probs: torch.Tensor, logprobs: torch.Tensor,
697 - sampling_metadata: SamplingMetadata, sampling_tensors: SamplingTensors,
698 - include_gpu_probs_tensor: bool, modify_greedy_probs: bool
752 + probs: torch.Tensor,
753 + logprobs: torch.Tensor,
754 + sampling_metadata: SamplingMetadata,
755 + sampling_tensors: SamplingTensors,
756 + include_gpu_probs_tensor: bool,
757 + modify_greedy_probs: bool,
758 ) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
759 """
760 Args:
@@ -713,6 +772,7 @@ def _sample(
772 probs,
773 logprobs,
774 sampling_metadata,
775 + sampling_tensors,
776 include_gpu_probs_tensor=include_gpu_probs_tensor,
777 modify_greedy_probs=modify_greedy_probs,
778 )