@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9a7e2d0534
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Allow vllm to still work if triton is not installed. (#6786)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Jul 29, 2024 at 23:51 UTC 9a7e2d053405da076ff0975660e82c4bc08a62be
13 files changed +65 -37
requirements-cpu.txt
-1
@@ -4,4 +4,3 @@
4 # Dependencies for x86_64 CPUs
5 torch == 2.4.0; platform_machine != "ppc64le"
6 torchvision; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
7 -triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
requirements-openvino.txt
-2
@@ -5,5 +5,3 @@
5 torch >= 2.1.2
6 openvino ~= 2024.3.0.dev
7 optimum-intel[openvino] >= 1.18.1
8 -
9 -triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
requirements-tpu.txt
-1
@@ -5,4 +5,3 @@
5 # Currently, the TPU backend uses a nightly version of PyTorch XLA.
6 # You can install the dependencies in Dockerfile.tpu.
7 ray
8 -triton # To avoid import errors
tests/kernels/test_sampler.py
+4 -3
@@ -5,11 +5,12 @@ import torch
5 import triton
6 import triton.language as tl
7
8 -from vllm.model_executor.layers.ops.sample import (
9 - MAX_TRITON_N_COLS, _uniform_to_exponential, get_num_triton_sampler_splits,
10 - sample)
8 +from vllm.model_executor.layers.ops.sample import (_uniform_to_exponential,
9 + sample)
10 from vllm.model_executor.sampling_metadata import SamplingTensors
11 from vllm.model_executor.utils import set_random_seed
12 +from vllm.triton_utils.sample import (MAX_TRITON_N_COLS,
13 + get_num_triton_sampler_splits)
14
15 SINGLE_SPLIT_VOCAB_SIZE = 32000 # llama/mistral/mixtral vocab size
16 MULTI_SPLIT_VOCAB_SIZE = MAX_TRITON_N_COLS + 100
vllm/attention/ops/paged_attn.py
+4 -1
@@ -4,7 +4,10 @@ from typing import List, Optional, Tuple
4 import torch
5
6 from vllm import _custom_ops as ops
7 -from vllm.attention.ops.prefix_prefill import context_attention_fwd
7 +from vllm.triton_utils import HAS_TRITON
8 +
9 +if HAS_TRITON:
10 + from vllm.attention.ops.prefix_prefill import context_attention_fwd
11
12 # Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
13 _PARTITION_SIZE = 512
vllm/model_executor/layers/fused_moe/__init__.py
+15 -7
@@ -1,14 +1,22 @@
1 -from vllm.model_executor.layers.fused_moe.fused_moe import (
2 - fused_experts, fused_moe, fused_topk, get_config_file_name, grouped_topk)
1 from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
2 FusedMoEMethodBase)
3 +from vllm.triton_utils import HAS_TRITON
4
5 __all__ = [
7 - "fused_moe",
8 - "fused_topk",
9 - "fused_experts",
10 - "get_config_file_name",
11 - "grouped_topk",
6 "FusedMoE",
7 "FusedMoEMethodBase",
8 ]
9 +
10 +if HAS_TRITON:
11 +
12 + from vllm.model_executor.layers.fused_moe.fused_moe import (
13 + fused_experts, fused_moe, fused_topk, get_config_file_name,
14 + grouped_topk)
15 +
16 + __all__ += [
17 + "fused_moe",
18 + "fused_topk",
19 + "fused_experts",
20 + "get_config_file_name",
21 + "grouped_topk",
22 + ]
vllm/model_executor/layers/ops/sample.py
+1 -13
@@ -1,4 +1,3 @@
1 -import math
1 from typing import Optional, Tuple
2
3 import torch
@@ -6,21 +5,10 @@ import triton
5 import triton.language as tl
6
7 from vllm.model_executor.layers.ops.rand import seeded_uniform
8 +from vllm.triton_utils.sample import get_num_triton_sampler_splits
9
10 _EPS = 1e-6
11
12 -# This is a hardcoded limit in Triton (max block size).
13 -MAX_TRITON_N_COLS = 131072
14 -
15 -
16 -def get_num_triton_sampler_splits(n_cols: int) -> int:
17 - """Get the number of splits to use for Triton sampling.
18 -
19 - Triton has a limit on the number of columns it can handle, so we need to
20 - split the tensor and call the kernel multiple times if it's too large.
21 - """
22 - return math.ceil(n_cols / MAX_TRITON_N_COLS)
23 -
12
13 def _multi_split_sample(
14 probs: torch.Tensor,
vllm/model_executor/layers/quantization/fp8.py
+2 -2
@@ -6,8 +6,7 @@ from torch.nn.parameter import Parameter
6
7 from vllm import _custom_ops as ops
8 from vllm.logger import init_logger
9 -from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
10 - fused_moe)
9 +from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
10 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 UnquantizedLinearMethod)
12 from vllm.model_executor.layers.quantization.base_config import (
@@ -404,6 +403,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
403 num_expert_group: Optional[int] = None,
404 topk_group: Optional[int] = None) -> torch.Tensor:
405
406 + from vllm.model_executor.layers.fused_moe import fused_moe
407 return fused_moe(x,
408 layer.w13_weight,
409 layer.w2_weight,
vllm/model_executor/layers/sampler.py
+5 -1
@@ -6,7 +6,11 @@ from typing import Dict, List, Optional, Tuple
6 import torch
7 import torch.nn as nn
8
9 -from vllm.model_executor.layers.ops.sample import sample as sample_triton
9 +from vllm.triton_utils import HAS_TRITON
10 +
11 +if HAS_TRITON:
12 + from vllm.model_executor.layers.ops.sample import sample as sample_triton
13 +
14 from vllm.model_executor.sampling_metadata import (SamplingMetadata,
15 SamplingTensors,
16 SequenceGroupToSample)
vllm/model_executor/sampling_metadata.py
+1 -1
@@ -5,9 +5,9 @@ from typing import Dict, List, Optional, Tuple
5
6 import torch
7
8 -from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
8 from vllm.sampling_params import SamplingParams, SamplingType
9 from vllm.sequence import SequenceData, SequenceGroupMetadata
10 +from vllm.triton_utils.sample import get_num_triton_sampler_splits
11 from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
12 make_tensor_with_pad, maybe_expand_dim)
13
vllm/triton_utils/__init__.py
+9 -5
@@ -1,6 +1,10 @@
1 -from vllm.triton_utils.custom_cache_manager import (
2 - maybe_set_triton_cache_manager)
1 +from vllm.triton_utils.importing import HAS_TRITON
2
4 -__all__ = [
5 - "maybe_set_triton_cache_manager",
6 -]
3 +__all__ = ["HAS_TRITON"]
4 +
5 +if HAS_TRITON:
6 +
7 + from vllm.triton_utils.custom_cache_manager import (
8 + maybe_set_triton_cache_manager)
9 +
10 + __all__ += ["maybe_set_triton_cache_manager"]
vllm/triton_utils/importing.py new
+11
@@ -0,0 +1,11 @@
1 +from importlib.util import find_spec
2 +
3 +from vllm.logger import init_logger
4 +
5 +logger = init_logger(__name__)
6 +
7 +HAS_TRITON = find_spec("triton") is not None
8 +
9 +if not HAS_TRITON:
10 + logger.info("Triton not installed; certain GPU-related functions"
11 + " will be not be available.")
vllm/triton_utils/sample.py new
+13
@@ -0,0 +1,13 @@
1 +import math
2 +
3 +# This is a hardcoded limit in Triton (max block size).
4 +MAX_TRITON_N_COLS = 131072
5 +
6 +
7 +def get_num_triton_sampler_splits(n_cols: int) -> int:
8 + """Get the number of splits to use for Triton sampling.
9 +
10 + Triton has a limit on the number of columns it can handle, so we need to
11 + split the tensor and call the kernel multiple times if it's too large.
12 + """
13 + return math.ceil(n_cols / MAX_TRITON_N_COLS)