A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Allow vllm to still work if triton is not installed. (#6786)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jul 29, 2024 at 23:51 UTC
9a7e2d053405da076ff0975660e82c4bc08a62be
13 files changed
+65
-37
requirements-cpu.txt
-1
@@ -4,4 +4,3 @@
4
# Dependencies for x86_64 CPUs
5
torch == 2.4.0; platform_machine != "ppc64le"
6
torchvision; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
7
-triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
requirements-openvino.txt
-2
@@ -5,5 +5,3 @@
5
torch >= 2.1.2
6
openvino ~= 2024.3.0.dev
7
optimum-intel[openvino] >= 1.18.1
8
-
9
-triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
requirements-tpu.txt
-1
@@ -5,4 +5,3 @@
5
# Currently, the TPU backend uses a nightly version of PyTorch XLA.
6
# You can install the dependencies in Dockerfile.tpu.
7
ray
8
-triton # To avoid import errors
tests/kernels/test_sampler.py
+4
-3
@@ -5,11 +5,12 @@ import torch
5
import triton
6
import triton.language as tl
7
8
-from vllm.model_executor.layers.ops.sample import (
9
- MAX_TRITON_N_COLS, _uniform_to_exponential, get_num_triton_sampler_splits,
10
- sample)
8
+from vllm.model_executor.layers.ops.sample import (_uniform_to_exponential,
9
+ sample)
10
from vllm.model_executor.sampling_metadata import SamplingTensors
11
from vllm.model_executor.utils import set_random_seed
12
+from vllm.triton_utils.sample import (MAX_TRITON_N_COLS,
13
+ get_num_triton_sampler_splits)
14
15
SINGLE_SPLIT_VOCAB_SIZE = 32000 # llama/mistral/mixtral vocab size
16
MULTI_SPLIT_VOCAB_SIZE = MAX_TRITON_N_COLS + 100
vllm/attention/ops/paged_attn.py
+4
-1
@@ -4,7 +4,10 @@ from typing import List, Optional, Tuple
4
import torch
5
6
from vllm import _custom_ops as ops
7
-from vllm.attention.ops.prefix_prefill import context_attention_fwd
7
+from vllm.triton_utils import HAS_TRITON
8
+
9
+if HAS_TRITON:
10
+ from vllm.attention.ops.prefix_prefill import context_attention_fwd
11
12
# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
13
_PARTITION_SIZE = 512
vllm/model_executor/layers/fused_moe/__init__.py
+15
-7
@@ -1,14 +1,22 @@
1
-from vllm.model_executor.layers.fused_moe.fused_moe import (
2
- fused_experts, fused_moe, fused_topk, get_config_file_name, grouped_topk)
1
from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
2
FusedMoEMethodBase)
3
+from vllm.triton_utils import HAS_TRITON
4
5
__all__ = [
7
- "fused_moe",
8
- "fused_topk",
9
- "fused_experts",
10
- "get_config_file_name",
11
- "grouped_topk",
6
"FusedMoE",
7
"FusedMoEMethodBase",
8
]
9
+
10
+if HAS_TRITON:
11
+
12
+ from vllm.model_executor.layers.fused_moe.fused_moe import (
13
+ fused_experts, fused_moe, fused_topk, get_config_file_name,
14
+ grouped_topk)
15
+
16
+ __all__ += [
17
+ "fused_moe",
18
+ "fused_topk",
19
+ "fused_experts",
20
+ "get_config_file_name",
21
+ "grouped_topk",
22
+ ]
vllm/model_executor/layers/ops/sample.py
+1
-13
@@ -1,4 +1,3 @@
1
-import math
1
from typing import Optional, Tuple
2
3
import torch
@@ -6,21 +5,10 @@ import triton
5
import triton.language as tl
6
7
from vllm.model_executor.layers.ops.rand import seeded_uniform
8
+from vllm.triton_utils.sample import get_num_triton_sampler_splits
9
10
_EPS = 1e-6
11
12
-# This is a hardcoded limit in Triton (max block size).
13
-MAX_TRITON_N_COLS = 131072
14
-
15
-
16
-def get_num_triton_sampler_splits(n_cols: int) -> int:
17
- """Get the number of splits to use for Triton sampling.
18
-
19
- Triton has a limit on the number of columns it can handle, so we need to
20
- split the tensor and call the kernel multiple times if it's too large.
21
- """
22
- return math.ceil(n_cols / MAX_TRITON_N_COLS)
23
-
12
13
def _multi_split_sample(
14
probs: torch.Tensor,
vllm/model_executor/layers/quantization/fp8.py
+2
-2
@@ -6,8 +6,7 @@ from torch.nn.parameter import Parameter
6
7
from vllm import _custom_ops as ops
8
from vllm.logger import init_logger
9
-from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
10
- fused_moe)
9
+from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
10
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
UnquantizedLinearMethod)
12
from vllm.model_executor.layers.quantization.base_config import (
@@ -404,6 +403,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
403
num_expert_group: Optional[int] = None,
404
topk_group: Optional[int] = None) -> torch.Tensor:
405
406
+ from vllm.model_executor.layers.fused_moe import fused_moe
407
return fused_moe(x,
408
layer.w13_weight,
409
layer.w2_weight,
vllm/model_executor/layers/sampler.py
+5
-1
@@ -6,7 +6,11 @@ from typing import Dict, List, Optional, Tuple
6
import torch
7
import torch.nn as nn
8
9
-from vllm.model_executor.layers.ops.sample import sample as sample_triton
9
+from vllm.triton_utils import HAS_TRITON
10
+
11
+if HAS_TRITON:
12
+ from vllm.model_executor.layers.ops.sample import sample as sample_triton
13
+
14
from vllm.model_executor.sampling_metadata import (SamplingMetadata,
15
SamplingTensors,
16
SequenceGroupToSample)
vllm/model_executor/sampling_metadata.py
+1
-1
@@ -5,9 +5,9 @@ from typing import Dict, List, Optional, Tuple
5
6
import torch
7
8
-from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
8
from vllm.sampling_params import SamplingParams, SamplingType
9
from vllm.sequence import SequenceData, SequenceGroupMetadata
10
+from vllm.triton_utils.sample import get_num_triton_sampler_splits
11
from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
12
make_tensor_with_pad, maybe_expand_dim)
13
vllm/triton_utils/__init__.py
+9
-5
@@ -1,6 +1,10 @@
1
-from vllm.triton_utils.custom_cache_manager import (
2
- maybe_set_triton_cache_manager)
1
+from vllm.triton_utils.importing import HAS_TRITON
2
4
-__all__ = [
5
- "maybe_set_triton_cache_manager",
6
-]
3
+__all__ = ["HAS_TRITON"]
4
+
5
+if HAS_TRITON:
6
+
7
+ from vllm.triton_utils.custom_cache_manager import (
8
+ maybe_set_triton_cache_manager)
9
+
10
+ __all__ += ["maybe_set_triton_cache_manager"]
vllm/triton_utils/importing.py
new
+11
@@ -0,0 +1,11 @@
1
+from importlib.util import find_spec
2
+
3
+from vllm.logger import init_logger
4
+
5
+logger = init_logger(__name__)
6
+
7
+HAS_TRITON = find_spec("triton") is not None
8
+
9
+if not HAS_TRITON:
10
+ logger.info("Triton not installed; certain GPU-related functions"
11
+ " will be not be available.")
vllm/triton_utils/sample.py
new
+13
@@ -0,0 +1,13 @@
1
+import math
2
+
3
+# This is a hardcoded limit in Triton (max block size).
4
+MAX_TRITON_N_COLS = 131072
5
+
6
+
7
+def get_num_triton_sampler_splits(n_cols: int) -> int:
8
+ """Get the number of splits to use for Triton sampling.
9
+
10
+ Triton has a limit on the number of columns it can handle, so we need to
11
+ split the tensor and call the kernel multiple times if it's too large.
12
+ """
13
+ return math.ceil(n_cols / MAX_TRITON_N_COLS)