609b533cb6 · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/kernels/benchmark_bitblas.py

+3 -1

@@ -3,6 +3,8 @@

3 # Copyright (c) Microsoft Corporation.

4 # Licensed under the MIT License.

5

6 +from packaging import version

7 +

8 from vllm.model_executor.layers.quantization.utils.bitblas_utils import (

9 MINIMUM_BITBLAS_VERSION,

10 )

@@ -10,7 +12,7 @@ from vllm.model_executor.layers.quantization.utils.bitblas_utils import (

12 try:

13 import bitblas

14

13 - if bitblas.__version__ < MINIMUM_BITBLAS_VERSION:

15 + if version.parse(bitblas.__version__) < version.parse(MINIMUM_BITBLAS_VERSION):

16 raise ImportError(

17 "bitblas version is wrong. Please "

18 f"install bitblas>={MINIMUM_BITBLAS_VERSION}"

docs/design/arch_overview.md

+2 -1

@@ -200,7 +200,8 @@ vision-language model.

200 lora_config = vllm_config.lora_config

201 super().__init__(config, cache_config, quant_config, lora_config, prefix)

202

203 - if __version__ >= "0.6.4":

203 + from packaging import version

204 + if version.parse(__version__) >= version.parse("0.6.4"):

205 MyModel = MyNewModel

206 else:

207 MyModel = MyOldModel

vllm/attention/ops/triton_decode_attention.py

+3 -1

@@ -31,6 +31,8 @@ It supports page size >= 1.

31

32 import logging

33

34 +from packaging import version

35 +

36 from vllm.platforms import current_platform

37 from vllm.triton_utils import tl, triton

38

@@ -40,7 +42,7 @@ logger = logging.getLogger(__name__)

42

43 # Only print the following warnings when triton version < 3.2.0.

44 # The issue won't affect performance or accuracy.

43 -if triton.__version__ < '3.2.0':

45 +if version.parse(triton.__version__) < version.parse('3.2.0'):

46 logger.warning(

47 "The following error message 'operation scheduled before its operands' "

48 "can be ignored.")

vllm/model_executor/layers/quantization/bitblas.py

+3 -1

@@ -3,6 +3,7 @@

3 from typing import Any, Optional

4

5 import torch

6 +from packaging import version

7

8 from vllm.logger import init_logger

9 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase

@@ -45,7 +46,8 @@ class BitBLASConfig(QuantizationConfig):

46 ) -> None:

47 try:

48 import bitblas

48 - if bitblas.__version__ < MINIMUM_BITBLAS_VERSION:

49 + if version.parse(bitblas.__version__) < version.parse(

50 + MINIMUM_BITBLAS_VERSION):

51 raise ImportError(

52 "bitblas version is wrong. Please "

53 f"install bitblas>={MINIMUM_BITBLAS_VERSION}")

vllm/model_executor/layers/quantization/bitsandbytes.py

+5 -2

@@ -4,6 +4,7 @@

4 from typing import Any, Callable, Optional, Union

5

6 import torch

7 +from packaging import version

8

9 from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,

10 FusedMoEMethodBase)

@@ -169,7 +170,8 @@ class BitsAndBytesLinearMethod(LinearMethodBase):

170 def __init__(self, quant_config: BitsAndBytesConfig):

171 try:

172 import bitsandbytes

172 - if bitsandbytes.__version__ < "0.46.1":

173 + if version.parse(

174 + bitsandbytes.__version__) < version.parse("0.46.1"):

175 raise ImportError("bitsandbytes version is wrong. Please "

176 "install bitsandbytes>=0.46.1.")

177 except ImportError as err:

@@ -412,7 +414,8 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase):

414 def __init__(self, quant_config: BitsAndBytesConfig):

415 try:

416 import bitsandbytes

415 - if bitsandbytes.__version__ < "0.46.1":

417 + if version.parse(

418 + bitsandbytes.__version__) < version.parse("0.46.1"):

419 raise ImportError("bitsandbytes version is wrong. Please "

420 "install bitsandbytes>=0.46.1.")

421 except ImportError as err:

vllm/model_executor/layers/quantization/deepspeedfp.py

+2 -1

@@ -6,6 +6,7 @@ from typing import Any, Optional

6 import torch

7 import torch.nn as nn

8 import torch.nn.functional as F

9 +from packaging import version

10

11 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase

12 from vllm.model_executor.layers.quantization import QuantizationMethods

@@ -145,7 +146,7 @@ class DeepSpeedFPParameter(nn.Parameter):

146 quant_config: DeepSpeedFPConfig):

147 try:

148 import deepspeed

148 - if deepspeed.__version__ < "0.14.2":

149 + if version.parse(deepspeed.__version__) < version.parse("0.14.2"):

150 raise ImportError("deepspeed version is wrong. Please "

151 "install deepspeed>=0.14.2.")

152 from deepspeed.ops.fp_quantizer import FP_Quantize

vllm/model_executor/layers/quantization/gptq_bitblas.py

+3 -1

@@ -3,6 +3,7 @@

3 from typing import Any, Optional

4

5 import torch

6 +from packaging import version

7 from torch.nn.parameter import Parameter

8

9 from vllm.logger import init_logger

@@ -63,7 +64,8 @@ class GPTQBitBLASConfig(QuantizationConfig):

64

65 try:

66 import bitblas

66 - if bitblas.__version__ < MINIMUM_BITBLAS_VERSION:

67 + if version.parse(bitblas.__version__) < version.parse(

68 + MINIMUM_BITBLAS_VERSION):

69 raise ImportError(

70 "bitblas version is wrong. Please "

71 f"install bitblas>={MINIMUM_BITBLAS_VERSION}")

vllm/model_executor/layers/quantization/ipex_quant.py

+5 -2

@@ -4,6 +4,7 @@

4 from typing import Any, Optional

5

6 import torch

7 +from packaging import version

8

9 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,

10 UnquantizedLinearMethod)

@@ -135,7 +136,8 @@ class IPEXGPTQLinearMethod(GPTQLinearMethod):

136

137 try:

138 import intel_extension_for_pytorch as ipex

138 - if ipex.__version__ < MIN_IPEX_VERSION:

139 + if version.parse(

140 + ipex.__version__) < version.parse(MIN_IPEX_VERSION):

141 raise ImportError(

142 "intel_extension_for_pytorch version is "

143 "wrong. Please install "

@@ -199,7 +201,8 @@ class IPEXAWQLinearMethod(AWQLinearMethod):

201

202 try:

203 import intel_extension_for_pytorch as ipex

202 - if ipex.__version__ < MIN_IPEX_VERSION:

204 + if version.parse(

205 + ipex.__version__) < version.parse(MIN_IPEX_VERSION):

206 raise ImportError(

207 "intel_extension_for_pytorch version is "

208 "wrong. Please install "

vllm/model_executor/layers/quantization/kernels/mixed_precision/bitblas.py

+3 -1

@@ -4,6 +4,7 @@

4 from typing import Optional

5

6 import torch

7 +from packaging import version

8

9 from vllm.logger import init_logger

10 from vllm.model_executor.layers.quantization.base_config import (

@@ -110,7 +111,8 @@ class BitBLASLinearKernel(MPLinearKernel):

111

112 try:

113 import bitblas

113 - if bitblas.__version__ < MINIMUM_BITBLAS_VERSION:

114 + if version.parse(bitblas.__version__) < version.parse(

115 + MINIMUM_BITBLAS_VERSION):

116 raise ImportError(

117 "bitblas version is wrong. Please "

118 f"install bitblas>={MINIMUM_BITBLAS_VERSION}")

vllm/model_executor/layers/quantization/utils/bitblas_utils.py

+3 -1

@@ -3,6 +3,7 @@

3 from typing import Optional

4

5 import torch

6 +from packaging import version

7

8 from vllm.platforms import current_platform

9 from vllm.scalar_type import ScalarType, scalar_types

@@ -75,7 +76,8 @@ def _check_bitblas_supported(

76 # Finally, check if bitblas is installed

77 try:

78 import bitblas

78 - if bitblas.__version__ < MINIMUM_BITBLAS_VERSION:

79 + if version.parse(

80 + bitblas.__version__) < version.parse(MINIMUM_BITBLAS_VERSION):

81 raise ImportError("bitblas version is wrong. Please "

82 f"install bitblas>={MINIMUM_BITBLAS_VERSION}")

83 except ImportError:

vllm/model_executor/layers/quantization/utils/w8a8_utils.py

+3 -2

@@ -4,6 +4,7 @@

4 from typing import Callable, Optional, Union

5

6 import torch

7 +from packaging import version

8

9 from vllm import _custom_ops as ops

10 from vllm import envs

@@ -21,8 +22,8 @@ TORCH_DEVICE_IDENTITY = None

22 # torch._scaled_mm rowwise feature.

23 # The condition is determined once as the operations

24 # are time consuming.

24 -USE_ROWWISE_TORCH_SCALED_MM = (current_platform.is_rocm()

25 - and torch.__version__[0:3] >= "2.7"

25 +USE_ROWWISE_TORCH_SCALED_MM = (current_platform.is_rocm() and version.parse(

26 + torch.__version__) >= version.parse("2.7")

27 and current_platform.has_device_capability(94))

28

29

vllm/model_executor/model_loader/bitsandbytes_loader.py

+3 -1

@@ -12,6 +12,7 @@ from typing import Any, Callable, Optional

12 import numpy as np

13 import torch

14 from huggingface_hub import HfApi

15 +from packaging import version

16 from torch import nn

17 from transformers.utils import SAFE_WEIGHTS_INDEX_NAME

18

@@ -193,7 +194,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):

194 try:

195 import bitsandbytes

196

196 - if bitsandbytes.__version__ < "0.46.1":

197 + if version.parse(

198 + bitsandbytes.__version__) < version.parse("0.46.1"):

199 raise ImportError("bitsandbytes version is wrong. Please "

200 "install bitsandbytes>=0.46.1.")

201 except ImportError as err:

vllm/v1/sample/ops/topk_topp_sampler.py

+2 -1

@@ -5,6 +5,7 @@ from typing import Optional

5

6 import torch

7 import torch.nn as nn

8 +from packaging import version

9

10 from vllm import envs

11 from vllm.logger import init_logger

@@ -32,7 +33,7 @@ class TopKTopPSampler(nn.Module):

33 if current_platform.is_cuda():

34 if is_flashinfer_available:

35 flashinfer_version = flashinfer.__version__

35 - if flashinfer_version < "0.2.3":

36 + if version.parse(flashinfer_version) < version.parse("0.2.3"):

37 logger.warning_once(

38 "FlashInfer version >= 0.2.3 required. "

39 "Falling back to default sampling implementation.")

[Bugfix] Add proper comparison for package versions (#22314)