A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 19:30 UTC
Fix CuPy runtime deps and restore humming (#43530)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
Mohammad Miadh Angkad committed
May 26, 2026 at 20:59 UTC
a970fb5a1a5800c552c74cf3278d6ee7c1c3fca1
5 files changed
+23
-16
docker/Dockerfile
+4
-2
@@ -220,7 +220,8 @@ COPY use_existing_torch.py use_existing_torch.py
220
COPY pyproject.toml pyproject.toml
221
RUN --mount=type=cache,target=/opt/uv/cache \
222
if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \
223
- sed -i 's/^nvidia-cutlass-dsl\[cu13\]>=/nvidia-cutlass-dsl>=/' requirements/cuda.txt; \
223
+ sed -i 's/^nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' requirements/cuda.txt; \
224
+ sed -i 's/^humming-kernels\[cu13\]/humming-kernels[cu12]/' requirements/cuda.txt; \
225
fi \
226
&& if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
227
echo "Installing torch nightly..." \
@@ -746,7 +747,8 @@ COPY requirements/common.txt /tmp/common.txt
747
COPY requirements/cuda.txt /tmp/requirements-cuda.txt
748
RUN --mount=type=cache,target=/opt/uv/cache \
749
if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \
749
- sed -i 's/^nvidia-cutlass-dsl\[cu13\]>=/nvidia-cutlass-dsl>=/' /tmp/requirements-cuda.txt; \
750
+ sed -i 's/^nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' /tmp/requirements-cuda.txt; \
751
+ sed -i 's/^humming-kernels\[cu13\]/humming-kernels[cu12]/' /tmp/requirements-cuda.txt; \
752
fi && \
753
uv pip install --system -r /tmp/requirements-cuda.txt \
754
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') && \
requirements/cuda.txt
+4
-1
@@ -25,4 +25,7 @@ nvidia-cutlass-dsl[cu13]==4.5.0
25
quack-kernels>=0.3.3
26
27
# Tokenspeed_MLA for faster mla with spec decode
28
-tokenspeed-mla==0.1.2
\ No newline at end of file
28
+tokenspeed-mla==0.1.2
29
+
30
+# Humming kernels for quantization gemm
31
+humming-kernels[cu13]==0.1.2
requirements/kv_connectors.txt
+3
@@ -1,3 +1,6 @@
1
lmcache >= 0.3.9
2
+# CuPy 14.1.0 imports pytest from cupy.testing._random. Use <14.1.0
3
+# until a fixed newer release is verified for runtime images.
4
+cupy-cuda13x < 14.1.0
5
nixl >= 1.1.0 # Required for disaggregated prefill
6
mooncake-transfer-engine >= 0.3.8
setup.py
+2
@@ -1017,6 +1017,8 @@ def get_requirements() -> list[str]:
1017
if "nvidia-cutlass-dsl[cu13]" in req and cuda_major == "12":
1018
# [cu13] extra is the default; strip it on CUDA 12 builds.
1019
req = req.replace("nvidia-cutlass-dsl[cu13]", "nvidia-cutlass-dsl")
1020
+ if "humming-kernels[cu13]" in req and cuda_major == "12":
1021
+ req = req.replace("humming-kernels[cu13]", "humming-kernels[cu12]")
1022
modified_requirements.append(req)
1023
requirements = modified_requirements
1024
elif _is_hip():
vllm/model_executor/layers/quantization/humming.py
+10
-13
@@ -43,12 +43,9 @@ from vllm.model_executor.parameter import (
43
RowvLLMParameter,
44
)
45
from vllm.model_executor.utils import set_weight_attrs
46
+from vllm.platforms import current_platform
47
47
-if TYPE_CHECKING:
48
- from vllm.model_executor.models.utils import WeightsMapper
49
-
50
-
51
-try:
48
+if current_platform.is_cuda():
49
from humming.dtypes import DataType
50
from humming.layer import HummingMethod
51
from humming.schema import (
@@ -65,16 +62,17 @@ try:
62
HummingIndexedExperts,
63
get_humming_moe_gemm_type,
64
)
68
-except ModuleNotFoundError:
69
- HummingMethod = None
65
71
-
72
-def assert_humming_available():
73
- assert HummingMethod is not None, (
74
- "humming is not available, please run "
75
- "'pip install git+https://github.com/inclusionAI/humming' to install it."
66
+if TYPE_CHECKING:
67
+ from humming.schema import (
68
+ BaseInputSchema,
69
+ BaseWeightSchema,
70
+ HummingInputSchema,
71
+ HummingWeightSchema,
72
)
73
74
+ from vllm.model_executor.models.utils import WeightsMapper
75
+
76
77
def prepare_padded_shape(shape, x):
78
padded_shape = math.ceil(shape / x) * x
@@ -186,7 +184,6 @@ class HummingConfig(QuantizationConfig):
184
packed_modules_mapping: dict[str, list[str]] = {}
185
186
def __init__(self, full_config: dict[str, Any] | None = None):
189
- assert_humming_available()
187
self.full_config: dict[str, Any] = full_config or {}
188
189
@classmethod