A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix undefined symbol: cutlass_moe_mm_sm100 (#26098)
Signed-off-by: Jun Jiang <jasl9187@hotmail.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Jun Jiang committed
Oct 3, 2025 at 23:48 UTC
4f8f47e87e4f65195cf77b0de93feb63fc5a5b2f
3 files changed
+23
-3
CMakeLists.txt
+1
-1
@@ -667,7 +667,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
667
endif()
668
669
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
670
- cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f" "${CUDA_ARCHS}")
670
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f;11.0f" "${CUDA_ARCHS}")
671
else()
672
cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a" "${CUDA_ARCHS}")
673
endif()
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+2
-2
@@ -254,7 +254,7 @@ void cutlass_moe_mm(
254
bool per_act_token, bool per_out_ch) {
255
int32_t version_num = get_sm_version_num();
256
#if defined ENABLE_CUTLASS_MOE_SM100 && ENABLE_CUTLASS_MOE_SM100
257
- if (version_num >= 100) {
257
+ if (version_num >= 100 && version_num < 110) {
258
cutlass_moe_mm_sm100(out_tensors, a_tensors, b_tensors, a_scales, b_scales,
259
expert_offsets, problem_sizes, a_strides, b_strides,
260
c_strides, per_act_token, per_out_ch);
@@ -262,7 +262,7 @@ void cutlass_moe_mm(
262
}
263
#endif
264
#if defined ENABLE_CUTLASS_MOE_SM90 && ENABLE_CUTLASS_MOE_SM90
265
- if (version_num >= 90) {
265
+ if (version_num >= 90 && version_num < 100) {
266
cutlass_moe_mm_sm90(out_tensors, a_tensors, b_tensors, a_scales, b_scales,
267
expert_offsets, problem_sizes, a_strides, b_strides,
268
c_strides, per_act_token, per_out_ch);
vllm/utils/__init__.py
+20
@@ -2747,6 +2747,8 @@ class MemorySnapshot:
2747
self.measure()
2748
2749
def measure(self):
2750
+ from vllm.platforms import current_platform
2751
+
2752
# we measure the torch peak memory usage via allocated_bytes,
2753
# rather than `torch.cuda.memory_reserved()` .
2754
# After `torch.cuda.reset_peak_memory_stats()`,
@@ -2756,6 +2758,24 @@ class MemorySnapshot:
2758
"allocated_bytes.all.peak", 0)
2759
2760
self.free_memory, self.total_memory = torch.cuda.mem_get_info()
2761
+ shared_sysmem_device_mem_sms = (
2762
+ (8, 7), (11, 0), (12, 1)) # Orin, Thor, Spark
2763
+ if current_platform.is_cuda() and \
2764
+ current_platform.get_device_capability() in \
2765
+ shared_sysmem_device_mem_sms:
2766
+ # On UMA (Orin, Thor and Spark) platform,
2767
+ # where both CPU and GPU rely on system memory,
2768
+ # the cudaMemGetInfo function shows the amount of free system memory
2769
+ # rather than what’s actually available.
2770
+ # In the case,
2771
+ # torch.cuda.mem_get_info() only reports "free" memory,
2772
+ # which can be lower than what is actually
2773
+ # available due to not including cache memory.
2774
+ # There’s also a comprehensive reference page
2775
+ # that explains how you can compute the proper value yourself.
2776
+ # https://docs.nvidia.com/cuda/cuda-for-tegra-appnote/#estimating-total-allocatable-device-memory-on-an-integrated-gpu-device
2777
+ self.free_memory = psutil.virtual_memory().available
2778
+
2779
self.cuda_memory = self.total_memory - self.free_memory
2780
2781
# torch.cuda.memory_reserved() is how many bytes