A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 18:15 UTC
[CI/Build][CPU] Fix failed pooling tests and macos smoke test (#32907)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Signed-off-by: Li, Jiang <bigpyj64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Li, Jiang committed
Jan 23, 2026 at 18:48 UTC
5da4c7d789fe0c4ca2c49913441f99df24715a97
2 files changed
+7
-1
csrc/cpu/torch_bindings.cpp
+1
-1
@@ -230,7 +230,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
230
#endif
231
232
// SHM CCL
233
-#if defined(__AVX512F__) || defined(__aarch64__)
233
+#if defined(__AVX512F__) || (defined(__aarch64__) && !defined(__APPLE__))
234
ops.def("init_shm_manager(str name, int group_size, int rank) -> int",
235
&init_shm_manager);
236
ops.def("join_shm_manager(int handle, str name) -> str", &join_shm_manager);
vllm/model_executor/layers/utils.py
+6
@@ -216,8 +216,14 @@ def dispatch_cpu_unquantized_gemm(
216
layer: torch.nn.Module,
217
remove_weight: bool,
218
) -> None:
219
+ # skip for missing layers
220
+ if layer.weight.is_meta:
221
+ layer.cpu_linear = torch.nn.functional.linear
222
+ return
223
+
224
N, K = layer.weight.size()
225
dtype = layer.weight.dtype
226
+
227
if envs.VLLM_CPU_SGL_KERNEL and check_cpu_sgl_kernel(N, K, dtype):
228
packed_weight = torch.ops._C.convert_weight_packed(layer.weight)
229
if getattr(layer, "bias", None) is not None: