A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix Stream usage in CPU model runner and OneDNN kernel check (#25046)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Sep 17, 2025 at 20:54 UTC
9fccd04e308b0b8a625dd78b7dfa4feed8131102
3 files changed
+14
-1
csrc/cpu/dnnl_kernels.cpp
+1
-1
@@ -523,7 +523,7 @@ void onednn_mm(torch::Tensor& c, // [M, OC], row-major
523
CPU_KERNEL_GUARD_IN(onednn_mm)
524
TORCH_CHECK(a.dim() == 2);
525
TORCH_CHECK(a.stride(-1) == 1);
526
- TORCH_CHECK(c.is_contiguous());
526
+ TORCH_CHECK(c.stride(-1) == 1);
527
MatMulPrimitiveHandler* ptr =
528
reinterpret_cast<MatMulPrimitiveHandler*>(handler);
529
vllm/platforms/cpu.py
+5
@@ -185,6 +185,11 @@ class CpuPlatform(Platform):
185
parallel_config.distributed_executor_backend = "mp"
186
if parallel_config.worker_cls == "auto":
187
parallel_config.worker_cls = "vllm.v1.worker.cpu_worker.CPUWorker"
188
+ # Disable DBO
189
+ if parallel_config.enable_dbo:
190
+ logger.warning(
191
+ "Dual-Batch Overlap is not supported on CPU, disabled.")
192
+ parallel_config.enable_dbo = False
193
194
# Note: workaround for v1 gpu_model_runner
195
from vllm.config import CompilationLevel
vllm/v1/worker/cpu_model_runner.py
+8
@@ -145,12 +145,20 @@ def _torch_cuda_wrapper():
145
self.record = lambda: None
146
self.synchronize = lambda: None
147
148
+ class _StreamPlaceholder:
149
+
150
+ def __init__(self, *args, **kwargs) -> None:
151
+ pass
152
+
153
cuda_event = torch.cuda.Event
154
+ cuda_stream = torch.cuda.Stream
155
try:
156
torch.cuda.Event = _EventPlaceholder
157
+ torch.cuda.Stream = _StreamPlaceholder
158
yield
159
finally:
160
torch.cuda.Event = cuda_event
161
+ torch.cuda.Stream = cuda_stream
162
163
164
@contextmanager