A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 02:00 UTC
[CPU][Bugfix] Fix _to_list in CPU model runner (#28824)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Nov 17, 2025 at 15:47 UTC
577bb34fffc83598d3e4940f8492c122d9e3318d
2 files changed
+8
-3
csrc/cpu/torch_bindings.cpp
+8
@@ -100,6 +100,9 @@ void cpu_attention_with_kv_cache(
100
const torch::Tensor& scheduler_metadata,
101
const std::optional<torch::Tensor>& s_aux);
102
103
+// Note: just for avoiding importing errors
104
+void placeholder_op() { TORCH_CHECK(false, "Unimplemented"); }
105
+
106
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
107
// vLLM custom ops
108
@@ -275,6 +278,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
278
"sliding_window_left, SymInt sliding_window_right, Tensor block_table, "
279
"float softcap, Tensor sheduler_metadata, Tensor? s_aux) -> ()",
280
&cpu_attention_with_kv_cache);
281
+
282
+ // placeholders
283
+ ops.def("static_scaled_fp8_quant() -> ()", placeholder_op);
284
+ ops.def("dynamic_scaled_fp8_quant() -> ()", placeholder_op);
285
+ ops.def("dynamic_per_token_scaled_fp8_quant() -> ()", placeholder_op);
286
}
287
288
TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _utils), utils) {
vllm/v1/worker/cpu_model_runner.py
-3
@@ -80,9 +80,6 @@ class CPUModelRunner(GPUModelRunner):
80
def _sync_device(self) -> None:
81
pass
82
83
- def _to_list(self, sampled_token_ids: torch.Tensor) -> list[list[int]]:
84
- return sampled_token_ids.tolist()
85
-
83
def get_dp_padding(self, num_tokens: int) -> tuple[int, torch.Tensor | None]:
84
# Note: For CPU backend, dp padding is not required for now.
85
return 0, None