@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a5286cc04
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix][Hardware][CPU] Fix CPU `input_positions` creation for text-only inputs with mrope (#11434)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Dec 24, 2024 at 17:59 UTC 7a5286cc047112c7cc52bad8da8c17aedc880ef5
1 file changed +7 -9
vllm/worker/cpu_model_runner.py
+7 -9
@@ -114,8 +114,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
114 def __init__(self, use_mrope: bool):
115 self.use_mrope = use_mrope
116 self.input_tokens: List[int] = []
117 - self.input_positions: Optional[
118 - List[int]] = [] if not self.use_mrope else None
117 + self.input_positions: List[int] = []
118 self.token_type_ids: Optional[List[int]] = []
119 self.seq_lens: List[int] = []
120 self.query_lens: List[int] = []
@@ -130,9 +129,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
129 self.multi_modal_placeholder_maps: Dict[
130 str, MultiModalPlaceholderMap] = defaultdict(
131 MultiModalPlaceholderMap)
133 - self.input_mrope_positions: Optional[List[List[int]]] = [
134 - [] for _ in range(3)
135 - ] if self.use_mrope else None
132 + self.input_mrope_positions: List[List[int]] = [[]
133 + for _ in range(3)]
134
135 def __init__(self,
136 runner: "CPUModelRunner",
@@ -167,7 +165,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
165 device="cpu")
166 input_positions = torch.tensor(
167 input_data.input_positions
170 - if not input_data.use_mrope else input_data.input_mrope_positions,
168 + if not any(input_data.input_mrope_positions) else
169 + input_data.input_mrope_positions,
170 dtype=torch.long,
171 device="cpu")
172 token_type_ids = torch.tensor(input_data.token_type_ids,
@@ -236,7 +235,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
235 block_table = block_table[start_block:]
236
237 # For MRotaryEmbedding
239 - if data.input_positions is None:
238 + if seq_data.mrope_position_delta is not None:
239 next_pos = MRotaryEmbedding.get_next_input_positions(
240 seq_data.mrope_position_delta,
241 context_len,
@@ -309,8 +308,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
308 data.slot_mapping.extend(slot_mapping)
309
310 # The MROPE positions are prepared in _compute_multi_modal_input
312 - if data.input_positions is not None:
313 - data.input_positions.extend(token_positions)
311 + data.input_positions.extend(token_positions)
312
313 if data.token_type_ids is not None:
314 data.token_type_ids.extend(token_types if token_types else [])