A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix][Hardware][CPU] Fix CPU `input_positions` creation for text-only inputs with mrope (#11434)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Dec 24, 2024 at 17:59 UTC
7a5286cc047112c7cc52bad8da8c17aedc880ef5
1 file changed
+7
-9
vllm/worker/cpu_model_runner.py
+7
-9
@@ -114,8 +114,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
114
def __init__(self, use_mrope: bool):
115
self.use_mrope = use_mrope
116
self.input_tokens: List[int] = []
117
- self.input_positions: Optional[
118
- List[int]] = [] if not self.use_mrope else None
117
+ self.input_positions: List[int] = []
118
self.token_type_ids: Optional[List[int]] = []
119
self.seq_lens: List[int] = []
120
self.query_lens: List[int] = []
@@ -130,9 +129,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
129
self.multi_modal_placeholder_maps: Dict[
130
str, MultiModalPlaceholderMap] = defaultdict(
131
MultiModalPlaceholderMap)
133
- self.input_mrope_positions: Optional[List[List[int]]] = [
134
- [] for _ in range(3)
135
- ] if self.use_mrope else None
132
+ self.input_mrope_positions: List[List[int]] = [[]
133
+ for _ in range(3)]
134
135
def __init__(self,
136
runner: "CPUModelRunner",
@@ -167,7 +165,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
165
device="cpu")
166
input_positions = torch.tensor(
167
input_data.input_positions
170
- if not input_data.use_mrope else input_data.input_mrope_positions,
168
+ if not any(input_data.input_mrope_positions) else
169
+ input_data.input_mrope_positions,
170
dtype=torch.long,
171
device="cpu")
172
token_type_ids = torch.tensor(input_data.token_type_ids,
@@ -236,7 +235,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
235
block_table = block_table[start_block:]
236
237
# For MRotaryEmbedding
239
- if data.input_positions is None:
238
+ if seq_data.mrope_position_delta is not None:
239
next_pos = MRotaryEmbedding.get_next_input_positions(
240
seq_data.mrope_position_delta,
241
context_len,
@@ -309,8 +308,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
308
data.slot_mapping.extend(slot_mapping)
309
310
# The MROPE positions are prepared in _compute_multi_modal_input
312
- if data.input_positions is not None:
313
- data.input_positions.extend(token_positions)
311
+ data.input_positions.extend(token_positions)
312
313
if data.token_type_ids is not None:
314
data.token_type_ids.extend(token_types if token_types else [])