A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Embedding model pooling_type equals ALL and multi input's bug (#10494)
Xiaoyu Zhang committed
Nov 21, 2024 at 22:40 UTC
4d676f085295d92a9248c4944433b4ade52a8ff3
1 file changed
+17
-12
vllm/model_executor/layers/pooler.py
+17
-12
@@ -94,14 +94,10 @@ class Pooler(nn.Module):
94
pooled_data = hidden_states[last_token_flat_indices]
95
elif self.pooling_type == PoolingType.ALL:
96
offset = 0
97
- pooled_data_lst = []
97
+ pooled_data = []
98
for prompt_len in prompt_lens:
99
- pooled_data_i = hidden_states[offset:offset + prompt_len]
100
-
101
- pooled_data_lst.append(pooled_data_i)
99
+ pooled_data.append(hidden_states[offset:offset + prompt_len])
100
offset += prompt_len
103
-
104
- pooled_data = torch.stack(pooled_data_lst)
101
elif self.pooling_type == PoolingType.MEAN:
102
# Calculate mean pooling
103
cumsum = torch.cumsum(hidden_states, dim=0)
@@ -121,7 +117,7 @@ class Pooler(nn.Module):
117
step_tag_id = self.step_tag_id
118
119
offset = 0
124
- pooled_data_lst = []
120
+ pooled_data = []
121
for prompt_len, seq_data_i in zip(
122
prompt_lens, pooling_metadata.seq_data.values()):
123
pooled_data_i = hidden_states[offset:offset + prompt_len]
@@ -130,17 +126,26 @@ class Pooler(nn.Module):
126
pooled_data_i = pooled_data_i[token_ids == step_tag_id]
127
128
offset += prompt_len
133
- pooled_data_lst.append(pooled_data_i)
134
-
135
- pooled_data = torch.stack(pooled_data_lst)
129
+ pooled_data.append(pooled_data_i)
130
else:
131
raise ValueError(f"Invalid pooling type: {self.pooling_type}")
132
133
if self.normalize:
140
- pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
134
+ if isinstance(pooled_data, list):
135
+ pooled_data = [
136
+ nn.functional.normalize(data, p=2, dim=1)
137
+ for data in pooled_data
138
+ ]
139
+ else:
140
+ pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
141
142
if self.softmax:
143
- pooled_data = nn.functional.softmax(pooled_data, dim=-1)
143
+ if isinstance(pooled_data, list):
144
+ pooled_data = [
145
+ nn.functional.softmax(data, dim=-1) for data in pooled_data
146
+ ]
147
+ else:
148
+ pooled_data = nn.functional.softmax(pooled_data, dim=-1)
149
150
pooled_outputs = [
151
EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data