@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4d676f0852
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Embedding model pooling_type equals ALL and multi input's bug (#10494)

Xiaoyu Zhang committed Nov 21, 2024 at 22:40 UTC 4d676f085295d92a9248c4944433b4ade52a8ff3
1 file changed +17 -12
vllm/model_executor/layers/pooler.py
+17 -12
@@ -94,14 +94,10 @@ class Pooler(nn.Module):
94 pooled_data = hidden_states[last_token_flat_indices]
95 elif self.pooling_type == PoolingType.ALL:
96 offset = 0
97 - pooled_data_lst = []
97 + pooled_data = []
98 for prompt_len in prompt_lens:
99 - pooled_data_i = hidden_states[offset:offset + prompt_len]
100 -
101 - pooled_data_lst.append(pooled_data_i)
99 + pooled_data.append(hidden_states[offset:offset + prompt_len])
100 offset += prompt_len
103 -
104 - pooled_data = torch.stack(pooled_data_lst)
101 elif self.pooling_type == PoolingType.MEAN:
102 # Calculate mean pooling
103 cumsum = torch.cumsum(hidden_states, dim=0)
@@ -121,7 +117,7 @@ class Pooler(nn.Module):
117 step_tag_id = self.step_tag_id
118
119 offset = 0
124 - pooled_data_lst = []
120 + pooled_data = []
121 for prompt_len, seq_data_i in zip(
122 prompt_lens, pooling_metadata.seq_data.values()):
123 pooled_data_i = hidden_states[offset:offset + prompt_len]
@@ -130,17 +126,26 @@ class Pooler(nn.Module):
126 pooled_data_i = pooled_data_i[token_ids == step_tag_id]
127
128 offset += prompt_len
133 - pooled_data_lst.append(pooled_data_i)
134 -
135 - pooled_data = torch.stack(pooled_data_lst)
129 + pooled_data.append(pooled_data_i)
130 else:
131 raise ValueError(f"Invalid pooling type: {self.pooling_type}")
132
133 if self.normalize:
140 - pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
134 + if isinstance(pooled_data, list):
135 + pooled_data = [
136 + nn.functional.normalize(data, p=2, dim=1)
137 + for data in pooled_data
138 + ]
139 + else:
140 + pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
141
142 if self.softmax:
143 - pooled_data = nn.functional.softmax(pooled_data, dim=-1)
143 + if isinstance(pooled_data, list):
144 + pooled_data = [
145 + nn.functional.softmax(data, dim=-1) for data in pooled_data
146 + ]
147 + else:
148 + pooled_data = nn.functional.softmax(pooled_data, dim=-1)
149
150 pooled_outputs = [
151 EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data