A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix embedding to support 2D inputs (#5829)
Woosuk Kwon committed
Jun 26, 2024 at 00:15 UTC
6806998bf9c7f24d710d9017c901e9e9a30757d5
1 file changed
+2
-2
vllm/model_executor/layers/vocab_parallel_embedding.py
+2
-2
@@ -306,11 +306,11 @@ class VocabParallelEmbedding(torch.nn.Module):
306
self.shard_indices.added_vocab_end_index)
307
else:
308
masked_input = input_
309
- # Get the embeddings.
309
+ # Get the embeddings.
310
output_parallel = F.embedding(masked_input.long(), self.weight)
311
# Mask the output embedding.
312
if self.tp_size > 1:
313
- output_parallel.masked_fill_(input_mask.unsqueeze(1), 0)
313
+ output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
314
# Reduce across all the model parallel GPUs.
315
output = tensor_model_parallel_all_reduce(output_parallel)
316
return output