| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import torch |
| 5 | |
| 6 | from vllm import LLM |
| 7 | from vllm.config import PoolerConfig |
| 8 | from vllm.inputs import TextPrompt |
| 9 | from vllm.multimodal.utils import fetch_image |
| 10 | |
| 11 | |
| 12 | def main(): |
| 13 | # Initialize model |
| 14 | model = LLM( |
| 15 | model="jinaai/jina-embeddings-v4-vllm-text-matching", |
| 16 | pooler_config=PoolerConfig(task="token_embed"), |
| 17 | runner="pooling", |
| 18 | max_model_len=1024, |
| 19 | gpu_memory_utilization=0.8, |
| 20 | ) |
| 21 | |
| 22 | # Create text prompts |
| 23 | text1 = "Ein wunderschöner Sonnenuntergang am Strand" |
| 24 | text1_prompt = TextPrompt(prompt=f"Query: {text1}") |
| 25 | |
| 26 | text2 = "浜辺に沈む美しい夕日" |
| 27 | text2_prompt = TextPrompt(prompt=f"Query: {text2}") |
| 28 | |
| 29 | # Create image prompt |
| 30 | image = fetch_image( |
| 31 | "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/eskimo.jpg" # noqa: E501 |
| 32 | ) |
| 33 | image_prompt = TextPrompt( |
| 34 | prompt="<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe the image.<|im_end|>\n", # noqa: E501 |
| 35 | multi_modal_data={"image": image}, |
| 36 | ) |
| 37 | |
| 38 | # Encode all prompts |
| 39 | prompts = [text1_prompt, text2_prompt, image_prompt] |
| 40 | outputs = model.encode(prompts, pooling_task="token_embed") |
| 41 | |
| 42 | def get_embeddings(outputs): |
| 43 | VISION_START_TOKEN_ID, VISION_END_TOKEN_ID = 151652, 151653 |
| 44 | |
| 45 | embeddings = [] |
| 46 | for output in outputs: |
| 47 | if VISION_START_TOKEN_ID in output.prompt_token_ids: |
| 48 | # Gather only vision tokens |
| 49 | img_start_pos = torch.where( |
| 50 | torch.tensor(output.prompt_token_ids) == VISION_START_TOKEN_ID |
| 51 | )[0][0] |
| 52 | img_end_pos = torch.where( |
| 53 | torch.tensor(output.prompt_token_ids) == VISION_END_TOKEN_ID |
| 54 | )[0][0] |
| 55 | embeddings_tensor = output.outputs.data.detach().clone()[ |
| 56 | img_start_pos : img_end_pos + 1 |
| 57 | ] |
| 58 | else: |
| 59 | # Use all tokens for text-only prompts |
| 60 | embeddings_tensor = output.outputs.data.detach().clone() |
| 61 | |
| 62 | # Pool and normalize embeddings |
| 63 | pooled_output = ( |
| 64 | embeddings_tensor.sum(dim=0, dtype=torch.float32) |
| 65 | / embeddings_tensor.shape[0] |
| 66 | ) |
| 67 | embeddings.append(torch.nn.functional.normalize(pooled_output, dim=-1)) |
| 68 | return embeddings |
| 69 | |
| 70 | embeddings = get_embeddings(outputs) |
| 71 | |
| 72 | for embedding in embeddings: |
| 73 | print(embedding.shape) |
| 74 | |
| 75 | |
| 76 | if __name__ == "__main__": |
| 77 | main() |