A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Doc] Fix prompt embedding examples (#18350)
Signed-off-by: wangli <wangli858794774@gmail.com>
Li Wang committed
May 19, 2025 at 21:48 UTC
c5bb0ebdc656bdf89013936b4a29666b143eb6a5
1 file changed
+2
docs/source/serving/prompt_embeds.md
+2
@@ -36,6 +36,7 @@ llm = LLM(model=model_name, enable_prompt_embeds=True)
36
chat = [{"role": "user", "content": "Please tell me about the capital of France."}]
37
token_ids = tokenizer.apply_chat_template(chat, add_generation_prompt=True, return_tensors='pt')
38
39
+embedding_layer = transformers_model.get_input_embeddings()
40
prompt_embeds = embedding_layer(token_ids).squeeze(0)
41
42
# Single prompt inference
@@ -116,6 +117,7 @@ transformers_model = transformers.AutoModelForCausalLM.from_pretrained(model_nam
117
chat = [{"role": "user", "content": "Please tell me about the capital of France."}]
118
token_ids = tokenizer.apply_chat_template(chat, add_generation_prompt=True, return_tensors='pt')
119
120
+embedding_layer = transformers_model.get_input_embeddings()
121
prompt_embeds = embedding_layer(token_ids).squeeze(0)
122
123
# Prompt embeddings