@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 316a41ac1d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix encoding_format in examples/openai_embedding_client.py (#6755)

Chang Su committed Jul 24, 2024 at 22:48 UTC 316a41ac1de4e6e46933cadb39b9b7af65b01abd
2 files changed +8 -6
examples/openai_embedding_client.py
+8 -5
@@ -13,11 +13,14 @@ client = OpenAI(
13 models = client.models.list()
14 model = models.data[0].id
15
16 -responses = client.embeddings.create(input=[
17 - "Hello my name is",
18 - "The best thing about vLLM is that it supports many different models"
19 -],
20 - model=model)
16 +responses = client.embeddings.create(
17 + input=[
18 + "Hello my name is",
19 + "The best thing about vLLM is that it supports many different models"
20 + ],
21 + model=model,
22 + encoding_format="float",
23 +)
24
25 for data in responses.data:
26 print(data.embedding) # list of float of len 4096
tests/entrypoints/openai/test_embedding.py
-1
@@ -18,7 +18,6 @@ def embedding_server():
18 "--enforce-eager",
19 "--max-model-len",
20 "8192",
21 - "--enforce-eager",
21 ]
22
23 with RemoteOpenAIServer(EMBEDDING_MODEL_NAME, args) as remote_server: