| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Example Python client for embedding API using vLLM API server |
| 4 | NOTE: |
| 5 | start a supported embeddings model server with `vllm serve`, e.g. |
| 6 | vllm serve intfloat/e5-small |
| 7 | """ |
| 8 | |
| 9 | from openai import OpenAI |
| 10 | |
| 11 | # Modify OpenAI's API key and API base to use vLLM's API server. |
| 12 | openai_api_key = "EMPTY" |
| 13 | openai_api_base = "http://localhost:8000/v1" |
| 14 | |
| 15 | |
| 16 | def main(): |
| 17 | client = OpenAI( |
| 18 | # defaults to os.environ.get("OPENAI_API_KEY") |
| 19 | api_key=openai_api_key, |
| 20 | base_url=openai_api_base, |
| 21 | ) |
| 22 | |
| 23 | models = client.models.list() |
| 24 | model = models.data[0].id |
| 25 | |
| 26 | responses = client.embeddings.create( |
| 27 | # ruff: noqa: E501 |
| 28 | input=[ |
| 29 | "Hello my name is", |
| 30 | "The best thing about vLLM is that it supports many different models", |
| 31 | ], |
| 32 | model=model, |
| 33 | ) |
| 34 | |
| 35 | for data in responses.data: |
| 36 | print(data.embedding) # List of float of len 4096 |
| 37 | |
| 38 | |
| 39 | if __name__ == "__main__": |
| 40 | main() |