@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0920ab9131
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Doc] Reorganize online pooling APIs (#11172)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 14, 2024 at 00:22 UTC 0920ab9131274df143cfc49245409378a009b3c6
10 files changed +368 -288
docs/source/models/pooling_models.rst
+7 -7
@@ -50,10 +50,10 @@ It returns the extracted hidden states directly, which is useful for reward mode
50 .. code-block:: python
51
52 llm = LLM(model="Qwen/Qwen2.5-Math-RM-72B", task="reward")
53 - output, = llm.encode("Hello, my name is")
53 + (output,) = llm.encode("Hello, my name is")
54
55 data = output.outputs.data
56 - print(f"Prompt: {prompt!r} | Data: {data!r}")
56 + print(f"Data: {data!r}")
57
58 ``LLM.embed``
59 ^^^^^^^^^^^^^
@@ -64,7 +64,7 @@ It is primarily designed for embedding models.
64 .. code-block:: python
65
66 llm = LLM(model="intfloat/e5-mistral-7b-instruct", task="embed")
67 - output, = llm.embed("Hello, my name is")
67 + (output,) = llm.embed("Hello, my name is")
68
69 embeds = output.outputs.embedding
70 print(f"Embeddings: {embeds!r} (size={len(embeds)})")
@@ -80,7 +80,7 @@ It is primarily designed for classification models.
80 .. code-block:: python
81
82 llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", task="classify")
83 - output, = llm.classify("Hello, my name is")
83 + (output,) = llm.classify("Hello, my name is")
84
85 probs = output.outputs.probs
86 print(f"Class Probabilities: {probs!r} (size={len(probs)})")
@@ -102,8 +102,8 @@ These types of models serve as rerankers between candidate query-document pairs
102 .. code-block:: python
103
104 llm = LLM(model="BAAI/bge-reranker-v2-m3", task="score")
105 - output, = llm.score("What is the capital of France?",
106 - "The capital of Brazil is Brasilia.")
105 + (output,) = llm.score("What is the capital of France?",
106 + "The capital of Brazil is Brasilia.")
107
108 score = output.outputs.score
109 print(f"Score: {score}")
@@ -119,7 +119,7 @@ Please click on the above link for more details on how to launch the server.
119 Embeddings API
120 ^^^^^^^^^^^^^^
121
122 -Our Embeddings API is similar to ``LLM.encode``, accepting both text and :ref:`multi-modal inputs <multimodal_inputs>`.
122 +Our Embeddings API is similar to ``LLM.embed``, accepting both text and :ref:`multi-modal inputs <multimodal_inputs>`.
123
124 The text-only API is compatible with `OpenAI Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`__
125 so that you can use OpenAI client to interact with it.
docs/source/serving/openai_compatible_server.md
+267 -195
@@ -1,13 +1,13 @@
1 # OpenAI Compatible Server
2
3 -vLLM provides an HTTP server that implements OpenAI's [Completions](https://platform.openai.com/docs/api-reference/completions) and [Chat](https://platform.openai.com/docs/api-reference/chat) API.
3 +vLLM provides an HTTP server that implements OpenAI's [Completions](https://platform.openai.com/docs/api-reference/completions) and [Chat](https://platform.openai.com/docs/api-reference/chat) API, and more!
4
5 -You can start the server using Python, or using [Docker](deploying_with_docker.rst):
5 +You can start the server via the [`vllm serve`](#vllm-serve) command, or through [Docker](deploying_with_docker.rst):
6 ```bash
7 vllm serve NousResearch/Meta-Llama-3-8B-Instruct --dtype auto --api-key token-abc123
8 ```
9
10 -To call the server, you can use the official OpenAI Python client library, or any other HTTP client.
10 +To call the server, you can use the [official OpenAI Python client](https://github.com/openai/openai-python), or any other HTTP client.
11 ```python
12 from openai import OpenAI
13 client = OpenAI(
@@ -25,166 +25,76 @@ completion = client.chat.completions.create(
25 print(completion.choices[0].message)
26 ```
27
28 -## API Reference
28 +## Supported APIs
29
30 We currently support the following OpenAI APIs:
31
32 -- [Completions API](https://platform.openai.com/docs/api-reference/completions)
32 +- [Completions API](#completions-api) (`/v1/completions`)
33 + - Only applicable to [text generation models](../models/generative_models.rst) (`--task generate`).
34 - *Note: `suffix` parameter is not supported.*
34 -- [Chat Completions API](https://platform.openai.com/docs/api-reference/chat)
35 +- [Chat Completions API](#chat-api) (`/v1/chat/completions`)
36 + - Only applicable to [text generation models](../models/generative_models.rst) (`--task generate`) with a [chat template](#chat-template).
37 - [Vision](https://platform.openai.com/docs/guides/vision)-related parameters are supported; see [Multimodal Inputs](../usage/multimodal_inputs.rst).
38 - *Note: `image_url.detail` parameter is not supported.*
39 - We also support `audio_url` content type for audio files.
40 - Refer to [vllm.entrypoints.chat_utils](https://github.com/vllm-project/vllm/tree/main/vllm/entrypoints/chat_utils.py) for the exact schema.
41 - *TODO: Support `input_audio` content type as defined [here](https://github.com/openai/openai-python/blob/v1.52.2/src/openai/types/chat/chat_completion_content_part_input_audio_param.py).*
42 - *Note: `parallel_tool_calls` and `user` parameters are ignored.*
41 -- [Embeddings API](https://platform.openai.com/docs/api-reference/embeddings)
42 - - Instead of `inputs`, you can pass in a list of `messages` (same schema as Chat Completions API),
43 - which will be treated as a single prompt to the model according to its chat template.
44 - - This enables multi-modal inputs to be passed to embedding models, see [this page](../usage/multimodal_inputs.rst) for details.
45 - - *Note: You should run `vllm serve` with `--task embedding` to ensure that the model is being run in embedding mode.*
46 -
47 -## Score API for Cross Encoder Models
43 +- [Embeddings API](#embeddings-api) (`/v1/embeddings`)
44 + - Only applicable to [embedding models](../models/pooling_models.rst) (`--task embed`).
45
49 -vLLM supports *cross encoders models* at the **/v1/score** endpoint, which is not an OpenAI API standard endpoint. You can find the documentation for these kind of models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html).
46 +In addition, we have the following custom APIs:
47
51 -A ***Cross Encoder*** takes exactly two sentences / texts as input and either predicts a score or label for this sentence pair. It can for example predict the similarity of the sentence pair on a scale of 0 … 1.
48 +- [Tokenizer API](#tokenizer-api) (`/tokenize`, `/detokenize`)
49 + - Applicable to any model with a tokenizer.
50 +- [Score API](#score-api) (`/score`)
51 + - Only applicable to [cross-encoder models](../models/pooling_models.rst) (`--task score`).
52
53 -### Example of usage for a pair of a string and a list of texts
53 +(chat-template)=
54 +## Chat Template
55
55 -In this case, the model will compare the first given text to each of the texts containing the list.
56 +In order for the language model to support chat protocol, vLLM requires the model to include
57 +a chat template in its tokenizer configuration. The chat template is a Jinja2 template that
58 +specifies how are roles, messages, and other chat-specific tokens are encoded in the input.
59
57 -```bash
58 -curl -X 'POST' \
59 - 'http://127.0.0.1:8000/v1/score' \
60 - -H 'accept: application/json' \
61 - -H 'Content-Type: application/json' \
62 - -d '{
63 - "model": "BAAI/bge-reranker-v2-m3",
64 - "text_1": "What is the capital of France?",
65 - "text_2": [
66 - "The capital of Brazil is Brasilia.",
67 - "The capital of France is Paris."
68 - ]
69 -}'
70 -```
60 +An example chat template for `NousResearch/Meta-Llama-3-8B-Instruct` can be found [here](https://github.com/meta-llama/llama3?tab=readme-ov-file#instruction-tuned-models)
61
72 -Response:
62 +Some models do not provide a chat template even though they are instruction/chat fine-tuned. For those model,
63 +you can manually specify their chat template in the `--chat-template` parameter with the file path to the chat
64 +template, or the template in string form. Without a chat template, the server will not be able to process chat
65 +and all chat requests will error.
66
67 ```bash
75 -{
76 - "id": "score-request-id",
77 - "object": "list",
78 - "created": 693570,
79 - "model": "BAAI/bge-reranker-v2-m3",
80 - "data": [
81 - {
82 - "index": 0,
83 - "object": "score",
84 - "score": [
85 - 0.001094818115234375
86 - ]
87 - },
88 - {
89 - "index": 1,
90 - "object": "score",
91 - "score": [
92 - 1
93 - ]
94 - }
95 - ],
96 - "usage": {}
97 -}
68 +vllm serve <model> --chat-template ./path-to-chat-template.jinja
69 ```
70
100 -### Example of usage for a pair of two lists of texts
101 -
102 -In this case, the model will compare the one by one, making pairs by same index correspondent in each list.
71 +vLLM community provides a set of chat templates for popular models. You can find them in the examples
72 +directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
73
104 -```bash
105 -curl -X 'POST' \
106 - 'http://127.0.0.1:8000/v1/score' \
107 - -H 'accept: application/json' \
108 - -H 'Content-Type: application/json' \
109 - -d '{
110 - "model": "BAAI/bge-reranker-v2-m3",
111 - "encoding_format": "float",
112 - "text_1": [
113 - "What is the capital of Brazil?",
114 - "What is the capital of France?"
115 - ],
116 - "text_2": [
117 - "The capital of Brazil is Brasilia.",
118 - "The capital of France is Paris."
74 +With the inclusion of multi-modal chat APIs, the OpenAI spec now accepts chat messages in a new format which specifies
75 +both a `type` and a `text` field. An example is provided below:
76 +```python
77 +completion = client.chat.completions.create(
78 + model="NousResearch/Meta-Llama-3-8B-Instruct",
79 + messages=[
80 + {"role": "user", "content": [{"type": "text", "text": "Classify this sentiment: vLLM is wonderful!"}]}
81 ]
120 -}'
121 -```
122 -
123 -Response:
124 -
125 -```bash
126 -{
127 - "id": "score-request-id",
128 - "object": "list",
129 - "created": 693447,
130 - "model": "BAAI/bge-reranker-v2-m3",
131 - "data": [
132 - {
133 - "index": 0,
134 - "object": "score",
135 - "score": [
136 - 1
137 - ]
138 - },
139 - {
140 - "index": 1,
141 - "object": "score",
142 - "score": [
143 - 1
144 - ]
145 - }
146 - ],
147 - "usage": {}
148 -}
82 +)
83 ```
84
151 -### Example of usage for a pair of two strings
152 -
153 -In this case, the model will compare the strings of texts.
154 -
155 -```bash
156 -curl -X 'POST' \
157 - 'http://127.0.0.1:8000/v1/score' \
158 - -H 'accept: application/json' \
159 - -H 'Content-Type: application/json' \
160 - -d '{
161 - "model": "BAAI/bge-reranker-v2-m3",
162 - "encoding_format": "float",
163 - "text_1": "What is the capital of France?",
164 - "text_2": "The capital of France is Paris."
165 -}'
166 -```
85 +Most chat templates for LLMs expect the `content` field to be a string, but there are some newer models like
86 +`meta-llama/Llama-Guard-3-1B` that expect the content to be formatted according to the OpenAI schema in the
87 +request. vLLM provides best-effort support to detect this automatically, which is logged as a string like
88 +*"Detected the chat template content format to be..."*, and internally converts incoming requests to match
89 +the detected format, which can be one of:
90
168 -Response:
91 +- `"string"`: A string.
92 + - Example: `"Hello world"`
93 +- `"openai"`: A list of dictionaries, similar to OpenAI schema.
94 + - Example: `[{"type": "text", "text": "Hello world!"}]`
95
170 -```bash
171 -{
172 - "id": "score-request-id",
173 - "object": "list",
174 - "created": 693447,
175 - "model": "BAAI/bge-reranker-v2-m3",
176 - "data": [
177 - {
178 - "index": 0,
179 - "object": "score",
180 - "score": [
181 - 1
182 - ]
183 - }
184 - ],
185 - "usage": {}
186 -}
187 -```
96 +If the result is not what you expect, you can set the `--chat-template-content-format` CLI argument
97 +to override which format to use.
98
99 ## Extra Parameters
100
@@ -204,7 +114,7 @@ completion = client.chat.completions.create(
114 )
115 ```
116
207 -### Extra HTTP Headers
117 +## Extra HTTP Headers
118
119 Only `X-Request-Id` HTTP request header is supported for now.
120
@@ -230,7 +140,53 @@ completion = client.completions.create(
140 print(completion._request_id)
141 ```
142
233 -### Extra Parameters for Completions API
143 +## CLI Reference
144 +
145 +(vllm-serve)=
146 +### `vllm serve`
147 +
148 +The `vllm serve` command is used to launch the OpenAI-compatible server.
149 +
150 +```{argparse}
151 +:module: vllm.entrypoints.openai.cli_args
152 +:func: create_parser_for_docs
153 +:prog: vllm serve
154 +```
155 +
156 +#### Configuration file
157 +
158 +You can load CLI arguments via a [YAML](https://yaml.org/) config file.
159 +The argument names must be the long form of those outlined [above](#vllm-serve).
160 +
161 +For example:
162 +
163 +```yaml
164 +# config.yaml
165 +
166 +host: "127.0.0.1"
167 +port: 6379
168 +uvicorn-log-level: "info"
169 +```
170 +
171 +To use the above config file:
172 +
173 +```bash
174 +$ vllm serve SOME_MODEL --config config.yaml
175 +```
176 +
177 +```{note}
178 +In case an argument is supplied simultaneously using command line and the config file, the value from the command line will take precedence.
179 +The order of priorities is `command line > config file values > defaults`.
180 +```
181 +
182 +## API Reference
183 +
184 +(completions-api)=
185 +### Completions API
186 +
187 +Refer to [OpenAI's API reference](https://platform.openai.com/docs/api-reference/completions) for more details.
188 +
189 +#### Extra parameters
190
191 The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
192
@@ -248,7 +204,12 @@ The following extra parameters are supported:
204 :end-before: end-completion-extra-params
205 ```
206
251 -### Extra Parameters for Chat Completions API
207 +(chat-api)=
208 +### Chat Completions API
209 +
210 +Refer to [OpenAI's API reference](https://platform.openai.com/docs/api-reference/chat) for more details.
211 +
212 +#### Extra parameters
213
214 The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
215
@@ -266,7 +227,19 @@ The following extra parameters are supported:
227 :end-before: end-chat-completion-extra-params
228 ```
229
269 -### Extra Parameters for Embeddings API
230 +(embeddings-api)=
231 +### Embeddings API
232 +
233 +Refer to [OpenAI's API reference](https://platform.openai.com/docs/api-reference/embeddings) for more details.
234 +
235 +If the model has a [chat template](#chat-template), you can replace `inputs` with a list of `messages` (same schema as [Chat Completions API](#chat-api))
236 +which will be treated as a single prompt to the model.
237 +
238 +```{tip}
239 +This enables multi-modal inputs to be passed to embedding models, see [this page](../usage/multimodal_inputs.rst) for details.
240 +```
241 +
242 +#### Extra parameters
243
244 The following [pooling parameters (click through to see documentation)](../dev/pooling_params.rst) are supported.
245
@@ -276,7 +249,7 @@ The following [pooling parameters (click through to see documentation)](../dev/p
249 :end-before: end-embedding-pooling-params
250 ```
251
279 -The following extra parameters are supported:
252 +The following extra parameters are supported by default:
253
254 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
255 :language: python
@@ -284,80 +257,179 @@ The following extra parameters are supported:
257 :end-before: end-embedding-extra-params
258 ```
259
287 -## Chat Template
260 +For chat-like input (i.e. if `messages` is passed), these extra parameters are supported instead:
261
289 -In order for the language model to support chat protocol, vLLM requires the model to include
290 -a chat template in its tokenizer configuration. The chat template is a Jinja2 template that
291 -specifies how are roles, messages, and other chat-specific tokens are encoded in the input.
262 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
263 +:language: python
264 +:start-after: begin-chat-embedding-extra-params
265 +:end-before: end-chat-embedding-extra-params
266 +```
267
293 -An example chat template for `NousResearch/Meta-Llama-3-8B-Instruct` can be found [here](https://github.com/meta-llama/llama3?tab=readme-ov-file#instruction-tuned-models)
268 +(tokenizer-api)=
269 +### Tokenizer API
270
295 -Some models do not provide a chat template even though they are instruction/chat fine-tuned. For those model,
296 -you can manually specify their chat template in the `--chat-template` parameter with the file path to the chat
297 -template, or the template in string form. Without a chat template, the server will not be able to process chat
298 -and all chat requests will error.
271 +The Tokenizer API is a simple wrapper over [HuggingFace-style tokenizers](https://huggingface.co/docs/transformers/en/main_classes/tokenizer).
272 +It consists of two endpoints:
273 +
274 +- `/tokenize` corresponds to calling `tokenizer.encode()`.
275 +- `/detokenize` corresponds to calling `tokenizer.decode()`.
276 +
277 +(score-api)=
278 +### Score API
279 +
280 +The Score API applies a cross-encoder model to predict scores for sentence pairs.
281 +Usually, the score for a sentence pair refers to the similarity between two sentences, on a scale of 0 to 1.
282 +
283 +You can find the documentation for these kind of models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html).
284 +
285 +#### Single inference
286 +
287 +You can pass a string to both `text_1` and `text_2`, forming a single sentence pair.
288 +
289 +Request:
290
291 ```bash
301 -vllm serve <model> --chat-template ./path-to-chat-template.jinja
292 +curl -X 'POST' \
293 + 'http://127.0.0.1:8000/score' \
294 + -H 'accept: application/json' \
295 + -H 'Content-Type: application/json' \
296 + -d '{
297 + "model": "BAAI/bge-reranker-v2-m3",
298 + "encoding_format": "float",
299 + "text_1": "What is the capital of France?",
300 + "text_2": "The capital of France is Paris."
301 +}'
302 ```
303
304 -vLLM community provides a set of chat templates for popular models. You can find them in the examples
305 -directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
304 +Response:
305
307 -With the inclusion of multi-modal chat APIs, the OpenAI spec now accepts chat messages in a new format which specifies
308 -both a `type` and a `text` field. An example is provided below:
309 -```python
310 -completion = client.chat.completions.create(
311 - model="NousResearch/Meta-Llama-3-8B-Instruct",
312 - messages=[
313 - {"role": "user", "content": [{"type": "text", "text": "Classify this sentiment: vLLM is wonderful!"}]}
314 - ]
315 -)
306 +```bash
307 +{
308 + "id": "score-request-id",
309 + "object": "list",
310 + "created": 693447,
311 + "model": "BAAI/bge-reranker-v2-m3",
312 + "data": [
313 + {
314 + "index": 0,
315 + "object": "score",
316 + "score": 1
317 + }
318 + ],
319 + "usage": {}
320 +}
321 ```
322
318 -Most chat templates for LLMs expect the `content` field to be a string, but there are some newer models like
319 -`meta-llama/Llama-Guard-3-1B` that expect the content to be formatted according to the OpenAI schema in the
320 -request. vLLM provides best-effort support to detect this automatically, which is logged as a string like
321 -*"Detected the chat template content format to be..."*, and internally converts incoming requests to match
322 -the detected format, which can be one of:
323 +#### Batch inference
324
324 -- `"string"`: A string.
325 - - Example: `"Hello world"`
326 -- `"openai"`: A list of dictionaries, similar to OpenAI schema.
327 - - Example: `[{"type": "text", "text": "Hello world!"}]`
325 +You can pass a string to `text_1` and a list to `text_2`, forming multiple sentence pairs
326 +where each pair is built from `text_1` and a string in `text_2`.
327 +The total number of pairs is `len(text_2)`.
328
329 -If the result is not what you expect, you can set the `--chat-template-content-format` CLI argument
330 -to override which format to use.
329 +Request:
330
332 -## Command line arguments for the server
331 +```bash
332 +curl -X 'POST' \
333 + 'http://127.0.0.1:8000/score' \
334 + -H 'accept: application/json' \
335 + -H 'Content-Type: application/json' \
336 + -d '{
337 + "model": "BAAI/bge-reranker-v2-m3",
338 + "text_1": "What is the capital of France?",
339 + "text_2": [
340 + "The capital of Brazil is Brasilia.",
341 + "The capital of France is Paris."
342 + ]
343 +}'
344 +```
345
334 -```{argparse}
335 -:module: vllm.entrypoints.openai.cli_args
336 -:func: create_parser_for_docs
337 -:prog: vllm serve
346 +Response:
347 +
348 +```bash
349 +{
350 + "id": "score-request-id",
351 + "object": "list",
352 + "created": 693570,
353 + "model": "BAAI/bge-reranker-v2-m3",
354 + "data": [
355 + {
356 + "index": 0,
357 + "object": "score",
358 + "score": 0.001094818115234375
359 + },
360 + {
361 + "index": 1,
362 + "object": "score",
363 + "score": 1
364 + }
365 + ],
366 + "usage": {}
367 +}
368 ```
369
370 +You can pass a list to both `text_1` and `text_2`, forming multiple sentence pairs
371 +where each pair is built from a string in `text_1` and the corresponding string in `text_2` (similar to `zip()`).
372 +The total number of pairs is `len(text_2)`.
373 +
374 +Request:
375 +
376 +```bash
377 +curl -X 'POST' \
378 + 'http://127.0.0.1:8000/score' \
379 + -H 'accept: application/json' \
380 + -H 'Content-Type: application/json' \
381 + -d '{
382 + "model": "BAAI/bge-reranker-v2-m3",
383 + "encoding_format": "float",
384 + "text_1": [
385 + "What is the capital of Brazil?",
386 + "What is the capital of France?"
387 + ],
388 + "text_2": [
389 + "The capital of Brazil is Brasilia.",
390 + "The capital of France is Paris."
391 + ]
392 +}'
393 +```
394
341 -### Config file
395 +Response:
396
343 -The `serve` module can also accept arguments from a config file in
344 -`yaml` format. The arguments in the yaml must be specified using the
345 -long form of the argument outlined [here](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#command-line-arguments-for-the-server):
397 +```bash
398 +{
399 + "id": "score-request-id",
400 + "object": "list",
401 + "created": 693447,
402 + "model": "BAAI/bge-reranker-v2-m3",
403 + "data": [
404 + {
405 + "index": 0,
406 + "object": "score",
407 + "score": 1
408 + },
409 + {
410 + "index": 1,
411 + "object": "score",
412 + "score": 1
413 + }
414 + ],
415 + "usage": {}
416 +}
417 +```
418
347 -For example:
419 +#### Extra parameters
420
349 -```yaml
350 -# config.yaml
421 +The following [pooling parameters (click through to see documentation)](../dev/pooling_params.rst) are supported.
422
352 -host: "127.0.0.1"
353 -port: 6379
354 -uvicorn-log-level: "info"
423 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
424 +:language: python
425 +:start-after: begin-score-pooling-params
426 +:end-before: end-score-pooling-params
427 ```
428
357 -```bash
358 -$ vllm serve SOME_MODEL --config config.yaml
429 +The following extra parameters are supported:
430 +
431 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
432 +:language: python
433 +:start-after: begin-score-extra-params
434 +:end-before: end-score-extra-params
435 ```
360 ----
361 -**NOTE**
362 -In case an argument is supplied simultaneously using command line and the config file, the value from the commandline will take precedence.
363 -The order of priorities is `command line > config file values > defaults`.
docs/source/usage/multimodal_inputs.rst
+4 -4
@@ -345,12 +345,12 @@ Here is an end-to-end example using VLM2Vec. To serve the model:
345
346 .. code-block:: bash
347
348 - vllm serve TIGER-Lab/VLM2Vec-Full --task embedding \
348 + vllm serve TIGER-Lab/VLM2Vec-Full --task embed \
349 --trust-remote-code --max-model-len 4096 --chat-template examples/template_vlm2vec.jinja
350
351 .. important::
352
353 - Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass ``--task embedding``
353 + Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass ``--task embed``
354 to run this model in embedding mode instead of text generation mode.
355
356 The custom chat template is completely different from the original one for this model,
@@ -386,12 +386,12 @@ Below is another example, this time using the ``MrLight/dse-qwen2-2b-mrl-v1`` mo
386
387 .. code-block:: bash
388
389 - vllm serve MrLight/dse-qwen2-2b-mrl-v1 --task embedding \
389 + vllm serve MrLight/dse-qwen2-2b-mrl-v1 --task embed \
390 --trust-remote-code --max-model-len 8192 --chat-template examples/template_dse_qwen2_vl.jinja
391
392 .. important::
393
394 - Like with VLM2Vec, we have to explicitly pass ``--task embedding``.
394 + Like with VLM2Vec, we have to explicitly pass ``--task embed``.
395
396 Additionally, ``MrLight/dse-qwen2-2b-mrl-v1`` requires an EOS token for embeddings, which is handled
397 by `this custom chat template <https://github.com/vllm-project/vllm/blob/main/examples/template_dse_qwen2_vl.jinja>`__.
examples/offline_inference_openai.md
+46 -46
@@ -1,45 +1,48 @@
1 # Offline Inference with the OpenAI Batch file format
2
3 - **NOTE:** This is a guide to performing batch inference using the OpenAI batch file format, **NOT** the complete Batch (REST) API.
4 -
5 - ## File Format
6 -
7 - The OpenAI batch file format consists of a series of json objects on new lines.
3 +```{important}
4 +This is a guide to performing batch inference using the OpenAI batch file format, **not** the complete Batch (REST) API.
5 +```
6 +
7 +## File Format
8
9 - [See here for an example file.](https://github.com/vllm-project/vllm/blob/main/examples/openai_example_batch.jsonl)
9 +The OpenAI batch file format consists of a series of json objects on new lines.
10
11 - Each line represents a separate request. See the [OpenAI package reference](https://platform.openai.com/docs/api-reference/batch/requestInput) for more details.
11 +[See here for an example file.](https://github.com/vllm-project/vllm/blob/main/examples/openai_example_batch.jsonl)
12
13 - **NOTE:** We currently only support `/v1/chat/completions` and `/v1/embeddings` endpoints (completions coming soon).
13 +Each line represents a separate request. See the [OpenAI package reference](https://platform.openai.com/docs/api-reference/batch/requestInput) for more details.
14
15 - ## Pre-requisites
15 +```{note}
16 +We currently only support `/v1/chat/completions` and `/v1/embeddings` endpoints (completions coming soon).
17 +```
18
17 -* Ensure you are using `vllm >= 0.4.3`. You can check by running `python -c "import vllm; print(vllm.__version__)"`.
19 +## Pre-requisites
20 +
21 * The examples in this document use `meta-llama/Meta-Llama-3-8B-Instruct`.
22 - Create a [user access token](https://huggingface.co/docs/hub/en/security-tokens)
23 - Install the token on your machine (Run `huggingface-cli login`).
24 - Get access to the gated model by [visiting the model card](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) and agreeing to the terms and conditions.
25
26
24 - ## Example 1: Running with a local file
25 -
26 - ### Step 1: Create your batch file
27 -
28 - To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
29 -
30 - ```
31 - wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/openai_example_batch.jsonl
32 - ```
33 -
34 - Once you've created your batch file it should look like this
35 -
36 - ```
37 - $ cat openai_example_batch.jsonl
27 +## Example 1: Running with a local file
28 +
29 +### Step 1: Create your batch file
30 +
31 +To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
32 +
33 +```
34 +wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/openai_example_batch.jsonl
35 +```
36 +
37 +Once you've created your batch file it should look like this
38 +
39 +```
40 +$ cat openai_example_batch.jsonl
41 {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
42 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
40 - ```
41 -
42 - ### Step 2: Run the batch
43 +```
44 +
45 +### Step 2: Run the batch
46
47 The batch running tool is designed to be used from the command line.
48
@@ -85,18 +88,18 @@ To integrate with cloud blob storage, we recommend using presigned urls.
88 ### Step 1: Upload your input script
89
90 To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
88 -
89 - ```
90 - wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/openai_example_batch.jsonl
91 - ```
92 -
93 - Once you've created your batch file it should look like this
94 -
95 - ```
96 - $ cat openai_example_batch.jsonl
91 +
92 +```
93 +wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/openai_example_batch.jsonl
94 +```
95 +
96 +Once you've created your batch file it should look like this
97 +
98 +```
99 +$ cat openai_example_batch.jsonl
100 {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
101 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
99 - ```
102 +```
103
104 Now upload your batch file to your S3 bucket.
105
@@ -104,7 +107,6 @@ Now upload your batch file to your S3 bucket.
107 aws s3 cp openai_example_batch.jsonl s3://MY_BUCKET/MY_INPUT_FILE.jsonl
108 ```
109
107 -
110 ### Step 2: Generate your presigned urls
111
112 Presigned urls can only be generated via the SDK. You can run the following python script to generate your presigned urls. Be sure to replace the `MY_BUCKET`, `MY_INPUT_FILE.jsonl`, and `MY_OUTPUT_FILE.jsonl` placeholders with your bucket and file names.
@@ -179,21 +181,19 @@ aws s3 cp s3://MY_BUCKET/MY_OUTPUT_FILE.jsonl -
181
182 ### Step 1: Create your batch file
183
182 - Add embedding requests to your batch file. The following is an example:
184 +Add embedding requests to your batch file. The following is an example:
185
184 - ```
185 - {"custom_id": "request-1", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/e5-mistral-7b-instruct", "input": "You are a helpful assistant."}}
186 +```
187 +{"custom_id": "request-1", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/e5-mistral-7b-instruct", "input": "You are a helpful assistant."}}
188 {"custom_id": "request-2", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/e5-mistral-7b-instruct", "input": "You are an unhelpful assistant."}}
189 ```
188 -
189 - You can even mix chat completion and embedding requests in the batch file, as long as the model you are using supports both chat completion and embeddings (note that all requests must use the same model).
190
191 +You can even mix chat completion and embedding requests in the batch file, as long as the model you are using supports both chat completion and embeddings (note that all requests must use the same model).
192
192 - ### Step 2: Run the batch
193 +### Step 2: Run the batch
194
195 You can run the batch using the same command as in earlier examples.
196
196 -
197 ### Step 3: Check your results
198
199 You can check your results by running `cat results.jsonl`
@@ -201,5 +201,5 @@ You can check your results by running `cat results.jsonl`
201 ```
202 $ cat results.jsonl
203 {"id":"vllm-db0f71f7dec244e6bce530e0b4ef908b","custom_id":"request-1","response":{"status_code":200,"request_id":"vllm-batch-3580bf4d4ae54d52b67eee266a6eab20","body":{"id":"embd-33ac2efa7996430184461f2e38529746","object":"list","created":444647,"model":"intfloat/e5-mistral-7b-instruct","data":[{"index":0,"object":"embedding","embedding":[0.016204833984375,0.0092010498046875,0.0018358230590820312,-0.0028228759765625,0.001422882080078125,-0.0031147003173828125,...]}],"usage":{"prompt_tokens":8,"total_tokens":8,"completion_tokens":0}}},"error":null}
204 -...```
204 +...
205 ```
examples/openai_chat_embedding_client_for_multimodal.py
+1 -1
@@ -99,7 +99,7 @@ def dse_qwen2_vl(inp: dict):
99 if __name__ == '__main__':
100 parser = argparse.ArgumentParser(
101 "Script to call a specified VLM through the API. Make sure to serve "
102 - "the model with --task embedding before running this.")
102 + "the model with --task embed before running this.")
103 parser.add_argument("model",
104 type=str,
105 choices=["vlm2vec", "dse_qwen2_vl"],
examples/openai_cross_encoder_score.py
+18 -17
@@ -1,14 +1,15 @@
1 -"""Examples Python client Score for Cross Encoder Models
1 """
2 +Example online usage of Score API.
3
4 +Run `vllm serve <model> --task score` to start up the server in vLLM.
5 +"""
6 import argparse
5 -import json
7 import pprint
8
9 import requests
10
11
11 -def post_http_request(prompt: json, api_url: str) -> requests.Response:
12 +def post_http_request(prompt: dict, api_url: str) -> requests.Response:
13 headers = {"User-Agent": "Test Client"}
14 response = requests.post(api_url, headers=headers, json=prompt)
15 return response
@@ -20,20 +21,29 @@ if __name__ == "__main__":
21 parser.add_argument("--port", type=int, default=8000)
22 parser.add_argument("--model", type=str, default="BAAI/bge-reranker-v2-m3")
23 args = parser.parse_args()
23 - api_url = f"http://{args.host}:{args.port}/v1/score"
24 + api_url = f"http://{args.host}:{args.port}/score"
25
26 model_name = args.model
27
28 + text_1 = "What is the capital of Brazil?"
29 + text_2 = "The capital of Brazil is Brasilia."
30 + prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
31 + score_response = post_http_request(prompt=prompt, api_url=api_url)
32 + print("Prompt when text_1 and text_2 are both strings:")
33 + pprint.pprint(prompt)
34 + print("Score Response:")
35 + pprint.pprint(score_response.json())
36 +
37 text_1 = "What is the capital of France?"
38 text_2 = [
39 "The capital of Brazil is Brasilia.", "The capital of France is Paris."
40 ]
41 prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
42 score_response = post_http_request(prompt=prompt, api_url=api_url)
33 - print("Prompt for text_1 is string and text_2 is a list:")
43 + print("Prompt when text_1 is string and text_2 is a list:")
44 pprint.pprint(prompt)
45 print("Score Response:")
36 - pprint.pprint(score_response.data)
46 + pprint.pprint(score_response.json())
47
48 text_1 = [
49 "What is the capital of Brazil?", "What is the capital of France?"
@@ -43,16 +53,7 @@ if __name__ == "__main__":
53 ]
54 prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
55 score_response = post_http_request(prompt=prompt, api_url=api_url)
46 - print("Prompt for text_1 and text_2 are lists:")
47 - pprint.pprint(prompt)
48 - print("Score Response:")
49 - pprint.pprint(score_response.data)
50 -
51 - text_1 = "What is the capital of Brazil?"
52 - text_2 = "The capital of Brazil is Brasilia."
53 - prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
54 - score_response = post_http_request(prompt=prompt, api_url=api_url)
55 - print("Prompt for text_1 and text_2 are strings:")
56 + print("Prompt when text_1 and text_2 are both lists:")
57 pprint.pprint(prompt)
58 print("Score Response:")
58 - pprint.pprint(score_response.data)
\ No newline at end of file
59 + pprint.pprint(score_response.json())
tests/entrypoints/openai/test_score.py
+3 -3
@@ -27,7 +27,7 @@ async def test_text_1_str_text_2_list(server: RemoteOpenAIServer,
27 "The capital of Brazil is Brasilia.", "The capital of France is Paris."
28 ]
29
30 - score_response = requests.post(server.url_for("v1/score"),
30 + score_response = requests.post(server.url_for("score"),
31 json={
32 "model": model_name,
33 "text_1": text_1,
@@ -55,7 +55,7 @@ async def test_text_1_list_text_2_list(server: RemoteOpenAIServer,
55 "The capital of Brazil is Brasilia.", "The capital of France is Paris."
56 ]
57
58 - score_response = requests.post(server.url_for("v1/score"),
58 + score_response = requests.post(server.url_for("score"),
59 json={
60 "model": model_name,
61 "text_1": text_1,
@@ -78,7 +78,7 @@ async def test_text_1_str_text_2_str(server: RemoteOpenAIServer,
78 text_1 = "What is the capital of France?"
79 text_2 = "The capital of France is Paris."
80
81 - score_response = requests.post(server.url_for("v1/score"),
81 + score_response = requests.post(server.url_for("score"),
82 json={
83 "model": model_name,
84 "text_1": text_1,
vllm/entrypoints/openai/api_server.py
+10 -1
@@ -406,7 +406,7 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
406 assert_never(generator)
407
408
409 -@router.post("/v1/score")
409 +@router.post("/score")
410 async def create_score(request: ScoreRequest, raw_request: Request):
411 handler = score(raw_request)
412 if handler is None:
@@ -423,6 +423,15 @@ async def create_score(request: ScoreRequest, raw_request: Request):
423 assert_never(generator)
424
425
426 +@router.post("/v1/score")
427 +async def create_score_v1(request: ScoreRequest, raw_request: Request):
428 + logger.warning(
429 + "To indicate that Score API is not part of standard OpenAI API, we "
430 + "have moved it to `/score`. Please update your client accordingly.")
431 +
432 + return await create_score(request, raw_request)
433 +
434 +
435 if envs.VLLM_TORCH_PROFILER_DIR:
436 logger.warning(
437 "Torch Profiler is enabled in the API server. This should ONLY be "
vllm/entrypoints/openai/protocol.py
+5 -2
@@ -812,10 +812,11 @@ class ScoreRequest(OpenAIBaseModel):
812 text_2: Union[List[str], str]
813 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
814
815 - # doc: begin-chat-embedding-pooling-params
815 + # doc: begin-score-pooling-params
816 additional_data: Optional[Any] = None
817 - # doc: end-chat-embedding-pooling-params
817 + # doc: end-score-pooling-params
818
819 + # doc: begin-score-extra-params
820 priority: int = Field(
821 default=0,
822 description=(
@@ -823,6 +824,8 @@ class ScoreRequest(OpenAIBaseModel):
824 "default: 0). Any priority other than 0 will raise an error "
825 "if the served model does not use priority scheduling."))
826
827 + # doc: end-score-extra-params
828 +
829 def to_pooling_params(self):
830 return PoolingParams(additional_data=self.additional_data)
831
vllm/outputs.py
+7 -12
@@ -1,12 +1,11 @@
1 import time
2 -import warnings
2 from dataclasses import dataclass
3 from typing import Dict, Generic, List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Union
6
7 import torch
9 -from typing_extensions import TypeVar
8 +from typing_extensions import TypeVar, deprecated
9
10 from vllm.lora.request import LoRARequest
11 from vllm.multimodal.inputs import MultiModalPlaceholderDict
@@ -73,13 +72,11 @@ class PoolingOutput:
72 (self.data == other.data).all()))
73
74 @property
75 + @deprecated("`LLM.encode()` now stores raw outputs in the `data` "
76 + "attribute. To return embeddings, use `LLM.embed()`. "
77 + "To return class probabilities, use `LLM.classify()` "
78 + "and access the `probs` attribute. ")
79 def embedding(self) -> list[float]:
77 - msg = ("`LLM.encode()` now returns raw outputs. "
78 - "To return embeddings, use `LLM.embed()`. "
79 - "To return class probabilities, use `LLM.classify()` "
80 - "and access the `probs` attribute. ")
81 - warnings.warn(msg, DeprecationWarning, stacklevel=2)
82 -
80 return self.data.tolist()
81
82
@@ -491,11 +488,9 @@ class ScoringOutput:
488 return f"ScoringOutput(score={self.score})"
489
490 @property
491 + @deprecated("`LLM.score()` now returns scalar scores. "
492 + "Please access it via the `score` attribute. ")
493 def embedding(self) -> list[float]:
495 - msg = ("`LLM.score()` now returns scalar scores. "
496 - "Please access it via the `score` attribute. ")
497 - warnings.warn(msg, DeprecationWarning, stacklevel=2)
498 -
494 return [self.score]
495
496