A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Doc] Add model development API Reference (#11884)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 9, 2025 at 17:43 UTC
65097ca0af5c1d7caa3d9d8224fa8b4790a5f7bc
9 files changed
+54
-9
.buildkite/test-pipeline.yaml
+1
-1
@@ -38,7 +38,7 @@ steps:
38
- pip install -r requirements-docs.txt
39
- SPHINXOPTS=\"-W\" make html
40
# Check API reference (if it fails, you may have missing mock imports)
41
- - grep \"sig sig-object py\" build/html/api/params.html
41
+ - grep \"sig sig-object py\" build/html/api/inference_params.html
42
43
- label: Async Engine, Inputs, Utils, Worker Test # 24min
44
fast_check: true
docs/source/api/inference_params.md
renamed
+2
-3
@@ -1,6 +1,6 @@
1
-# Optional Parameters
1
+# Inference Parameters
2
3
-Optional parameters for vLLM APIs.
3
+Inference parameters for vLLM APIs.
4
5
(sampling-params)=
6
@@ -19,4 +19,3 @@ Optional parameters for vLLM APIs.
19
.. autoclass:: vllm.PoolingParams
20
:members:
21
```
22
-
docs/source/api/model/adapters.md
new
+9
@@ -0,0 +1,9 @@
1
+# Model Adapters
2
+
3
+## Module Contents
4
+
5
+```{eval-rst}
6
+.. automodule:: vllm.model_executor.models.adapters
7
+ :members:
8
+ :member-order: bysource
9
+```
docs/source/api/model/index.md
new
+12
@@ -0,0 +1,12 @@
1
+# Model Development
2
+
3
+## Submodules
4
+
5
+```{toctree}
6
+:maxdepth: 1
7
+
8
+interfaces_base
9
+interfaces
10
+adapters
11
+```
12
+
docs/source/api/model/interfaces.md
new
+9
@@ -0,0 +1,9 @@
1
+# Optional Interfaces
2
+
3
+## Module Contents
4
+
5
+```{eval-rst}
6
+.. automodule:: vllm.model_executor.models.interfaces
7
+ :members:
8
+ :member-order: bysource
9
+```
docs/source/api/model/interfaces_base.md
new
+9
@@ -0,0 +1,9 @@
1
+# Base Model Interfaces
2
+
3
+## Module Contents
4
+
5
+```{eval-rst}
6
+.. automodule:: vllm.model_executor.models.interfaces_base
7
+ :members:
8
+ :member-order: bysource
9
+```
docs/source/index.md
+2
-1
@@ -139,8 +139,9 @@ community/sponsors
139
140
api/offline_inference/index
141
api/engine/index
142
+api/inference_params
143
api/multimodal/index
143
-api/params
144
+api/model/index
145
```
146
147
% Design Documents: Details about vLLM internals
vllm/model_executor/models/interfaces.py
+7
-4
@@ -38,13 +38,15 @@ class SupportsMultiModal(Protocol):
38
to be merged with text embeddings.
39
40
The output embeddings must be one of the following formats:
41
+
42
- A list or tuple of 2D tensors, where each tensor corresponds to
42
- each input multimodal data item (e.g, image).
43
+ each input multimodal data item (e.g, image).
44
- A single 3D tensor, with the batch dimension grouping the 2D tensors.
45
45
- NOTE: The returned multimodal embeddings must be in the same order as
46
- the appearances of their corresponding multimodal data item in the
47
- input prompt.
46
+ Note:
47
+ The returned multimodal embeddings must be in the same order as
48
+ the appearances of their corresponding multimodal data item in the
49
+ input prompt.
50
"""
51
...
52
@@ -59,6 +61,7 @@ class SupportsMultiModal(Protocol):
61
) -> torch.Tensor:
62
...
63
64
+ @overload
65
def get_input_embeddings(
66
self,
67
input_ids: torch.Tensor,
vllm/model_executor/models/interfaces_base.py
+3
@@ -35,6 +35,7 @@ T_co = TypeVar("T_co", default=torch.Tensor, covariant=True)
35
36
@runtime_checkable
37
class VllmModel(Protocol[C_co, T_co]):
38
+ """The interface required for all models in vLLM."""
39
40
def __init__(
41
self,
@@ -97,6 +98,7 @@ def is_vllm_model(
98
99
@runtime_checkable
100
class VllmModelForTextGeneration(VllmModel[C_co, T], Protocol[C_co, T]):
101
+ """The interface required for all generative models in vLLM."""
102
103
def compute_logits(
104
self,
@@ -142,6 +144,7 @@ def is_text_generation_model(
144
145
@runtime_checkable
146
class VllmModelForPooling(VllmModel[C_co, T], Protocol[C_co, T]):
147
+ """The interface required for all pooling models in vLLM."""
148
149
def pooler(
150
self,