@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65097ca0af
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Doc] Add model development API Reference (#11884)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 9, 2025 at 17:43 UTC 65097ca0af5c1d7caa3d9d8224fa8b4790a5f7bc
9 files changed +54 -9
.buildkite/test-pipeline.yaml
+1 -1
@@ -38,7 +38,7 @@ steps:
38 - pip install -r requirements-docs.txt
39 - SPHINXOPTS=\"-W\" make html
40 # Check API reference (if it fails, you may have missing mock imports)
41 - - grep \"sig sig-object py\" build/html/api/params.html
41 + - grep \"sig sig-object py\" build/html/api/inference_params.html
42
43 - label: Async Engine, Inputs, Utils, Worker Test # 24min
44 fast_check: true
docs/source/api/inference_params.md renamed
+2 -3
@@ -1,6 +1,6 @@
1 -# Optional Parameters
1 +# Inference Parameters
2
3 -Optional parameters for vLLM APIs.
3 +Inference parameters for vLLM APIs.
4
5 (sampling-params)=
6
@@ -19,4 +19,3 @@ Optional parameters for vLLM APIs.
19 .. autoclass:: vllm.PoolingParams
20 :members:
21 ```
22 -
docs/source/api/model/adapters.md new
+9
@@ -0,0 +1,9 @@
1 +# Model Adapters
2 +
3 +## Module Contents
4 +
5 +```{eval-rst}
6 +.. automodule:: vllm.model_executor.models.adapters
7 + :members:
8 + :member-order: bysource
9 +```
docs/source/api/model/index.md new
+12
@@ -0,0 +1,12 @@
1 +# Model Development
2 +
3 +## Submodules
4 +
5 +```{toctree}
6 +:maxdepth: 1
7 +
8 +interfaces_base
9 +interfaces
10 +adapters
11 +```
12 +
docs/source/api/model/interfaces.md new
+9
@@ -0,0 +1,9 @@
1 +# Optional Interfaces
2 +
3 +## Module Contents
4 +
5 +```{eval-rst}
6 +.. automodule:: vllm.model_executor.models.interfaces
7 + :members:
8 + :member-order: bysource
9 +```
docs/source/api/model/interfaces_base.md new
+9
@@ -0,0 +1,9 @@
1 +# Base Model Interfaces
2 +
3 +## Module Contents
4 +
5 +```{eval-rst}
6 +.. automodule:: vllm.model_executor.models.interfaces_base
7 + :members:
8 + :member-order: bysource
9 +```
docs/source/index.md
+2 -1
@@ -139,8 +139,9 @@ community/sponsors
139
140 api/offline_inference/index
141 api/engine/index
142 +api/inference_params
143 api/multimodal/index
143 -api/params
144 +api/model/index
145 ```
146
147 % Design Documents: Details about vLLM internals
vllm/model_executor/models/interfaces.py
+7 -4
@@ -38,13 +38,15 @@ class SupportsMultiModal(Protocol):
38 to be merged with text embeddings.
39
40 The output embeddings must be one of the following formats:
41 +
42 - A list or tuple of 2D tensors, where each tensor corresponds to
42 - each input multimodal data item (e.g, image).
43 + each input multimodal data item (e.g, image).
44 - A single 3D tensor, with the batch dimension grouping the 2D tensors.
45
45 - NOTE: The returned multimodal embeddings must be in the same order as
46 - the appearances of their corresponding multimodal data item in the
47 - input prompt.
46 + Note:
47 + The returned multimodal embeddings must be in the same order as
48 + the appearances of their corresponding multimodal data item in the
49 + input prompt.
50 """
51 ...
52
@@ -59,6 +61,7 @@ class SupportsMultiModal(Protocol):
61 ) -> torch.Tensor:
62 ...
63
64 + @overload
65 def get_input_embeddings(
66 self,
67 input_ids: torch.Tensor,
vllm/model_executor/models/interfaces_base.py
+3
@@ -35,6 +35,7 @@ T_co = TypeVar("T_co", default=torch.Tensor, covariant=True)
35
36 @runtime_checkable
37 class VllmModel(Protocol[C_co, T_co]):
38 + """The interface required for all models in vLLM."""
39
40 def __init__(
41 self,
@@ -97,6 +98,7 @@ def is_vllm_model(
98
99 @runtime_checkable
100 class VllmModelForTextGeneration(VllmModel[C_co, T], Protocol[C_co, T]):
101 + """The interface required for all generative models in vLLM."""
102
103 def compute_logits(
104 self,
@@ -142,6 +144,7 @@ def is_text_generation_model(
144
145 @runtime_checkable
146 class VllmModelForPooling(VllmModel[C_co, T], Protocol[C_co, T]):
147 + """The interface required for all pooling models in vLLM."""
148
149 def pooler(
150 self,