@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef9676a1f1

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Doc] ruff format some Python examples (#26767)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Oct 14, 2025 at 18:21 UTC ef9676a1f1af444d35160794b79c85594c517a6c

20 files changed +341 -290

docs/configuration/conserving_memory.md

+23 -21

@@ -11,8 +11,7 @@ The following code splits the model across 2 GPUs.

11 ```python

12 from vllm import LLM

14 -llm = LLM(model="ibm-granite/granite-3.1-8b-instruct",

15 - tensor_parallel_size=2)

14 +llm = LLM(model="ibm-granite/granite-3.1-8b-instruct", tensor_parallel_size=2)

15 ```

17 !!! warning

@@ -43,9 +42,7 @@ and the maximum batch size (`max_num_seqs` option).

42 ```python

43 from vllm import LLM

46 -llm = LLM(model="adept/fuyu-8b",

47 - max_model_len=2048,

48 - max_num_seqs=2)

45 +llm = LLM(model="adept/fuyu-8b", max_model_len=2048, max_num_seqs=2)

46 ```

48 ## Reduce CUDA Graphs

@@ -78,8 +75,7 @@ You can disable graph capturing completely via the `enforce_eager` flag:

75 ```python

76 from vllm import LLM

81 -llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct",

82 - enforce_eager=True)

78 +llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct", enforce_eager=True)

79 ```

81 ## Adjust cache size

@@ -97,8 +93,10 @@ You can allow a smaller number of multi-modal items per prompt to reduce the mem

93 from vllm import LLM

95 # Accept up to 3 images and 1 video per prompt

100 -llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

101 - limit_mm_per_prompt={"image": 3, "video": 1})

96 +llm = LLM(

97 + model="Qwen/Qwen2.5-VL-3B-Instruct",

98 + limit_mm_per_prompt={"image": 3, "video": 1},

99 +)

100 ```

101

102 You can go a step further and disable unused modalities completely by setting its limit to zero.

@@ -108,8 +106,10 @@ For example, if your application only accepts image input, there is no need to a

106 from vllm import LLM

107

108 # Accept any number of images but no videos

111 -llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

112 - limit_mm_per_prompt={"video": 0})

109 +llm = LLM(

110 + model="Qwen/Qwen2.5-VL-3B-Instruct",

111 + limit_mm_per_prompt={"video": 0},

112 +)

113 ```

114

115 You can even run a multi-modal model for text-only inference:

@@ -118,8 +118,10 @@ You can even run a multi-modal model for text-only inference:

118 from vllm import LLM

119

120 # Don't accept images. Just text.

121 -llm = LLM(model="google/gemma-3-27b-it",

122 - limit_mm_per_prompt={"image": 0})

121 +llm = LLM(

122 + model="google/gemma-3-27b-it",

123 + limit_mm_per_prompt={"image": 0},

124 +)

125 ```

126

127 ### Configurable options

@@ -173,14 +175,14 @@ Here are some examples:

175 from vllm import LLM

176

177 # Available for Qwen2-VL series models

176 -llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

177 - mm_processor_kwargs={

178 - "max_pixels": 768 * 768, # Default is 1280 * 28 * 28

179 - })

178 +llm = LLM(

179 + model="Qwen/Qwen2.5-VL-3B-Instruct",

180 + mm_processor_kwargs={"max_pixels": 768 * 768}, # Default is 1280 * 28 * 28

181 +)

182

183 # Available for InternVL series models

182 -llm = LLM(model="OpenGVLab/InternVL2-2B",

183 - mm_processor_kwargs={

184 - "max_dynamic_patch": 4, # Default is 12

185 - })

184 +llm = LLM(

185 + model="OpenGVLab/InternVL2-2B",

186 + mm_processor_kwargs={"max_dynamic_patch": 4}, # Default is 12

187 +)

188 ```

docs/configuration/optimization.md

+15 -9

@@ -100,7 +100,7 @@ from vllm import LLM

100 llm = LLM(

101 model="meta-llama/Llama-3.3-70B-Instruct,

102 tensor_parallel_size=4,

103 - pipeline_parallel_size=2

103 + pipeline_parallel_size=2,

104 )

105 ```

106

@@ -257,18 +257,24 @@ Examples:

257

258 ```python

259 # Use a larger cache

260 -llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

261 - mm_processor_cache_gb=8)

260 +llm = LLM(

261 + model="Qwen/Qwen2.5-VL-3B-Instruct",

262 + mm_processor_cache_gb=8,

263 +)

264

265 # Use a shared-memory based IPC cache

264 -llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

265 - tensor_parallel_size=2,

266 - mm_processor_cache_type="shm",

267 - mm_processor_cache_gb=8)

266 +llm = LLM(

267 + model="Qwen/Qwen2.5-VL-3B-Instruct",

268 + tensor_parallel_size=2,

269 + mm_processor_cache_type="shm",

270 + mm_processor_cache_gb=8,

271 +)

272

273 # Disable the cache

270 -llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

271 - mm_processor_cache_gb=0)

274 +llm = LLM(

275 + model="Qwen/Qwen2.5-VL-3B-Instruct",

276 + mm_processor_cache_gb=0,

277 +)

278 ```

279

280 ### Cache Placement

docs/contributing/model/basic.md

+2 -2

@@ -73,8 +73,8 @@ def forward(

73 self,

74 input_ids: torch.Tensor,

75 positions: torch.Tensor,

76 - intermediate_tensors: Optional[IntermediateTensors] = None,

77 - inputs_embeds: Optional[torch.Tensor] = None,

76 + intermediate_tensors: IntermediateTensors | None = None,

77 + inputs_embeds: torch.Tensor | None = None,

78 ) -> torch.Tensor:

79 ...

80 ```

docs/contributing/model/multimodal.md

+23 -19

@@ -16,7 +16,7 @@ Further update the model as follows:

16 ...

18 @classmethod

19 - def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

19 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:

20 if modality.startswith("image"):

21 return "<image>"

@@ -45,14 +45,14 @@ Further update the model as follows:

45 ...

47 def _process_image_input(self, image_input: YourModelImageInputs) -> torch.Tensor:

48 -

48 assert self.vision_encoder is not None

49 image_features = self.vision_encoder(image_input)

50 return self.multi_modal_projector(image_features)

52 def get_multimodal_embeddings(

54 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

55 -

53 + self,

54 + **kwargs: object,

55 + ) -> MultiModalEmbeddings | None:

56 # Validate the multimodal input keyword arguments

57 image_input = self._parse_and_validate_image_input(**kwargs)

58 if image_input is None:

@@ -110,7 +110,7 @@ to return the maximum number of input items for each modality supported by the m

110 For example, if the model supports any number of images but only one video per prompt:

111

112 ```python

113 -def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:

113 +def get_supported_mm_limits(self) -> Mapping[str, int | None]:

114 return {"image": None, "video": 1}

115 ```

116

@@ -258,7 +258,7 @@ Assuming that the memory usage increases with the number of tokens, the dummy in

258 self,

259 seq_len: int,

260 mm_counts: Mapping[str, int],

261 - mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

261 + mm_options: Mapping[str, BaseDummyOptions] | None = None,

262 ) -> MultiModalDataDict:

263 num_images = mm_counts.get("image", 0)

264

@@ -421,8 +421,10 @@ Assuming that the memory usage increases with the number of tokens, the dummy in

421 ```python

422 def get_image_size_with_most_features(self) -> ImageSize:

423 image_processor = self.get_image_processor()

424 - return ImageSize(width=image_processor.size["width"],

425 - height=image_processor.size["height"])

424 + return ImageSize(

425 + width=image_processor.size["width"],

426 + height=image_processor.size["height"],

427 + )

428 ```

429

430 Fuyu does not expect image placeholders in the inputs to HF processor, so

@@ -452,10 +454,12 @@ Assuming that the memory usage increases with the number of tokens, the dummy in

454

455 return {

456 "image":

455 - self._get_dummy_images(width=target_width,

456 - height=target_height,

457 - num_images=num_images,

458 - overrides=image_overrides)

457 + self._get_dummy_images(

458 + width=target_width,

459 + height=target_height,

460 + num_images=num_images,

461 + overrides=image_overrides,

462 + )

463 }

464 ```

465

@@ -744,8 +748,7 @@ Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies

748 image_width=image_size.width,

749 image_height=image_size.height,

750 )

747 - image_tokens = ([_IMAGE_TOKEN_ID] * ncols +

748 - [_NEWLINE_TOKEN_ID]) * nrows

751 + image_tokens = ([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows

752

753 return PromptUpdateDetails.select_token_id(

754 image_tokens + [bos_token_id],

@@ -781,8 +784,7 @@ Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies

784 image_width=image_size.width,

785 image_height=image_size.height,

786 )

784 - image_tokens = ([_IMAGE_TOKEN_ID] * ncols +

785 - [_NEWLINE_TOKEN_ID]) * nrows

787 + image_tokens = ([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows

788

789 return PromptUpdateDetails.select_token_id(

790 image_tokens + [bos_token_id],

@@ -810,9 +812,11 @@ to register them to the multi-modal registry:

812 from vllm.model_executor.models.interfaces import SupportsMultiModal

813 + from vllm.multimodal import MULTIMODAL_REGISTRY

814

813 -+ @MULTIMODAL_REGISTRY.register_processor(YourMultiModalProcessor,

814 -+ info=YourProcessingInfo,

815 -+ dummy_inputs=YourDummyInputsBuilder)

815 ++ @MULTIMODAL_REGISTRY.register_processor(

816 ++ YourMultiModalProcessor,

817 ++ info=YourProcessingInfo,

818 ++ dummy_inputs=YourDummyInputsBuilder,

819 ++ )

820 class YourModelForImage2Seq(nn.Module, SupportsMultiModal):

821 ```

822

docs/contributing/model/registration.md

+1 -1

@@ -42,7 +42,7 @@ def register():

43 ModelRegistry.register_model(

44 "YourModelForCausalLM",

45 - "your_code:YourModelForCausalLM"

45 + "your_code:YourModelForCausalLM",

46 )

47 ```

docs/contributing/model/transcription.md

+11 -1

@@ -15,6 +15,7 @@ Declare supported languages and capabilities:

15 - Set `supports_transcription_only=True` if the model should not serve text generation (eg Whisper).

17 ??? code "supported_languages and supports_transcription_only"

18 +

19 ```python

20 from typing import ClassVar, Mapping, Literal

21 import numpy as np

@@ -43,6 +44,7 @@ Provide an ASR configuration via [get_speech_to_text_config][vllm.model_executor

44 This is for controlling general behavior of the API when serving your model:

46 ??? code "get_speech_to_text_config()"

47 +

48 ```python

49 class YourASRModel(nn.Module, SupportsTranscription):

50 ...

@@ -71,6 +73,7 @@ Implement the prompt construction via [get_generation_prompt][vllm.model_executo

73 Return a dict containing `multi_modal_data` with the audio, and either a `prompt` string or `prompt_token_ids`:

75 ??? code "get_generation_prompt()"

76 +

77 ```python

78 class YourASRModel(nn.Module, SupportsTranscription):

79 ...

@@ -107,6 +110,7 @@ Return a dict containing `multi_modal_data` with the audio, and either a `prompt

110 Return a dict with separate `encoder_prompt` and `decoder_prompt` entries:

111

112 ??? code "get_generation_prompt()"

113 +

114 ```python

115 class YourASRModel(nn.Module, SupportsTranscription):

116 ...

@@ -148,12 +152,16 @@ Language validation via [validate_language][vllm.model_executor.models.interface

152 If your model requires a language and you want a default, override this method (see Whisper):

153

154 ??? code "validate_language()"

155 +

156 ```python

157 @classmethod

158 def validate_language(cls, language: str | None) -> str | None:

159 if language is None:

160 logger.warning(

156 - "Defaulting to language='en'. If you wish to transcribe audio in a different language, pass the `language` field.")

161 + "Defaulting to language='en'. If you wish to transcribe "

162 + "audio in a different language, pass the `language` field "

163 + "in the TranscriptionRequest."

164 + )

165 language = "en"

166 return super().validate_language(language)

167 ```

@@ -165,6 +173,7 @@ Token accounting for streaming via [get_num_audio_tokens][vllm.model_executor.mo

173 Provide a fast duration→token estimate to improve streaming usage statistics:

174

175 ??? code "get_num_audio_tokens()"

176 +

177 ```python

178 class YourASRModel(nn.Module, SupportsTranscription):

179 ...

@@ -191,6 +200,7 @@ The API server takes care of basic audio I/O and optional chunking before buildi

200 Relevant server logic:

201

202 ??? code "_preprocess_speech_to_text()"

203 +

204 ```python

205 # vllm/entrypoints/openai/speech_to_text.py

206 async def _preprocess_speech_to_text(...):

docs/deployment/frameworks/cerebrium.md

+2 -2

@@ -63,7 +63,7 @@ If successful, you should be returned a CURL command that you can call inference

64 ??? console "Command"

66 - ```python

66 + ```bash

67 curl -X POST https://api.cortex.cerebrium.ai/v4/p-xxxxxx/vllm/run \

68 -H 'Content-Type: application/json' \

69 -H 'Authorization: <JWT TOKEN>' \

@@ -81,7 +81,7 @@ You should get a response like:

82 ??? console "Response"

84 - ```python

84 + ```json

85 {

86 "run_id": "52911756-3066-9ae8-bcc9-d9129d1bd262",

87 "result": {

docs/deployment/frameworks/dstack.md

+2 -2

@@ -83,7 +83,7 @@ After the provisioning, you can interact with the model by using the OpenAI SDK:

84 client = OpenAI(

85 base_url="https://gateway.<gateway domain>",

86 - api_key="<YOUR-DSTACK-SERVER-ACCESS-TOKEN>"

86 + api_key="<YOUR-DSTACK-SERVER-ACCESS-TOKEN>",

87 )

89 completion = client.chat.completions.create(

@@ -93,7 +93,7 @@ After the provisioning, you can interact with the model by using the OpenAI SDK:

93 "role": "user",

94 "content": "Compose a poem that explains the concept of recursion in programming.",

95 }

96 - ]

96 + ],

97 )

99 print(completion.choices[0].message.content)

docs/deployment/frameworks/haystack.md

+1 -1

@@ -34,7 +34,7 @@ pip install vllm haystack-ai

34 api_key=Secret.from_token("VLLM-PLACEHOLDER-API-KEY"),

35 model="mistralai/Mistral-7B-Instruct-v0.1",

36 api_base_url="http://{your-vLLM-host-ip}:{your-vLLM-host-port}/v1",

37 - generation_kwargs = {"max_tokens": 512}

37 + generation_kwargs={"max_tokens": 512},

38 )

40 response = generator.run(

docs/deployment/frameworks/hf_inference_endpoints.md

+18 -18

@@ -32,28 +32,28 @@ This is the easiest way to get started with vLLM on Hugging Face Inference Endpo

32 import os

34 client = OpenAI(

35 - base_url = DEPLOYMENT_URL,

36 - api_key = os.environ["HF_TOKEN"] # https://huggingface.co/settings/tokens

35 + base_url=DEPLOYMENT_URL,

36 + api_key=os.environ["HF_TOKEN"], # https://huggingface.co/settings/tokens

37 )

39 chat_completion = client.chat.completions.create(

40 - model = "HuggingFaceTB/SmolLM3-3B",

41 - messages = [

40 + model="HuggingFaceTB/SmolLM3-3B",

41 + messages=[

42 {

43 "role": "user",

44 "content": [

45 {

46 "type": "text",

47 - "text": "Give me a brief explanation of gravity in simple terms."

47 + "text": "Give me a brief explanation of gravity in simple terms.",

48 }

49 - ]

49 + ],

50 }

51 ],

52 - stream = True

52 + stream=True,

53 )

55 for message in chat_completion:

56 - print(message.choices[0].delta.content, end = "")

56 + print(message.choices[0].delta.content, end="")

57 ```

59 !!! note

@@ -86,34 +86,34 @@ This method applies to models with the [`transformers` library tag](https://hugg

86 import os

88 client = OpenAI(

89 - base_url = DEPLOYMENT_URL,

90 - api_key = os.environ["HF_TOKEN"] # https://huggingface.co/settings/tokens

89 + base_url=DEPLOYMENT_URL,

90 + api_key=os.environ["HF_TOKEN"], # https://huggingface.co/settings/tokens

91 )

93 chat_completion = client.chat.completions.create(

94 - model = "ibm-granite/granite-docling-258M",

95 - messages = [

94 + model="ibm-granite/granite-docling-258M",

95 + messages=[

96 {

97 "role": "user",

98 "content": [

99 {

100 "type": "image_url",

101 "image_url": {

102 - "url": "https://huggingface.co/ibm-granite/granite-docling-258M/resolve/main/assets/new_arxiv.png"

103 - }

102 + "url": "https://huggingface.co/ibm-granite/granite-docling-258M/resolve/main/assets/new_arxiv.png",

103 + },

104 },

105 {

106 "type": "text",

107 - "text": "Convert this page to docling."

108 - }

107 + "text": "Convert this page to docling.",

108 + },

109 ]

110 }

111 ],

112 - stream = True

112 + stream=True,

113 )

114

115 for message in chat_completion:

116 - print(message.choices[0].delta.content, end = "")

116 + print(message.choices[0].delta.content, end="")

117 ```

118

119 !!! note

docs/deployment/frameworks/litellm.md

+7 -6

@@ -36,15 +36,16 @@ pip install vllm litellm

36 ```python

37 import litellm

39 - messages = [{ "content": "Hello, how are you?","role": "user"}]

39 + messages = [{"content": "Hello, how are you?", "role": "user"}]

41 # hosted_vllm is prefix key word and necessary

42 response = litellm.completion(

43 - model="hosted_vllm/qwen/Qwen1.5-0.5B-Chat", # pass the vllm model name

44 - messages=messages,

45 - api_base="http://{your-vllm-server-host}:{your-vllm-server-port}/v1",

46 - temperature=0.2,

47 - max_tokens=80)

43 + model="hosted_vllm/qwen/Qwen1.5-0.5B-Chat", # pass the vllm model name

44 + messages=messages,

45 + api_base="http://{your-vllm-server-host}:{your-vllm-server-port}/v1",

46 + temperature=0.2,

47 + max_tokens=80,

48 + )

50 print(response)

51 ```

docs/deployment/frameworks/retrieval_augmented_generation.md

+2 -2

@@ -40,7 +40,7 @@ pip install -U vllm \

41 1. Run the script

43 - ```python

43 + ```bash

44 python retrieval_augmented_generation_with_langchain.py

45 ```

@@ -78,6 +78,6 @@ pip install vllm \

79 1. Run the script:

81 - ```python

81 + ```bash

82 python retrieval_augmented_generation_with_llamaindex.py

83 ```

docs/design/cuda_graphs.md

+9 -7

@@ -106,9 +106,11 @@ The dispatch code looks like:

106 batch_descriptor=BatchDescriptor(num_tokens=num_input_tokens, uniform_decode=...)

107 runtime_mode, batch_descriptor = cudagraphdispatcher.dispatch(batch_descriptor)

108 # execution

109 -with set_forward_context(...,

110 - cudagraph_runtime_mode=runtime_mode,

111 - batch_descriptor=batch_descriptor):

109 +with set_forward_context(

110 + ...,

111 + cudagraph_runtime_mode=runtime_mode,

112 + batch_descriptor=batch_descriptor,

113 +):

114 output = self.model(...)

115 ```

116

@@ -202,10 +204,10 @@ from vllm.config import CUDAGraphMode

204

205 compilation_config = {"level": 3, "cudagraph_mode": "FULL_AND_PIECEWISE"}

206 model = vllm.LLM(

205 - model="meta-llama/Llama-3.1-8B-Instruct",

206 - dtype='auto',

207 - compilation_config = compilation_config,

208 - )

207 + model="meta-llama/Llama-3.1-8B-Instruct",

208 + dtype="auto",

209 + compilation_config=compilation_config,

210 +)

211 sampling_params = vllm.SamplingParams(

212 temperature=0, # greedy decoding

213 max_tokens=1024,

docs/design/io_processor_plugins.md

+15 -12

@@ -9,8 +9,8 @@ When performing an inference with IO Processor plugins, the prompt type is defin

9 IO Processor plugins implement the `IOProcessor` interface (<gh-file:vllm/plugins/io_processors/interface.py>):

11 ```python

12 -IOProcessorInput = TypeVar('IOProcessorInput')

13 -IOProcessorOutput = TypeVar('IOProcessorOutput')

12 +IOProcessorInput = TypeVar("IOProcessorInput")

13 +IOProcessorOutput = TypeVar("IOProcessorOutput")

15 class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):

@@ -21,30 +21,32 @@ class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):

21 def pre_process(

22 self,

23 prompt: IOProcessorInput,

24 - request_id: Optional[str] = None,

24 + request_id: str | None = None,

25 **kwargs,

26 - ) -> Union[PromptType, Sequence[PromptType]]:

26 + ) -> PromptType | Sequence[PromptType]:

27 raise NotImplementedError

29 async def pre_process_async(

30 self,

31 prompt: IOProcessorInput,

32 - request_id: Optional[str] = None,

32 + request_id: str | None = None,

33 **kwargs,

34 - ) -> Union[PromptType, Sequence[PromptType]]:

34 + ) -> PromptType | Sequence[PromptType]:

35 return self.pre_process(prompt, request_id, **kwargs)

37 @abstractmethod

38 - def post_process(self,

39 - model_output: Sequence[PoolingRequestOutput],

40 - request_id: Optional[str] = None,

41 - **kwargs) -> IOProcessorOutput:

38 + def post_process(

39 + self,

40 + model_output: Sequence[PoolingRequestOutput],

41 + request_id: str | None = None,

42 + **kwargs,

43 + ) -> IOProcessorOutput:

44 raise NotImplementedError

46 async def post_process_async(

47 self,

48 model_output: AsyncGenerator[tuple[int, PoolingRequestOutput]],

47 - request_id: Optional[str] = None,

49 + request_id: str | None = None,

50 **kwargs,

51 ) -> IOProcessorOutput:

52 collected_output = [item async for i, item in model_output]

@@ -56,7 +58,8 @@ class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):

59 @abstractmethod

60 def output_to_response(

59 - self, plugin_output: IOProcessorOutput) -> IOProcessorResponse:

61 + self, plugin_output: IOProcessorOutput

62 + ) -> IOProcessorResponse:

63 raise NotImplementedError

64 ```

docs/design/metrics.md

+7 -5

@@ -478,15 +478,17 @@ us with:

478

479 ```python

480 if seq_group.is_finished():

481 - if (seq_group.metrics.first_scheduled_time is not None and

482 - seq_group.metrics.first_token_time is not None):

481 + if (

482 + seq_group.metrics.first_scheduled_time is not None

483 + and seq_group.metrics.first_token_time is not None

484 + ):

485 time_queue_requests.append(

486 seq_group.metrics.first_scheduled_time -

485 - seq_group.metrics.arrival_time)

487 + seq_group.metrics.arrival_time

488 + )

489 ...

490 if seq_group.metrics.time_in_queue is not None:

488 - time_in_queue_requests.append(

489 - seq_group.metrics.time_in_queue)

491 + time_in_queue_requests.append(seq_group.metrics.time_in_queue)

492 ```

493

494 This seems duplicative, and one of them should be removed. The latter

docs/design/prefix_caching.md

+2 -2

@@ -112,8 +112,8 @@ class KVCacheBlock:

112 ref_cnt: int

113

114 # The pointers to form a doubly linked list for the free queue.

115 - prev_free_block: Optional["KVCacheBlock"] = None

116 - next_free_block: Optional["KVCacheBlock"] = None

115 + prev_free_block: "KVCacheBlock | None" = None

116 + next_free_block: "KVCacheBlock | None" = None

117 ```

118

119 There are two design points to highlight:

docs/features/lora.md

+4 -7

@@ -32,7 +32,7 @@ the third parameter is the path to the LoRA adapter.

32 sampling_params = SamplingParams(

33 temperature=0,

34 max_tokens=256,

35 - stop=["[/assistant]"]

35 + stop=["[/assistant]"],

36 )

38 prompts = [

@@ -43,7 +43,7 @@ the third parameter is the path to the LoRA adapter.

43 outputs = llm.generate(

44 prompts,

45 sampling_params,

46 - lora_request=LoRARequest("sql_adapter", 1, sql_lora_path)

46 + lora_request=LoRARequest("sql_adapter", 1, sql_lora_path),

47 )

48 ```

@@ -197,7 +197,7 @@ Alternatively, follow these example steps to implement your own plugin:

197 lora_request = LoRARequest(

198 lora_name=lora_name,

199 lora_path=local_path,

200 - lora_int_id=abs(hash(lora_name))

200 + lora_int_id=abs(hash(lora_name)),

201 )

202 return lora_request

203 ```

@@ -296,10 +296,7 @@ To this end, we allow registration of default multimodal LoRAs to handle this au

296 if has_audio:

297 question = f"<|audio|>{question}"

298 chat = [

299 - {

300 - "role": "user",

301 - "content": question

302 - }

299 + {"role": "user", "content": question},

300 ]

301 return tokenizer.apply_chat_template(chat, tokenize=False)

302

docs/features/multimodal_inputs.md

+147 -130

@@ -154,9 +154,7 @@ To substitute multiple images inside the same text prompt, you can pass in a lis

154

155 outputs = llm.generate({

156 "prompt": prompt,

157 - "multi_modal_data": {

158 - "image": [image1, image2]

159 - },

157 + "multi_modal_data": {"image": [image1, image2]},

158 })

159

160 for o in outputs:

@@ -183,21 +181,24 @@ conversation = [

181 {"role": "assistant", "content": "Hello! How can I assist you today?"},

182 {

183 "role": "user",

186 - "content": [{

187 - "type": "image_url",

188 - "image_url": {

189 - "url": image_url

190 - }

191 - },{

192 - "type": "image_pil",

193 - "image_pil": image_pil

194 - }, {

195 - "type": "image_embeds",

196 - "image_embeds": image_embeds

197 - }, {

198 - "type": "text",

199 - "text": "What's in these images?"

200 - }],

184 + "content": [

185 + {

186 + "type": "image_url",

187 + "image_url": {"url": image_url},

188 + },

189 + {

190 + "type": "image_pil",

191 + "image_pil": image_pil,

192 + },

193 + {

194 + "type": "image_embeds",

195 + "image_embeds": image_embeds,

196 + },

197 + {

198 + "type": "text",

199 + "text": "What's in these images?",

200 + },

201 + ],

202 },

203 ]

204

@@ -224,7 +225,10 @@ Multi-image input can be extended to perform video captioning. We show this with

225 message = {

226 "role": "user",

227 "content": [

227 - {"type": "text", "text": "Describe this set of frames. Consider the frames to be a part of the same video."},

228 + {

229 + "type": "text",

230 + "text": "Describe this set of frames. Consider the frames to be a part of the same video.",

231 + },

232 ],

233 }

234 for i in range(len(video_frames)):

@@ -255,13 +259,13 @@ When loading RGBA images (images with transparency), vLLM converts them to RGB f

259 # Custom black background for dark theme

260 llm = LLM(

261 model="llava-hf/llava-1.5-7b-hf",

258 - media_io_kwargs={"image": {"rgba_background_color": [0, 0, 0]}}

262 + media_io_kwargs={"image": {"rgba_background_color": [0, 0, 0]}},

263 )

264

265 # Custom brand color background (e.g., blue)

266 llm = LLM(

267 model="llava-hf/llava-1.5-7b-hf",

264 - media_io_kwargs={"image": {"rgba_background_color": [0, 0, 255]}}

268 + media_io_kwargs={"image": {"rgba_background_color": [0, 0, 255]}},

269 )

270 ```

271

@@ -294,20 +298,23 @@ Instead of NumPy arrays, you can also pass `'torch.Tensor'` instances, as shown

298 limit_mm_per_prompt={"video": 1},

299 )

300

297 - sampling_params = SamplingParams(

298 - max_tokens=1024,

299 - )

301 + sampling_params = SamplingParams(max_tokens=1024)

302

303 video_messages = [

302 - {"role": "system", "content": "You are a helpful assistant."},

303 - {"role": "user", "content": [

304 + {

305 + "role": "system",

306 + "content": "You are a helpful assistant.",

307 + },

308 + {

309 + "role": "user",

310 + "content": [

311 {"type": "text", "text": "describe this video."},

312 {

313 "type": "video",

314 "video": video_path,

315 "total_pixels": 20480 * 28 * 28,

309 - "min_pixels": 16 * 28 * 28

310 - }

316 + "min_pixels": 16 * 28 * 28,

317 + },

318 ]

319 },

320 ]

@@ -465,21 +472,24 @@ Then, you can use the OpenAI client as follows:

472

473 chat_response = client.chat.completions.create(

474 model="microsoft/Phi-3.5-vision-instruct",

468 - messages=[{

469 - "role": "user",

470 - "content": [

471 - # NOTE: The prompt formatting with the image token `<image>` is not needed

472 - # since the prompt will be processed automatically by the API server.

473 - {"type": "text", "text": "What’s in this image?"},

474 - {

475 - "type": "image_url",

476 - "image_url": {

477 - url": image_url

475 + messages=[

476 + {

477 + "role": "user",

478 + "content": [

479 + # NOTE: The prompt formatting with the image token `<image>` is not needed

480 + # since the prompt will be processed automatically by the API server.

481 + {

482 + "type": "text",

483 + "text": "What’s in this image?",

484 },

479 - "uuid": image_url # Optional

480 - },

481 - ],

482 - }],

485 + {

486 + "type": "image_url",

487 + "image_url": {"url": image_url},

488 + "uuid": image_url, # Optional

489 + },

490 + ],

491 + }

492 + ],

493 )

494 print("Chat completion output:", chat_response.choices[0].message.content)

495

@@ -489,26 +499,27 @@ Then, you can use the OpenAI client as follows:

499

500 chat_response = client.chat.completions.create(

501 model="microsoft/Phi-3.5-vision-instruct",

492 - messages=[{

493 - "role": "user",

494 - "content": [

495 - {"type": "text", "text": "What are the animals in these images?"},

496 - {

497 - "type": "image_url",

498 - "image_url": {

499 - "url": image_url_duck

502 + messages=[

503 + {

504 + "role": "user",

505 + "content": [

506 + {

507 + "type": "text",

508 + "text": "What are the animals in these images?",

509 },

501 - "uuid": image_url_duck # Optional

502 - },

503 - {

504 - "type": "image_url",

505 - "image_url": {

506 - "url": image_url_lion

510 + {

511 + "type": "image_url",

512 + "image_url": {"url": image_url_duck},

513 + "uuid": image_url_duck, # Optional

514 },

508 - "uuid": image_url_lion # Optional

509 - },

510 - ],

511 - }],

515 + {

516 + "type": "image_url",

517 + "image_url": {"url": image_url_lion},

518 + "uuid": image_url_lion, # Optional

519 + },

520 + ],

521 + }

522 + ],

523 )

524 print("Chat completion output:", chat_response.choices[0].message.content)

525 ```

@@ -560,23 +571,22 @@ Then, you can use the OpenAI client as follows:

571

572 ## Use video url in the payload

573 chat_completion_from_url = client.chat.completions.create(

563 - messages=[{

564 - "role":

565 - "user",

566 - "content": [

567 - {

568 - "type": "text",

569 - "text": "What's in this video?"

570 - },

571 - {

572 - "type": "video_url",

573 - "video_url": {

574 - "url": video_url

574 + messages=[

575 + {

576 + "role": "user",

577 + "content": [

578 + {

579 + "type": "text",

580 + "text": "What's in this video?",

581 },

576 - "uuid": video_url # Optional

577 - },

578 - ],

579 - }],

582 + {

583 + "type": "video_url",

584 + "video_url": {"url": video_url},

585 + "uuid": video_url, # Optional

586 + },

587 + ],

588 + }

589 + ],

590 model=model,

591 max_completion_tokens=64,

592 )

@@ -652,23 +662,25 @@ Then, you can use the OpenAI client as follows:

662 audio_base64 = encode_base64_content_from_url(audio_url)

663

664 chat_completion_from_base64 = client.chat.completions.create(

655 - messages=[{

656 - "role": "user",

657 - "content": [

658 - {

659 - "type": "text",

660 - "text": "What's in this audio?"

661 - },

662 - {

663 - "type": "input_audio",

664 - "input_audio": {

665 - "data": audio_base64,

666 - "format": "wav"

665 + messages=[

666 + {

667 + "role": "user",

668 + "content": [

669 + {

670 + "type": "text",

671 + "text": "What's in this audio?",

672 },

668 - "uuid": audio_url # Optional

669 - },

670 - ],

671 - }],

673 + {

674 + "type": "input_audio",

675 + "input_audio": {

676 + "data": audio_base64,

677 + "format": "wav",

678 + },

679 + "uuid": audio_url, # Optional

680 + },

681 + ],

682 + },

683 + ],

684 model=model,

685 max_completion_tokens=64,

686 )

@@ -683,22 +695,22 @@ Alternatively, you can pass `audio_url`, which is the audio counterpart of `imag

695

696 ```python

697 chat_completion_from_url = client.chat.completions.create(

686 - messages=[{

687 - "role": "user",

688 - "content": [

689 - {

690 - "type": "text",

691 - "text": "What's in this audio?"

692 - },

693 - {

694 - "type": "audio_url",

695 - "audio_url": {

696 - "url": audio_url

698 + messages=[

699 + {

700 + "role": "user",

701 + "content": [

702 + {

703 + "type": "text",

704 + "text": "What's in this audio?",

705 },

698 - "uuid": audio_url # Optional

699 - },

700 - ],

701 - }],

706 + {

707 + "type": "audio_url",

708 + "audio_url": {"url": audio_url},

709 + "uuid": audio_url, # Optional

710 + },

711 + ],

712 + }

713 + ],

714 model=model,

715 max_completion_tokens=64,

716 )

@@ -747,43 +759,48 @@ The following example demonstrates how to pass image embeddings to the OpenAI se

759

760 # Basic usage - this is equivalent to the LLaVA example for offline inference

761 model = "llava-hf/llava-1.5-7b-hf"

750 - embeds = {

762 + embeds = {

763 "type": "image_embeds",

764 "image_embeds": f"{base64_image_embedding}",

753 - "uuid": image_url # Optional

765 + "uuid": image_url, # Optional

766 }

767

768 # Pass additional parameters (available to Qwen2-VL and MiniCPM-V)

769 model = "Qwen/Qwen2-VL-2B-Instruct"

758 - embeds = {

770 + embeds = {

771 "type": "image_embeds",

772 "image_embeds": {

761 - "image_embeds": f"{base64_image_embedding}" , # Required

762 - "image_grid_thw": f"{base64_image_grid_thw}" # Required by Qwen/Qwen2-VL-2B-Instruct

773 + "image_embeds": f"{base64_image_embedding}", # Required

774 + "image_grid_thw": f"{base64_image_grid_thw}", # Required by Qwen/Qwen2-VL-2B-Instruct

775 },

764 - "uuid": image_url # Optional

776 + "uuid": image_url, # Optional

777 }

778 model = "openbmb/MiniCPM-V-2_6"

767 - embeds = {

779 + embeds = {

780 "type": "image_embeds",

781 "image_embeds": {

770 - "image_embeds": f"{base64_image_embedding}" , # Required

771 - "image_sizes": f"{base64_image_sizes}" # Required by openbmb/MiniCPM-V-2_6

782 + "image_embeds": f"{base64_image_embedding}", # Required

783 + "image_sizes": f"{base64_image_sizes}", # Required by openbmb/MiniCPM-V-2_6

784 },

773 - "uuid": image_url # Optional

785 + "uuid": image_url, # Optional

786 }

787 chat_completion = client.chat.completions.create(

788 messages=[

777 - {"role": "system", "content": "You are a helpful assistant."},

778 - {"role": "user", "content": [

789 {

780 - "type": "text",

781 - "text": "What's in this image?",

790 + "role": "system",

791 + "content": "You are a helpful assistant.",

792 },

783 - embeds,

784 - ],

785 - },

786 - ],

793 + {

794 + "role": "user",

795 + "content": [

796 + {

797 + "type": "text",

798 + "text": "What's in this image?",

799 + },

800 + embeds,

801 + ],

802 + },

803 + ],

804 model=model,

805 )

806 ```

@@ -802,22 +819,22 @@ For Online Serving, you can also skip sending media if you expect cache hits wit

819 {

820 "type": "image_embeds",

821 "image_embeds": None,

805 - "uuid": image_uuid

822 + "uuid": image_uuid,

823 },

824

825 # input_audio:

826 {

827 "type": "input_audio",

828 "input_audio": None,

812 - "uuid": audio_uuid

829 + "uuid": audio_uuid,

830 },

831

832 # PIL Image:

833 {

834 "type": "image_pil",

818 - "image_pil": None

819 - "uuid": image_uuid

820 - }

835 + "image_pil": None,

836 + "uuid": image_uuid,

837 + },

838

839 ```

840

docs/features/reasoning_outputs.md

+31 -25

@@ -117,9 +117,11 @@ OpenAI Python client library does not officially support `reasoning_content` att

117 # For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`

118 # For Qwen3 series, if you want to disable thinking in reasoning mode, add:

119 # extra_body={"chat_template_kwargs": {"enable_thinking": False}}

120 - stream = client.chat.completions.create(model=model,

121 - messages=messages,

122 - stream=True)

120 + stream = client.chat.completions.create(

121 + model=model,

122 + messages=messages,

123 + stream=True,

124 + )

125

126 print("client: Start streaming chat completions...")

127 printed_reasoning_content = False

@@ -159,27 +161,29 @@ The reasoning content is also available when both tool calling and the reasoning

161

162 client = OpenAI(base_url="http://localhost:8000/v1", api_key="dummy")

163

162 - tools = [{

163 - "type": "function",

164 - "function": {

165 - "name": "get_weather",

166 - "description": "Get the current weather in a given location",

167 - "parameters": {

168 - "type": "object",

169 - "properties": {

170 - "location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},

171 - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}

172 - },

173 - "required": ["location", "unit"]

174 - }

164 + tools = [

165 + {

166 + "type": "function",

167 + "function": {

168 + "name": "get_weather",

169 + "description": "Get the current weather in a given location",

170 + "parameters": {

171 + "type": "object",

172 + "properties": {

173 + "location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},

174 + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},

175 + },

176 + "required": ["location", "unit"],

177 + }

178 + },

179 }

176 - }]

180 + ]

181

182 response = client.chat.completions.create(

183 model=client.models.list().data[0].id,

184 messages=[{"role": "user", "content": "What's the weather like in San Francisco?"}],

185 tools=tools,

182 - tool_choice="auto"

186 + tool_choice="auto",

187 )

188

189 print(response)

@@ -225,7 +229,7 @@ You can add a new `ReasoningParser` similar to <gh-file:vllm/reasoning/deepseek_

229 previous_token_ids: Sequence[int],

230 current_token_ids: Sequence[int],

231 delta_token_ids: Sequence[int],

228 - ) -> Union[DeltaMessage, None]:

232 + ) -> DeltaMessage | None:

233 """

234 Instance method that should be implemented for extracting reasoning

235 from an incomplete response; for use when handling reasoning calls and

@@ -235,8 +239,10 @@ You can add a new `ReasoningParser` similar to <gh-file:vllm/reasoning/deepseek_

239 """

240

241 def extract_reasoning_content(

238 - self, model_output: str, request: ChatCompletionRequest

239 - ) -> tuple[Optional[str], Optional[str]]:

242 + self,

243 + model_output: str,

244 + request: ChatCompletionRequest | ResponsesRequest,

245 + ) -> tuple[str | None, str | None]:

246 """

247 Extract reasoning content from a complete model-generated string.

248

@@ -274,10 +280,10 @@ Additionally, to enable structured output, you'll need to create a new `Reasoner

280

281 @classmethod

282 def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:

277 - return cls(start_token_id=tokenizer.encode(

278 - "<think>", add_special_tokens=False)[0],

279 - end_token_id=tokenizer.encode("</think>",

280 - add_special_tokens=False)[0])

283 + return cls(

284 + start_token_id=tokenizer.encode("<think>", add_special_tokens=False)[0],

285 + end_token_id=tokenizer.encode("</think>", add_special_tokens=False)[0],

286 + )

287

288 def is_reasoning_end(self, input_ids: list[int]) -> bool:

289 return self.end_token_id in input_ids

docs/features/tool_calling.md

+19 -18

@@ -27,27 +27,29 @@ Next, make a request that triggers the model to use the available tools:

27 return f"Getting the weather for {location} in {unit}..."

28 tool_functions = {"get_weather": get_weather}

30 - tools = [{

31 - "type": "function",

32 - "function": {

33 - "name": "get_weather",

34 - "description": "Get the current weather in a given location",

35 - "parameters": {

36 - "type": "object",

37 - "properties": {

38 - "location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},

39 - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}

30 + tools = [

31 + {

32 + "type": "function",

33 + "function": {

34 + "name": "get_weather",

35 + "description": "Get the current weather in a given location",

36 + "parameters": {

37 + "type": "object",

38 + "properties": {

39 + "location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},

40 + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}

41 + },

42 + "required": ["location", "unit"],

43 },

41 - "required": ["location", "unit"]

42 - }

43 - }

44 - }]

44 + },

45 + },

46 + ]

48 response = client.chat.completions.create(

49 model=client.models.list().data[0].id,

50 messages=[{"role": "user", "content": "What's the weather like in San Francisco?"}],

51 tools=tools,

50 - tool_choice="auto"

52 + tool_choice="auto",

53 )

55 tool_call = response.choices[0].message.tool_calls[0].function

@@ -402,8 +404,7 @@ Here is a summary of a plugin file:

404

405 # adjust request. e.g.: set skip special tokens

406 # to False for tool call output.

405 - def adjust_request(

406 - self, request: ChatCompletionRequest) -> ChatCompletionRequest:

407 + def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:

408 return request

409

410 # implement the tool call parse for stream call

@@ -416,7 +417,7 @@ Here is a summary of a plugin file:

417 current_token_ids: Sequence[int],

418 delta_token_ids: Sequence[int],

419 request: ChatCompletionRequest,

419 - ) -> Union[DeltaMessage, None]:

420 + ) -> DeltaMessage | None:

421 return delta

422

423 # implement the tool parse for non-stream call