1bd007f234 · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/benchmark_block_pool.py

+1 -1

@@ -57,7 +57,7 @@ def invoke_main() -> None:

57 "--num-iteration",

58 type=int,

59 default=1000,

60 - help="Number of iterations to run to stablize final data readings",

60 + help="Number of iterations to run to stabilize final data readings",

61 )

62 parser.add_argument(

63 "--allocate-blocks",

benchmarks/benchmark_ngram_proposer.py

+1 -1

@@ -77,7 +77,7 @@ def invoke_main() -> None:

77 "--num-iteration",

78 type=int,

79 default=100,

80 - help="Number of iterations to run to stablize final data readings",

80 + help="Number of iterations to run to stabilize final data readings",

81 )

82 parser.add_argument(

83 "--num-req", type=int, default=128, help="Number of requests in the batch"

csrc/quantization/cutlass_w4a8/w4a8_mm_entry.cu

+1 -1

@@ -181,7 +181,7 @@ struct W4A8GemmKernel {

181 auto A_ptr = static_cast<MmaType const*>(A.const_data_ptr());

182 auto B_ptr = static_cast<QuantType const*>(B.const_data_ptr());

183 auto D_ptr = static_cast<ElementD*>(D.data_ptr());

184 - // can we avoid harcode the 8 here

184 + // can we avoid hardcode the 8 here

185 auto S_ptr =

186 static_cast<cutlass::Array<ElementScale, ScalePackSize> const*>(

187 group_scales.const_data_ptr());

docs/configuration/optimization.md

+2 -2

@@ -210,7 +210,7 @@ vllm serve Qwen/Qwen2.5-VL-3B-Instruct --api-server-count 4 -dp 2

210

211 !!! note

212 API server scale-out disables [multi-modal IPC caching](#ipc-caching)

213 - because it requires a one-to-one correspondance between API and engine core processes.

213 + because it requires a one-to-one correspondence between API and engine core processes.

214

215 This does not impact [multi-modal processor caching](#processor-caching).

216

@@ -227,7 +227,7 @@ to avoid repeatedly processing the same multi-modal inputs in `BaseMultiModalPro

227 ### IPC Caching

228

229 Multi-modal IPC caching is automatically enabled when

230 -there is a one-to-one correspondance between API (`P0`) and engine core (`P1`) processes,

230 +there is a one-to-one correspondence between API (`P0`) and engine core (`P1`) processes,

231 to avoid repeatedly transferring the same multi-modal inputs between them.

232

233 ### Configuration

docs/design/io_processor_plugins.md

+1 -1

@@ -2,7 +2,7 @@

2

3 IO Processor plugins are a feature that allows pre and post processing of the model input and output for pooling models. The idea is that users are allowed to pass a custom input to vLLM that is converted into one or more model prompts and fed to the model `encode` method. One potential use-case of such plugins is that of using vLLM for generating multi-modal data. Say users feed an image to vLLM and get an image in output.

4

5 -When performing an inference with IO Processor plugins, the prompt type is defined by the plugin and the same is valid for the final request output. vLLM does not perform any validation of input/output data, and it is up to the plugin to ensure the correct data is being fed to the model and returned to the user. As of now these plugins support only pooling models and can be triggerd via the `encode` method in `LLM` and `AsyncLLM`, or in online serving mode via the `/pooling` endpoint.

5 +When performing an inference with IO Processor plugins, the prompt type is defined by the plugin and the same is valid for the final request output. vLLM does not perform any validation of input/output data, and it is up to the plugin to ensure the correct data is being fed to the model and returned to the user. As of now these plugins support only pooling models and can be triggered via the `encode` method in `LLM` and `AsyncLLM`, or in online serving mode via the `/pooling` endpoint.

6

7 ## Writing an IO Processor Plugin

8

examples/offline_inference/prithvi_geospatial_mae_io_processor.py

+1 -1

@@ -12,7 +12,7 @@ from vllm.pooling_params import PoolingParams

12 # multimodal data. In this specific case this example will take a geotiff

13 # image as input, process it using the multimodal data processor, and

14 # perform inference.

15 -# Reuirement - install plugin at:

15 +# Requirement - install plugin at:

16 # https://github.com/christian-pinto/prithvi_io_processor_plugin

17

18

examples/online_serving/prithvi_geospatial_mae.py

+1 -1

@@ -10,7 +10,7 @@ import requests

10 # multimodal data. In this specific case this example will take a geotiff

11 # image as input, process it using the multimodal data processor, and

12 # perform inference.

13 -# Reuirements :

13 +# Requirements :

14 # - install plugin at:

15 # https://github.com/christian-pinto/prithvi_io_processor_plugin

16 # - start vllm in serving mode with the below args

tests/compile/piecewise/test_multiple_graphs.py

+1 -1

@@ -134,7 +134,7 @@ class SimpleModelWithTwoGraphs(ParentModel):

134 # Test will fail without set_model_tag here with error:

135 # "ValueError: too many values to unpack (expected 3)"

136 # This is because CompiledAttention and CompiledAttentionTwo

137 - # have different implmentations but the same torch.compile

137 + # have different implementations but the same torch.compile

138 # cache dir will be used as default prefix is 'model_tag'

139 with set_model_tag("attn_one"):

140 self.attn_one = CompiledAttention(

tests/kernels/moe/test_mxfp4_moe.py

+1 -1

@@ -224,7 +224,7 @@ def tg_mxfp4_moe(

224 assert (w2_bias.dim() == 2 and w2_bias.shape[0] == num_experts

225 and w2_bias.shape[1] == hidden_size)

226

227 - # Swap w1 and w3 as the defenition of

227 + # Swap w1 and w3 as the definition of

228 # swiglu is different in the trtllm-gen

229 w13_weight_scale_ = w13_weight_scale.clone()

230 w13_weight_ = w13_weight.clone()

tests/models/multimodal/processing/test_mllama4.py

+1 -1

@@ -52,7 +52,7 @@ def test_profiling(model_id: str, max_model_len: int):

52 chunks_per_image = prod(mm_data["patches_per_image"])

53 total_num_patches = chunks_per_image * tokens_per_patch

54 num_tiles = mm_data["aspect_ratios"][0][0] * mm_data["aspect_ratios"][0][

55 - 1] # x-y seperator tokens

55 + 1] # x-y separator tokens

56 total_tokens = total_num_patches.item() + num_tiles.item(

57 ) + 3 # image start, image, image end

58

tests/quantization/test_modelopt.py

+1 -1

@@ -27,7 +27,7 @@ def use_v0_only(monkeypatch):

27 reason="ModelOpt FP8 is not supported on this GPU type.")

28 def test_modelopt_fp8_checkpoint_setup(vllm_runner):

29 """Test ModelOpt FP8 checkpoint loading and structure validation."""

30 - # TODO: provide a small publically available test checkpoint

30 + # TODO: provide a small publicly available test checkpoint

31 model_path = ("/home/scratch.omniml_data_1/zhiyu/ckpts/test_ckpts/"

32 "TinyLlama-1.1B-Chat-v1.0-fp8-0710")

33

tests/samplers/test_beam_search.py

+1 -1

@@ -82,7 +82,7 @@ def test_beam_search_with_concurrency_limit(

82 beam_width: int,

83 ) -> None:

84 # example_prompts[1]&[3]&[7] fails due to unknown reason even without

85 - # concurency limit. skip them for now.

85 + # concurrency limit. skip them for now.

86 example_prompts = (example_prompts[:8])

87 concurrency_limit = 2

88 assert len(example_prompts) > concurrency_limit

tests/v1/attention/test_chunked_local_attention.py

+1 -1

@@ -160,7 +160,7 @@ def test_local_attention_virtual_batches(test_data: LocalAttentionTestData):

160 # Use torch.arange instead of torch.randint so we can assert on

161 # block table tensor values. The block table will have shape

162 # (num_batches, cdiv(max_seq_len, block_size)) and the values will be

163 - # aranged from 0 to cdiv(max_seq_len, block_size)-1

163 + # arranged from 0 to cdiv(max_seq_len, block_size)-1

164 arange_block_indices=True,

165 )

166

tests/v1/kv_connector/unit/test_shared_storage_connector.py

+7 -7

@@ -33,7 +33,7 @@ def _check_path_len(path):

33

34

35 def _list_path(path):

36 - """Return the list of foldername (hashes generatd) under the path"""

36 + """Return the list of foldername (hashes generated) under the path"""

37 return list(path.iterdir())

38

39

@@ -41,7 +41,7 @@ def run_test(tmp_path, processor, llm: LLM, question: str,

41 image_urls: list[Image], expected_len: int, info: str):

42 """

43 One individual test to process the prompt and output base on 1 set of input

44 - Then check if the length in the strorage path matches the expected length

44 + Then check if the length in the storage path matches the expected length

45 `info` introduces details or purpose of the individual test

46 """

47 print(f"***info: {info}***")

@@ -115,7 +115,7 @@ def test_shared_storage_connector_hashes(tmp_path):

115 """

116 Tests that SharedStorageConnector saves KV to the storage locations

117 with proper hashes; that are unique for inputs with identical text but

118 - differnt images (same size), or same multiple images but different orders.

118 + different images (same size), or same multiple images but different orders.

119 """

120 # Using tmp_path as the storage path to store KV

121 print(f"KV storage path at: {str(tmp_path)}")

@@ -171,12 +171,12 @@ def test_shared_storage_connector_hashes(tmp_path):

171 img=[image_1],

172 expected_len=2,

173 info=("image_1 single input the 2nd time. "

174 - "It should not form aother new hash.")),

174 + "It should not form another new hash.")),

175 InputCase(text=TEXT_PROMPTS[0],

176 img=[image_2],

177 expected_len=2,

178 info=("image_2 single input the 2nd time. "

179 - "It should not form aother new hash.")),

179 + "It should not form another new hash.")),

180 InputCase(text=TEXT_PROMPTS[0],

181 img=[image_1, image_2],

182 expected_len=3,

@@ -189,12 +189,12 @@ def test_shared_storage_connector_hashes(tmp_path):

189 img=[image_1, image_2],

190 expected_len=4,

191 info=("[image_1, image_2] input the 2nd time. "

192 - "It should not form aother new hash.")),

192 + "It should not form another new hash.")),

193 InputCase(text=TEXT_PROMPTS[0],

194 img=[image_2, image_1],

195 expected_len=4,

196 info=("[image_2, image_1] input the 2nd time. "

197 - "It should not form aother new hash.")),

197 + "It should not form another new hash.")),

198 InputCase(text=TEXT_PROMPTS[0],

199 img=[],

200 expected_len=5,

tests/v1/logits_processors/test_custom_offline.py

+1 -1

@@ -81,7 +81,7 @@ def _run_test(kwargs: dict, logitproc_loaded: bool) -> None:

81 target_token = params.extra_args[DUMMY_LOGITPROC_ARG]

82 if not all(x == target_token for x in lp_toks):

83 raise AssertionError(

84 - f"Request {bdx} generated {lp_toks}, shoud all be "

84 + f"Request {bdx} generated {lp_toks}, should all be "

85 f"{target_token}")

86 else:

87 # This request does not exercise custom logitproc (or custom

vllm/benchmarks/serve.py

+1 -1

@@ -189,7 +189,7 @@ async def get_request(

189 # NOTE: If we simply accumulate the random delta values

190 # from the gamma distribution, their sum would have 1-2% gap

191 # from target_total_delay_s. The purpose of the following logic is to

192 - # close the gap for stablizing the throughput data

192 + # close the gap for stabilizing the throughput data

193 # from different random seeds.

194 target_total_delay_s = total_requests / request_rate

195 normalize_factor = target_total_delay_s / delay_ts[-1]

vllm/config/compilation.py

+1 -1

@@ -234,7 +234,7 @@ class CompilationConfig:

234 - FULL_AND_PIECEWISE.

235

236 PIECEWISE mode build piecewise cudagraph only, keeping the cudagraph

237 - incompatiable ops (i.e. some attention ops) outside the cudagraph

237 + incompatible ops (i.e. some attention ops) outside the cudagraph

238 for general flexibility.

239 This is the default mode.

240

vllm/config/parallel.py

+1 -1

@@ -87,7 +87,7 @@ class ParallelConfig:

87 data_parallel_external_lb: bool = False

88 """Whether to use "external" DP LB mode. Applies only to online serving

89 and when data_parallel_size > 0. This is useful for a "one-pod-per-rank"

90 - wide-EP setup in Kuberentes. Set implicitly when --data-parallel-rank

90 + wide-EP setup in Kubernetes. Set implicitly when --data-parallel-rank

91 is provided explicitly to vllm serve."""

92 data_parallel_hybrid_lb: bool = False

93 """Whether to use "hybrid" DP LB mode. Applies only to online serving

vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py

+1 -1

@@ -787,7 +787,7 @@ class NixlConnectorWorker:

787 self.src_xfer_side_handle = self.nixl_wrapper.prep_xfer_dlist(

788 "NIXL_INIT_AGENT", descs)

789

790 - # TODO(mgoin): Hybrid memory allocator is currently diabled for

790 + # TODO(mgoin): Hybrid memory allocator is currently disabled for

791 # models with local attention (Llama 4). Can remove this once enabled.

792 if self.vllm_config.model_config.hf_config.model_type == "llama4":

793 from transformers import Llama4TextConfig

vllm/entrypoints/openai/serving_responses.py

+1 -1

@@ -717,7 +717,7 @@ class OpenAIServingResponses(OpenAIServing):

717 prev_msgs.append(msg)

718 messages.extend(prev_msgs)

719 # Append the new input.

720 - # Reponses API supports simple text inputs without chat format.

720 + # Responses API supports simple text inputs without chat format.

721 if isinstance(request.input, str):

722 messages.append(get_user_message(request.input))

723 else:

vllm/model_executor/layers/activation.py

+1 -1

@@ -362,7 +362,7 @@ class ReLUSquaredActivation(CustomOp):

362 return torch.square(F.relu(x))

363

364 def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:

365 - #TODO : implement cuda kenrels

365 + #TODO : implement cuda kernels

366 return self.forward_native(x)

367

368

vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py

+1 -1

@@ -83,7 +83,7 @@ class HadamardTransform(torch.nn.Module):

83 # do not fold into weight in order to utilize FWHT

84 self.scales[part_id] = 1 / math.sqrt(data.size(0))

85

86 - # FUTURE: avoid runtime tranpose by processing weights

86 + # FUTURE: avoid runtime transpose by processing weights

87 # prior to apply

88

89 def forward(self, value: Tensor, part_id: int = 0) -> Tensor:

vllm/model_executor/layers/quantization/mxfp4.py

+1 -1

@@ -310,7 +310,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):

310 w13_bias = layer.w13_bias.data.to(torch.float32)

311 w2_bias = layer.w2_bias.data.to(torch.float32)

312

313 - # Swap w1 and w3 as the defenition of

313 + # Swap w1 and w3 as the definition of

314 # swiglu is different in the trtllm-gen

315 def swap_every_two_rows(x, axis=-1):

316 shape = x.shape

vllm/model_executor/models/gemma3n_mm.py

+1 -1

@@ -179,7 +179,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]

179 ) -> BatchFeature:

180

181 # HF Transformers audio processor no longer accepts `audios` key.

182 - # We pop `audios` and replace it with `audio` key to surpress

182 + # We pop `audios` and replace it with `audio` key to suppress

183 # the warning.

184 if 'audios' in mm_data:

185 mm_data['audio'] = mm_data.pop('audios')

vllm/model_executor/models/interns1.py

+1 -1

@@ -492,7 +492,7 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,

492

493 @classmethod

494 def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

495 - # transformers InternVLProcessor uses <IMG_CONTEXT> as the seperator

495 + # transformers InternVLProcessor uses <IMG_CONTEXT> as the separator

496 # refer to https://github.com/huggingface/transformers/blob/f90de364c2484c7c325bbe05befdcf487bd75b63/src/transformers/models/internvl/processing_internvl.py#L116

497 if modality.startswith("image"):

498 return '<IMG_CONTEXT>'

vllm/third_party/pynvml.py

+1 -1

@@ -3533,7 +3533,7 @@ def nvmlDeviceGetMPSComputeRunningProcesses_v3(handle):

3533 return []

3534 elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):

3535 # typical case

3536 - # oversize the array incase more processes are created

3536 + # oversize the array in case more processes are created

3537 c_count.value = c_count.value * 2 + 5

3538 proc_array = c_nvmlProcessInfo_v3_t * c_count.value

3539 c_procs = proc_array()

vllm/v1/attention/backends/flash_attn.py

+1 -1

@@ -167,7 +167,7 @@ class FlashAttentionMetadataBuilder(

167 # work for mixed prefill-decode and uniform-decode. But for non-spec decodes

168 # the graphs would not work for mixed prefill-decode; sorta the inverse

169 # of UNIFORM_SINGLE_TOKEN_DECODE.

170 - # Theres probably a better way to describe this using `AttentionCGSupport`

170 + # There's probably a better way to describe this using `AttentionCGSupport`

171 # but for now just set it to `UNIFORM_BATCH` to get use to drop down

172 # to FULL_AND_PIECEWISE.

173 # TODO(luka, lucas): audit FA2 as part of:

vllm/v1/attention/backends/flashinfer.py

+1 -1

@@ -291,7 +291,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):

291 paged_kv_indices_buffer=paged_kv_indices,

292 paged_kv_last_page_len_buffer=paged_kv_last_page_len,

293 # Tensor cores are enabled by default because the perf would be

294 - # atleast as good as cuda cores for all attention ops in latest

294 + # at least as good as cuda cores for all attention ops in latest

295 # gpus.

296 use_tensor_cores=True,

297 )

vllm/v1/core/kv_cache_utils.py

+1 -1

@@ -217,7 +217,7 @@ class FreeKVCacheBlockQueue:

217 # Create a fake head and a tail block for the doubly linked list to

218 # reduce branching in the code

219 #

220 - # The implementation garenteed that the fake head and tail

220 + # The implementation guaranteed that the fake head and tail

221 # are NEVER got popped, so we could safely assume each real blocks

222 # in the queue has prev and next blocks.

223 self.fake_free_list_head = KVCacheBlock(block_id=-1)

vllm/v1/worker/gpu_input_batch.py

+1 -1

@@ -584,7 +584,7 @@ class InputBatch:

584

585 if self.is_pooling_model:

586 last_req_index -= 1

587 - # Samping state not used by pooling models.

587 + # Sampling state not used by pooling models.

588 continue

589

590 # Autoregressive models require detailed tracking of condense

vllm/v1/worker/gpu_model_runner.py

+1 -1

@@ -2776,7 +2776,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

2776 self.attn_groups.append(

2777 create_attn_groups(attn_backends, kv_cache_spec))

2778

2779 - # Calculate reorder batch threshold (if neeeded)

2779 + # Calculate reorder batch threshold (if needed)

2780 self.calculate_reorder_batch_threshold()

2781

2782 def initialize_cudagraph_capture(self) -> None:

vllm/v1/worker/kv_connector_model_runner_mixin.py

+1 -1

@@ -82,7 +82,7 @@ class KVConnectorModelRunnerMixin:

82 scheduler_output) if has_kv_transfer_group() else nullcontext()

83

84 # This context manager must be used within an active forward context.

85 - # It encapsulates the entire KV conector lifecycle within execute_model

85 + # It encapsulates the entire KV connector lifecycle within execute_model

86 @staticmethod

87 @contextmanager

88 def _get_kv_connector_output(

fix some typos (#24071)