@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 35bf193864

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Doc]: fix typos in Python comments (#24294)

Signed-off-by: Didier Durand <durand.didier@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>

Didier Durand committed Sep 6, 2025 at 04:41 UTC 35bf19386489c204189fe920aa6e1b420fe75928

12 files changed +17 -17

csrc/quantization/machete/generate.py

+1 -1

@@ -417,7 +417,7 @@ def create_sources(impl_configs: list[ImplConfig], num_impl_files=8):

417 ))

418

419 def prepacked_type_key(prepack_type: PrepackTypeConfig):

420 - # For now we we can just use the first accumulator type seen since

420 + # For now, we can just use the first accumulator type seen since

421 # the tensor core shapes/layouts don't vary based on accumulator

422 # type so we can generate less code this way

423 return (prepack_type.a, prepack_type.b_num_bits, prepack_type.convert)

docs/getting_started/installation/cpu.md

+1 -1

@@ -180,7 +180,7 @@ Inference batch size is an important parameter for the performance. Larger batch

180 - Offline Inference: `256 * world_size`

181 - Online Serving: `128 * world_size`

182

183 -vLLM CPU supports data parallel (DP), tensor parallel (TP) and pipeline parallel (PP) to leverage multiple CPU sockets and memory nodes. For more details of tuning DP, TP and PP, please refer to [Optimization and Tuning](../../configuration/optimization.md). For vLLM CPU, it is recommend to use DP, TP and PP together if there are enough CPU sockets and memory nodes.

183 +vLLM CPU supports data parallel (DP), tensor parallel (TP) and pipeline parallel (PP) to leverage multiple CPU sockets and memory nodes. For more details of tuning DP, TP and PP, please refer to [Optimization and Tuning](../../configuration/optimization.md). For vLLM CPU, it is recommended to use DP, TP and PP together if there are enough CPU sockets and memory nodes.

184

185 ### Which quantization configs does vLLM CPU support?

186

tests/models/multimodal/generation/vlm_utils/core.py

+1 -1

@@ -42,7 +42,7 @@ def run_test(

42 tensor_parallel_size: int = 1,

43 vllm_embeddings: Optional[torch.Tensor] = None,

44 ):

45 - """Modality agnostic test test executor for comparing HF/vLLM outputs."""

45 + """Modality agnostic test executor for comparing HF/vLLM outputs."""

46 # In the case of embeddings, vLLM takes separate input tensors

47 vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs

vllm/distributed/device_communicators/custom_all_reduce.py

+2 -2

@@ -60,7 +60,7 @@ class CustomAllreduce:

60 group: the process group to work on. If None, it will use the

61 default process group.

62 device: the device to bind the CustomAllreduce to. If None,

63 - it will be bind to f"cuda:{local_rank}".

63 + it will be bound to f"cuda:{local_rank}".

64 It is the caller's responsibility to make sure each communicator

65 is bind to a unique device, and all communicators in this group

66 are in the same node.

@@ -158,7 +158,7 @@ class CustomAllreduce:

158

159 self.disabled = False

160 # Buffers memory are owned by this Python class and passed to C++.

161 - # Meta data composes of two parts: meta data for synchronization and a

161 + # Metadata composes of two parts: metadata for synchronization and a

162 # temporary buffer for storing intermediate allreduce results.

163 self.meta_ptrs = self.create_shared_buffer(ops.meta_size() + max_size,

164 group=group,

vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py

+3 -3

@@ -35,7 +35,7 @@ class Internlm2ToolParser(ToolParser):

35 self, request: ChatCompletionRequest) -> ChatCompletionRequest:

36 if request.tools and request.tool_choice != 'none':

37 # do not skip special tokens because internlm use the special

38 - # tokens to indicated the start and end of the tool calls

38 + # tokens to indicate the start and end of the tool calls

39 # information.

40 request.skip_special_tokens = False

41 return request

@@ -60,8 +60,8 @@ class Internlm2ToolParser(ToolParser):

60 if '<|action_start|>' not in current_text:

61 self.position = len(current_text)

62 return DeltaMessage(content=delta_text)

63 - # if the tool call is sended, return a empty delta message

64 - # to make sure the finish_reason will be send correctly.

63 + # if the tool call is sended, return an empty delta message

64 + # to make sure the finish_reason will be sent correctly.

65 if self.current_tool_id > 0:

66 return DeltaMessage(content='')

vllm/envs.py

+1 -1

@@ -1064,7 +1064,7 @@ environment_variables: dict[str, Callable[[], Any]] = {

1064 # vllm should use flashinfer fused allreduce. The variable should be a

1065 # JSON with the following format:

1066 # { <world size>: <max size in mb> }

1067 - # Unspecified world sizes will fallback to

1067 + # Unspecified world sizes will fall back to

1068 # { 2: 64, 4: 1, <everything else>: 0.5 }

1069 "VLLM_FLASHINFER_ALLREDUCE_FUSION_THRESHOLDS_MB":

1070 lambda: json.loads(os.getenv(

vllm/model_executor/layers/fused_moe/fused_moe.py

+1 -1

@@ -534,7 +534,7 @@ def invoke_fused_moe_kernel(A: torch.Tensor,

534 EM = sorted_token_ids.size(0)

535 if A.size(0) < config["BLOCK_SIZE_M"]:

536 # optimize for small batch_size.

537 - # We assume that top_ids of each token is unique, so

537 + # We assume that top_ids of each token is unique,

538 # so num_valid_experts <= batch_size <= BLOCK_SIZE_M,

539 # and we can skip some invalid blocks.

540 EM = min(sorted_token_ids.size(0),

vllm/model_executor/layers/fused_moe/layer.py

+2 -2

@@ -710,7 +710,7 @@ def determine_expert_map(

710

711 # Create a tensor of size num_experts filled with -1

712 expert_map = torch.full((global_num_experts, ), -1, dtype=torch.int32)

713 - # Create a expert map for the local experts

713 + # Create an expert map for the local experts

714 start_idx = ep_rank * base_experts + min(ep_rank, remainder)

715 expert_map[start_idx:start_idx + local_num_experts] = torch.arange(

716 0, local_num_experts, dtype=torch.int32)

@@ -806,7 +806,7 @@ class FusedMoE(CustomOp):

806

807 self.global_num_experts = num_experts + num_redundant_experts

808

809 - # we padding globally so EP buffer allocation works

809 + # we are padding globally so EP buffer allocation works

810 if quant_config and quant_config.get_name() == "mxfp4":

811 from vllm.model_executor.layers.quantization.mxfp4 import ( # noqa: E501

812 should_use_flashinfer_mxfp4)

vllm/model_executor/layers/quantization/gptq_marlin.py

+2 -2

@@ -469,7 +469,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):

469 )

470 layer.register_parameter("w2_scales", w2_scales)

471 set_weight_attrs(w2_scales, extra_weight_attrs)

472 - # dont shard the w2 scales when running act order

472 + # don't shard the w2 scales when running act order

473 set_weight_attrs(w2_scales,

474 {"load_full_w2": self.quant_config.desc_act})

475 # up_proj scales

@@ -493,7 +493,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):

493 )

494 layer.register_parameter("w2_qzeros", w2_qzeros)

495 set_weight_attrs(w2_qzeros, extra_weight_attrs)

496 - # dont shard the w2 scales when running act order

496 + # don't shard the w2 scales when running act order

497 set_weight_attrs(w2_qzeros,

498 {"load_full_w2": self.quant_config.desc_act})

499 w13_g_idx = torch.nn.Parameter(

vllm/v1/attention/backends/flashinfer.py

+1 -1

@@ -687,7 +687,7 @@ class FlashInferImpl(AttentionImpl):

687 else:

688 raise ValueError(f"Unsupported output dtype: {output.dtype}")

689

690 - # TRTLLM attn kernel requires o scale to pass as a host scalar,

690 + # TRTLLM attn kernel requires to scale to pass as a host scalar,

691 # store the o scale as a host scalar in warmup run with cuda graph

692 # not enabled

693 if layer._o_scale_float is None:

vllm/v1/engine/core.py

+1 -1

@@ -439,7 +439,7 @@ class EngineCore:

439 """

440 # Note on thread safety: no race condition.

441 # `mm_receiver_cache` is reset at the end of LLMEngine init,

442 - # and will only accessed in the input processing thread afterwards.

442 + # and will only be accessed in the input processing thread afterwards.

443 if self.mm_receiver_cache is not None and request.mm_features:

444 request.mm_features = (

445 self.mm_receiver_cache.get_and_update_features(

vllm/v1/worker/gpu_model_runner.py

+1 -1

@@ -2826,7 +2826,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

2826 # Disable cudagraph capturing globally, so any unexpected cudagraph

2827 # capturing will be detected and raise an error after here.

2828 # Note: We don't put it into graph_capture context manager because

2829 - # we may doing lazy capturing in future that still allows capturing

2829 + # we may do lazy capturing in future that still allows capturing

2830 # after here.

2831 set_cudagraph_capturing_enabled(False)

2832