04149cce27 · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/benchmark_serving.py

+2 -2

@@ -921,7 +921,7 @@ if __name__ == "__main__":

921 "--percentile-metrics",

922 type=str,

923 default="ttft,tpot,itl",

924 - help="Comma-seperated list of selected metrics to report percentils. "

924 + help="Comma-separated list of selected metrics to report percentils. "

925 "This argument specifies the metrics to report percentiles. "

926 "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". "

927 "Default value is \"ttft,tpot,itl\".")

@@ -929,7 +929,7 @@ if __name__ == "__main__":

929 "--metric-percentiles",

930 type=str,

931 default="99",

932 - help="Comma-seperated list of percentiles for selected metrics. "

932 + help="Comma-separated list of percentiles for selected metrics. "

933 "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "

934 "Default value is \"99\". "

935 "Use \"--percentile-metrics\" to select metrics.",

benchmarks/benchmark_serving_structured_output.py

+2 -2

@@ -963,7 +963,7 @@ if __name__ == "__main__":

963 "--percentile-metrics",

964 type=str,

965 default="ttft,tpot,itl",

966 - help="Comma-seperated list of selected metrics to report percentils. "

966 + help="Comma-separated list of selected metrics to report percentils. "

967 "This argument specifies the metrics to report percentiles. "

968 "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". "

969 "Default value is \"ttft,tpot,itl\".")

@@ -971,7 +971,7 @@ if __name__ == "__main__":

971 "--metric-percentiles",

972 type=str,

973 default="99",

974 - help="Comma-seperated list of percentiles for selected metrics. "

974 + help="Comma-separated list of percentiles for selected metrics. "

975 "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "

976 "Default value is \"99\". "

977 "Use \"--percentile-metrics\" to select metrics.",

csrc/mamba/causal_conv1d/causal_conv1d.cu

+1 -1

@@ -422,7 +422,7 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {

422 int final_state_position = ((seqlen - (kWidth - 1)) - (n_chunks - 1) * kChunkSize);

423 // in case the final state is separated between the last "smem_exchange" and

424 // and the one before it (chunk = n_chunks - 1 and chunk = n_chunks - 2),

425 - // (which occurs when `final_state_position` is a non-positivie index)

425 + // (which occurs when `final_state_position` is a non-positive index)

426 // we load the correct data from smem_exchange from both chunks, the last chunk iteration and the one before it

427 if (conv_states != nullptr && final_state_position < 0 && seqlen > kWidth){

428 input_t vals_load[kNElts] = {0};

vllm/attention/backends/flash_attn.py

+1 -1

@@ -326,7 +326,7 @@ class FlashAttentionMetadata(AttentionMetadata):

326 assert self.use_cuda_graph

327

328 if turn_prefills_into_decodes:

329 - # When Mutli-Step is enabled with Chunked-Prefill, prefills and

329 + # When Multi-Step is enabled with Chunked-Prefill, prefills and

330 # decodes are scheduled together. In the first step, all the

331 # prefills turn into decodes. This update reflects that

332 # conversion.

vllm/attention/backends/hpu_attn.py

+3 -3

@@ -152,11 +152,11 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):

152 logger.warning("Could not import HPU FusedSDPA kernel. "

153 "vLLM will use native implementation.")

154

155 - suppored_head_sizes = HPUPagedAttention.get_supported_head_sizes()

156 - if head_size not in suppored_head_sizes:

155 + supported_head_sizes = HPUPagedAttention.get_supported_head_sizes()

156 + if head_size not in supported_head_sizes:

157 raise ValueError(

158 f"Head size {head_size} is not supported by PagedAttention. "

159 - f"Supported head sizes are: {suppored_head_sizes}.")

159 + f"Supported head sizes are: {supported_head_sizes}.")

160

161 if attn_type != AttentionType.DECODER:

162 raise NotImplementedError("Encoder self-attention and "

vllm/attention/backends/mla/common.py

+3 -3

@@ -83,8 +83,8 @@ spda_o = scaled_dot_product_attention(

83 return spda_o @ W_O

84

85 NOTE: in the actual code,

86 - `kv_b_proj` is [W_UK; W_UV] concatnated per head

87 - `q_b_proj` is [W_UQ; W_QR] concatnated per head

86 + `kv_b_proj` is [W_UK; W_UV] concatenated per head

87 + `q_b_proj` is [W_UQ; W_QR] concatenated per head

88 `out_proj` is W_O

89

90

@@ -667,7 +667,7 @@ class MLACommonMetadata(AttentionMetadata):

667 assert num_seqs > num_queries

668

669 if turn_prefills_into_decodes:

670 - # When Mutli-Step is enabled with Chunked-Prefill, prefills and

670 + # When Multi-Step is enabled with Chunked-Prefill, prefills and

671 # decodes are scheduled together. In the first step, all the

672 # prefills turn into decodes. This update reflects that

673 # conversion.

vllm/attention/backends/xformers.py

+3 -3

@@ -414,11 +414,11 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):

414 assert self.num_heads % self.num_kv_heads == 0

415 self.num_queries_per_kv = self.num_heads // self.num_kv_heads

416

417 - suppored_head_sizes = PagedAttention.get_supported_head_sizes()

418 - if head_size not in suppored_head_sizes:

417 + supported_head_sizes = PagedAttention.get_supported_head_sizes()

418 + if head_size not in supported_head_sizes:

419 raise ValueError(

420 f"Head size {head_size} is not supported by PagedAttention. "

421 - f"Supported head sizes are: {suppored_head_sizes}.")

421 + f"Supported head sizes are: {supported_head_sizes}.")

422

423 self.attn_type = attn_type

424

vllm/attention/ops/nki_flash_attn.py

+1 -1

@@ -446,7 +446,7 @@ def flash_paged_attention(

446 IO tensor dtypes:

447 - This kernel assumes all IO tensors have the same dtype except for

448 block_tables (int32) and mask (int32)

449 - - If mixed_percision is True, then all Tensor Engine operation will be

449 + - If mixed_precision is True, then all Tensor Engine operation will be

450 performed in bfloat16 and accumulation will be performed in float32.

451 Otherwise the intermediates will be in the same type as the inputs.

452

vllm/benchmarks/serve.py

+2 -2

@@ -724,14 +724,14 @@ def add_cli_args(parser: argparse.ArgumentParser):

724 "--percentile-metrics",

725 type=str,

726 default="ttft,tpot,itl",

727 - help="Comma-seperated list of selected metrics to report percentils. "

727 + help="Comma-separated list of selected metrics to report percentils. "

728 "This argument specifies the metrics to report percentiles. "

729 "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". ")

730 parser.add_argument(

731 "--metric-percentiles",

732 type=str,

733 default="99",

734 - help="Comma-seperated list of percentiles for selected metrics. "

734 + help="Comma-separated list of percentiles for selected metrics. "

735 "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "

736 "Use \"--percentile-metrics\" to select metrics.",

737 )

vllm/engine/output_processor/multi_step.py

+1 -1

@@ -93,7 +93,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):

93 externally (before the next schedule() call)

94 """

95 # Sequences can be in RUNNING or FINISHED_ABORTED state

96 - # once scheduled, as a sequence is moved to FINSIHED_ABORTED

96 + # once scheduled, as a sequence is moved to FINISHED_ABORTED

97 # if a client disconnects from the api server.

98 seqs = sequence_group.get_seqs(status=SequenceStatus.RUNNING)

99 if seqs is None:

vllm/entrypoints/openai/tool_parsers/utils.py

+1 -1

@@ -98,7 +98,7 @@ def find_all_indices(string: str, substring: str) -> list[int]:

98

99

100 # partial_json_parser doesn't support extra data and

101 -# JSONDecorder.raw_decode doesn't support partial JSON

101 +# JSONDecoder.raw_decode doesn't support partial JSON

102 def partial_json_loads(input_str: str, flags: Allow) -> tuple[Any, int]:

103 try:

104 return (partial_json_parser.loads(input_str, flags), len(input_str))

vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py

+1 -1

@@ -29,7 +29,7 @@ def choose_scaled_mm_linear_kernel(

29 compute_capability: Optional[int] = None

30 ) -> Type[ScaledMMLinearKernel]:

31 """

32 - Choose an ScalledMMLinearKernel that can implement the given config for the

32 + Choose an ScaledMMLinearKernel that can implement the given config for the

33 given compute capability. Attempts to choose the best kernel in terms of

34 performance.

35

vllm/platforms/cpu.py

+3 -3

@@ -69,12 +69,12 @@ class CpuPlatform(Platform):

69

70 cache_config = vllm_config.cache_config

71

72 - ipex_avaliable = find_spec("intel_extension_for_pytorch") is not None

72 + ipex_available = find_spec("intel_extension_for_pytorch") is not None

73

74 if cache_config and cache_config.block_size is None:

75 - cache_config.block_size = 128 if ipex_avaliable else 16

75 + cache_config.block_size = 128 if ipex_available else 16

76

77 - if not ipex_avaliable and cache_config.block_size != 16:

77 + if not ipex_available and cache_config.block_size != 16:

78 raise RuntimeError(

79 f"--block-size={cache_config.block_size} requires"

80 " intel_extension_for_pytorch")

vllm/platforms/interface.py

+1 -1

@@ -231,7 +231,7 @@ class Platform:

231 parser: Optional[FlexibleArgumentParser] = None

232 ) -> None:

233 """

234 - Do some pre-registeration or update action for the current platform.

234 + Do some pre-registration or update action for the current platform.

235

236 This function is called before global VllmConfig is initialized or cli

237 arguments are parsed. It's used for out-of-tree platforms to register or

vllm/reasoning/granite_reasoning_parser.py

+1 -1

@@ -60,7 +60,7 @@ class GraniteReasoningParser(ReasoningParser):

60

61 Args:

62 model_output (str): Output of the model to be parsed.

63 - request (ChatCompletionReqest): Request being processed.

63 + request (ChatCompletionRequest): Request being processed.

64

65 Returns:

66 tuple[Optional[str], Optional[str]]: Tuple pair containing the

vllm/sampling_params.py

+1 -1

@@ -101,7 +101,7 @@ class RequestOutputKind(Enum):

101 CUMULATIVE = 0

102 # Return only deltas in each RequestOutput

103 DELTA = 1

104 - # Do not return intermediate RequestOuputs

104 + # Do not return intermediate RequestOutput

105 FINAL_ONLY = 2

106

107

vllm/third_party/pynvml.py

+1 -1

@@ -1119,7 +1119,7 @@ class _PrintableStructure(Structure):

1119 e.g. class that has _field_ 'hex_value', c_uint could be formatted with

1120 _fmt_ = {"hex_value" : "%08X"}

1121 to produce nicer output.

1122 - Default fomratting string for all fields can be set with key "<default>" like:

1122 + Default formatting string for all fields can be set with key "<default>" like:

1123 _fmt_ = {"<default>" : "%d MHz"} # e.g all values are numbers in MHz.

1124 If not set it's assumed to be just "%s"

1125

vllm/v1/attention/backends/mla/common.py

+2 -2

@@ -83,8 +83,8 @@ spda_o = scaled_dot_product_attention(

83 return spda_o @ W_O

84

85 NOTE: in the actual code,

86 - `kv_b_proj` is [W_UK; W_UV] concatnated per head

87 - `q_b_proj` is [W_UQ; W_QR] concatnated per head

86 + `kv_b_proj` is [W_UK; W_UV] concatenated per head

87 + `q_b_proj` is [W_UQ; W_QR] concatenated per head

88 `out_proj` is W_O

89

90

vllm/v1/executor/multiproc_executor.py

+1 -1

@@ -326,7 +326,7 @@ class WorkerProc:

326 logger.debug("Worker interrupted.")

327

328 except Exception:

329 - # worker_busy_loop sends exceptions exceptons to Executor

329 + # worker_busy_loop sends exceptions to Executor

330 # for shutdown, but if there is an error in startup or an

331 # error with IPC itself, we need to alert the parent.

332 psutil.Process().parent().send_signal(signal.SIGUSR1)

vllm/v1/worker/gpu_model_runner.py

+1 -1

@@ -998,7 +998,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):

998 ) -> Union[ModelRunnerOutput, torch.Tensor]:

999 self._update_states(scheduler_output)

1000 if not scheduler_output.total_num_scheduled_tokens:

1001 - # Return empty ModelRunnerOuptut if there's no work to do.

1001 + # Return empty ModelRunnerOutput if there's no work to do.

1002 return EMPTY_MODEL_RUNNER_OUTPUT

1003

1004 if self.is_multimodal_model:

vllm/v1/worker/tpu_model_runner.py

+1 -1

@@ -652,7 +652,7 @@ class TPUModelRunner:

652 # Update cached state

653 self._update_states(scheduler_output)

654 if not scheduler_output.total_num_scheduled_tokens:

655 - # Return empty ModelRunnerOuptut if there's no work to do.

655 + # Return empty ModelRunnerOutput if there's no work to do.

656 return EMPTY_MODEL_RUNNER_OUTPUT

657

658 if self.is_multimodal_model:

[BugFix] fix some typos found by typos. (#16314)