6c9fdbf725 · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/multi_turn/benchmark_serving_multi_turn.py

+1 -1

@@ -1251,7 +1251,7 @@ async def main() -> None:

1251 default=None,

1252 help="The model name used in the API. "

1253 "If not specified, the model name will be the "

1254 - "same as the ``--model`` argument. ",

1254 + "same as the `--model` argument. ",

1255 )

1256

1257 parser.add_argument(

docs/models/extensions/fastsafetensor.md

+1 -1

@@ -3,4 +3,4 @@ Loading Model weights with fastsafetensors

3

4 Using fastsafetensors library enables loading model weights to GPU memory by leveraging GPU direct storage. See [their GitHub repository](https://github.com/foundation-model-stack/fastsafetensors) for more details.

5

6 -To enable this feature, use the ``--load-format fastsafetensors`` command-line argument

6 +To enable this feature, use the `--load-format fastsafetensors` command-line argument

tests/models/registry.py

+3 -3

@@ -67,17 +67,17 @@ class _HfExamplesInfo:

67

68 is_available_online: bool = True

69 """

70 - Set this to ``False`` if the name of this architecture no longer exists on

70 + Set this to `False` if the name of this architecture no longer exists on

71 the HF repo. To maintain backwards compatibility, we have not removed them

72 from the main model registry, so without this flag the registry tests will

73 fail.

74 """

75

76 trust_remote_code: bool = False

77 - """The ``trust_remote_code`` level required to load the model."""

77 + """The `trust_remote_code` level required to load the model."""

78

79 hf_overrides: dict[str, Any] = field(default_factory=dict)

80 - """The ``hf_overrides`` required to load the model."""

80 + """The `hf_overrides` required to load the model."""

81

82 max_model_len: int | None = None

83 """

tests/models/utils.py

+1 -1

@@ -162,7 +162,7 @@ def check_logprobs_close(

162

163 # Test prompt logprobs closeness

164 if prompt_logprobs_0 is not None and prompt_logprobs_1 is not None:

165 - # Both sequences' prompt logprobs lists are not `None``

165 + # Both sequences' prompt logprobs lists are not `None`

166 # (although individual list elements may be `None`);

167 # for each token's logprobs:

168 for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(

tools/check_init_lazy_imports.py

+1 -1

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 """Ensure we perform lazy loading in vllm/__init__.py.

4 -i.e: appears only within the ``if typing.TYPE_CHECKING:`` guard,

4 +i.e: appears only within the `if typing.TYPE_CHECKING:` guard,

5 **except** for a short whitelist.

6 """

7

vllm/assets/base.py

+1 -1

@@ -21,7 +21,7 @@ def get_cache_dir() -> Path:

21 @lru_cache

22 def get_vllm_public_assets(filename: str, s3_prefix: str | None = None) -> Path:

23 """

24 - Download an asset file from ``s3://vllm-public-assets``

24 + Download an asset file from `s3://vllm-public-assets`

25 and return the path to the downloaded file.

26 """

27 asset_directory = get_cache_dir() / "vllm_public_assets"

vllm/benchmarks/serve.py

+1 -1

@@ -1231,7 +1231,7 @@ def add_cli_args(parser: argparse.ArgumentParser):

1231 default=None,

1232 help="The model name used in the API. "

1233 "If not specified, the model name will be the "

1234 - "same as the ``--model`` argument. ",

1234 + "same as the `--model` argument. ",

1235 )

1236

1237 parser.add_argument(

vllm/compilation/decorators.py

+2 -2

@@ -138,8 +138,8 @@ def support_torch_compile(

138 """

139

140 def cls_decorator_helper(cls: _T) -> _T:

141 - # helper to pass `dynamic_arg_dims`` to `_support_torch_compile``

142 - # to avoid too much indentation for `_support_torch_compile``

141 + # helper to pass `dynamic_arg_dims` to `_support_torch_compile`

142 + # to avoid too much indentation for `_support_torch_compile`

143 if not hasattr(cls, "forward"):

144 raise TypeError("decorated class should have a forward method.")

145 sig = inspect.signature(cls.forward)

vllm/config/pooler.py

+3 -3

@@ -66,15 +66,15 @@ class PoolerConfig:

66 """

67 step_tag_id: int | None = None

68 """

69 - If set, only the score corresponding to the ``step_tag_id`` in the

69 + If set, only the score corresponding to the `step_tag_id` in the

70 generated sentence should be returned. Otherwise, the scores for all tokens

71 are returned.

72 """

73 returned_token_ids: list[int] | None = None

74 """

75 A list of indices for the vocabulary dimensions to be extracted,

76 - such as the token IDs of ``good_token`` and ``bad_token`` in the

77 - ``math-shepherd-mistral-7b-prm`` model.

76 + such as the token IDs of `good_token` and `bad_token` in the

77 + `math-shepherd-mistral-7b-prm` model.

78 """

79

80 def compute_hash(self) -> str:

vllm/distributed/kv_events.py

+1 -1

@@ -117,7 +117,7 @@ class ZmqEventPublisher(EventPublisher):

117 Parameters

118 ----------

119 endpoint:

120 - PUB address. Use ``tcp://*:5557`` to bind or ``tcp://host:5557`` to

120 + PUB address. Use `tcp://*:5557` to bind or `tcp://host:5557` to

121 connect.

122 replay_endpoint:

123 Optional ROUTER address for replay requests. When given, subscribers can

vllm/entrypoints/context.py

+1 -1

@@ -515,7 +515,7 @@ class StreamingHarmonyContext(HarmonyContext):

515

516 def render_for_completion(self) -> list[int]:

517 # now this list of tokens as next turn's starting tokens

518 - # `<|start|>assistant``,

518 + # `<|start|>assistant`,

519 # we need to process them in parser.

520 rendered_tokens = super().render_for_completion()

521

vllm/entrypoints/llm.py

+1 -1

@@ -1504,7 +1504,7 @@ class LLM:

1504 """Return a snapshot of aggregated metrics from Prometheus.

1505

1506 Returns:

1507 - A ``MetricSnapshot`` instance capturing the current state

1507 + A `MetricSnapshot` instance capturing the current state

1508 of all aggregated metrics from Prometheus.

1509

1510 Note:

vllm/entrypoints/renderer.py

+11 -11

@@ -26,12 +26,12 @@ class RenderConfig:

26

27 max_length: int | None = None

28 """Maximum allowable total input token length. If provided,

29 - token inputs longer than this raise ``ValueError``."""

29 + token inputs longer than this raise `ValueError`."""

30

31 truncate_prompt_tokens: int | None = None

32 - """Number of tokens to keep. ``None`` means no truncation.

33 - ``0`` yields an empty list (and skips embeds).

34 - ``-1`` maps to ``model_config.max_model_len``."""

32 + """Number of tokens to keep. `None` means no truncation.

33 + `0` yields an empty list (and skips embeds).

34 + `-1` maps to `model_config.max_model_len`."""

35

36 add_special_tokens: bool | None = True

37 """Whether to add model-specific special tokens during tokenization."""

@@ -107,10 +107,10 @@ class BaseRenderer(ABC):

107

108 Args:

109 prompt_or_prompts: One of:

110 - - ``str``: Single text prompt.

111 - - ``list[str]``: Batch of text prompts.

112 - - ``list[int]``: Single pre-tokenized sequence.

113 - - ``list[list[int]]``: Batch of pre-tokenized sequences.

110 + - `str`: Single text prompt.

111 + - `list[str]`: Batch of text prompts.

112 + - `list[int]`: Single pre-tokenized sequence.

113 + - `list[list[int]]`: Batch of pre-tokenized sequences.

114 config: Render configuration controlling how prompts are prepared

115 (e.g., tokenization and length handling).

116

@@ -134,9 +134,9 @@ class BaseRenderer(ABC):

134 Convert text/token and/or base64-encoded embeddings inputs into

135 engine-ready prompt objects using a unified RenderConfig.

136

137 - At least one of ``prompt_or_prompts`` or ``prompt_embeds`` must be

137 + At least one of `prompt_or_prompts` or `prompt_embeds` must be

138 provided and non-empty. If both are omitted or empty (e.g., empty

139 - string and empty list), a ``ValueError`` is raised.

139 + string and empty list), a `ValueError` is raised.

140

141 Args:

142 prompt_or_prompts: Text or token inputs to include.

@@ -150,7 +150,7 @@ class BaseRenderer(ABC):

150 Engine-ready prompt objects.

151

152 Raises:

153 - ValueError: If both ``prompt_or_prompts`` and ``prompt_embeds``

153 + ValueError: If both `prompt_or_prompts` and `prompt_embeds`

154 are omitted or empty (decoder prompt cannot be empty), or if

155 length limits are exceeded.

156 """

vllm/inputs/data.py

+1 -1

@@ -327,7 +327,7 @@ def zip_enc_dec_prompts(

327 [`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]

328 instances.

329

330 - ``mm_processor_kwargs`` may also be provided; if a dict is passed, the same

330 + `mm_processor_kwargs` may also be provided; if a dict is passed, the same

331 dictionary will be used for every encoder/decoder prompt. If an iterable is

332 provided, it will be zipped with the encoder/decoder prompts.

333 """

vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py

+1 -1

@@ -27,7 +27,7 @@ __all__ = [

27

28

29 def is_flashinfer_fp4_cutlass_moe_available() -> bool:

30 - """Return ``True`` when FlashInfer CUTLASS NV-FP4 kernels can be used."""

30 + """Return `True` when FlashInfer CUTLASS NV-FP4 kernels can be used."""

31 return (

32 envs.VLLM_USE_FLASHINFER_MOE_FP4

33 and has_flashinfer_cutlass_fused_moe()

vllm/model_executor/layers/quantization/utils/fp8_utils.py

+5 -5

@@ -887,11 +887,11 @@ def requant_weight_ue8m0_inplace(

887 UE8M0 (power-of-two) format expected by the new DeepGEMM kernels inplace.

888

889 Args:

890 - weight: Block-quantised weight tensor stored in ``torch.float8_e4m3fn``.

891 - Expected shape ``(..., M, K)``.

892 - weight_scale: Corresponding per-block scale tensor (``torch.float32``)

893 - with shape ``(..., M // block_size[0], K // block_size[1])``.

894 - block_size: 2-element iterable ``[block_m, block_k]`` describing the

890 + weight: Block-quantised weight tensor stored in `torch.float8_e4m3fn`.

891 + Expected shape `(..., M, K)`.

892 + weight_scale: Corresponding per-block scale tensor (`torch.float32`)

893 + with shape `(..., M // block_size[0], K // block_size[1])`.

894 + block_size: 2-element iterable `[block_m, block_k]` describing the

895 block quantisation granularity.

896 """

897 if weight.numel() == 0:

vllm/model_executor/models/olmo.py

+3 -3

@@ -64,7 +64,7 @@ from .utils import (

64 class OlmoAttention(nn.Module):

65 """

66 This is the attention block where the output is computed as

67 - ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``

67 + `Attention(LN(x))` in `MLP(LN(x + Attention(LN(x))))`

68 (plus another skip connection).

69 """

70

@@ -144,7 +144,7 @@ class OlmoAttention(nn.Module):

144 class OlmoMLP(nn.Module):

145 """

146 This is the MLP block where the output is computed as

147 - ``MLP(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``

147 + `MLP(LN(x))` in `MLP(LN(x + Attention(LN(x))))`

148 (plus another skip connection).

149 """

150

@@ -193,7 +193,7 @@ class OlmoMLP(nn.Module):

193 class OlmoDecoderLayer(nn.Module):

194 """

195 This is a typical transformer block where the output is

196 - computed as ``MLP(LN(x + Attention(LN(x))))``

196 + computed as `MLP(LN(x + Attention(LN(x))))`

197 (plus another skip connection).

198 """

199

vllm/model_executor/models/olmo2.py

+3 -3

@@ -69,7 +69,7 @@ from vllm.transformers_utils.configs import Olmo3Config

69 class Olmo2Attention(nn.Module):

70 """

71 This is the attention block where the output is computed as

72 - ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``

72 + `Attention(LN(x))` in `MLP(LN(x + Attention(LN(x))))`

73 (plus another skip connection).

74 """

75

@@ -190,7 +190,7 @@ class Olmo2Attention(nn.Module):

190 class Olmo2MLP(nn.Module):

191 """

192 This is the MLP block where the output is computed as

193 - ``MLP(x)`` in ``LN(MLP(x + LN(Attention(x))))``

193 + `MLP(x)` in `LN(MLP(x + LN(Attention(x))))`

194 (plus another skip connection).

195 """

196

@@ -235,7 +235,7 @@ class Olmo2MLP(nn.Module):

235 class Olmo2DecoderLayer(nn.Module):

236 """

237 This is a typical transformer block where the output is

238 - computed as ``MLP(LN(x + Attention(LN(x))))``

238 + computed as `MLP(LN(x + Attention(LN(x))))`

239 (plus another skip connection).

240 """

241

vllm/model_executor/models/ovis.py

+1 -1

@@ -166,7 +166,7 @@ class VisualTokenizer(torch.nn.Module):

166 # e.g., for hidden_stride=2, this leads to a token length reduction:

167 # 1024 -> 256 for aimv2

168 if self.config.hidden_stride > 1:

169 - # this `d` maybe different from the above `d``

169 + # this `d` maybe different from the above `d`

170 n, L, d = features.shape

171 sqrt_l = int(L**0.5)

172 assert sqrt_l**2 == L, (

vllm/model_executor/models/utils.py

+16 -16

@@ -99,13 +99,13 @@ class AutoWeightsLoader:

99 the weights only once.

100

101 The weight loading logic for individual modules can be overridden

102 - by defining a ``load_weights`` method.

102 + by defining a `load_weights` method.

103

104 Similarly, the weight loading logic for individual parameters can be

105 - overridden by defining a ``weight_loader`` method.

105 + overridden by defining a `weight_loader` method.

106

107 Detailed weight loading information can be viewed by setting the

108 - environment variable ``VLLM_LOGGING_LEVEL=DEBUG``.

108 + environment variable `VLLM_LOGGING_LEVEL=DEBUG`.

109 """

110

111 # Models trained using early version ColossalAI

@@ -372,9 +372,9 @@ def flatten_bn(

372 concat: bool = False,

373 ) -> list[torch.Tensor] | torch.Tensor:

374 """

375 - Flatten the ``B`` and ``N`` dimensions of batched multimodal inputs.

375 + Flatten the `B` and `N` dimensions of batched multimodal inputs.

376

377 - The input tensor should have shape ``(B, N, ...)```.

377 + The input tensor should have shape `(B, N, ...)`.

378 """

379 if isinstance(x, torch.Tensor):

380 return x.flatten(0, 1)

@@ -424,12 +424,12 @@ def _merge_multimodal_embeddings(

424 is_multimodal: torch.Tensor,

425 ) -> torch.Tensor:

426 """

427 - Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the

428 - positions in ``inputs_embeds`` corresponding to placeholder tokens in

429 - ``input_ids``.

427 + Merge `multimodal_embeddings` into `inputs_embeds` by overwriting the

428 + positions in `inputs_embeds` corresponding to placeholder tokens in

429 + `input_ids`.

430

431 Note:

432 - This updates ``inputs_embeds`` in place.

432 + This updates `inputs_embeds` in place.

433 """

434 if len(multimodal_embeddings) == 0:

435 return inputs_embeds

@@ -475,14 +475,14 @@ def merge_multimodal_embeddings(

475 placeholder_token_id: int | list[int],

476 ) -> torch.Tensor:

477 """

478 - Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the

479 - positions in ``inputs_embeds`` corresponding to placeholder tokens in

480 - ``input_ids``.

478 + Merge `multimodal_embeddings` into `inputs_embeds` by overwriting the

479 + positions in `inputs_embeds` corresponding to placeholder tokens in

480 + `input_ids`.

481

482 - ``placeholder_token_id`` can be a list of token ids (e.g, token ids

482 + `placeholder_token_id` can be a list of token ids (e.g, token ids

483 of img_start, img_break, and img_end tokens) when needed: This means

484 - the order of these tokens in the ``input_ids`` MUST MATCH the order of

485 - their embeddings in ``multimodal_embeddings`` since we need to

484 + the order of these tokens in the `input_ids` MUST MATCH the order of

485 + their embeddings in `multimodal_embeddings` since we need to

486 slice-merge instead of individually scattering.

487

488 For example, if input_ids is "TTTTTSIIIBIIIBIIIETTT", where

@@ -497,7 +497,7 @@ def merge_multimodal_embeddings(

497 input_ids for a correct embedding merge.

498

499 Note:

500 - This updates ``inputs_embeds`` in place.

500 + This updates `inputs_embeds` in place.

501 """

502 if isinstance(placeholder_token_id, list):

503 is_multimodal = isin_list(input_ids, placeholder_token_id)

vllm/model_executor/parameter.py

+1 -1

@@ -70,7 +70,7 @@ class BasevLLMParameter(Parameter):

70 # NOTE(@ksayers) some models such as mamba_mixer2 override the

71 # weight loader to support custom loading. In the future, model-specific

72 # weight loading should be implemented via Model.load_weights. In the

73 - # meantime, support deleting and overriding `weight_loader`` attribute

73 + # meantime, support deleting and overriding `weight_loader` attribute

74 if self._weight_loader is None:

75 raise AttributeError(

76 f"{self.__class__.__name__} weight_loader attribute has been deleted"

vllm/multimodal/processing.py

+7 -7

@@ -332,8 +332,8 @@ class PromptInsertion(PromptUpdate):

332

333 Example:

334

335 - For each image, insert a number of ``<image>`` feature placeholders

336 - equal to the feature size of the vision encoder after the ``<s>`` token:

335 + For each image, insert a number of `<image>` feature placeholders

336 + equal to the feature size of the vision encoder after the `<s>` token:

337

338 ```python

339 PromptInsertion(

@@ -353,7 +353,7 @@ class PromptInsertion(PromptUpdate):

353 )

354 ```

355

356 - Insert these tokens after a prefix ``Images:``:

356 + Insert these tokens after a prefix `Images:`:

357

358 ```python

359 PromptInsertion(

@@ -401,8 +401,8 @@ class PromptReplacement(PromptUpdate):

401

402 Example:

403

404 - For each image, replace one ``<image>`` input placeholder in the prompt

405 - with a number of ``<image>`` feature placeholders

404 + For each image, replace one `<image>` input placeholder in the prompt

405 + with a number of `<image>` feature placeholders

406 equal to the feature size of the vision encoder:

407

408 ```python

@@ -413,8 +413,8 @@ class PromptReplacement(PromptUpdate):

413 )

414 ```

415

416 - As above, but further pad the feature placeholders with ``<image_bos>``

417 - and `<image_eos>``, which are not supposed to be passed to the vision

416 + As above, but further pad the feature placeholders with `<image_bos>`

417 + and `<image_eos>`, which are not supposed to be passed to the vision

418 encoder:

419

420 ```python

vllm/multimodal/registry.py

+2 -2

@@ -307,7 +307,7 @@ class MultiModalRegistry:

307 """

308 Create dummy data for profiling the memory usage of a model.

309

310 - The model is identified by ``model_config``.

310 + The model is identified by `model_config`.

311 """

312 processor = self.create_processor(model_config, cache=cache)

313 profiler: MultiModalProfiler = MultiModalProfiler(processor)

@@ -340,7 +340,7 @@ class MultiModalRegistry:

340 """

341 Create dummy data for profiling the memory usage of a model.

342

343 - The model is identified by ``model_config``.

343 + The model is identified by `model_config`.

344 """

345 processor = self.create_processor(model_config, cache=cache)

346 profiler: MultiModalProfiler = MultiModalProfiler(processor)

vllm/platforms/rocm.py

+1 -1

@@ -75,7 +75,7 @@ _ROCM_DEVICE_ID_NAME_MAP: dict[str, str] = {

75 "0x74bd": "AMD_Instinct_MI300X_HF",

76 }

77

78 -# Prevent use of clashing `{CUDA/HIP}_VISIBLE_DEVICES``

78 +# Prevent use of clashing `{CUDA/HIP}_VISIBLE_DEVICES`

79 if "HIP_VISIBLE_DEVICES" in os.environ:

80 val = os.environ["HIP_VISIBLE_DEVICES"]

81 if cuda_val := os.environ.get("CUDA_VISIBLE_DEVICES", None):

vllm/platforms/xpu.py

+1 -1

@@ -168,7 +168,7 @@ class XPUPlatform(Platform):

168 parallel_config.distributed_executor_backend = "uni"

169 elif parallel_config.distributed_executor_backend == "mp":

170 # FIXME(kunshang):

171 - # spawn needs calling `if __name__ == '__main__':``

171 + # spawn needs calling `if __name__ == '__main__':`

172 # fork is not supported for xpu start new process.

173 if envs.VLLM_WORKER_MULTIPROC_METHOD != "spawn":

174 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

vllm/sampling_params.py

+4 -4

@@ -306,10 +306,10 @@ class SamplingParams(

306 )

307

308 def __post_init__(self) -> None:

309 - # how we deal with `best_of``:

310 - # if `best_of`` is not set, we default to `n`;

311 - # if `best_of`` is set, we set `n`` to `best_of`,

312 - # and set `_real_n`` to the original `n`.

309 + # how we deal with `best_of`:

310 + # if `best_of` is not set, we default to `n`;

311 + # if `best_of` is set, we set `n` to `best_of`,

312 + # and set `_real_n` to the original `n`.

313 # when we return the result, we will check

314 # if we need to return `n` or `_real_n` results

315 if self.best_of:

vllm/utils/deep_gemm.py

+4 -4

@@ -21,7 +21,7 @@ from vllm.utils import cdiv, has_deep_gemm

21

22 @functools.cache

23 def is_deep_gemm_supported() -> bool:

24 - """Return ``True`` if DeepGEMM is supported on the current platform.

24 + """Return `True` if DeepGEMM is supported on the current platform.

25 Currently, only Hopper and Blackwell GPUs are supported.

26 """

27 is_supported_arch = current_platform.is_cuda() and (

@@ -33,7 +33,7 @@ def is_deep_gemm_supported() -> bool:

33

34 @functools.cache

35 def is_deep_gemm_e8m0_used() -> bool:

36 - """Return ``True`` if vLLM is configured to use DeepGEMM "

36 + """Return `True` if vLLM is configured to use DeepGEMM "

37 "E8M0 scale on a Hopper or Blackwell-class GPU.

38 """

39 if not is_deep_gemm_supported():

@@ -311,9 +311,9 @@ def calc_diff(x: torch.Tensor, y: torch.Tensor):

311 """Return a global difference metric for unit tests.

312

313 DeepGEMM kernels on Blackwell/B200 currently exhibit noticeable per-element

314 - error, causing ``torch.testing.assert_close`` to fail. Instead of checking

314 + error, causing `torch.testing.assert_close` to fail. Instead of checking

315 every element, we compute a cosine-style similarity over the whole tensor

316 - and report ``1 - sim``. Once kernel accuracy improves this helper can be

316 + and report `1 - sim`. Once kernel accuracy improves this helper can be

317 removed.

318 """

319

vllm/utils/flashinfer.py

+10 -10

@@ -34,7 +34,7 @@ FLASHINFER_CUBINS_REPOSITORY = os.environ.get(

34

35 @functools.cache

36 def has_flashinfer() -> bool:

37 - """Return ``True`` if FlashInfer is available."""

37 + """Return `True` if FlashInfer is available."""

38 # Use find_spec to check if the module exists without importing it

39 # This avoids potential CUDA initialization side effects

40 if importlib.util.find_spec("flashinfer") is None:

@@ -114,13 +114,13 @@ autotune = _lazy_import_wrapper(

114

115 @functools.cache

116 def has_flashinfer_comm() -> bool:

117 - """Return ``True`` if FlashInfer comm module is available."""

117 + """Return `True` if FlashInfer comm module is available."""

118 return has_flashinfer() and importlib.util.find_spec("flashinfer.comm") is not None

119

120

121 @functools.cache

122 def has_flashinfer_all2all() -> bool:

123 - """Return ``True`` if FlashInfer mnnvl all2all is available."""

123 + """Return `True` if FlashInfer mnnvl all2all is available."""

124 if not has_flashinfer_comm():

125 return False

126

@@ -141,7 +141,7 @@ def has_flashinfer_all2all() -> bool:

141

142 @functools.cache

143 def has_flashinfer_moe() -> bool:

144 - """Return ``True`` if FlashInfer MoE module is available."""

144 + """Return `True` if FlashInfer MoE module is available."""

145 return (

146 has_flashinfer()

147 and importlib.util.find_spec("flashinfer.fused_moe") is not None

@@ -150,7 +150,7 @@ def has_flashinfer_moe() -> bool:

150

151 @functools.cache

152 def has_flashinfer_cutlass_fused_moe() -> bool:

153 - """Return ``True`` if FlashInfer CUTLASS fused MoE is available."""

153 + """Return `True` if FlashInfer CUTLASS fused MoE is available."""

154 if not has_flashinfer_moe():

155 return False

156

@@ -171,7 +171,7 @@ def has_flashinfer_cutlass_fused_moe() -> bool:

171

172 @functools.cache

173 def has_nvidia_artifactory() -> bool:

174 - """Return ``True`` if NVIDIA's artifactory is accessible.

174 + """Return `True` if NVIDIA's artifactory is accessible.

175

176 This checks connectivity to the kernel inference library artifactory

177 which is required for downloading certain cubin kernels like TRTLLM FHMA.

@@ -218,9 +218,9 @@ def _force_use_trtllm_attention(env_value: bool | None) -> bool | None:

218

219 def force_use_trtllm_attention() -> bool | None:

220 """

221 - Return ``None`` if VLLM_USE_TRTLLM_ATTENTION is not set,

222 - return ``True`` if TRTLLM attention is forced to be used,

223 - return ``False`` if TRTLLM attention is forced to be not used.

221 + Return `None` if VLLM_USE_TRTLLM_ATTENTION is not set,

222 + return `True` if TRTLLM attention is forced to be used,

223 + return `False` if TRTLLM attention is forced to be not used.

224 """

225 return _force_use_trtllm_attention(envs.VLLM_USE_TRTLLM_ATTENTION)

226

@@ -244,7 +244,7 @@ def use_trtllm_attention(

244 has_sinks: bool = False,

245 has_spec: bool = False,

246 ) -> bool:

247 - """Return ``True`` if TRTLLM attention is used."""

247 + """Return `True` if TRTLLM attention is used."""

248 force_use_trtllm = force_use_trtllm_attention()

249

250 # Environment variable is set to 0 - respect it

vllm/v1/core/kv_cache_utils.py

+7 -7

@@ -26,17 +26,17 @@ from vllm.v1.kv_cache_interface import (

26 from vllm.v1.request import Request

27

28 # BlockHash represents the hash of a single KV-cache block used for

29 -# prefix caching. Treating it as a distinct type from ``bytes`` helps

29 +# prefix caching. Treating it as a distinct type from `bytes` helps

30 # catch accidental misuse when passing around raw byte strings.

31 BlockHash = NewType("BlockHash", bytes)

32

33 -# ``BlockHashWithGroupId`` combines a ``BlockHash`` with its KV cache group ID.

33 +# `BlockHashWithGroupId` combines a `BlockHash` with its KV cache group ID.

34 # It is represented as raw bytes for compactness and efficiency. The helper

35 -# functions below pack/unpack the ``BlockHash`` and group id into/from the key.

35 +# functions below pack/unpack the `BlockHash` and group id into/from the key.

36 BlockHashWithGroupId = NewType("BlockHashWithGroupId", bytes)

37

38 # ExternalBlockHash is used for reproducible prefix-cache block hashing.

39 -# It's a union of ``bytes`` and ``int`` to keep backward compatibility

39 +# It's a union of `bytes` and `int` to keep backward compatibility

40 # after we default block hashing to use sha256 bytes.

41 ExternalBlockHash: TypeAlias = bytes | int

42

@@ -44,7 +44,7 @@ ExternalBlockHash: TypeAlias = bytes | int

44 def make_block_hash_with_group_id(

45 block_hash: BlockHash, group_id: int

46 ) -> BlockHashWithGroupId:

47 - """Pack a ``BlockHash`` and group id into a ``BlockHashWithGroupId``.

47 + """Pack a `BlockHash` and group id into a `BlockHashWithGroupId`.

48

49 The group id is encoded using 4 bytes in big-endian order and appended to

50 the block hash bytes. This representation avoids creating tuples while

@@ -54,12 +54,12 @@ def make_block_hash_with_group_id(

54

55

56 def get_block_hash(key: BlockHashWithGroupId) -> BlockHash:

57 - """Extract the ``BlockHash`` from a ``BlockHashWithGroupId``."""

57 + """Extract the `BlockHash` from a `BlockHashWithGroupId`."""

58 return BlockHash(key[:-4])

59

60

61 def get_group_id(key: BlockHashWithGroupId) -> int:

62 - """Extract the group id from a ``BlockHashWithGroupId``."""

62 + """Extract the group id from a `BlockHashWithGroupId`."""

63 return int.from_bytes(key[-4:], "big", signed=False)

64

65

vllm/v1/worker/cpu_worker.py

+1 -1

@@ -128,7 +128,7 @@ class CPUWorker(Worker):

128 "Please try to bind threads manually."

129 )

130

131 - # Get CPUs on NUMA node `allowed_numa_nodes[local_rank]``

131 + # Get CPUs on NUMA node `allowed_numa_nodes[local_rank]`

132 selected_numa_node = allowed_numa_nodes[self.local_rank] # type: ignore

133 logical_cpu_list = [

134 x for x in logical_cpu_list if x.numa_node == selected_numa_node

vllm/v1/worker/tpu_worker.py

+2 -2

@@ -182,8 +182,8 @@ class TPUWorker:

182 if isinstance(layer_spec, AttentionSpec):

183 dtype = layer_spec.dtype

184

185 - # Use an empty tensor instead of `None`` to force Dynamo to pass

186 - # it by reference, rather by specializing on the value ``None``.

185 + # Use an empty tensor instead of `None` to force Dynamo to pass

186 + # it by reference, rather by specializing on the value `None`.

187 tpu_kv_cache = torch.tensor([], dtype=dtype).to(self.device)

188 kv_caches[layer_name] = tpu_kv_cache

189 else:

[Docs] Replace `rst` style double-backtick with `md` single-backtick (#27091)