31b25f6516 · tduncan/Nvidia-Optimization-Stack-VLLM

.buildkite/scripts/generate-nightly-index.py

+6 -6

@@ -112,7 +112,7 @@ def parse_from_filename(file: str) -> WheelFileInfo:

112

113 def generate_project_list(subdir_names: list[str], comment: str = "") -> str:

114 """

115 - Generate project list HTML content linking to each project & variant sub-directory.

115 + Generate project list HTML content linking to each project & variant subdirectory.

116 """

117 href_tags = []

118 for name in sorted(subdir_names):

@@ -168,23 +168,23 @@ def generate_index_and_metadata(

168 comment (str | None): Optional comment to include in the generated HTML files.

169

170 First, parse all wheel files to extract metadata.

171 - We need to collect all wheel files for each variant, and generate an index for it (in a sub-directory).

171 + We need to collect all wheel files for each variant, and generate an index for it (in a subdirectory).

172 The index for the default variant (if any) is generated in the root index directory.

173

174 If `default_variant` is provided, all wheels must have variant suffixes, and the default variant index

175 is purely a copy of the corresponding variant index, with only the links adjusted.

176 Otherwise, all wheels without variant suffixes are treated as the default variant.

177

178 - If `alias_to_default` is provided, an additional alias sub-directory is created, it has the same content

178 + If `alias_to_default` is provided, an additional alias subdirectory is created, it has the same content

179 as the default variant index, but the links are adjusted accordingly.

180

181 Index directory structure:

182 index_base_dir/ (hosted at wheels.vllm.ai/{nightly,$commit,$version}/)

183 - index.html # project list, linking to "vllm/" and other packages, and all variant sub-directories

183 + index.html # project list, linking to "vllm/" and other packages, and all variant subdirectories

184 vllm/

185 index.html # package index, pointing to actual files in wheel_base_dir (relative path)

186 metadata.json # machine-readable metadata for all wheels in this package

187 - cpu/ # cpu variant sub-directory

187 + cpu/ # cpu variant subdirectory

188 index.html

189 vllm/

190 index.html

@@ -194,7 +194,7 @@ def generate_index_and_metadata(

194 vllm/

195 index.html

196 metadata.json

197 - cu130/ # cu130 variant sub-directory

197 + cu130/ # cu130 variant subdirectory

198 index.html

199 vllm/

200 index.html

csrc/cpu/cpu_wna16.cpp

+1 -1

@@ -116,7 +116,7 @@ class Dequantizer4b {

116 scalar_vec_t output_vec_0(wb_0);

117 scalar_vec_t output_vec_1(wb_1);

118

119 - // AMX needs to interlave K elements to pack as 32 bits

119 + // AMX needs to interleave K elements to pack as 32 bits

120 if constexpr (isa == ISA::AMX) {

121 vec_op::interleave_save(output_vec_0, output_vec_1, curr_weight);

122 } else {

csrc/cpu/torch_bindings.cpp

+1 -1

@@ -292,7 +292,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {

292 "value_cache, Tensor(a3!) output, Tensor query_start_loc, Tensor "

293 "seq_lens, float scale, bool causal, Tensor? alibi_slopes, SymInt "

294 "sliding_window_left, SymInt sliding_window_right, Tensor block_table, "

295 - "float softcap, Tensor sheduler_metadata, Tensor? s_aux) -> ()",

295 + "float softcap, Tensor scheduler_metadata, Tensor? s_aux) -> ()",

296 &cpu_attention_with_kv_cache);

297

298 // placeholders

docs/design/custom_op.md

+1 -1

@@ -28,7 +28,7 @@ Furthermore, vLLM decides whether to enable or disable a `CustomOp` based on `co

28 !!! note

29 Note that `all` and `none` cannot coexist in `compilation_config.custom_ops`.

30

31 -By default, if `compilation_config.backend == "inductor"` and `compilation_config.mode != CompilationMode.NONE`, a `none` will be appended into `compilation_config.custom_ops`, otherwise a `all` will be appended. In other words, this means `CustomOp` will be disabled in some platforms (i.e., those use `inductor` as dafault backend for `torch.compile`) when running with torch compile mode. In this case, Inductor generates (fused) Triton kernels for those disabled custom ops.

31 +By default, if `compilation_config.backend == "inductor"` and `compilation_config.mode != CompilationMode.NONE`, a `none` will be appended into `compilation_config.custom_ops`, otherwise a `all` will be appended. In other words, this means `CustomOp` will be disabled in some platforms (i.e., those use `inductor` as default backend for `torch.compile`) when running with torch compile mode. In this case, Inductor generates (fused) Triton kernels for those disabled custom ops.

32

33 !!! note

34 For multi-modal models, vLLM has enforced the enabling of some custom ops to use device-specific deep-optimized kernels for better performance in ViT part, such as `MMEncoderAttention` and `ApplyRotaryEmb`. We can also pass a `enforce_enable=True` param to the `__init__()` method of the `CustomOp` to enforce enable itself at object-level.

docs/design/debug_vllm_compile.md

+1 -1

@@ -211,7 +211,7 @@ LLM(model, compilation_config=CompilationConfig(

211 These modes are stricter and reduce or eliminate the need of dynamic shapes guarding, which can help isolate issues:

212

213 - `unbacked`: Uses unbacked symints which don't allow guards, making it easier to identify where guards are being incorrectly added

214 -- `backed_size_oblivious`: Uses a mode that is more strict about guarding.

214 +- `backed_size_oblivious`: Uses a mode that is stricter about guarding.

215

216 For more details on dynamic shapes modes, see [Dynamic shapes and vLLM guard dropping](torch_compile.md#dynamic-shapes-and-vllm-guard-dropping).

217

docs/design/plugin_system.md

+2 -2

@@ -100,7 +100,7 @@ Every plugin has three parts:

100 - `_enum`: This property is the device enumeration from [PlatformEnum][vllm.platforms.interface.PlatformEnum]. Usually, it should be `PlatformEnum.OOT`, which means the platform is out-of-tree.

101 - `device_type`: This property should return the type of the device which pytorch uses. For example, `"cpu"`, `"cuda"`, etc.

102 - `device_name`: This property is set the same as `device_type` usually. It's mainly used for logging purposes.

103 - - `check_and_update_config`: This function is called very early in the vLLM's initialization process. It's used for plugins to update the vllm configuration. For example, the block size, graph mode config, etc, can be updated in this function. The most important thing is that the **worker_cls** should be set in this function to let vLLM know which worker class to use for the worker process.

103 + - `check_and_update_config`: This function is called very early in the vLLM's initialization process. It's used for plugins to update the vllm configuration. For example, the block size, graph mode config, etc., can be updated in this function. The most important thing is that the **worker_cls** should be set in this function to let vLLM know which worker class to use for the worker process.

104 - `get_attn_backend_cls`: This function should return the attention backend class's fully qualified name.

105 - `get_device_communicator_cls`: This function should return the device communicator class's fully qualified name.

106

@@ -126,7 +126,7 @@ Every plugin has three parts:

126

127 5. Implement the attention backend class `MyDummyAttention` in `my_dummy_attention.py`. The attention backend class should inherit from [AttentionBackend][vllm.v1.attention.backend.AttentionBackend]. It's used to calculate attentions with your device. Take `vllm.v1.attention.backends` as examples, it contains many attention backend implementations.

128

129 -6. Implement custom ops for high performance. Most ops can be ran by pytorch native implementation, while the performance may not be good. In this case, you can implement specific custom ops for your plugins. Currently, there are kinds of custom ops vLLM supports:

129 +6. Implement custom ops for high performance. Most ops can be run by pytorch native implementation, while the performance may not be good. In this case, you can implement specific custom ops for your plugins. Currently, there are kinds of custom ops vLLM supports:

130

131 - pytorch ops

132 there are 3 kinds of pytorch ops:

docs/models/pooling_models.md

+1 -1

@@ -327,7 +327,7 @@ curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{

327 }'

328 ```

329

330 -Due to limitations in the the output schema, the output consists of a list of

330 +Due to limitations in the output schema, the output consists of a list of

331 token scores for each token for each input. This means that you'll have to call

332 `/tokenize` as well to be able to pair tokens with scores.

333 Refer to the tests in `tests/models/language/pooling/test_bge_m3.py` to see how

docs/serving/context_parallel_deployment.md

+1 -1

@@ -9,7 +9,7 @@ Context parallel mainly solves the problem of serving long context requests. As

9

10 During prefill, for a long request with `T` new tokens, we need to compute query/key/value tensors for these new tokens. Say we have `N` GPUs, we can split the request into `N` chunks, and each GPU computes one chunk of the query/key/value tensors.

11

12 -Depending on the use case, there're two possible strategies:

12 +Depending on the use case, there are two possible strategies:

13

14 1. Partial query, full key/value: If the request token length is moderately long (we can afford holding the full key/value tensors), and the goal is to accelerate the prefill (and amortize the computation time of the prefill across query tokens), then we can gather the key/value tensors from all GPUs and let each GPU compute the attention output corresponding to the query tokens of its chunk.

15 2. Partial query, partial key/value: If the request token length is too long, we cannot afford holding the full key/value tensors anymore, then we can only compute one chunk of query/key/value tensors for each GPU, and use techniques like [ring-attention](http://arxiv.org/abs/2310.01889) to send/recv key/value tensors chunk by chunk.

vllm/entrypoints/openai/responses/serving.py

+1 -1

@@ -693,7 +693,7 @@ class OpenAIServingResponses(OpenAIServing):

693 except ValueError as e:

694 return self.create_error_response(e)

695

696 - # NOTE: Implementation of stauts is still WIP, but for now

696 + # NOTE: Implementation of status is still WIP, but for now

697 # we guarantee that if the status is not "completed", it is accurate.

698 # "completed" is implemented as the "catch-all" for now.

699 status: ResponseStatus = "completed"

vllm/model_executor/layers/fused_moe/shared_fused_moe.py

+1 -1

@@ -30,7 +30,7 @@ class SharedFusedMoE(FusedMoE):

30

31 # Disable shared expert overlap if:

32 # - we are using eplb with non-default backend, because of correctness issues

33 - # - we are using flashinfer with DP, since there nothint to gain

33 + # - we are using flashinfer with DP, since there nothing to gain

34 # - we are using marlin kernels

35 backend = self.moe_parallel_config.all2all_backend

36 self.use_overlapped = (

vllm/model_executor/layers/quantization/kernels/scaled_mm/ScaledMMLinearKernel.py

+1 -1

@@ -22,7 +22,7 @@ class ScaledMMLinearLayerConfig:

22

23 @dataclass

24 class Int8ScaledMMLinearLayerConfig(ScaledMMLinearLayerConfig):

25 - # TODO: Chnage to QuantKey like FP8ScaledMMLinearLayerConfig

25 + # TODO: Change to QuantKey like FP8ScaledMMLinearLayerConfig

26 is_static_input_scheme: bool

27 is_channelwise: bool

28 input_symmetric: bool

vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py

+1 -1

@@ -119,7 +119,7 @@ def choose_scaled_mm_linear_kernel(

119 config (_KernelConfigT): Description of the linear layer

120 to be implemented.

121 possible_kernels (dict[PlatformEnum, list[_KernelT]]): A

122 - dictionary of platforms and their list list of possible kernels.

122 + dictionary of platforms and their list of possible kernels.

123 compute_capability (Optional[int], optional): The compute capability of

124 the target device, if None uses `current_platform` to get the

125 compute capability. Defaults to None.

vllm/utils/system_utils.py

+1 -1

@@ -278,7 +278,7 @@ def find_loaded_library(lib_name: str) -> str | None:

278 According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html,

279 the file `/proc/self/maps` contains the memory maps of the process, which includes the

280 shared libraries loaded by the process. We can use this file to find the path of the

281 - a loaded library.

281 + loaded library.

282 """ # noqa

283 found_line = None

284 with open("/proc/self/maps") as f:

vllm/v1/core/sched/scheduler.py

+1 -1

@@ -1159,7 +1159,7 @@ class Scheduler(SchedulerInterface):

1159 break

1160

1161 # Calculate the number of embeddings to schedule in the current range

1162 - # of scheduled encoder placholder tokens.

1162 + # of scheduled encoder placeholder tokens.

1163 start_idx_rel = max(0, num_computed_tokens - start_pos)

1164 end_idx_rel = min(

1165 num_encoder_tokens, num_computed_tokens + num_new_tokens - start_pos

[Doc]: fixing multiple typos in diverse files (#33256)