@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f1c19b245
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 06:00 UTC

[CI] change spell checker from codespell to typos (#18711)

Signed-off-by: Andy Xie <andy.xning@gmail.com>

Ning Xie committed Jun 12, 2025 at 10:57 UTC 2f1c19b2456d4fb15f3475c9db5b077777feab76
57 files changed +335 -163
.gitignore
+1 -1
@@ -200,5 +200,5 @@ benchmarks/**/*.json
200 actionlint
201 shellcheck*/
202
203 -# Ingore moe/marlin_moe gen code
203 +# Ignore moe/marlin_moe gen code
204 csrc/moe/marlin_moe_wna16/kernel_*
.pre-commit-config.yaml
+3 -5
@@ -20,12 +20,10 @@ repos:
20 args: [--output-format, github, --fix]
21 - id: ruff-format
22 files: ^(.buildkite|benchmarks|examples)/.*
23 -- repo: https://github.com/codespell-project/codespell
24 - rev: v2.4.1
23 +- repo: https://github.com/crate-ci/typos
24 + rev: v1.32.0
25 hooks:
26 - - id: codespell
27 - additional_dependencies: ['tomli']
28 - args: ['--toml', 'pyproject.toml']
26 + - id: typos
27 - repo: https://github.com/PyCQA/isort
28 rev: 6.0.1
29 hooks:
csrc/cpu/attention.cpp
+3 -3
@@ -137,8 +137,8 @@ FORCE_INLINE std::pair<T, T> reduceSoftmaxAlibi(T* data, const int size,
137 }
138
139 template <typename T>
140 -FORCE_INLINE void reducePartitonSoftmax(const T* max_data, T* sum_data,
141 - const int size) {
140 +FORCE_INLINE void reducePartitionSoftmax(const T* max_data, T* sum_data,
141 + const int size) {
142 T max = max_data[0];
143 for (int i = 1; i < size; ++i) {
144 max = max >= max_data[i] ? max : max_data[i];
@@ -634,7 +634,7 @@ struct paged_attention_v2_impl {
634
635 if (partition_num == 1) continue;
636
637 - reducePartitonSoftmax(
637 + reducePartitionSoftmax(
638 max_logits + seq_idx * num_heads * max_num_partitions +
639 head_idx * max_num_partitions,
640 exp_sums + seq_idx * num_heads * max_num_partitions +
csrc/cpu/cpu_types_x86.hpp
+5 -5
@@ -83,7 +83,7 @@ struct FP16Vec16 : public Vec<FP16Vec16> {
83 explicit FP16Vec16(const void* ptr)
84 : reg((__m256i)_mm256_loadu_si256((__m256i*)ptr)) {}
85
86 - // non-temproal load
86 + // non-temporal load
87 explicit FP16Vec16(bool, void* ptr)
88 : reg(_mm256_stream_load_si256((__m256i*)ptr)) {}
89
@@ -120,7 +120,7 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
120 explicit BF16Vec16(const void* ptr)
121 : reg((__m256i)_mm256_loadu_si256((__m256i*)ptr)) {}
122
123 - // non-temproal load
123 + // non-temporal load
124 explicit BF16Vec16(bool, void* ptr)
125 : reg(_mm256_stream_load_si256((__m256i*)ptr)) {}
126
@@ -327,7 +327,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
327 // normal load
328 explicit FP32Vec16(const float* ptr) : reg(_mm512_loadu_ps(ptr)) {}
329
330 - // non-temproal load
330 + // non-temporal load
331 explicit FP32Vec16(bool, void* ptr)
332 : reg((__m512)_mm512_stream_load_si512(ptr)) {}
333
@@ -576,7 +576,7 @@ struct INT8Vec64 : public Vec<INT8Vec64> {
576 // normal load
577 explicit INT8Vec64(void* ptr) : reg(_mm512_loadu_epi8(ptr)) {}
578
579 - // non-temproal load
579 + // non-temporal load
580 explicit INT8Vec64(bool, void* ptr) : reg(_mm512_stream_load_si512(ptr)) {}
581
582 void save(void* ptr) const { _mm512_storeu_epi8(ptr, reg); }
@@ -587,7 +587,7 @@ struct INT8Vec64 : public Vec<INT8Vec64> {
587 _mm512_mask_storeu_epi8(ptr, mask, reg);
588 }
589
590 - // non-temproal save
590 + // non-temporal save
591 void nt_save(int8_t* ptr) { _mm512_stream_si512((__m512i*)ptr, reg); }
592 };
593 #endif
csrc/moe/moe_permute_unpermute_op.cu
+8 -8
@@ -12,7 +12,7 @@ void moe_permute(
12 const torch::Tensor& input, // [n_token, hidden]
13 const torch::Tensor& topk_weights, //[n_token, topk]
14 torch::Tensor& topk_ids, // [n_token, topk]
15 - const torch::Tensor& token_expert_indicies, // [n_token, topk]
15 + const torch::Tensor& token_expert_indices, // [n_token, topk]
16 const std::optional<torch::Tensor>& expert_map, // [n_expert]
17 int64_t n_expert, int64_t n_local_expert, int64_t topk,
18 const std::optional<int64_t>& align_block_size,
@@ -27,15 +27,15 @@ void moe_permute(
27 "expert_first_token_offset must be int64");
28 TORCH_CHECK(topk_ids.scalar_type() == at::ScalarType::Int,
29 "topk_ids must be int32");
30 - TORCH_CHECK(token_expert_indicies.scalar_type() == at::ScalarType::Int,
31 - "token_expert_indicies must be int32");
30 + TORCH_CHECK(token_expert_indices.scalar_type() == at::ScalarType::Int,
31 + "token_expert_indices must be int32");
32 TORCH_CHECK(src_row_id2dst_row_id_map.scalar_type() == at::ScalarType::Int,
33 "src_row_id2dst_row_id_map must be int32");
34 TORCH_CHECK(expert_first_token_offset.size(0) == n_local_expert + 1,
35 "expert_first_token_offset shape != n_local_expert+1")
36 TORCH_CHECK(
37 - src_row_id2dst_row_id_map.sizes() == token_expert_indicies.sizes(),
38 - "token_expert_indicies shape must be same as src_row_id2dst_row_id_map");
37 + src_row_id2dst_row_id_map.sizes() == token_expert_indices.sizes(),
38 + "token_expert_indices shape must be same as src_row_id2dst_row_id_map");
39 auto n_token = input.sizes()[0];
40 auto n_hidden = input.sizes()[1];
41 auto align_block_size_value =
@@ -71,7 +71,7 @@ void moe_permute(
71 expert_map_ptr, n_expert, stream);
72 }
73 // expert sort topk expert id and scan expert id get expert_first_token_offset
74 - sortAndScanExpert(get_ptr<int>(topk_ids), get_ptr<int>(token_expert_indicies),
74 + sortAndScanExpert(get_ptr<int>(topk_ids), get_ptr<int>(token_expert_indices),
75 get_ptr<int>(permuted_experts_id),
76 get_ptr<int>(dst_row_id2src_row_id_map),
77 get_ptr<int64_t>(expert_first_token_offset), n_token,
@@ -190,7 +190,7 @@ void shuffle_rows(const torch::Tensor& input_tensor,
190
191 void moe_permute(const torch::Tensor& input, const torch::Tensor& topk_weights,
192 torch::Tensor& topk_ids,
193 - const torch::Tensor& token_expert_indicies,
193 + const torch::Tensor& token_expert_indices,
194 const std::optional<torch::Tensor>& expert_map,
195 int64_t n_expert, int64_t n_local_expert, int64_t topk,
196 const std::optional<int64_t>& align_block_size,
@@ -203,7 +203,7 @@ void moe_permute(const torch::Tensor& input, const torch::Tensor& topk_weights,
203
204 void moe_unpermute(const torch::Tensor& input,
205 const torch::Tensor& topk_weights, torch::Tensor& topk_ids,
206 - const torch::Tensor& token_expert_indicies,
206 + const torch::Tensor& token_expert_indices,
207 const std::optional<torch::Tensor>& expert_map,
208 int64_t n_expert, int64_t n_local_expert, int64_t topk,
209 const std::optional<int64_t>& align_block_size,
csrc/moe/topk_softmax_kernels.cu
+3 -3
@@ -425,7 +425,7 @@ void topkGatingSoftmaxLauncherHelper(const float* input, const bool* finished, f
425
426 #define LAUNCH_SOFTMAX(NUM_EXPERTS, WARPS_PER_TB) \
427 topkGatingSoftmaxLauncherHelper<NUM_EXPERTS, WARPS_PER_TB>( \
428 - gating_output, nullptr, topk_weights, topk_indicies, \
428 + gating_output, nullptr, topk_weights, topk_indices, \
429 token_expert_indices, num_tokens, topk, 0, num_experts, \
430 stream);
431
@@ -433,7 +433,7 @@ template <typename IndType>
433 void topkGatingSoftmaxKernelLauncher(
434 const float* gating_output,
435 float* topk_weights,
436 - IndType* topk_indicies,
436 + IndType* topk_indices,
437 int* token_expert_indices,
438 float* softmax_workspace,
439 const int num_tokens,
@@ -476,7 +476,7 @@ void topkGatingSoftmaxKernelLauncher(
476 moeSoftmax<TPB><<<num_tokens, TPB, 0, stream>>>(
477 gating_output, nullptr, softmax_workspace, num_experts);
478 moeTopK<TPB><<<num_tokens, TPB, 0, stream>>>(
479 - softmax_workspace, nullptr, topk_weights, topk_indicies, token_expert_indices,
479 + softmax_workspace, nullptr, topk_weights, topk_indices, token_expert_indices,
480 num_experts, topk, 0, num_experts);
481 }
482 }
csrc/moe/torch_bindings.cpp
+1 -1
@@ -66,7 +66,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
66
67 m.def(
68 "moe_permute(Tensor input, Tensor topk_weight, Tensor! topk_ids,"
69 - "Tensor token_expert_indicies, Tensor? expert_map, int n_expert,"
69 + "Tensor token_expert_indices, Tensor? expert_map, int n_expert,"
70 "int n_local_expert,"
71 "int topk, int? align_block_size,Tensor! permuted_input, Tensor! "
72 "expert_first_token_offset, Tensor! src_row_id2dst_row_id_map, Tensor! "
csrc/quantization/machete/machete_mainloop.cuh
+3 -3
@@ -1003,7 +1003,7 @@ struct MacheteCollectiveMma {
1003 static constexpr int A_CPY_VEC =
1004 decltype(max_common_vector(tCsA, tCrA_load)){};
1005
1006 - static constexpr int COVERSION_WIDTH =
1006 + static constexpr int CONVERSION_WIDTH =
1007 std::min(A_CPY_VEC, int(size<0>(tCrA_mma)));
1008
1009 auto load_A_to_registers = [&](int read_stage) {
@@ -1026,8 +1026,8 @@ struct MacheteCollectiveMma {
1026 // PIPELINED MAIN LOOP
1027 //
1028
1029 - auto convert_A = [&, a_vec = Int<COVERSION_WIDTH>{}](int k_block,
1030 - int read_stage) {
1029 + auto convert_A = [&, a_vec = Int<CONVERSION_WIDTH>{}](int k_block,
1030 + int read_stage) {
1031 load_extra_info_to_registers(partitioned_extra_info,
1032 copy_partitions_extra_info, k_block,
1033 read_stage);
csrc/rocm/skinny_gemms.cu
+7 -7
@@ -320,7 +320,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
320 // Goal is to bring the activation matrix A to the LDS
321 // and use it across the lifetime of the work group
322 // TODO: When activation matrix is larger than 64 KB
323 - // then this is not goint to work!
323 + // then this is not going to work!
324 //----------------------------------------------------
325 __shared__ scalar_t s[max_lds_len];
326
@@ -581,7 +581,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
581 // Goal is to bring the activation matrix A to the LDS
582 // and use it across the lifetime of the work group
583 // TODO: When activation matrix is larger than 64 KB
584 - // then this is not goint to work!
584 + // then this is not going to work!
585 //----------------------------------------------------
586 __shared__ scalar_t s[max_lds_len];
587
@@ -601,7 +601,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
601 // int _WvPrGrp = mindiv(N, CuCount * YTILE, WvPrGrp);
602 uint32_t m = (blockIdx.x * _WvPrGrp + threadIdx.y) * YTILE;
603
604 - // Check whether there will be fragmenation!
604 + // Check whether there will be fragmentation!
605 // This will happen only for the last wave!
606 if (m < M && (m + YTILE) >= M) {
607 uint32_t startColumn = M - YTILE;
@@ -827,7 +827,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
827
828 m += CuCount * _WvPrGrp * YTILE;
829
830 - // Check whether there will be fragmenation!
830 + // Check whether there will be fragmentation!
831 // This will happen only for the last wave!
832 if (m < M && (m + YTILE) >= M) {
833 uint32_t startColumn = M - YTILE;
@@ -882,7 +882,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
882 // Goal is to bring the activation matrix A to the LDS
883 // and use it across the lifetime of the work group
884 // TODO: When activation matrix is larger than 64 KB
885 - // then this is not goint to work!
885 + // then this is not going to work!
886 //----------------------------------------------------
887 __shared__ scalar_t s[max_lds_len];
888
@@ -904,7 +904,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
904 //----------------------------------------------------
905 uint32_t m = (blockIdx.x * _WvPrGrp + threadIdx.y) * YTILE;
906
907 - // Check whether there will be fragmenation!
907 + // Check whether there will be fragmentation!
908 // This will happen only for the last wave!
909 if (m < M && (m + YTILE) >= M) {
910 uint32_t startColumn = M - YTILE;
@@ -1176,7 +1176,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
1176 m += CuCount * _WvPrGrp * YTILE;
1177 kBase = 0;
1178
1179 - // Check whether there will be fragmenation!
1179 + // Check whether there will be fragmentation!
1180 // This will happen only for the last wave!
1181 if (m < M && (m + YTILE) >= M) {
1182 uint32_t startColumn = M - YTILE;
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu
+1 -1
@@ -277,7 +277,7 @@ CompressorResult cutlass_sparse_compress_sm90(torch::Tensor const& a) {
277 uint32_t const m = 1; // Set M to 1 for compression
278 uint32_t const n = a.size(1);
279
280 - // Note: For correctess, the compressed format must be invariant in:
280 + // Note: For correctness, the compressed format must be invariant in:
281 // - M, the flattened number of tokens
282 // - Whether output dtype is fp16 or bf16
283 // - CUTLASS epilogues
pyproject.toml
-4
@@ -137,10 +137,6 @@ exclude = [
137 'vllm/attention/ops/.*\.py$'
138 ]
139
140 -[tool.codespell]
141 -ignore-words-list = "dout, te, indicies, subtile, ElementE"
142 -skip = "tests/models/fixtures/*,tests/prompts/*,benchmarks/sonnet.txt,tests/lora/data/*,build/*,vllm/third_party/*"
143 -
140 [tool.isort]
141 skip_glob = [
142 ".buildkite/*",
tests/compile/test_async_tp.py
+2 -2
@@ -223,7 +223,7 @@ def test_async_tp_pass_correctness(
223 "VLLM_USE_V1": "1",
224 }
225
226 - aysnc_tp_args = [
226 + async_tp_args = [
227 *common_args,
228 "--tensor-parallel-size",
229 str(tp_size),
@@ -242,7 +242,7 @@ def test_async_tp_pass_correctness(
242 ]
243
244 compare_two_settings(model_id,
245 - aysnc_tp_args,
245 + async_tp_args,
246 tp_args,
247 async_tp_env,
248 tp_env,
tests/core/block/e2e/test_correctness.py
+2 -2
@@ -437,8 +437,8 @@ def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
437 "enable_prefix_caching": True,
438 }])
439 @pytest.mark.parametrize("seed", [1])
440 -def test_auto_prefix_caching_after_evition_start(baseline_llm_generator,
441 - test_llm_generator):
440 +def test_auto_prefix_caching_after_eviction_start(baseline_llm_generator,
441 + test_llm_generator):
442 """Verify block manager v2 with auto prefix caching could works normal
443 even when eviction started.
444 With APC enabled, all blocks are held by native block at the beginning.
tests/core/block/e2e/test_correctness_sliding_window.py
+3 -3
@@ -33,8 +33,8 @@ BLOCK_SIZE = 16
33 @pytest.mark.parametrize("batch_size", [5])
34 @pytest.mark.parametrize("seed", [1])
35 @pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
36 -def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
37 - batch_size, seed, backend, monkeypatch):
36 +def test_sliding_window_retrieval(baseline_llm_generator, test_llm_generator,
37 + batch_size, seed, backend, monkeypatch):
38 """
39 The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
40 asks for value of one of them (which is outside the sliding window).
@@ -100,7 +100,7 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
100 def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed,
101 backend, monkeypatch):
102 """
103 - This is similar to test_sliding_window_retrival, however, it doesn't
103 + This is similar to test_sliding_window_retrieval, however, it doesn't
104 compare against the v1 block manager since v1 doesn't support
105 chunked prefill with sliding window.
106
tests/core/test_scheduler.py
+2 -2
@@ -594,8 +594,8 @@ def test_decode_schedule_preempted():
594 # should be preempted. 1 will also be preempted.
595 budget = create_token_budget()
596 output = scheduler._schedule_running(budget, curr_loras)
597 - remainig_running = scheduler.running
598 - assert len(remainig_running) == 0
597 + remaining_running = scheduler.running
598 + assert len(remaining_running) == 0
599 assert len(output.decode_seq_groups) == 1
600 assert len(output.prefill_seq_groups) == 0
601 assert output.decode_seq_groups[0].seq_group.request_id == "0"
tests/entrypoints/openai/test_chat_template.py
+2 -2
@@ -16,7 +16,7 @@ chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
16 assert chatml_jinja_path.exists()
17
18 # Define models, templates, and their corresponding expected outputs
19 -MODEL_TEMPLATE_GENERATON_OUTPUT = [
19 +MODEL_TEMPLATE_GENERATION_OUTPUT = [
20 ("facebook/opt-125m", chatml_jinja_path, True, False, """<|im_start|>user
21 Hello<|im_end|>
22 <|im_start|>assistant
@@ -91,7 +91,7 @@ def test_no_load_chat_template_literallike():
91
92 @pytest.mark.parametrize(
93 "model,template,add_generation_prompt,continue_final_message,expected_output",
94 - MODEL_TEMPLATE_GENERATON_OUTPUT)
94 + MODEL_TEMPLATE_GENERATION_OUTPUT)
95 def test_get_gen_prompt(model, template, add_generation_prompt,
96 continue_final_message, expected_output):
97 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
tests/kernels/attention/test_cache.py
+8 -8
@@ -72,8 +72,8 @@ def test_copy_blocks(
72 # destination blocks.
73 assert 2 * num_mappings <= num_blocks
74 src_blocks = random.sample(range(num_blocks), num_mappings)
75 - remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
76 - dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)
75 + remaining_blocks = list(set(range(num_blocks)) - set(src_blocks))
76 + dst_blocks = random.sample(remaining_blocks, 2 * num_mappings)
77 block_mapping: list[tuple[int, int]] = []
78 for i in range(num_mappings):
79 src = src_blocks[i]
@@ -189,12 +189,12 @@ def test_reshape_and_cache(
189
190 # Run the reference implementation.
191 reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
192 - block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
193 - block_indicies_lst = block_indicies.cpu().tolist()
192 + block_indices = torch.div(slot_mapping, block_size, rounding_mode="floor")
193 + block_indices_lst = block_indices.cpu().tolist()
194 block_offsets = slot_mapping % block_size
195 block_offsets_lst = block_offsets.cpu().tolist()
196 for i in range(num_tokens):
197 - block_idx = block_indicies_lst[i]
197 + block_idx = block_indices_lst[i]
198 block_offset = block_offsets_lst[i]
199 cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
200 cloned_value_cache[block_idx, :, :, block_offset] = value[i]
@@ -322,12 +322,12 @@ def test_reshape_and_cache_flash(
322 kv_dtype=kv_cache_dtype)
323
324 # Run the reference implementation.
325 - block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
326 - block_indicies_lst = block_indicies.cpu().tolist()
325 + block_indices = torch.div(slot_mapping, block_size, rounding_mode="floor")
326 + block_indices_lst = block_indices.cpu().tolist()
327 block_offsets = slot_mapping % block_size
328 block_offsets_lst = block_offsets.cpu().tolist()
329 for i in range(num_tokens):
330 - block_idx = block_indicies_lst[i]
330 + block_idx = block_indices_lst[i]
331 block_offset = block_offsets_lst[i]
332 if kv_cache_layout == "NHD":
333 cloned_key_cache[block_idx, block_offset, :, :] = key[i]
tests/kernels/attention/test_encoder_decoder_attn.py
+1 -1
@@ -46,7 +46,7 @@ CUDA_DEVICE = "cuda:0"
46 MAX_DEC_SEQ_LENS = [128]
47 MAX_ENC_SEQ_LENS = [128]
48
49 -# Narrow teest-cases for unsupported-scenario
49 +# Narrow test-cases for unsupported-scenario
50 # tests
51 HEAD_SIZES_FOR_UNSUPP = [HEAD_SIZES[0]]
52
tests/kernels/core/test_rotary_embedding.py
+4 -4
@@ -39,10 +39,10 @@ def rotary_embedding_opcheck(rot,
39 @pytest.mark.parametrize("head_size", [32, 108])
40 @pytest.mark.parametrize("seq_len", [11, 1024])
41 @pytest.mark.parametrize("use_key", [True, False])
42 -@pytest.mark.parametrize("head_stride_is_contingous", [True, False])
42 +@pytest.mark.parametrize("head_stride_is_contiguous", [True, False])
43 def test_rotary_embedding_opcheck(dist_init, device, max_position,
44 is_neox_style, rotary_dim, head_size,
45 - seq_len, use_key, head_stride_is_contingous):
45 + seq_len, use_key, head_stride_is_contiguous):
46 batch_size = 1
47 base = 10000
48 num_heads = 7
@@ -52,7 +52,7 @@ def test_rotary_embedding_opcheck(dist_init, device, max_position,
52 positions = torch.randint(0,
53 max_position, (batch_size, seq_len),
54 device=device)
55 - head_stride = head_size + (64 if head_stride_is_contingous else 0)
55 + head_stride = head_size + (64 if head_stride_is_contiguous else 0)
56
57 query = torch.randn(batch_size,
58 seq_len,
@@ -72,7 +72,7 @@ def test_rotary_embedding_opcheck(dist_init, device, max_position,
72
73 # if we have a contiguous head stride, test the alternate
74 # [..., num_heads * head_dim] shape/layout
75 - if head_stride_is_contingous:
75 + if head_stride_is_contiguous:
76 rotary_embedding_opcheck(
77 rot, positions, query.flatten(start_dim=-2),
78 key.flatten(start_dim=-2) if use_key else None)
tests/kernels/mamba/test_mamba_ssm_ssd.py
+13 -14
@@ -107,15 +107,15 @@ def generate_random_inputs(batch_size,
107 return A, dt, X, B, C
108
109
110 -def generate_continous_batched_examples(example_lens_by_batch,
111 - num_examples,
112 - full_length,
113 - last_taken,
114 - exhausted,
115 - n_heads,
116 - d_head,
117 - itype,
118 - device='cuda'):
110 +def generate_continuous_batched_examples(example_lens_by_batch,
111 + num_examples,
112 + full_length,
113 + last_taken,
114 + exhausted,
115 + n_heads,
116 + d_head,
117 + itype,
118 + device='cuda'):
119
120 # this function generates a random examples of certain length
121 # and then cut according to "example_lens_by_batch" and feed
@@ -269,11 +269,10 @@ def test_mamba_chunk_scan_cont_batch(d_head, n_heads, seq_len_chunk_size_cases,
269 exhausted: dict = {} # map: eg -> boolean indicating example is exhausted
270
271 states = None
272 - for Y_min, cu_seqlens, seq_idx, (A, dt, X, B,
273 - C) in generate_continous_batched_examples(
274 - cases, num_examples, seqlen,
275 - last_taken, exhausted, n_heads,
276 - d_head, itype):
272 + for Y_min, cu_seqlens, seq_idx, (
273 + A, dt, X, B, C) in generate_continuous_batched_examples(
274 + cases, num_examples, seqlen, last_taken, exhausted, n_heads,
275 + d_head, itype):
276
277 chunk_indices, chunk_offsets = \
278 _query_start_loc_to_chunk_indices_offsets(
tests/lora/test_transformers_model.py renamed
tests/models/language/generation/test_bart.py
+1 -1
@@ -118,7 +118,7 @@ def run_test(
118 # default to enforce_eager=True if enforce_eager
119 # is left unspecified. However, the
120 # VllmRunner test fixture (which wraps around the LLM class) defaults to
121 - # enforce_eager=False (a behavior which a number of already-exisitng
121 + # enforce_eager=False (a behavior which a number of already-existing
122 # decoder-only unit tests expect), so when testing an encoder/decoder
123 # model we must explicitly specify enforce_eager=True in the VllmRunner
124 # constructor.
tests/samplers/test_typical_acceptance_sampler.py
+1 -1
@@ -248,7 +248,7 @@ def test_temperature_zero_target_distribution(seed: int, device: str):
248 size=(batch_size, 1),
249 dtype=torch.int64)
250 # The target probaility distribution is a temperature zero distribution
251 - # with zero entroy. Since our draft token ids don't match the probability
251 + # with zero entropy. Since our draft token ids don't match the probability
252 # 1.0 tokens in the target distribution we will reject all of them and
253 # fallback to the greedy sampling for selecting 1 token for each sequence.
254 # Verify the same.
tests/spec_decode/e2e/test_eagle_correctness.py
+1 -1
@@ -18,7 +18,7 @@ However, we still need to verify below scenario could be passed:
18 * Test greedy equality under various number of speculative tokens.
19
20 With those tests, we can say at least, EAGLE would not break the
21 -correctess for the target model outputs.
21 +correctness for the target model outputs.
22 """
23
24 import pytest
tests/spec_decode/e2e/test_medusa_correctness.py
+1 -1
@@ -18,7 +18,7 @@ However, we still need to verify below scenario could be passed:
18 * Test greedy equality under various number of speculative tokens.
19
20 With those tests, we can say at least, Medusa would not break the
21 -correctess for the target model outputs.
21 +correctness for the target model outputs.
22 """
23
24 import pytest
tests/spec_decode/e2e/test_mtp_correctness.py
+1 -1
@@ -18,7 +18,7 @@ However, we still need to verify below scenario could be passed:
18 * Test greedy equality under various number of speculative tokens.
19
20 With those tests, we can say at least, mtp would not break the
21 -correctess for the target model outputs.
21 +correctness for the target model outputs.
22 """
23
24 import pytest
tests/spec_decode/e2e/test_ngram_correctness.py
+2 -2
@@ -22,8 +22,8 @@ However, we still need to verify below scenario could be passed:
22 * Test greedy equality under preemption
23 * Test greedy equality under various ngram sizes / speculative sizes
24
25 -With those tests, we can say at least, ngram spec would not break the correctess
26 -for the target model outputs.
25 +With those tests, we can say at least, ngram spec would not break the
26 +correctness for the target model outputs.
27 """
28
29 import pytest
tests/v1/e2e/test_correctness_sliding_window.py
+1 -1
@@ -30,7 +30,7 @@ model_config = {
30 ])
31 @pytest.mark.parametrize("batch_size", [5])
32 @pytest.mark.parametrize("seed", [1])
33 -def test_sliding_window_retrival(monkeypatch, model, batch_size, seed):
33 +def test_sliding_window_retrieval(monkeypatch, model, batch_size, seed):
34 """
35 The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
36 asks for value of one of them (which is outside the sliding window).
tests/v1/kv_connector/unit/test_nixl_connector.py
+2 -2
@@ -7,7 +7,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector import (
7 from .utils import create_request, create_scheduler, create_vllm_config
8
9
10 -def test_basic_inferface():
10 +def test_basic_interface():
11 """Unit test for basic NixlConnector interface functionality."""
12
13 vllm_config = create_vllm_config()
@@ -25,7 +25,7 @@ def test_basic_inferface():
25
26 scheduler.add_request(request)
27
28 - # Remote Prefill, triggers NixlConnectorMetdata.
28 + # Remote Prefill, triggers NixlConnectorMetadata.
29 scheduler_output = scheduler.schedule()
30 kv_connector_metadata = scheduler_output.kv_connector_metadata
31 assert kv_connector_metadata is not None
tests/v1/sample/test_logprobs_e2e.py
+1 -1
@@ -32,7 +32,7 @@ def test_prompt_logprobs_e2e():
32 ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
33
34
35 -def test_promt_logprobs_e2e_server():
35 +def test_prompt_logprobs_e2e_server():
36 with RemoteOpenAIServer(MODEL, SERVER_ARGS) as remote_server:
37 url = f"{remote_server.url_for('v1')}/completions"
38
tests/worker/test_model_input.py
+13 -13
@@ -209,32 +209,32 @@ def test_multi_step_model_runner_input():
209 received_model_input = (StatefulModelInput.from_broadcasted_tensor_dict(
210 tensor_dict, attn_backend=attn_backend))
211
212 - receieved_frozen_input = received_model_input.frozen_model_input
212 + received_frozen_input = received_model_input.frozen_model_input
213
214 # Check that received copy has correct values.
215 assert isinstance(received_model_input, StatefulModelInput)
216 - assert receieved_frozen_input.input_tokens is not None
217 - assert (receieved_frozen_input.input_tokens ==
216 + assert received_frozen_input.input_tokens is not None
217 + assert (received_frozen_input.input_tokens ==
218 frozen_model_input.input_tokens).all()
219 - assert receieved_frozen_input.input_positions is not None
220 - assert (receieved_frozen_input.input_positions ==
219 + assert received_frozen_input.input_positions is not None
220 + assert (received_frozen_input.input_positions ==
221 frozen_model_input.input_positions).all()
222 - assert receieved_frozen_input.multi_modal_kwargs is None
222 + assert received_frozen_input.multi_modal_kwargs is None
223 assert (frozen_model_input.multi_modal_kwargs ==
224 frozen_model_input.multi_modal_kwargs)
225 - assert receieved_frozen_input.lora_requests is None
226 - assert (receieved_frozen_input.lora_requests ==
225 + assert received_frozen_input.lora_requests is None
226 + assert (received_frozen_input.lora_requests ==
227 frozen_model_input.lora_requests)
228 - assert receieved_frozen_input.lora_mapping is None
228 + assert received_frozen_input.lora_mapping is None
229 assert (
230 - receieved_frozen_input.lora_mapping == frozen_model_input.lora_mapping)
230 + received_frozen_input.lora_mapping == frozen_model_input.lora_mapping)
231 for field in dataclasses.fields(AttentionMetadata):
232 - assert getattr(receieved_frozen_input.attn_metadata, field.name,
232 + assert getattr(received_frozen_input.attn_metadata, field.name,
233 None) == getattr(attn_metadata, field.name, None)
234 # For sampling metadata, only selected_token_indices is copied.
235 - assert (receieved_frozen_input.sampling_metadata.selected_token_indices ==
235 + assert (received_frozen_input.sampling_metadata.selected_token_indices ==
236 sampling_metadata.selected_token_indices)
237 - assert receieved_frozen_input.sampling_metadata.seq_groups is None
237 + assert received_frozen_input.sampling_metadata.seq_groups is None
238
239 # check non frozen fields
240 assert received_model_input.is_last_step == model_input.is_last_step
tools/report_build_time_ninja.py
+1 -1
@@ -116,7 +116,7 @@ def ReadTargets(log, show_all):
116 # If ninja.exe is rudely halted then the .ninja_log file may be
117 # corrupt. Silently continue.
118 continue
119 - start, end, _, name, cmdhash = parts # Ignore restat.
119 + start, end, _, name, cmdhash = parts # Ignore restart.
120 # Convert from integral milliseconds to float seconds.
121 start = int(start) / 1000.0
122 end = int(end) / 1000.0
typos.toml new
+179
@@ -0,0 +1,179 @@
1 +[files]
2 +# these files may be written in non english words
3 +extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*",
4 + "benchmarks/sonnet.txt", "tests/lora/data/*", "build/*",
5 + "vllm/third_party/*"]
6 +ignore-hidden = true
7 +ignore-files = true
8 +ignore-dot = true
9 +ignore-vcs = true
10 +ignore-global = true
11 +ignore-parent = true
12 +
13 +[default]
14 +binary = false
15 +check-filename = false
16 +check-file = true
17 +unicode = true
18 +ignore-hex = true
19 +identifier-leading-digits = false
20 +locale = "en"
21 +extend-ignore-identifiers-re = ["NVML_*", ".*Unc.*", ".*_thw",
22 + ".*UE8M0.*", ".*[UE4M3|ue4m3].*", ".*eles.*", ".*fo.*", ".*ba.*",
23 + ".*ot.*", ".*[Tt]h[rR].*"]
24 +extend-ignore-words-re = []
25 +extend-ignore-re = []
26 +
27 +[default.extend-identifiers]
28 +bbc5b7ede = "bbc5b7ede"
29 +womens_doubles = "womens_doubles"
30 +v_2nd = "v_2nd"
31 +splitted_input = "splitted_input"
32 +NOOPs = "NOOPs"
33 +typ = "typ"
34 +nin_shortcut = "nin_shortcut"
35 +UperNetDecoder = "UperNetDecoder"
36 +subtile = "subtile"
37 +cudaDevAttrMaxSharedMemoryPerBlockOptin = "cudaDevAttrMaxSharedMemoryPerBlockOptin"
38 +SFOuput = "SFOuput"
39 +# huggingface transformers repo uses these words
40 +depthwise_seperable_out_channel = "depthwise_seperable_out_channel"
41 +DepthWiseSeperableConv1d = "DepthWiseSeperableConv1d"
42 +depthwise_seperable_CNN = "depthwise_seperable_CNN"
43 +
44 +[default.extend-words]
45 +iy = "iy"
46 +tendencias = "tendencias"
47 +# intel cpu features
48 +tme = "tme"
49 +dout = "dout"
50 +Pn = "Pn"
51 +arange = "arange"
52 +
53 +[type.py]
54 +extend-glob = []
55 +extend-ignore-identifiers-re = []
56 +extend-ignore-words-re = []
57 +extend-ignore-re = []
58 +
59 +[type.py.extend-identifiers]
60 +arange = "arange"
61 +NDArray = "NDArray"
62 +EOFError = "EOFError"
63 +
64 +[type.py.extend-words]
65 +
66 +[type.cpp]
67 +extend-glob = []
68 +extend-ignore-identifiers-re = []
69 +extend-ignore-words-re = []
70 +extend-ignore-re = []
71 +
72 +[type.cpp.extend-identifiers]
73 +countr_one = "countr_one"
74 +
75 +[type.cpp.extend-words]
76 +
77 +[type.rust]
78 +extend-glob = []
79 +extend-ignore-identifiers-re = []
80 +extend-ignore-words-re = []
81 +extend-ignore-re = []
82 +
83 +[type.rust.extend-identifiers]
84 +flate2 = "flate2"
85 +
86 +[type.rust.extend-words]
87 +ser = "ser"
88 +
89 +[type.lock]
90 +extend-glob = []
91 +check-file = false
92 +extend-ignore-identifiers-re = []
93 +extend-ignore-words-re = []
94 +extend-ignore-re = []
95 +
96 +[type.lock.extend-identifiers]
97 +
98 +[type.lock.extend-words]
99 +
100 +[type.jl]
101 +extend-glob = []
102 +extend-ignore-identifiers-re = []
103 +extend-ignore-words-re = []
104 +extend-ignore-re = []
105 +
106 +[type.jl.extend-identifiers]
107 +
108 +[type.jl.extend-words]
109 +modul = "modul"
110 +egals = "egals"
111 +usig = "usig"
112 +egal = "egal"
113 +
114 +[type.go]
115 +extend-glob = []
116 +extend-ignore-identifiers-re = []
117 +extend-ignore-words-re = []
118 +extend-ignore-re = []
119 +
120 +[type.go.extend-identifiers]
121 +flate = "flate"
122 +
123 +[type.go.extend-words]
124 +
125 +[type.css]
126 +extend-glob = []
127 +extend-ignore-identifiers-re = []
128 +extend-ignore-words-re = []
129 +extend-ignore-re = []
130 +
131 +[type.css.extend-identifiers]
132 +nd = "nd"
133 +
134 +[type.css.extend-words]
135 +
136 +[type.man]
137 +extend-glob = []
138 +extend-ignore-identifiers-re = []
139 +extend-ignore-words-re = []
140 +extend-ignore-re = []
141 +
142 +[type.man.extend-identifiers]
143 +Nd = "Nd"
144 +
145 +[type.man.extend-words]
146 +
147 +[type.cert]
148 +extend-glob = []
149 +check-file = false
150 +extend-ignore-identifiers-re = []
151 +extend-ignore-words-re = []
152 +extend-ignore-re = []
153 +
154 +[type.cert.extend-identifiers]
155 +
156 +[type.cert.extend-words]
157 +
158 +[type.sh]
159 +extend-glob = []
160 +extend-ignore-identifiers-re = []
161 +extend-ignore-words-re = []
162 +extend-ignore-re = []
163 +
164 +[type.sh.extend-identifiers]
165 +stap = "stap"
166 +ot = "ot"
167 +
168 +[type.sh.extend-words]
169 +
170 +[type.vimscript]
171 +extend-glob = []
172 +extend-ignore-identifiers-re = []
173 +extend-ignore-words-re = []
174 +extend-ignore-re = []
175 +
176 +[type.vimscript.extend-identifiers]
177 +windo = "windo"
178 +
179 +[type.vimscript.extend-words]
vllm/_custom_ops.py
+3 -3
@@ -1550,10 +1550,10 @@ def moe_wna16_gemm(input: torch.Tensor, output: torch.Tensor,
1550
1551
1552 def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
1553 - token_expert_indicies: torch.Tensor,
1553 + token_expert_indices: torch.Tensor,
1554 gating_output: torch.Tensor) -> None:
1555 - torch.ops._moe_C.topk_softmax(topk_weights, topk_ids,
1556 - token_expert_indicies, gating_output)
1555 + torch.ops._moe_C.topk_softmax(topk_weights, topk_ids, token_expert_indices,
1556 + gating_output)
1557
1558
1559 def moe_wna16_marlin_gemm(input: torch.Tensor, output: Optional[torch.Tensor],
vllm/attention/backends/utils.py
+2 -2
@@ -373,7 +373,7 @@ class CommonAttentionState(AttentionState):
373 f"Expected attn_backend name to be either 'XFORMERS'," \
374 f"'ROCM_FLASH', or 'FLASH_ATTN', but " \
375 f"got '{self.runner.attn_backend.get_name()}'"
376 - self._add_additonal_input_buffers_for_enc_dec_model(
376 + self._add_additional_input_buffers_for_enc_dec_model(
377 attn_metadata=attn_metadata, input_buffers=input_buffers)
378 return input_buffers
379
@@ -427,7 +427,7 @@ class CommonAttentionState(AttentionState):
427 attn_metadata.max_encoder_seq_len = self.runner.max_seq_len_to_capture
428 attn_metadata.num_encoder_tokens = 0
429
430 - def _add_additonal_input_buffers_for_enc_dec_model(
430 + def _add_additional_input_buffers_for_enc_dec_model(
431 self, attn_metadata, input_buffers: Dict[str, Any]):
432 """
433 Saves additional input buffers specific to the encoder-decoder model
vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py
+4 -4
@@ -40,7 +40,7 @@ class Internlm2ToolParser(ToolParser):
40 request.skip_special_tokens = False
41 return request
42
43 - def get_argments(self, obj):
43 + def get_arguments(self, obj):
44 if "parameters" in obj:
45 return obj.get("parameters")
46 elif "arguments" in obj:
@@ -119,9 +119,9 @@ class Internlm2ToolParser(ToolParser):
119 # now we know we're on the same tool call and we're streaming
120 # arguments
121 else:
122 - prev_arguments = self.get_argments(
122 + prev_arguments = self.get_arguments(
123 self.prev_tool_call_arr[self.current_tool_id])
124 - cur_arguments = self.get_argments(tool_call_arr)
124 + cur_arguments = self.get_arguments(tool_call_arr)
125
126 # not arguments generated
127 if not cur_arguments and not prev_arguments:
@@ -170,7 +170,7 @@ class Internlm2ToolParser(ToolParser):
170 # check to see if the name is defined and has been sent. if so,
171 # stream the name - otherwise keep waiting
172 # finish by setting old and returning None as base case
173 - tool_call_arr["arguments"] = self.get_argments(tool_call_arr)
173 + tool_call_arr["arguments"] = self.get_arguments(tool_call_arr)
174 self.prev_tool_call_arr = [tool_call_arr]
175 return delta
176 except Exception:
vllm/lora/layers.py
+1 -1
@@ -1202,7 +1202,7 @@ class LinearScalingRotaryEmbeddingWithLoRA(BaseLayerWithLoRA):
1202 multiple LoRA adapters with a specialized kernel.
1203
1204 Replace LinearScalingRotaryEmbedding with MultiLinearScalingRotaryEmbedding
1205 - which can handle multi lora adapters in a specialied kernel.
1205 + which can handle multi lora adapters in a specialized kernel.
1206 """
1207
1208 def __init__(self, base_layer: RotaryEmbedding) -> None:
vllm/lora/punica_wrapper/utils.py
+2 -2
@@ -68,11 +68,11 @@ def convert_mapping(
68 LoRA indices.
69 sampler_indices: Tensor of shape [batch_size] mapping requests to
70 LoRA indices for sampler. For generation, this will be the
71 - same as base_indicies. For prefill, this will map requests
71 + same as base_indices. For prefill, this will map requests
72 to LoRA indices.
73 sampler_indices_padded: Tensor of shape [batch_size] mapping
74 requests to LoRA indices for sampler with padding.
75 - Same as sampler_indicies, but -1 is replaced with
75 + Same as sampler_indices, but -1 is replaced with
76 max_loras.
77 embeddings_indices: Tensor of shape [2, batch_size] mapping
78 requests to embedding indices. First row is for embeddings
vllm/model_executor/layers/mamba/mamba_mixer2.py
+2 -2
@@ -319,7 +319,7 @@ class MambaMixer2(CustomOp):
319 n_groups == 1, # if there was only one group
320 )
321 intermediate_settings = (intermediate_size, 0, False)
322 - head_setings = (self.num_heads, 0, False)
322 + head_settings = (self.num_heads, 0, False)
323
324 # - the weight already has a "weight_loader" attribute
325 # which set_weight_attrs will raise if we do not
@@ -372,7 +372,7 @@ class MambaMixer2(CustomOp):
372 intermediate_settings,
373 group_shard_settings,
374 group_shard_settings,
375 - head_setings, # for dt
375 + head_settings, # for dt
376 ],
377 self.tp_size,
378 tp_rank,
vllm/model_executor/layers/mamba/ops/ssd_chunk_state.py
+1 -1
@@ -516,7 +516,7 @@ def _chunk_state_varlen_kernel(
516 offs_n[None, :] * stride_chunk_states_dstate)
517 else:
518
519 - # - this seems repetitve, buts its to help the compiler
519 + # - this seems repetitive, buts its to help the compiler
520 if start_idx < pid_c * chunk_size:
521 past_states_ptrs = chunk_states_ptr + (
522 offs_m[:, None] * stride_chunk_states_hdim +
vllm/model_executor/layers/quantization/utils/int8_utils.py
+1 -1
@@ -219,7 +219,7 @@ def per_token_group_quant_int8(
219 quantized tensor along with the scaling factor used for quantization.
220
221 Args:
222 - x: The input tenosr with ndim >= 2.
222 + x: The input tensor with ndim >= 2.
223 group_size: The group size used for quantization.
224 eps: The minimum to avoid dividing zero.
225 dtype: The dype of output tensor. Note that only `torch.int8`
vllm/model_executor/model_loader/bitsandbytes_loader.py
+1 -1
@@ -401,7 +401,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
401 self.target_modules.append(
402 name.replace(rep_name, sub_name))
403 # Add original module name even if the module has stacked map,
404 - # in case model has a mixture of disk-merged and disk-splitted
404 + # in case model has a mixture of disk-merged and disk-split
405 # weights with same last name.
406 self.target_modules.append(name)
407
vllm/model_executor/models/baichuan.py
+3 -3
@@ -131,7 +131,7 @@ class BaiChuanAttention(nn.Module):
131 self.num_heads = (self.total_num_heads //
132 tensor_model_parallel_world_size)
133 self.head_dim = hidden_size // self.total_num_heads
134 - self.postion_embedding = position_embedding
134 + self.position_embedding = position_embedding
135 self.rope_theta = rope_theta
136 self.max_position_embeddings = max_position_embeddings
137
@@ -151,7 +151,7 @@ class BaiChuanAttention(nn.Module):
151 quant_config=quant_config,
152 )
153 # Create the alibi slopes and slice them.
154 - if self.postion_embedding == "ALIBI":
154 + if self.position_embedding == "ALIBI":
155 tp_rank = get_tensor_model_parallel_rank()
156 head_start = tp_rank * self.num_heads
157 head_end = (tp_rank + 1) * self.num_heads
@@ -187,7 +187,7 @@ class BaiChuanAttention(nn.Module):
187 ) -> torch.Tensor:
188 qkv, _ = self.W_pack(hidden_states)
189 q, k, v = qkv.chunk(chunks=3, dim=-1)
190 - if self.postion_embedding != "ALIBI":
190 + if self.position_embedding != "ALIBI":
191 q, k = self.rotary_emb(positions, q, k)
192 attn_output = self.attn(q, k, v)
193 output, _ = self.o_proj(attn_output)
vllm/model_executor/models/deepseek_vl2.py
+3 -3
@@ -344,7 +344,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
344 self.image_newline = nn.Parameter(
345 torch.randn(self.projector_config.n_embed) * embed_std)
346 # This is a typo in original implementation
347 - self.view_seperator = nn.Parameter(
347 + self.view_separator = nn.Parameter(
348 torch.randn(self.projector_config.n_embed) * embed_std)
349 else:
350 raise ValueError(
@@ -549,13 +549,13 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
549 if self.global_view_pos == "head":
550 global_local_features = torch.cat([
551 global_features,
552 - self.view_seperator[None, :],
552 + self.view_separator[None, :],
553 local_features,
554 ])
555 else:
556 global_local_features = torch.cat([
557 local_features,
558 - self.view_seperator[None, :],
558 + self.view_separator[None, :],
559 global_features,
560 ])
561
vllm/model_executor/models/eagle.py
+1 -1
@@ -197,7 +197,7 @@ class EAGLE(nn.Module):
197 return logits
198
199 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
200 - # This implementation is incompitable with https://huggingface.co/yuhuili/EAGLE-LLaMA3-Instruct-8B
200 + # This implementation is incompatible with https://huggingface.co/yuhuili/EAGLE-LLaMA3-Instruct-8B
201 # due to missing lm_head weights and its config being that of a
202 # Llama model. Here's a compatible version with the same weights:
203 # https://huggingface.co/abhigoyal/EAGLE-LLaMA3-Instruct-8B-vllm
vllm/model_executor/models/gemma3_mm.py
+4 -4
@@ -634,13 +634,13 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
634 kwargs["has_images"] = True
635 # NOTE(woosuk): Here, we distinguish the sequences by the position id 0.
636 # This is a HACK. Fix this.
637 - start_idices = (positions == 0).cpu().nonzero()
638 - num_seqs = len(start_idices)
637 + start_indices = (positions == 0).cpu().nonzero()
638 + num_seqs = len(start_indices)
639 seq_lens = []
640 for i in range(num_seqs):
641 - start_idx = start_idices[i].item()
641 + start_idx = start_indices[i].item()
642 if i < num_seqs - 1:
643 - end_idx = start_idices[i + 1].item()
643 + end_idx = start_indices[i + 1].item()
644 else:
645 end_idx = len(input_ids)
646 seq_lens.append(end_idx - start_idx)
vllm/model_executor/models/llama4.py
+1 -1
@@ -52,7 +52,7 @@ class Llama4MoE(nn.Module):
52 renormalize: bool,
53 ) -> tuple[torch.Tensor, torch.Tensor]:
54 router_scores, router_indices = fast_topk(gating_output, topk, dim=-1)
55 - # psuedo-standard is that the router scores are floats
55 + # pseudo-standard is that the router scores are floats
56 router_scores = torch.sigmoid(router_scores.float())
57 return (router_scores, router_indices.to(torch.int32))
58
vllm/model_executor/models/mixtral_quant.py
+5 -5
@@ -114,9 +114,9 @@ class MixtralMoE(nn.Module):
114 f"Tensor parallel size {self.tp_size} is greater than "
115 f"the number of experts {self.num_total_experts}.")
116 # Split experts equally between ranks
117 - self.expert_indicies = np.array_split(range(
118 - self.num_total_experts), self.tp_size)[self.rank].tolist()
119 - if not self.expert_indicies:
117 + self.expert_indices = np.array_split(range(self.num_total_experts),
118 + self.tp_size)[self.rank].tolist()
119 + if not self.expert_indices:
120 raise ValueError(
121 f"Rank {self.rank} has no experts assigned to it.")
122
@@ -125,7 +125,7 @@ class MixtralMoE(nn.Module):
125 config.hidden_size,
126 config.intermediate_size,
127 quant_config=quant_config)
128 - if idx in self.expert_indicies else None
128 + if idx in self.expert_indices else None
129 for idx in range(self.num_total_experts)
130 ])
131 self.gate = ReplicatedLinear(config.hidden_size,
@@ -146,7 +146,7 @@ class MixtralMoE(nn.Module):
146 routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
147
148 final_hidden_states = None
149 - for expert_idx in self.expert_indicies:
149 + for expert_idx in self.expert_indices:
150 expert_layer = self.experts[expert_idx]
151 expert_mask = (selected_experts == expert_idx)
152 expert_weights = (routing_weights * expert_mask).sum(dim=-1,
vllm/model_executor/models/ovis.py
+1 -1
@@ -283,7 +283,7 @@ class OvisProcessingInfo(BaseProcessingInfo):
283 def get_image_size_with_most_features(self) -> ImageSize:
284 height, width = self.get_hf_processor().get_image_size()
285 hs = self.get_hf_config().visual_tokenizer_config.hidden_stride
286 - # NOTE(Isotr0py): 9 is `max_partion` hardcoded in original code
286 + # NOTE(Isotr0py): 9 is `max_partition` hardcoded in original code
287 # https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/modeling_ovis.py#L96
288 return ImageSize(width=width * hs * 9, height=height * hs * 9)
289
vllm/model_executor/models/phi3_small.py
+5 -5
@@ -145,7 +145,7 @@ class Phi3SmallSelfAttention(nn.Module):
145 self.num_q_per_kv = self.num_heads // self.num_key_value_heads
146 if self.tp_size > 1:
147 assert self.num_key_value_heads % self.tp_size == 0
148 - self.num_kv_heads_per_partion = max(
148 + self.num_kv_heads_per_partition = max(
149 1, self.num_key_value_heads // self.tp_size)
150 self.num_heads_per_partition = self.num_heads // self.tp_size
151
@@ -212,7 +212,7 @@ class Phi3SmallSelfAttention(nn.Module):
212 bs_params = {
213 'max_seqlen': self.max_position_embeddings,
214 'num_heads': self.num_heads_per_partition,
215 - "num_kv_heads": self.num_kv_heads_per_partion,
215 + "num_kv_heads": self.num_kv_heads_per_partition,
216 "block_size": self.sparse_block_size,
217 "local_blocks": self.local_blocks,
218 "vert_stride": self.vert_stride,
@@ -222,7 +222,7 @@ class Phi3SmallSelfAttention(nn.Module):
222 self.attn = Attention(self.num_heads_per_partition,
223 self.head_dim,
224 self.scale,
225 - num_kv_heads=self.num_kv_heads_per_partion,
225 + num_kv_heads=self.num_kv_heads_per_partition,
226 cache_config=cache_config,
227 quant_config=quant_config,
228 blocksparse_params=bs_params,
@@ -243,8 +243,8 @@ class Phi3SmallSelfAttention(nn.Module):
243 # NOTE: this is required by RotaryEmbed, which indeed does not have to
244 # TODO: allow 3D QK for rotary forward
245 q = q.reshape(-1, self.head_dim * self.num_heads_per_partition)
246 - k = k.reshape(-1, self.head_dim * self.num_kv_heads_per_partion)
247 - v = v.reshape(-1, self.head_dim * self.num_kv_heads_per_partion)
246 + k = k.reshape(-1, self.head_dim * self.num_kv_heads_per_partition)
247 + v = v.reshape(-1, self.head_dim * self.num_kv_heads_per_partition)
248
249 q, k = self.rotary_emb(positions, q, k)
250 attn_output = self.attn(q, k, v)
vllm/model_executor/models/phi4mm_audio.py
+4 -4
@@ -41,7 +41,7 @@ class ConformerEncoderLayer(nn.Module):
41 for the last pointwise conv after swish activation.
42 depthwise_seperable_out_channel: int
43 if set different to 0, the number of
44 - depthwise_seperable_out_channel will be used as a
44 + depthwise_seperable_out_channel will be used as a
45 channel_out of the second conv1d layer.
46 otherwise, it equal to 0, the second conv1d layer is skipped.
47 depthwise_multiplier: int
@@ -126,7 +126,7 @@ class ConformerEncoderLayer(nn.Module):
126 (Multi-Head Attention),
127 1 = typical Multi-Head Attention,
128 1 < attn_group_sizes < attention_heads = Grouped-Query Attention
129 - attn_group_sizes = attenion_heads = Multi-Query Attention
129 + attn_group_sizes = attention_heads = Multi-Query Attention
130 """
131
132 def __init__(
@@ -318,7 +318,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
318 1 = typical Multi-Head Attention,
319 1 < attention_group_size < attention_heads = Grouped-Query
320 Attention
321 - attention_group_size = attenion_heads = Multi-Query Attention
321 + attention_group_size = attention_heads = Multi-Query Attention
322 """
323
324 def __init__(
@@ -744,7 +744,7 @@ class ConformerEncoder(TransformerEncoderBase):
744 1 = typical Multi-Head Attention,
745 1 < attention_group_size < attention_heads = Grouped-Query
746 Attention
747 - attention_group_size = attenion_heads = Multi-Query Attention
747 + attention_group_size = attention_heads = Multi-Query Attention
748 """
749
750 extra_multi_layer_output_idxs: list[int]
vllm/model_executor/models/phimoe.py
+3 -3
@@ -147,15 +147,15 @@ class mp(torch.autograd.Function):
147
148 grad_at_output = grad_at_output * multiplier
149
150 - grad_at_scores_expaned = masked_gates * grad_at_output.mul(-1)
151 - grad_at_scores_expaned.scatter_add_(
150 + grad_at_scores_expanded = masked_gates * grad_at_output.mul(-1)
151 + grad_at_scores_expanded.scatter_add_(
152 dim=-1,
153 index=selected_experts,
154 src=grad_at_output,
155 )
156
157 return (
158 - grad_at_scores_expaned,
158 + grad_at_scores_expanded,
159 None,
160 None,
161 None,
vllm/multimodal/utils.py
+1 -1
@@ -324,7 +324,7 @@ def merge_and_sort_multimodal_metadata(
324 Returns:
325 list[str]: List of item modalities in order of their positions in the
326 input sequence.
327 - list[PlaceholderRange]: Sorted list of all PlaceholdeRanges from
327 + list[PlaceholderRange]: Sorted list of all PlaceholderRanges from
328 mm_positions.
329 Optional[list[str]]: Sorted list of all hashes from mm_hashes if given,
330 None otherwise.
vllm/transformers_utils/processors/ovis.py
+1 -1
@@ -68,7 +68,7 @@ class OvisProcessor(ProcessorMixin):
68 """
69
70 attributes = ["image_processor", "tokenizer"]
71 - valid_kwargs = ["chat_template", "image_pad_token", "image_segement_len"]
71 + valid_kwargs = ["chat_template", "image_pad_token", "image_segment_len"]
72
73 image_processor_class = "AutoImageProcessor"
74 tokenizer_class = "AutoTokenizer"
vllm/worker/hpu_model_runner.py
+1 -1
@@ -886,7 +886,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
886 num_decode_tokens=0,
887 slot_mapping=slot_mapping,
888 multi_modal_placeholder_index_maps=
889 - None, # FIXME(kzawora): mutli-modality will not work here
889 + None, # FIXME(kzawora): multi-modality will not work here
890 enable_kv_scales_calculation=False,
891 )
892 multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
vllm/worker/multi_step_model_runner.py
+1 -1
@@ -277,7 +277,7 @@ class StatefulModelInput(BroadcastableModelInput):
277 assert fmi.input_tokens.shape[0] >= self.num_seqs
278 fmi_new_input_tokens: torch.Tensor = fmi.input_tokens[:self.num_seqs]
279
280 - # Update frozen_model_input::input_positons.
280 + # Update frozen_model_input::input_positions.
281 assert fmi.input_positions is not None
282 assert fmi.input_positions.shape[0] >= self.num_seqs
283 fmi_new_input_positions: torch.Tensor = fmi.input_positions[:self.
vllm/worker/tpu_model_runner.py
+8 -8
@@ -798,9 +798,9 @@ class ModelWrapper(nn.Module):
798 """
799 batch_size, seq_len = token_ids.shape
800 # Calculate the positions to sample from.
801 - start_indicies = torch.arange(
801 + start_indices = torch.arange(
802 batch_size, dtype=torch.int32, device=input_lens.device) * seq_len
803 - logits_indices = start_indicies + input_lens - 1
803 + logits_indices = start_indices + input_lens - 1
804 attn_metadata = get_forward_context().attn_metadata
805
806 # FIXME(woosuk): This is a temporary hack to avoid using the existing
@@ -822,14 +822,14 @@ class ModelWrapper(nn.Module):
822 num_kv_heads, num_blocks, block_size, _ = kv_caches[0][0].shape
823 slot_mapping = attn_metadata.slot_mapping
824 slot_mapping = slot_mapping.flatten()
825 - head_indicies = torch.arange(0,
826 - num_kv_heads,
827 - device=slot_mapping.device,
828 - dtype=slot_mapping.dtype)
829 - head_indicies *= block_size * num_blocks
825 + head_indices = torch.arange(0,
826 + num_kv_heads,
827 + device=slot_mapping.device,
828 + dtype=slot_mapping.dtype)
829 + head_indices *= block_size * num_blocks
830 slot_mapping = slot_mapping.repeat_interleave(num_kv_heads).view(
831 -1, num_kv_heads)
832 - slot_mapping = slot_mapping + head_indicies.view(1, -1)
832 + slot_mapping = slot_mapping + head_indices.view(1, -1)
833 slot_mapping = slot_mapping.flatten()
834 attn_metadata.slot_mapping = slot_mapping
835