A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix some typing issues found by `mypy==1.18.2` (#26596)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Oct 10, 2025 at 19:21 UTC
7c12763b241150916c59a31ed5f17798ab918ab5
6 files changed
+19
-18
tests/v1/core/test_scheduler.py
+2
@@ -807,8 +807,10 @@ def test_schedule_spec_decoding_stats(spec_tokens, output_tokens, expected):
807
engine_core_outputs[0].scheduler_stats if engine_core_outputs else None
808
)
809
if expected[0] == 0:
810
+ assert scheduler_stats is not None
811
assert scheduler_stats.spec_decoding_stats is None
812
else:
813
+ assert scheduler_stats is not None
814
assert scheduler_stats.spec_decoding_stats is not None
815
stats = scheduler_stats.spec_decoding_stats
816
assert stats.num_drafts == expected[0]
vllm/model_executor/layers/linear.py
+3
-3
@@ -1229,10 +1229,10 @@ class QKVParallelLinear(ColumnParallelLinear):
1229
1230
param_data = param_data.narrow(output_dim, shard_offset, shard_size)
1231
if loaded_shard_id == "q":
1232
- shard_id = self.tp_rank
1232
+ shard_rank = self.tp_rank
1233
else:
1234
- shard_id = self.tp_rank // self.num_kv_head_replicas
1235
- start_idx = shard_id * shard_size
1234
+ shard_rank = self.tp_rank // self.num_kv_head_replicas
1235
+ start_idx = shard_rank * shard_size
1236
1237
if not is_sharded_weight:
1238
loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+6
-6
@@ -49,16 +49,16 @@ class CompressedTensors24(CompressedTensorsScheme):
49
self.quantized = quantized
50
self.weight_quant = weight_quant
51
self.input_quant = input_quant
52
- self.model_compressor = (
53
- ModelCompressor.from_compression_config(model_compression_config)
54
- if model_compression_config is not None
55
- else None
52
+ model_compressor = ModelCompressor.from_compression_config(
53
+ model_compression_config
54
)
55
self.do_sparse_decompress = (
58
- self.model_compressor is not None
59
- and self.model_compressor.sparsity_config.format
56
+ model_compressor is not None
57
+ and model_compressor.sparsity_config.format
58
== CompressionFormat.sparse_24_bitmask.value
59
)
60
+ if self.do_sparse_decompress:
61
+ self.model_compressor = model_compressor
62
63
if (
64
quantized
vllm/model_executor/layers/resampler.py
+4
-6
@@ -200,12 +200,10 @@ class BaseResampler(nn.Module):
200
self.ln_q = norm_layer(embed_dim)
201
self.ln_kv = norm_layer(embed_dim)
202
self.do_post_projection = do_post_projection
203
- self.ln_post = norm_layer(embed_dim) if do_post_projection else None
204
- self.proj = (
205
- nn.Parameter((embed_dim**-0.5) * torch.empty(embed_dim, embed_dim))
206
- if do_post_projection
207
- else None
208
- )
203
+ if self.do_post_projection:
204
+ self.ln_post = norm_layer(embed_dim)
205
+ data = (embed_dim**-0.5) * torch.empty(embed_dim, embed_dim)
206
+ self.proj = nn.Parameter(data=data)
207
208
def _repeat(self, query, N: int):
209
return query.unsqueeze(1).repeat(1, N, 1)
vllm/model_executor/model_loader/bitsandbytes_loader.py
+2
-3
@@ -542,8 +542,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
542
)
543
544
quant_config = getattr(model_config.hf_config, "quantization_config", None)
545
- if quant_config is not None:
546
- quant_method = quant_config.get("quant_method")
545
+ if quant_config and (quant_method := quant_config.get("quant_method")):
546
if quant_method == "bitsandbytes":
547
self.pre_quant = True
548
else:
@@ -558,7 +557,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
557
"Prequant BitsAndBytes models with tensor parallelism is not "
558
"supported. Please try with pipeline parallelism."
559
)
561
- if self.pre_quant:
560
+ if quant_config and self.pre_quant:
561
self.load_8bit = quant_config.get("load_in_8bit", False)
562
563
def _initialize_loader_state(
vllm/transformers_utils/processors/ovis2_5.py
+2
@@ -397,6 +397,8 @@ class Ovis2_5Processor(ProcessorMixin):
397
images.append(image)
398
elif isinstance(video, list):
399
images = video
400
+ else:
401
+ raise ValueError("Either images or video should be provided.")
402
min_pixels = min(
403
max_pixels if max_pixels is not None else MAX_PIXELS,
404
min_pixels if min_pixels is not None else MIN_PIXELS,