A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Minor] Remove unnecessary error message (#27115)
Signed-off-by: Zhuohan Li <zhuohan123@gmail.com>
Zhuohan Li committed
Oct 17, 2025 at 13:02 UTC
d29483b58a036526307669c5d98bbb35a9e45b98
2 files changed
+19
-55
vllm/attention/layer.py
+8
-27
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
34
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
35
from vllm.model_executor.models.vision import get_vit_attn_backend
36
from vllm.platforms import current_platform
37
-from vllm.utils import GiB_bytes, direct_register_custom_op
37
+from vllm.utils import direct_register_custom_op
38
39
FP8_DTYPE = current_platform.fp8_dtype()
40
logger = init_logger(__name__)
@@ -281,25 +281,10 @@ class Attention(nn.Module, AttentionLayerBase):
281
)
282
]
283
284
- try:
285
- self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
286
- self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
287
- self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
288
- except torch.cuda.OutOfMemoryError as e:
289
- logger.error("Failed to initialize attention q/k/v range constants: %s", e)
290
- if torch.cuda.is_available():
291
- logger.debug("CUDA device: %s", torch.cuda.current_device())
292
- logger.debug(
293
- "Allocated: %.2f GiB", torch.cuda.memory_allocated() / GiB_bytes
294
- )
295
- logger.debug(
296
- "Reserved: %.2f GiB", torch.cuda.memory_reserved() / GiB_bytes
297
- )
298
- raise RuntimeError(
299
- "Failed to initialize q/k/v range constants. "
300
- "This may be caused by insufficient memory to allocate "
301
- "kv cache."
302
- ) from e
284
+ # Initialize q/k/v range constants.
285
+ self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
286
+ self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
287
+ self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
288
289
# for attn backends supporting query quantization
290
self.query_quant = None
@@ -668,13 +653,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
653
self.use_sparse = use_sparse
654
655
# Initialize q/k/v range constants.
671
- try:
672
- self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
673
- self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
674
- self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
675
- except torch.cuda.OutOfMemoryError:
676
- # Keep defaults if allocation fails; not critical for init.
677
- pass
656
+ self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
657
+ self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
658
+ self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
659
660
def forward(
661
self,
vllm/model_executor/layers/linear.py
+11
-28
@@ -34,7 +34,6 @@ from vllm.model_executor.parameter import (
34
)
35
from vllm.model_executor.utils import set_weight_attrs
36
from vllm.platforms import current_platform
37
-from vllm.utils import GiB_bytes
37
38
logger = init_logger(__name__)
39
@@ -211,33 +210,17 @@ class UnquantizedLinearMethod(LinearMethodBase):
210
# The weights are not quantized, and they are not sharded.
211
# The amount of memory allocated for the weights is
212
# sum(output_partition_sizes) * input_size_per_partition.
214
- try:
215
- weight_loader = extra_weight_attrs.pop("weight_loader")
216
- weight = ModelWeightParameter(
217
- data=torch.empty(
218
- sum(output_partition_sizes),
219
- input_size_per_partition,
220
- dtype=params_dtype,
221
- ),
222
- input_dim=1,
223
- output_dim=0,
224
- weight_loader=weight_loader,
225
- )
226
- except torch.cuda.OutOfMemoryError as e:
227
- logger.error("Failed to create unquantized linear weights: %s", e)
228
- if torch.cuda.is_available():
229
- logger.debug("CUDA device: %s", torch.cuda.current_device())
230
- logger.debug(
231
- "Allocated: %.2f GiB", torch.cuda.memory_allocated() / GiB_bytes
232
- )
233
- logger.debug(
234
- "Reserved: %.2f GiB", torch.cuda.memory_reserved() / GiB_bytes
235
- )
236
- raise RuntimeError(
237
- "Failed to create unquantized linear weights. "
238
- "This may be caused by insufficient memory to allocate "
239
- "the weight."
240
- ) from e
213
+ weight_loader = extra_weight_attrs.pop("weight_loader")
214
+ weight = ModelWeightParameter(
215
+ data=torch.empty(
216
+ sum(output_partition_sizes),
217
+ input_size_per_partition,
218
+ dtype=params_dtype,
219
+ ),
220
+ input_dim=1,
221
+ output_dim=0,
222
+ weight_loader=weight_loader,
223
+ )
224
225
layer.register_parameter("weight", weight)
226
set_weight_attrs(weight, extra_weight_attrs)