A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
Various cosmetic/comment fixes (#12089)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Jan 16, 2025 at 04:59 UTC
9aa1519f089e8dc2c2bd1b4c74a8ce47d386f0a9
14 files changed
+16
-29
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+1
-1
@@ -42,7 +42,7 @@ class CompressedTensors24(CompressedTensorsScheme):
42
43
if not sparse_cutlass_supported():
44
raise ValueError(
45
- "Sparse CUTLASS not supported. vLLM must be built with"
45
+ "Sparse CUTLASS not supported. vLLM must be built with "
46
"CUDA 12.2 or later to use this feature")
47
48
self.output_dtype = params_dtype
vllm/model_executor/models/aria.py
+1
-2
@@ -390,8 +390,7 @@ class AriaMoELMModel(LlamaModel):
390
continue
391
if (self.quant_config is not None and
392
(scale_name := self.quant_config.get_cache_scale(name))):
393
- # Loading kv cache scales for quark and
394
- # compressed-tensors quantization
393
+ # Loading kv cache quantization scales
394
param = params_dict[scale_name]
395
weight_loader = getattr(param, "weight_loader",
396
default_weight_loader)
vllm/model_executor/models/commandr.py
+1
-2
@@ -440,8 +440,7 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
440
441
if (self.quant_config is not None and
442
(scale_name := self.quant_config.get_cache_scale(name))):
443
- # Loading kv cache scales for quark and
444
- # compressed-tensors quantization
443
+ # Loading kv cache quantization scales
444
param = params_dict[scale_name]
445
weight_loader = getattr(param, "weight_loader",
446
default_weight_loader)
vllm/model_executor/models/dbrx.py
+1
-2
@@ -452,8 +452,7 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
452
for name, loaded_weight in weights:
453
if (self.quant_config is not None and
454
(scale_name := self.quant_config.get_cache_scale(name))):
455
- # Loading kv cache scales for quark and
456
- # compressed-tensors quantization
455
+ # Loading kv cache quantization scales
456
param = params_dict[scale_name]
457
weight_loader = getattr(param, "weight_loader",
458
default_weight_loader)
vllm/model_executor/models/exaone.py
+1
-2
@@ -533,8 +533,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
533
continue
534
if (self.quant_config is not None and
535
(scale_name := self.quant_config.get_cache_scale(name))):
536
- # Loading kv cache scales for quark and
537
- # compressed-tensors quantization
536
+ # Loading kv cache quantization scales
537
param = params_dict[scale_name]
538
weight_loader = getattr(param, "weight_loader",
539
default_weight_loader)
vllm/model_executor/models/gpt_j.py
+1
-2
@@ -316,8 +316,7 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
316
317
if (self.quant_config is not None and
318
(scale_name := self.quant_config.get_cache_scale(name))):
319
- # Loading kv cache scales for quark and
320
- # compressed-tensors quantization
319
+ # Loading kv cache quantization scales
320
param = params_dict[scale_name]
321
weight_loader = getattr(param, "weight_loader",
322
default_weight_loader)
vllm/model_executor/models/granite.py
+1
-2
@@ -475,8 +475,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
475
continue
476
if (self.quant_config is not None and
477
(scale_name := self.quant_config.get_cache_scale(name))):
478
- # Loading kv cache scales for quark and
479
- # compressed-tensors quantization
478
+ # Loading kv cache quantization scales
479
param = params_dict[scale_name]
480
weight_loader = getattr(param, "weight_loader",
481
default_weight_loader)
vllm/model_executor/models/llama.py
+3
-4
@@ -105,9 +105,9 @@ class LlamaAttention(nn.Module):
105
max_position_embeddings: int = 8192,
106
quant_config: Optional[QuantizationConfig] = None,
107
bias: bool = False,
108
+ bias_o_proj: bool = False,
109
cache_config: Optional[CacheConfig] = None,
109
- prefix: str = "",
110
- bias_o_proj: bool = False) -> None:
110
+ prefix: str = "") -> None:
111
super().__init__()
112
layer_idx = extract_layer_index(prefix)
113
self.hidden_size = hidden_size
@@ -397,8 +397,7 @@ class LlamaModel(nn.Module):
397
continue
398
if (self.quant_config is not None and
399
(scale_name := self.quant_config.get_cache_scale(name))):
400
- # Loading kv cache scales for quark and
401
- # compressed-tensors quantization
400
+ # Loading kv cache quantization scales
401
param = params_dict[scale_name]
402
weight_loader = getattr(param, "weight_loader",
403
default_weight_loader)
vllm/model_executor/models/mixtral.py
+1
-2
@@ -431,8 +431,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
431
432
if (self.quant_config is not None and
433
(scale_name := self.quant_config.get_cache_scale(name))):
434
- # Loading kv cache scales for quark and
435
- # compressed-tensors quantization
434
+ # Loading kv cache quantization scales
435
param = params_dict[scale_name]
436
weight_loader = getattr(param, "weight_loader",
437
default_weight_loader)
vllm/model_executor/models/mllama.py
+1
-2
@@ -1432,8 +1432,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1432
loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1433
if (self.quant_config is not None and
1434
(scale_name := self.quant_config.get_cache_scale(name))):
1435
- # Loading kv cache scales for quark and
1436
- # compressed-tensors quantization
1435
+ # Loading kv cache quantization scales
1436
param = params_dict[scale_name]
1437
weight_loader = getattr(param, "weight_loader",
1438
default_weight_loader)
vllm/model_executor/models/nemotron.py
+1
-2
@@ -492,8 +492,7 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
492
continue
493
if (self.quant_config is not None and
494
(scale_name := self.quant_config.get_cache_scale(name))):
495
- # Loading kv cache scales for quark and
496
- # compressed-tensors quantization
495
+ # Loading kv cache quantization scales
496
param = params_dict[scale_name]
497
weight_loader = getattr(param, "weight_loader",
498
default_weight_loader)
vllm/model_executor/models/phimoe.py
+1
-2
@@ -626,8 +626,7 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
626
627
if (self.quant_config is not None and
628
(scale_name := self.quant_config.get_cache_scale(name))):
629
- # Loading kv cache scales for quark and
630
- # compressed-tensors quantization
629
+ # Loading kv cache quantization scales
630
param = params_dict[scale_name]
631
weight_loader = getattr(param, "weight_loader",
632
default_weight_loader)
vllm/model_executor/models/qwen2.py
+1
-2
@@ -367,8 +367,7 @@ class Qwen2Model(nn.Module):
367
continue
368
if (self.quant_config is not None and
369
(scale_name := self.quant_config.get_cache_scale(name))):
370
- # Loading kv cache scales for quark and
371
- # compressed-tensors quantization
370
+ # Loading kv cache quantization scales
371
param = params_dict[scale_name]
372
weight_loader = getattr(param, "weight_loader",
373
default_weight_loader)
vllm/model_executor/models/solar.py
+1
-2
@@ -492,8 +492,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
492
continue
493
if (self.quant_config is not None and
494
(scale_name := self.quant_config.get_cache_scale(name))):
495
- # Loading kv cache scales for quark and
496
- # compressed-tensors quantization
495
+ # Loading kv cache quantization scales
496
param = params_dict[scale_name]
497
weight_loader = getattr(param, "weight_loader",
498
default_weight_loader)