@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9aa1519f08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

Various cosmetic/comment fixes (#12089)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Jan 16, 2025 at 04:59 UTC 9aa1519f089e8dc2c2bd1b4c74a8ce47d386f0a9
14 files changed +16 -29
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+1 -1
@@ -42,7 +42,7 @@ class CompressedTensors24(CompressedTensorsScheme):
42
43 if not sparse_cutlass_supported():
44 raise ValueError(
45 - "Sparse CUTLASS not supported. vLLM must be built with"
45 + "Sparse CUTLASS not supported. vLLM must be built with "
46 "CUDA 12.2 or later to use this feature")
47
48 self.output_dtype = params_dtype
vllm/model_executor/models/aria.py
+1 -2
@@ -390,8 +390,7 @@ class AriaMoELMModel(LlamaModel):
390 continue
391 if (self.quant_config is not None and
392 (scale_name := self.quant_config.get_cache_scale(name))):
393 - # Loading kv cache scales for quark and
394 - # compressed-tensors quantization
393 + # Loading kv cache quantization scales
394 param = params_dict[scale_name]
395 weight_loader = getattr(param, "weight_loader",
396 default_weight_loader)
vllm/model_executor/models/commandr.py
+1 -2
@@ -440,8 +440,7 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
440
441 if (self.quant_config is not None and
442 (scale_name := self.quant_config.get_cache_scale(name))):
443 - # Loading kv cache scales for quark and
444 - # compressed-tensors quantization
443 + # Loading kv cache quantization scales
444 param = params_dict[scale_name]
445 weight_loader = getattr(param, "weight_loader",
446 default_weight_loader)
vllm/model_executor/models/dbrx.py
+1 -2
@@ -452,8 +452,7 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
452 for name, loaded_weight in weights:
453 if (self.quant_config is not None and
454 (scale_name := self.quant_config.get_cache_scale(name))):
455 - # Loading kv cache scales for quark and
456 - # compressed-tensors quantization
455 + # Loading kv cache quantization scales
456 param = params_dict[scale_name]
457 weight_loader = getattr(param, "weight_loader",
458 default_weight_loader)
vllm/model_executor/models/exaone.py
+1 -2
@@ -533,8 +533,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
533 continue
534 if (self.quant_config is not None and
535 (scale_name := self.quant_config.get_cache_scale(name))):
536 - # Loading kv cache scales for quark and
537 - # compressed-tensors quantization
536 + # Loading kv cache quantization scales
537 param = params_dict[scale_name]
538 weight_loader = getattr(param, "weight_loader",
539 default_weight_loader)
vllm/model_executor/models/gpt_j.py
+1 -2
@@ -316,8 +316,7 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
316
317 if (self.quant_config is not None and
318 (scale_name := self.quant_config.get_cache_scale(name))):
319 - # Loading kv cache scales for quark and
320 - # compressed-tensors quantization
319 + # Loading kv cache quantization scales
320 param = params_dict[scale_name]
321 weight_loader = getattr(param, "weight_loader",
322 default_weight_loader)
vllm/model_executor/models/granite.py
+1 -2
@@ -475,8 +475,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
475 continue
476 if (self.quant_config is not None and
477 (scale_name := self.quant_config.get_cache_scale(name))):
478 - # Loading kv cache scales for quark and
479 - # compressed-tensors quantization
478 + # Loading kv cache quantization scales
479 param = params_dict[scale_name]
480 weight_loader = getattr(param, "weight_loader",
481 default_weight_loader)
vllm/model_executor/models/llama.py
+3 -4
@@ -105,9 +105,9 @@ class LlamaAttention(nn.Module):
105 max_position_embeddings: int = 8192,
106 quant_config: Optional[QuantizationConfig] = None,
107 bias: bool = False,
108 + bias_o_proj: bool = False,
109 cache_config: Optional[CacheConfig] = None,
109 - prefix: str = "",
110 - bias_o_proj: bool = False) -> None:
110 + prefix: str = "") -> None:
111 super().__init__()
112 layer_idx = extract_layer_index(prefix)
113 self.hidden_size = hidden_size
@@ -397,8 +397,7 @@ class LlamaModel(nn.Module):
397 continue
398 if (self.quant_config is not None and
399 (scale_name := self.quant_config.get_cache_scale(name))):
400 - # Loading kv cache scales for quark and
401 - # compressed-tensors quantization
400 + # Loading kv cache quantization scales
401 param = params_dict[scale_name]
402 weight_loader = getattr(param, "weight_loader",
403 default_weight_loader)
vllm/model_executor/models/mixtral.py
+1 -2
@@ -431,8 +431,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
431
432 if (self.quant_config is not None and
433 (scale_name := self.quant_config.get_cache_scale(name))):
434 - # Loading kv cache scales for quark and
435 - # compressed-tensors quantization
434 + # Loading kv cache quantization scales
435 param = params_dict[scale_name]
436 weight_loader = getattr(param, "weight_loader",
437 default_weight_loader)
vllm/model_executor/models/mllama.py
+1 -2
@@ -1432,8 +1432,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1432 loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1433 if (self.quant_config is not None and
1434 (scale_name := self.quant_config.get_cache_scale(name))):
1435 - # Loading kv cache scales for quark and
1436 - # compressed-tensors quantization
1435 + # Loading kv cache quantization scales
1436 param = params_dict[scale_name]
1437 weight_loader = getattr(param, "weight_loader",
1438 default_weight_loader)
vllm/model_executor/models/nemotron.py
+1 -2
@@ -492,8 +492,7 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
492 continue
493 if (self.quant_config is not None and
494 (scale_name := self.quant_config.get_cache_scale(name))):
495 - # Loading kv cache scales for quark and
496 - # compressed-tensors quantization
495 + # Loading kv cache quantization scales
496 param = params_dict[scale_name]
497 weight_loader = getattr(param, "weight_loader",
498 default_weight_loader)
vllm/model_executor/models/phimoe.py
+1 -2
@@ -626,8 +626,7 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
626
627 if (self.quant_config is not None and
628 (scale_name := self.quant_config.get_cache_scale(name))):
629 - # Loading kv cache scales for quark and
630 - # compressed-tensors quantization
629 + # Loading kv cache quantization scales
630 param = params_dict[scale_name]
631 weight_loader = getattr(param, "weight_loader",
632 default_weight_loader)
vllm/model_executor/models/qwen2.py
+1 -2
@@ -367,8 +367,7 @@ class Qwen2Model(nn.Module):
367 continue
368 if (self.quant_config is not None and
369 (scale_name := self.quant_config.get_cache_scale(name))):
370 - # Loading kv cache scales for quark and
371 - # compressed-tensors quantization
370 + # Loading kv cache quantization scales
371 param = params_dict[scale_name]
372 weight_loader = getattr(param, "weight_loader",
373 default_weight_loader)
vllm/model_executor/models/solar.py
+1 -2
@@ -492,8 +492,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
492 continue
493 if (self.quant_config is not None and
494 (scale_name := self.quant_config.get_cache_scale(name))):
495 - # Loading kv cache scales for quark and
496 - # compressed-tensors quantization
495 + # Loading kv cache quantization scales
496 param = params_dict[scale_name]
497 weight_loader = getattr(param, "weight_loader",
498 default_weight_loader)