A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Correct adapter usage for cohere and jamba (#8292)
Vladislav Kruglikov committed
Sep 9, 2024 at 21:20 UTC
f9b4a2d41587da0692d32797221df55a02d890a6
2 files changed
+6
-3
vllm/model_executor/models/commandr.py
+3
-2
@@ -47,6 +47,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.model_executor.utils import set_weight_attrs
48
from vllm.sequence import IntermediateTensors
49
50
+from .interfaces import SupportsLoRA
51
+
52
53
@torch.compile
54
def layer_norm_func(hidden_states, weight, variance_epsilon):
@@ -292,8 +294,7 @@ class CohereModel(nn.Module):
294
return hidden_states
295
296
295
-class CohereForCausalLM(nn.Module):
296
-
297
+class CohereForCausalLM(nn.Module, SupportsLoRA):
298
packed_modules_mapping = {
299
"qkv_proj": [
300
"q_proj",
vllm/model_executor/models/jamba.py
+3
-1
@@ -38,6 +38,8 @@ from vllm.sequence import IntermediateTensors
38
from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
39
_get_graph_batch_size)
40
41
+from .interfaces import SupportsLoRA
42
+
43
KVCache = Tuple[torch.Tensor, torch.Tensor]
44
45
@@ -539,7 +541,7 @@ class JambaModel(nn.Module):
541
return hidden_states
542
543
542
-class JambaForCausalLM(nn.Module, HasInnerState):
544
+class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
545
packed_modules_mapping = {
546
"qkv_proj": [
547
"q_proj",