@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f9b4a2d415
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Correct adapter usage for cohere and jamba (#8292)

Vladislav Kruglikov committed Sep 9, 2024 at 21:20 UTC f9b4a2d41587da0692d32797221df55a02d890a6
2 files changed +6 -3
vllm/model_executor/models/commandr.py
+3 -2
@@ -47,6 +47,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
47 from vllm.model_executor.utils import set_weight_attrs
48 from vllm.sequence import IntermediateTensors
49
50 +from .interfaces import SupportsLoRA
51 +
52
53 @torch.compile
54 def layer_norm_func(hidden_states, weight, variance_epsilon):
@@ -292,8 +294,7 @@ class CohereModel(nn.Module):
294 return hidden_states
295
296
295 -class CohereForCausalLM(nn.Module):
296 -
297 +class CohereForCausalLM(nn.Module, SupportsLoRA):
298 packed_modules_mapping = {
299 "qkv_proj": [
300 "q_proj",
vllm/model_executor/models/jamba.py
+3 -1
@@ -38,6 +38,8 @@ from vllm.sequence import IntermediateTensors
38 from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
39 _get_graph_batch_size)
40
41 +from .interfaces import SupportsLoRA
42 +
43 KVCache = Tuple[torch.Tensor, torch.Tensor]
44
45
@@ -539,7 +541,7 @@ class JambaModel(nn.Module):
541 return hidden_states
542
543
542 -class JambaForCausalLM(nn.Module, HasInnerState):
544 +class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
545 packed_modules_mapping = {
546 "qkv_proj": [
547 "q_proj",