A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Improve olmo and olmo2 (#23228)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Aug 20, 2025 at 20:47 UTC
c6d80a7a9620637ba5016dd3c0d6061e79eed73c
3 files changed
+36
-7
docs/models/supported_models.md
+2
-2
@@ -384,8 +384,8 @@ th {
384
| `MPTForCausalLM` | MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter | `mosaicml/mpt-7b`, `mosaicml/mpt-7b-storywriter`, `mosaicml/mpt-30b`, etc. | | ✅︎ | ✅︎ |
385
| `NemotronForCausalLM` | Nemotron-3, Nemotron-4, Minitron | `nvidia/Minitron-8B-Base`, `mgoin/Nemotron-4-340B-Base-hf-FP8`, etc. | ✅︎ | ✅︎ | ✅︎ |
386
| `NemotronHForCausalLM` | Nemotron-H | `nvidia/Nemotron-H-8B-Base-8K`, `nvidia/Nemotron-H-47B-Base-8K`, `nvidia/Nemotron-H-56B-Base-8K`, etc. | ✅︎ | ✅︎ | ✅︎ |
387
-| `OLMoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | | ✅︎ | ✅︎ |
388
-| `OLMo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | | ✅︎ | ✅︎ |
387
+| `OLMoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | ✅︎ | ✅︎ | ✅︎ |
388
+| `OLMo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ | ✅︎ |
389
| `OLMoEForCausalLM` | OLMoE | `allenai/OLMoE-1B-7B-0924`, `allenai/OLMoE-1B-7B-0924-Instruct`, etc. | | ✅︎ | ✅︎ |
390
| `OPTForCausalLM` | OPT, OPT-IML | `facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc. | | ✅︎ | ✅︎ |
391
| `OrionForCausalLM` | Orion | `OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc. | | ✅︎ | ✅︎ |
vllm/model_executor/models/olmo.py
+19
-3
@@ -47,7 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.sequence import IntermediateTensors
49
50
-from .interfaces import SupportsPP
50
+from .interfaces import SupportsLoRA, SupportsPP
51
from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
52
make_empty_intermediate_tensors_factory, make_layers,
53
maybe_prefix)
@@ -91,6 +91,7 @@ class OlmoAttention(nn.Module):
91
self.total_num_heads,
92
bias=config.attention_bias,
93
quant_config=quant_config,
94
+ prefix=f"{prefix}.qkv_proj",
95
)
96
97
# Rotary embeddings.
@@ -114,6 +115,7 @@ class OlmoAttention(nn.Module):
115
self.hidden_size,
116
bias=config.attention_bias,
117
quant_config=quant_config,
118
+ prefix=f"{prefix}.o_proj",
119
)
120
121
def forward(
@@ -142,6 +144,7 @@ class OlmoMLP(nn.Module):
144
self,
145
config: OlmoConfig,
146
quant_config: Optional[QuantizationConfig] = None,
147
+ prefix: str = "",
148
):
149
super().__init__()
150
self.config = config
@@ -154,6 +157,7 @@ class OlmoMLP(nn.Module):
157
[self.intermediate_size] * 2,
158
bias=False,
159
quant_config=quant_config,
160
+ prefix=f"{prefix}.gate_up_proj",
161
)
162
163
# Activation function.
@@ -165,6 +169,7 @@ class OlmoMLP(nn.Module):
169
self.hidden_size,
170
bias=False,
171
quant_config=quant_config,
172
+ prefix=f"{prefix}.down_proj",
173
)
174
175
def forward(
@@ -197,7 +202,7 @@ class OlmoDecoderLayer(nn.Module):
202
prefix=f"{prefix}.self_attn")
203
204
# MLP block.
200
- self.mlp = OlmoMLP(config, quant_config)
205
+ self.mlp = OlmoMLP(config, quant_config, prefix=f"{prefix}.mlp")
206
207
# LayerNorm
208
self.input_layernorm = nn.LayerNorm(config.hidden_size,
@@ -326,10 +331,21 @@ class OlmoModel(nn.Module):
331
return loaded_params
332
333
329
-class OlmoForCausalLM(nn.Module, SupportsPP):
334
+class OlmoForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
335
"""
336
Extremely barebones HF model wrapper.
337
"""
338
+ packed_modules_mapping = {
339
+ "qkv_proj": [
340
+ "q_proj",
341
+ "k_proj",
342
+ "v_proj",
343
+ ],
344
+ "gate_up_proj": [
345
+ "gate_proj",
346
+ "up_proj",
347
+ ],
348
+ }
349
350
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
351
super().__init__()
vllm/model_executor/models/olmo2.py
+15
-2
@@ -33,6 +33,7 @@ from torch import nn
33
from transformers import Olmo2Config
34
35
from vllm.attention import Attention
36
+from vllm.compilation.decorators import support_torch_compile
37
from vllm.config import VllmConfig
38
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
39
from vllm.distributed.communication_op import tensor_model_parallel_all_gather
@@ -48,7 +49,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
49
from vllm.model_executor.layers.vocab_parallel_embedding import (
50
ParallelLMHead, VocabParallelEmbedding)
51
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
-from vllm.model_executor.models.interfaces import SupportsPP
52
+from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
53
from vllm.model_executor.models.utils import (
54
AutoWeightsLoader, is_pp_missing_parameter,
55
make_empty_intermediate_tensors_factory, make_layers, maybe_prefix)
@@ -253,6 +254,7 @@ class Olmo2DecoderLayer(nn.Module):
254
return hidden_states
255
256
257
+@support_torch_compile
258
class Olmo2Model(nn.Module):
259
260
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -354,10 +356,21 @@ class Olmo2Model(nn.Module):
356
return loaded_params
357
358
357
-class Olmo2ForCausalLM(nn.Module, SupportsPP):
359
+class Olmo2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
360
"""
361
Extremely barebones HF model wrapper.
362
"""
363
+ packed_modules_mapping = {
364
+ "qkv_proj": [
365
+ "q_proj",
366
+ "k_proj",
367
+ "v_proj",
368
+ ],
369
+ "gate_up_proj": [
370
+ "gate_proj",
371
+ "up_proj",
372
+ ],
373
+ }
374
375
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
376
super().__init__()