A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[bugfix] fix aria model and add torch.compile (#10645)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 25, 2024 at 18:32 UTC
45ac4ff270b267765457159c0b75e1bb7ebf6d79
2 files changed
+14
-28
vllm/model_executor/models/aria.py
+4
-22
@@ -29,7 +29,7 @@ from vllm.model_executor.models.llama import (LlamaDecoderLayer, LlamaMLP,
29
LlamaModel)
30
from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper,
31
is_pp_missing_parameter,
32
- make_layers, maybe_prefix,
32
+ maybe_prefix,
33
merge_multimodal_embeddings)
34
from vllm.multimodal import MULTIMODAL_REGISTRY
35
from vllm.multimodal.base import MultiModalInputs
@@ -363,27 +363,9 @@ class AriaMoELMModel(LlamaModel):
363
"""
364
365
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
366
- super().__init__(vllm_config=vllm_config, prefix=prefix)
367
-
368
- config = vllm_config.model_config.hf_config
369
- cache_config = vllm_config.cache_config
370
- quant_config = vllm_config.quant_config
371
-
372
- # FIXME: this is a hack to disable the compilation of the model
373
- self.do_not_compile = True
374
-
375
- self.layers = None
376
-
377
- self.start_layer, self.end_layer, self.layers = make_layers(
378
- config.num_hidden_layers,
379
- lambda prefix: MoEDecoderLayer(
380
- config=config,
381
- cache_config=cache_config,
382
- quant_config=quant_config,
383
- prefix=prefix,
384
- ),
385
- prefix=f"{prefix}.layers",
386
- )
366
+ super().__init__(vllm_config=vllm_config,
367
+ prefix=prefix,
368
+ layer_type=MoEDecoderLayer)
369
370
# Adapted from LlamaModel.load_weights with the modification of adding
371
# the expert weights mapping to `stacked_params_mapping`
vllm/model_executor/models/llama.py
+10
-6
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only LLaMA model compatible with HuggingFace weights."""
23
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
23
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
24
25
import torch
26
from torch import nn
@@ -273,7 +273,11 @@ class LlamaDecoderLayer(nn.Module):
273
@support_torch_compile
274
class LlamaModel(nn.Module):
275
276
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
276
+ def __init__(self,
277
+ *,
278
+ vllm_config: VllmConfig,
279
+ prefix: str = "",
280
+ layer_type: Type[LlamaDecoderLayer] = LlamaDecoderLayer):
281
super().__init__()
282
283
config = vllm_config.model_config.hf_config
@@ -299,10 +303,10 @@ class LlamaModel(nn.Module):
303
self.embed_tokens = PPMissingLayer()
304
self.start_layer, self.end_layer, self.layers = make_layers(
305
config.num_hidden_layers,
302
- lambda prefix: LlamaDecoderLayer(config=config,
303
- cache_config=cache_config,
304
- quant_config=quant_config,
305
- prefix=prefix),
306
+ lambda prefix: layer_type(config=config,
307
+ cache_config=cache_config,
308
+ quant_config=quant_config,
309
+ prefix=prefix),
310
prefix=f"{prefix}.layers",
311
)
312
if get_pp_group().is_last_rank: