@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 45ac4ff270
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[bugfix] fix aria model and add torch.compile (#10645)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 25, 2024 at 18:32 UTC 45ac4ff270b267765457159c0b75e1bb7ebf6d79
2 files changed +14 -28
vllm/model_executor/models/aria.py
+4 -22
@@ -29,7 +29,7 @@ from vllm.model_executor.models.llama import (LlamaDecoderLayer, LlamaMLP,
29 LlamaModel)
30 from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper,
31 is_pp_missing_parameter,
32 - make_layers, maybe_prefix,
32 + maybe_prefix,
33 merge_multimodal_embeddings)
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 from vllm.multimodal.base import MultiModalInputs
@@ -363,27 +363,9 @@ class AriaMoELMModel(LlamaModel):
363 """
364
365 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
366 - super().__init__(vllm_config=vllm_config, prefix=prefix)
367 -
368 - config = vllm_config.model_config.hf_config
369 - cache_config = vllm_config.cache_config
370 - quant_config = vllm_config.quant_config
371 -
372 - # FIXME: this is a hack to disable the compilation of the model
373 - self.do_not_compile = True
374 -
375 - self.layers = None
376 -
377 - self.start_layer, self.end_layer, self.layers = make_layers(
378 - config.num_hidden_layers,
379 - lambda prefix: MoEDecoderLayer(
380 - config=config,
381 - cache_config=cache_config,
382 - quant_config=quant_config,
383 - prefix=prefix,
384 - ),
385 - prefix=f"{prefix}.layers",
386 - )
366 + super().__init__(vllm_config=vllm_config,
367 + prefix=prefix,
368 + layer_type=MoEDecoderLayer)
369
370 # Adapted from LlamaModel.load_weights with the modification of adding
371 # the expert weights mapping to `stacked_params_mapping`
vllm/model_executor/models/llama.py
+10 -6
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only LLaMA model compatible with HuggingFace weights."""
23 -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
23 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
24
25 import torch
26 from torch import nn
@@ -273,7 +273,11 @@ class LlamaDecoderLayer(nn.Module):
273 @support_torch_compile
274 class LlamaModel(nn.Module):
275
276 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
276 + def __init__(self,
277 + *,
278 + vllm_config: VllmConfig,
279 + prefix: str = "",
280 + layer_type: Type[LlamaDecoderLayer] = LlamaDecoderLayer):
281 super().__init__()
282
283 config = vllm_config.model_config.hf_config
@@ -299,10 +303,10 @@ class LlamaModel(nn.Module):
303 self.embed_tokens = PPMissingLayer()
304 self.start_layer, self.end_layer, self.layers = make_layers(
305 config.num_hidden_layers,
302 - lambda prefix: LlamaDecoderLayer(config=config,
303 - cache_config=cache_config,
304 - quant_config=quant_config,
305 - prefix=prefix),
306 + lambda prefix: layer_type(config=config,
307 + cache_config=cache_config,
308 + quant_config=quant_config,
309 + prefix=prefix),
310 prefix=f"{prefix}.layers",
311 )
312 if get_pp_group().is_last_rank: