@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb76538bbd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Hardwware][Neuron] Simplify model load for transformers-neuronx library (#9380)

Shashwat Srijan committed Oct 17, 2024 at 15:39 UTC bb76538bbdf6ad61c6359d3aaca4863541596685
1 file changed +1 -30
vllm/model_executor/model_loader/neuron.py
+1 -30
@@ -6,7 +6,6 @@ from typing import Dict, List, Optional, Tuple
6
7 import torch
8 import torch.nn as nn
9 -import transformers
9 from transformers import PretrainedConfig
10
11 from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
@@ -108,39 +107,11 @@ class NeuronCasualLM(nn.Module):
107 neuronx_module = importlib.import_module(neuronx_module_path)
108 neuronx_model_cls = getattr(neuronx_module, neuronx_model_cls_name)
109
111 - split_model_dir = f"{model_name_or_path}-split"
112 - if _is_pretrained_neuron_checkpoint(model_name_or_path):
113 - split_model_dir = model_name_or_path
114 - elif not os.path.exists(f"{model_name_or_path}-split"):
115 - hf_model_cls = getattr(transformers, hf_model_cls_name)
116 - from transformers_neuronx.module import save_pretrained_split
117 -
118 - hf_model = hf_model_cls.from_pretrained(model_name_or_path,
119 - low_cpu_mem_usage=True)
120 - save_pretrained_split(hf_model, f"{model_name_or_path}-split")
121 -
122 - self.model = neuronx_model_cls.from_pretrained(split_model_dir,
110 + self.model = neuronx_model_cls.from_pretrained(model_name_or_path,
111 **kwargs)
112 self.model.to_neuron()
113
114
127 -def _is_pretrained_neuron_checkpoint(model_name_or_path: str) -> bool:
128 - # Checking if the neuron checkpoint is saved in the old format.
129 - if os.path.isdir(os.path.join(model_name_or_path, "pytorch_model.bin")):
130 - return True
131 - # Checking if the neuron checkpoint is saved in the new format.
132 - pretrained_split_files = ["config.json", "generation_config.json"]
133 - pretrained_split_format = ".safetensors"
134 - for file in pretrained_split_files:
135 - file_path = os.path.join(model_name_or_path, file)
136 - if not os.path.isfile(file_path):
137 - return False
138 - for file in os.listdir(model_name_or_path):
139 - if file.endswith(pretrained_split_format):
140 - return True
141 - return False
142 -
143 -
115 def _get_model_architecture(config: PretrainedConfig) -> str:
116 architectures = getattr(config, "architectures", [])
117 for arch in architectures: