A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Hardwware][Neuron] Simplify model load for transformers-neuronx library (#9380)
Shashwat Srijan committed
Oct 17, 2024 at 15:39 UTC
bb76538bbdf6ad61c6359d3aaca4863541596685
1 file changed
+1
-30
vllm/model_executor/model_loader/neuron.py
+1
-30
@@ -6,7 +6,6 @@ from typing import Dict, List, Optional, Tuple
6
7
import torch
8
import torch.nn as nn
9
-import transformers
9
from transformers import PretrainedConfig
10
11
from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
@@ -108,39 +107,11 @@ class NeuronCasualLM(nn.Module):
107
neuronx_module = importlib.import_module(neuronx_module_path)
108
neuronx_model_cls = getattr(neuronx_module, neuronx_model_cls_name)
109
111
- split_model_dir = f"{model_name_or_path}-split"
112
- if _is_pretrained_neuron_checkpoint(model_name_or_path):
113
- split_model_dir = model_name_or_path
114
- elif not os.path.exists(f"{model_name_or_path}-split"):
115
- hf_model_cls = getattr(transformers, hf_model_cls_name)
116
- from transformers_neuronx.module import save_pretrained_split
117
-
118
- hf_model = hf_model_cls.from_pretrained(model_name_or_path,
119
- low_cpu_mem_usage=True)
120
- save_pretrained_split(hf_model, f"{model_name_or_path}-split")
121
-
122
- self.model = neuronx_model_cls.from_pretrained(split_model_dir,
110
+ self.model = neuronx_model_cls.from_pretrained(model_name_or_path,
111
**kwargs)
112
self.model.to_neuron()
113
114
127
-def _is_pretrained_neuron_checkpoint(model_name_or_path: str) -> bool:
128
- # Checking if the neuron checkpoint is saved in the old format.
129
- if os.path.isdir(os.path.join(model_name_or_path, "pytorch_model.bin")):
130
- return True
131
- # Checking if the neuron checkpoint is saved in the new format.
132
- pretrained_split_files = ["config.json", "generation_config.json"]
133
- pretrained_split_format = ".safetensors"
134
- for file in pretrained_split_files:
135
- file_path = os.path.join(model_name_or_path, file)
136
- if not os.path.isfile(file_path):
137
- return False
138
- for file in os.listdir(model_name_or_path):
139
- if file.endswith(pretrained_split_format):
140
- return True
141
- return False
142
-
143
-
115
def _get_model_architecture(config: PretrainedConfig) -> str:
116
architectures = getattr(config, "architectures", [])
117
for arch in architectures: