A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix a bug in tying OPT embeddings (#1)
Woosuk Kwon committed
Feb 24, 2023 at 16:29 UTC
cbf8779afafdaba2ddc6e2212d67c40f1b6e11fd
2 files changed
+25
-2
cacheflow/models/model_utils.py
+3
-2
@@ -25,7 +25,8 @@ def get_model(
25
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
26
else:
27
torch_dtype = dtype
28
- for model_class, model in MODEL_CLASSES.items():
28
+ for model_class, hf_model in MODEL_CLASSES.items():
29
if model_class in model_name:
30
- return model.from_pretrained(model_name, torch_dtype=torch_dtype)
30
+ model = hf_model.from_pretrained(model_name, torch_dtype=torch_dtype)
31
+ return model.eval()
32
raise ValueError(f'Invalid model name: {model_name}')
cacheflow/models/opt.py
+22
@@ -232,6 +232,28 @@ class OPTForCausalLM(OPTPreTrainedModel):
232
# Initialize weights and apply final processing
233
self.post_init()
234
235
+ # NOTE(woosuk): While the following methods are not called in the model code,
236
+ # they may be internally used by the transformers library.
237
+ # For example, tie_weights() does not work without these methods.
238
+ # Thus, do not delete these methods.
239
+ def get_input_embeddings(self):
240
+ return self.model.decoder.embed_tokens
241
+
242
+ def set_input_embeddings(self, value):
243
+ self.model.decoder.embed_tokens = value
244
+
245
+ def get_output_embeddings(self):
246
+ return self.lm_head
247
+
248
+ def set_output_embeddings(self, new_embeddings):
249
+ self.lm_head = new_embeddings
250
+
251
+ def set_decoder(self, decoder):
252
+ self.model.decoder = decoder
253
+
254
+ def get_decoder(self):
255
+ return self.model.decoder
256
+
257
def forward(
258
self,
259
input_ids: torch.LongTensor,