@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cbf8779afa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix a bug in tying OPT embeddings (#1)

Woosuk Kwon committed Feb 24, 2023 at 16:29 UTC cbf8779afafdaba2ddc6e2212d67c40f1b6e11fd
2 files changed +25 -2
cacheflow/models/model_utils.py
+3 -2
@@ -25,7 +25,8 @@ def get_model(
25 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
26 else:
27 torch_dtype = dtype
28 - for model_class, model in MODEL_CLASSES.items():
28 + for model_class, hf_model in MODEL_CLASSES.items():
29 if model_class in model_name:
30 - return model.from_pretrained(model_name, torch_dtype=torch_dtype)
30 + model = hf_model.from_pretrained(model_name, torch_dtype=torch_dtype)
31 + return model.eval()
32 raise ValueError(f'Invalid model name: {model_name}')
cacheflow/models/opt.py
+22
@@ -232,6 +232,28 @@ class OPTForCausalLM(OPTPreTrainedModel):
232 # Initialize weights and apply final processing
233 self.post_init()
234
235 + # NOTE(woosuk): While the following methods are not called in the model code,
236 + # they may be internally used by the transformers library.
237 + # For example, tie_weights() does not work without these methods.
238 + # Thus, do not delete these methods.
239 + def get_input_embeddings(self):
240 + return self.model.decoder.embed_tokens
241 +
242 + def set_input_embeddings(self, value):
243 + self.model.decoder.embed_tokens = value
244 +
245 + def get_output_embeddings(self):
246 + return self.lm_head
247 +
248 + def set_output_embeddings(self, new_embeddings):
249 + self.lm_head = new_embeddings
250 +
251 + def set_decoder(self, decoder):
252 + self.model.decoder = decoder
253 +
254 + def get_decoder(self):
255 + return self.model.decoder
256 +
257 def forward(
258 self,
259 input_ids: torch.LongTensor,