@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1efce68605
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Use runner_type instead of task in GritLM (#11144)

Signed-off-by: Pooya Davoodi <pooya.davoodi@parasail.io>

Pooya Davoodi committed Dec 12, 2024 at 20:09 UTC 1efce686053c15cd6f84361bb0bd1898fbb23a82
2 files changed +7 -7
tests/models/embedding/language/test_gritlm.py
+3 -3
@@ -35,7 +35,7 @@ def test_find_array(monkeypatch):
35 from vllm.model_executor.models.gritlm import GritLMPooler
36
37 # Create an LLM object to get the model config.
38 - llm = vllm.LLM(MODEL_NAME, task="embedding", max_model_len=MAX_MODEL_LEN)
38 + llm = vllm.LLM(MODEL_NAME, task="embed", max_model_len=MAX_MODEL_LEN)
39 pooler = GritLMPooler(model_config=llm.llm_engine.model_config)
40
41 arr = _arr([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
@@ -55,7 +55,7 @@ def server_embedding():
55 with pytest.MonkeyPatch.context() as mp:
56 mp.setenv("VLLM_ATTENTION_BACKEND", "XFORMERS")
57
58 - args = ["--task", "embedding", "--max_model_len", str(MAX_MODEL_LEN)]
58 + args = ["--task", "embed", "--max_model_len", str(MAX_MODEL_LEN)]
59 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
60 yield remote_server
61
@@ -141,7 +141,7 @@ def test_gritlm_offline_embedding(monkeypatch):
141
142 queries, q_instruction, documents, d_instruction = get_test_data()
143
144 - llm = vllm.LLM(MODEL_NAME, task="embedding", max_model_len=MAX_MODEL_LEN)
144 + llm = vllm.LLM(MODEL_NAME, task="embed", max_model_len=MAX_MODEL_LEN)
145
146 d_rep = run_llm_encode(
147 llm,
vllm/model_executor/models/gritlm.py
+4 -4
@@ -203,12 +203,12 @@ class GritLM(LlamaForCausalLM):
203 ) -> None:
204 super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
205
206 - self.task = vllm_config.model_config.task
206 + self.runner_type = vllm_config.model_config.runner_type
207
208 self._pooler = GritLMPooler(vllm_config.model_config)
209
210 for layer in self.model.layers:
211 - if self.task == "embedding" and hasattr(layer, "self_attn"):
211 + if self.runner_type == "pooling" and hasattr(layer, "self_attn"):
212 assert isinstance(layer.self_attn.attn.impl, XFormersImpl), (
213 "GritLM embedding is only supported by XFormers backend, "
214 "which can be forced by VLLM_ATTENTION_BACKEND=XFORMERS")
@@ -222,8 +222,8 @@ class GritLM(LlamaForCausalLM):
222 **kwargs,
223 ) -> Union[torch.Tensor, IntermediateTensors]:
224
225 - # Change attention to non-causal for embedding task.
226 - if self.task == "embedding":
225 + # Change attention to non-causal for pooling tasks.
226 + if self.runner_type == "pooling":
227 assert attn_metadata.prefill_metadata.attn_bias is None
228 attn_metadata.prefill_metadata.attn_bias = [
229 BlockDiagonalMask.from_seqlens(attn_metadata.seq_lens)