@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1fe554bac3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

treat do_lower_case in the same way as the sentence-transformers library (#11815)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>

Maximilien de Bayser committed Jan 9, 2025 at 00:05 UTC 1fe554bac32419a6d64a5c977849806a1efd9725
5 files changed +13 -5
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -35,6 +35,7 @@ class MockModelConfig:
35 logits_processor_pattern = None
36 diff_sampling_param: Optional[dict] = None
37 allowed_local_media_path: str = ""
38 + encoder_config = None
39
40 def get_diff_sampling_param(self):
41 return self.diff_sampling_param or {}
tests/models/embedding/language/test_embedding.py
+1
@@ -15,6 +15,7 @@ from ..utils import check_embeddings_close
15 # [Encoder-only]
16 pytest.param("BAAI/bge-base-en-v1.5",
17 marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
18 + pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
19 pytest.param("intfloat/multilingual-e5-large"),
20 # [Encoder-decoder]
21 pytest.param("intfloat/e5-mistral-7b-instruct",
vllm/entrypoints/openai/serving_engine.py
+5
@@ -160,6 +160,11 @@ class OpenAIServing:
160 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],
161 add_special_tokens: bool,
162 ) -> TextTokensPrompt:
163 + if (self.model_config.encoder_config is not None
164 + and self.model_config.encoder_config.get(
165 + "do_lower_case", False)):
166 + prompt = prompt.lower()
167 +
168 if truncate_prompt_tokens is None:
169 encoded = tokenizer(prompt, add_special_tokens=add_special_tokens)
170 else:
vllm/inputs/preprocess.py
+6
@@ -190,6 +190,12 @@ class InputPreprocessor:
190 # on the task and language of their request. Also needed to avoid
191 # appending an EOS token to the prompt which disrupts generation.
192 add_special_tokens = False
193 +
194 + if (self.model_config.encoder_config is not None
195 + and self.model_config.encoder_config.get(
196 + "do_lower_case", False)):
197 + prompt = prompt.lower()
198 +
199 return tokenizer.encode(request_id=request_id,
200 prompt=prompt,
201 lora_request=lora_request,
vllm/transformers_utils/tokenizer_group/__init__.py
-5
@@ -26,11 +26,6 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
26 trust_remote_code=model_config.trust_remote_code,
27 revision=model_config.tokenizer_revision)
28
29 - if (model_config.encoder_config is not None
30 - and "do_lower_case" in model_config.encoder_config):
31 - init_kwargs["do_lower_case"] = model_config.encoder_config[
32 - "do_lower_case"]
33 -
29 return get_tokenizer_group(parallel_config.tokenizer_pool_config,
30 **init_kwargs)
31