A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
treat do_lower_case in the same way as the sentence-transformers library (#11815)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Maximilien de Bayser committed
Jan 9, 2025 at 00:05 UTC
1fe554bac32419a6d64a5c977849806a1efd9725
5 files changed
+13
-5
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -35,6 +35,7 @@ class MockModelConfig:
35
logits_processor_pattern = None
36
diff_sampling_param: Optional[dict] = None
37
allowed_local_media_path: str = ""
38
+ encoder_config = None
39
40
def get_diff_sampling_param(self):
41
return self.diff_sampling_param or {}
tests/models/embedding/language/test_embedding.py
+1
@@ -15,6 +15,7 @@ from ..utils import check_embeddings_close
15
# [Encoder-only]
16
pytest.param("BAAI/bge-base-en-v1.5",
17
marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
18
+ pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
19
pytest.param("intfloat/multilingual-e5-large"),
20
# [Encoder-decoder]
21
pytest.param("intfloat/e5-mistral-7b-instruct",
vllm/entrypoints/openai/serving_engine.py
+5
@@ -160,6 +160,11 @@ class OpenAIServing:
160
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],
161
add_special_tokens: bool,
162
) -> TextTokensPrompt:
163
+ if (self.model_config.encoder_config is not None
164
+ and self.model_config.encoder_config.get(
165
+ "do_lower_case", False)):
166
+ prompt = prompt.lower()
167
+
168
if truncate_prompt_tokens is None:
169
encoded = tokenizer(prompt, add_special_tokens=add_special_tokens)
170
else:
vllm/inputs/preprocess.py
+6
@@ -190,6 +190,12 @@ class InputPreprocessor:
190
# on the task and language of their request. Also needed to avoid
191
# appending an EOS token to the prompt which disrupts generation.
192
add_special_tokens = False
193
+
194
+ if (self.model_config.encoder_config is not None
195
+ and self.model_config.encoder_config.get(
196
+ "do_lower_case", False)):
197
+ prompt = prompt.lower()
198
+
199
return tokenizer.encode(request_id=request_id,
200
prompt=prompt,
201
lora_request=lora_request,
vllm/transformers_utils/tokenizer_group/__init__.py
-5
@@ -26,11 +26,6 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
26
trust_remote_code=model_config.trust_remote_code,
27
revision=model_config.tokenizer_revision)
28
29
- if (model_config.encoder_config is not None
30
- and "do_lower_case" in model_config.encoder_config):
31
- init_kwargs["do_lower_case"] = model_config.encoder_config[
32
- "do_lower_case"]
33
-
29
return get_tokenizer_group(parallel_config.tokenizer_pool_config,
30
**init_kwargs)
31