A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] use right truncation for non-generative tasks (#12050)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Jan 15, 2025 at 09:31 UTC
edce722eaa5e9f0b97bea611531e3341ec2e2e71
3 files changed
+13
-1
tests/entrypoints/llm/test_encode.py
+7
@@ -105,3 +105,10 @@ def test_multiple_pooling_params(llm: LLM):
105
# pooling_params is None, default params should be applied
106
outputs = llm.encode(PROMPTS, pooling_params=None)
107
assert len(PROMPTS) == len(outputs)
108
+
109
+
110
+@pytest.mark.skip_global_cleanup
111
+def test_right_side_truncation(llm: LLM):
112
+ # Embeddings models should truncate the end of the prompt
113
+ tokenizer = llm.get_tokenizer()
114
+ assert tokenizer.truncation_side == "right"
vllm/config.py
+4
@@ -357,6 +357,10 @@ class ModelConfig:
357
supported_tasks, task = self._resolve_task(task, self.hf_config)
358
self.supported_tasks = supported_tasks
359
self.task: Final = task
360
+ if self.task in ("draft", "generate"):
361
+ self.truncation_side = "left"
362
+ else:
363
+ self.truncation_side = "right"
364
365
self.pooler_config = self._init_pooler_config(override_pooler_config)
366
self.logits_processor_pattern = logits_processor_pattern
vllm/transformers_utils/tokenizer_group/__init__.py
+2
-1
@@ -24,7 +24,8 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
24
max_input_length=None,
25
tokenizer_mode=model_config.tokenizer_mode,
26
trust_remote_code=model_config.trust_remote_code,
27
- revision=model_config.tokenizer_revision)
27
+ revision=model_config.tokenizer_revision,
28
+ truncation_side=model_config.truncation_side)
29
30
return get_tokenizer_group(parallel_config.tokenizer_pool_config,
31
**init_kwargs)