@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / edce722eaa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] use right truncation for non-generative tasks (#12050)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Jan 15, 2025 at 09:31 UTC edce722eaa5e9f0b97bea611531e3341ec2e2e71
3 files changed +13 -1
tests/entrypoints/llm/test_encode.py
+7
@@ -105,3 +105,10 @@ def test_multiple_pooling_params(llm: LLM):
105 # pooling_params is None, default params should be applied
106 outputs = llm.encode(PROMPTS, pooling_params=None)
107 assert len(PROMPTS) == len(outputs)
108 +
109 +
110 +@pytest.mark.skip_global_cleanup
111 +def test_right_side_truncation(llm: LLM):
112 + # Embeddings models should truncate the end of the prompt
113 + tokenizer = llm.get_tokenizer()
114 + assert tokenizer.truncation_side == "right"
vllm/config.py
+4
@@ -357,6 +357,10 @@ class ModelConfig:
357 supported_tasks, task = self._resolve_task(task, self.hf_config)
358 self.supported_tasks = supported_tasks
359 self.task: Final = task
360 + if self.task in ("draft", "generate"):
361 + self.truncation_side = "left"
362 + else:
363 + self.truncation_side = "right"
364
365 self.pooler_config = self._init_pooler_config(override_pooler_config)
366 self.logits_processor_pattern = logits_processor_pattern
vllm/transformers_utils/tokenizer_group/__init__.py
+2 -1
@@ -24,7 +24,8 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
24 max_input_length=None,
25 tokenizer_mode=model_config.tokenizer_mode,
26 trust_remote_code=model_config.trust_remote_code,
27 - revision=model_config.tokenizer_revision)
27 + revision=model_config.tokenizer_revision,
28 + truncation_side=model_config.truncation_side)
29
30 return get_tokenizer_group(parallel_config.tokenizer_pool_config,
31 **init_kwargs)