@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01d079fd8e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[LoRA] Change lora_tokenizers capacity (#10796)

Signed-off-by: Xin Yang <xyang19@gmail.com>

Xin Yang committed Dec 4, 2024 at 09:40 UTC 01d079fd8e65ed9a243ebbf6b771393607942907
7 files changed +31 -10
tests/lora/test_tokenizer_group.py
+20
@@ -17,6 +17,7 @@ async def test_tokenizer_group_lora(sql_lora_files, tokenizer_group_type):
17 tokenizer_id="gpt2",
18 enable_lora=True,
19 max_num_seqs=1,
20 + max_loras=1,
21 max_input_length=None,
22 )
23 lora_request = LoRARequest("1", 1, sql_lora_files)
@@ -53,3 +54,22 @@ def test_get_lora_tokenizer(sql_lora_files, tmp_path):
54 lora_request = LoRARequest("1", 1, str(tmp_path))
55 tokenizer = get_lora_tokenizer(lora_request)
56 assert not tokenizer
57 +
58 +
59 +@pytest.mark.parametrize("enable_lora", [True, False])
60 +@pytest.mark.parametrize("max_num_seqs", [1, 2])
61 +@pytest.mark.parametrize("max_loras", [1, 2])
62 +def test_lora_tokenizers(enable_lora, max_num_seqs, max_loras):
63 + tokenizer_group = get_tokenizer_group(
64 + get_tokenizer_pool_config(None),
65 + tokenizer_id="gpt2",
66 + enable_lora=enable_lora,
67 + max_num_seqs=max_num_seqs,
68 + max_loras=max_loras,
69 + max_input_length=None,
70 + )
71 + if enable_lora:
72 + assert tokenizer_group.lora_tokenizers.capacity == max(
73 + max_num_seqs, max_loras)
74 + else:
75 + assert tokenizer_group.lora_tokenizers.capacity == 0
vllm/engine/llm_engine.py
+1 -1
@@ -620,7 +620,7 @@ class LLMEngine:
620 model_config=self.model_config,
621 scheduler_config=self.scheduler_config,
622 parallel_config=self.parallel_config,
623 - enable_lora=bool(self.lora_config))
623 + lora_config=self.lora_config)
624
625 def _verify_args(self) -> None:
626 self.model_config.verify_with_parallel_config(self.parallel_config)
vllm/engine/multiprocessing/client.py
+1 -2
@@ -94,8 +94,7 @@ class MQLLMEngineClient(EngineClient):
94 model_config=self.model_config,
95 scheduler_config=engine_config.scheduler_config,
96 parallel_config=engine_config.parallel_config,
97 - enable_lora=bool(engine_config.lora_config),
98 - )
97 + lora_config=engine_config.lora_config)
98 self.input_preprocessor = InputPreprocessor(self.model_config,
99 self.tokenizer)
100
vllm/transformers_utils/tokenizer_group/__init__.py
+5 -4
@@ -1,7 +1,7 @@
1 from typing import Optional, Type
2
3 -from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
4 - TokenizerPoolConfig)
3 +from vllm.config import (LoRAConfig, ModelConfig, ParallelConfig,
4 + SchedulerConfig, TokenizerPoolConfig)
5 from vllm.executor.ray_utils import ray
6
7 from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
@@ -16,10 +16,11 @@ else:
16 def init_tokenizer_from_configs(model_config: ModelConfig,
17 scheduler_config: SchedulerConfig,
18 parallel_config: ParallelConfig,
19 - enable_lora: bool):
19 + lora_config: LoRAConfig):
20 init_kwargs = dict(tokenizer_id=model_config.tokenizer,
21 - enable_lora=enable_lora,
21 + enable_lora=bool(lora_config),
22 max_num_seqs=scheduler_config.max_num_seqs,
23 + max_loras=lora_config.max_loras if lora_config else 0,
24 max_input_length=None,
25 tokenizer_mode=model_config.tokenizer_mode,
26 trust_remote_code=model_config.trust_remote_code,
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+2 -1
@@ -21,8 +21,9 @@ class TokenizerGroup(BaseTokenizerGroup):
21 self.enable_lora = enable_lora
22 self.max_input_length = max_input_length
23 self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
24 + max_loras = tokenizer_config.get("max_loras", 0)
25 self.lora_tokenizers = LRUCache[AnyTokenizer](
25 - capacity=max_num_seqs if enable_lora else 0)
26 + capacity=max(max_loras, max_num_seqs) if enable_lora else 0)
27
28 @classmethod
29 def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
vllm/v1/engine/async_llm.py
+1 -1
@@ -51,7 +51,7 @@ class AsyncLLM(EngineClient):
51 model_config=vllm_config.model_config,
52 scheduler_config=vllm_config.scheduler_config,
53 parallel_config=vllm_config.parallel_config,
54 - enable_lora=bool(vllm_config.lora_config))
54 + lora_config=vllm_config.lora_config)
55 self.tokenizer.ping()
56
57 # Request streams (map of request_id -> AsyncStream).
vllm/v1/engine/llm_engine.py
+1 -1
@@ -46,7 +46,7 @@ class LLMEngine:
46 model_config=vllm_config.model_config,
47 scheduler_config=vllm_config.scheduler_config,
48 parallel_config=vllm_config.parallel_config,
49 - enable_lora=bool(vllm_config.lora_config))
49 + lora_config=vllm_config.lora_config)
50 self.tokenizer.ping()
51
52 # Processor (convert Inputs --> EngineCoreRequests)