A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc][LoRA] Ensure Lora Adapter requests return adapter name (#11094)
Signed-off-by: Jiaxin Shan <seedjeffwan@gmail.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Jiaxin Shan committed
Dec 12, 2024 at 01:25 UTC
85362f028c0324d8d00b0438f29c3d9f64737b9a
4 files changed
+33
-7
tests/entrypoints/openai/test_serving_engine.py
+11
@@ -9,6 +9,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse,
9
LoadLoraAdapterRequest,
10
UnloadLoraAdapterRequest)
11
from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
12
+from vllm.lora.request import LoRARequest
13
14
MODEL_NAME = "meta-llama/Llama-2-7b"
15
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
@@ -33,6 +34,16 @@ async def _async_serving_engine_init():
34
return serving_engine
35
36
37
+@pytest.mark.asyncio
38
+async def test_serving_model_name():
39
+ serving_engine = await _async_serving_engine_init()
40
+ assert serving_engine._get_model_name(None) == MODEL_NAME
41
+ request = LoRARequest(lora_name="adapter",
42
+ lora_path="/path/to/adapter2",
43
+ lora_int_id=1)
44
+ assert serving_engine._get_model_name(request) == request.lora_name
45
+
46
+
47
@pytest.mark.asyncio
48
async def test_load_lora_adapter_success():
49
serving_engine = await _async_serving_engine_init()
vllm/entrypoints/openai/serving_chat.py
+8
-6
@@ -123,6 +123,8 @@ class OpenAIServingChat(OpenAIServing):
123
prompt_adapter_request,
124
) = self._maybe_get_adapters(request)
125
126
+ model_name = self._get_model_name(lora_request)
127
+
128
tokenizer = await self.engine_client.get_tokenizer(lora_request)
129
130
tool_parser = self.tool_parser
@@ -238,13 +240,13 @@ class OpenAIServingChat(OpenAIServing):
240
# Streaming response
241
if request.stream:
242
return self.chat_completion_stream_generator(
241
- request, result_generator, request_id, conversation, tokenizer,
242
- request_metadata)
243
+ request, result_generator, request_id, model_name,
244
+ conversation, tokenizer, request_metadata)
245
246
try:
247
return await self.chat_completion_full_generator(
246
- request, result_generator, request_id, conversation, tokenizer,
247
- request_metadata)
248
+ request, result_generator, request_id, model_name,
249
+ conversation, tokenizer, request_metadata)
250
except ValueError as e:
251
# TODO: Use a vllm-specific Validation Error
252
return self.create_error_response(str(e))
@@ -259,11 +261,11 @@ class OpenAIServingChat(OpenAIServing):
261
request: ChatCompletionRequest,
262
result_generator: AsyncIterator[RequestOutput],
263
request_id: str,
264
+ model_name: str,
265
conversation: List[ConversationMessage],
266
tokenizer: AnyTokenizer,
267
request_metadata: RequestResponseMetadata,
268
) -> AsyncGenerator[str, None]:
266
- model_name = self.base_model_paths[0].name
269
created_time = int(time.time())
270
chunk_object_type: Final = "chat.completion.chunk"
271
first_iteration = True
@@ -604,12 +606,12 @@ class OpenAIServingChat(OpenAIServing):
606
request: ChatCompletionRequest,
607
result_generator: AsyncIterator[RequestOutput],
608
request_id: str,
609
+ model_name: str,
610
conversation: List[ConversationMessage],
611
tokenizer: AnyTokenizer,
612
request_metadata: RequestResponseMetadata,
613
) -> Union[ErrorResponse, ChatCompletionResponse]:
614
612
- model_name = self.base_model_paths[0].name
615
created_time = int(time.time())
616
final_res: Optional[RequestOutput] = None
617
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -85,7 +85,6 @@ class OpenAIServingCompletion(OpenAIServing):
85
return self.create_error_response(
86
"suffix is not currently supported")
87
88
- model_name = self.base_model_paths[0].name
88
request_id = f"cmpl-{self._base_request_id(raw_request)}"
89
created_time = int(time.time())
90
@@ -162,6 +161,7 @@ class OpenAIServingCompletion(OpenAIServing):
161
result_generator = merge_async_iterators(
162
*generators, is_cancelled=raw_request.is_disconnected)
163
164
+ model_name = self._get_model_name(lora_request)
165
num_prompts = len(engine_prompts)
166
167
# Similar to the OpenAI API, when n != best_of, we do not stream the
vllm/entrypoints/openai/serving_engine.py
+13
@@ -661,3 +661,16 @@ class OpenAIServing:
661
662
def _is_model_supported(self, model_name):
663
return any(model.name == model_name for model in self.base_model_paths)
664
+
665
+ def _get_model_name(self, lora: Optional[LoRARequest]):
666
+ """
667
+ Returns the appropriate model name depending on the availability
668
+ and support of the LoRA or base model.
669
+ Parameters:
670
+ - lora: LoRARequest that contain a base_model_name.
671
+ Returns:
672
+ - str: The name of the base model or the first available model path.
673
+ """
674
+ if lora is not None:
675
+ return lora.lora_name
676
+ return self.base_model_paths[0].name