@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 85362f028c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc][LoRA] Ensure Lora Adapter requests return adapter name (#11094)

Signed-off-by: Jiaxin Shan <seedjeffwan@gmail.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>

Jiaxin Shan committed Dec 12, 2024 at 01:25 UTC 85362f028c0324d8d00b0438f29c3d9f64737b9a
4 files changed +33 -7
tests/entrypoints/openai/test_serving_engine.py
+11
@@ -9,6 +9,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse,
9 LoadLoraAdapterRequest,
10 UnloadLoraAdapterRequest)
11 from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
12 +from vllm.lora.request import LoRARequest
13
14 MODEL_NAME = "meta-llama/Llama-2-7b"
15 BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
@@ -33,6 +34,16 @@ async def _async_serving_engine_init():
34 return serving_engine
35
36
37 +@pytest.mark.asyncio
38 +async def test_serving_model_name():
39 + serving_engine = await _async_serving_engine_init()
40 + assert serving_engine._get_model_name(None) == MODEL_NAME
41 + request = LoRARequest(lora_name="adapter",
42 + lora_path="/path/to/adapter2",
43 + lora_int_id=1)
44 + assert serving_engine._get_model_name(request) == request.lora_name
45 +
46 +
47 @pytest.mark.asyncio
48 async def test_load_lora_adapter_success():
49 serving_engine = await _async_serving_engine_init()
vllm/entrypoints/openai/serving_chat.py
+8 -6
@@ -123,6 +123,8 @@ class OpenAIServingChat(OpenAIServing):
123 prompt_adapter_request,
124 ) = self._maybe_get_adapters(request)
125
126 + model_name = self._get_model_name(lora_request)
127 +
128 tokenizer = await self.engine_client.get_tokenizer(lora_request)
129
130 tool_parser = self.tool_parser
@@ -238,13 +240,13 @@ class OpenAIServingChat(OpenAIServing):
240 # Streaming response
241 if request.stream:
242 return self.chat_completion_stream_generator(
241 - request, result_generator, request_id, conversation, tokenizer,
242 - request_metadata)
243 + request, result_generator, request_id, model_name,
244 + conversation, tokenizer, request_metadata)
245
246 try:
247 return await self.chat_completion_full_generator(
246 - request, result_generator, request_id, conversation, tokenizer,
247 - request_metadata)
248 + request, result_generator, request_id, model_name,
249 + conversation, tokenizer, request_metadata)
250 except ValueError as e:
251 # TODO: Use a vllm-specific Validation Error
252 return self.create_error_response(str(e))
@@ -259,11 +261,11 @@ class OpenAIServingChat(OpenAIServing):
261 request: ChatCompletionRequest,
262 result_generator: AsyncIterator[RequestOutput],
263 request_id: str,
264 + model_name: str,
265 conversation: List[ConversationMessage],
266 tokenizer: AnyTokenizer,
267 request_metadata: RequestResponseMetadata,
268 ) -> AsyncGenerator[str, None]:
266 - model_name = self.base_model_paths[0].name
269 created_time = int(time.time())
270 chunk_object_type: Final = "chat.completion.chunk"
271 first_iteration = True
@@ -604,12 +606,12 @@ class OpenAIServingChat(OpenAIServing):
606 request: ChatCompletionRequest,
607 result_generator: AsyncIterator[RequestOutput],
608 request_id: str,
609 + model_name: str,
610 conversation: List[ConversationMessage],
611 tokenizer: AnyTokenizer,
612 request_metadata: RequestResponseMetadata,
613 ) -> Union[ErrorResponse, ChatCompletionResponse]:
614
612 - model_name = self.base_model_paths[0].name
615 created_time = int(time.time())
616 final_res: Optional[RequestOutput] = None
617
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -85,7 +85,6 @@ class OpenAIServingCompletion(OpenAIServing):
85 return self.create_error_response(
86 "suffix is not currently supported")
87
88 - model_name = self.base_model_paths[0].name
88 request_id = f"cmpl-{self._base_request_id(raw_request)}"
89 created_time = int(time.time())
90
@@ -162,6 +161,7 @@ class OpenAIServingCompletion(OpenAIServing):
161 result_generator = merge_async_iterators(
162 *generators, is_cancelled=raw_request.is_disconnected)
163
164 + model_name = self._get_model_name(lora_request)
165 num_prompts = len(engine_prompts)
166
167 # Similar to the OpenAI API, when n != best_of, we do not stream the
vllm/entrypoints/openai/serving_engine.py
+13
@@ -661,3 +661,16 @@ class OpenAIServing:
661
662 def _is_model_supported(self, model_name):
663 return any(model.name == model_name for model in self.base_model_paths)
664 +
665 + def _get_model_name(self, lora: Optional[LoRARequest]):
666 + """
667 + Returns the appropriate model name depending on the availability
668 + and support of the LoRA or base model.
669 + Parameters:
670 + - lora: LoRARequest that contain a base_model_name.
671 + Returns:
672 + - str: The name of the base model or the first available model path.
673 + """
674 + if lora is not None:
675 + return lora.lora_name
676 + return self.base_model_paths[0].name