@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b311efd0bd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Misc] Fix import error in tensorizer tests and cleanup some code (#10349)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 15, 2024 at 17:34 UTC b311efd0bd84faffcb1fe47aaa27ffd8c53688be
7 files changed +67 -58
tests/tensorizer_loader/test_tensorizer.py
+36 -34
@@ -8,10 +8,12 @@ from unittest.mock import MagicMock, patch
8 import openai
9 import pytest
10 import torch
11 +from huggingface_hub import snapshot_download
12 from tensorizer import EncryptionParams
13
14 from vllm import SamplingParams
15 from vllm.engine.arg_utils import EngineArgs
16 +# yapf conflicts with isort for this docstring
17 # yapf: disable
18 from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
19 TensorSerializer,
@@ -20,13 +22,14 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
22 open_stream,
23 serialize_vllm_model,
24 tensorize_vllm_model)
25 +# yapf: enable
26 +from vllm.utils import import_from_path
27
28 from ..conftest import VllmRunner
25 -from ..utils import RemoteOpenAIServer
29 +from ..utils import VLLM_PATH, RemoteOpenAIServer
30 from .conftest import retry_until_skip
31
28 -# yapf conflicts with isort for this docstring
29 -
32 +EXAMPLES_PATH = VLLM_PATH / "examples"
33
34 prompts = [
35 "Hello, my name is",
@@ -94,8 +97,8 @@ def test_can_deserialize_s3(vllm_runner):
97 num_readers=1,
98 s3_endpoint="object.ord1.coreweave.com",
99 )) as loaded_hf_model:
97 - deserialized_outputs = loaded_hf_model.generate(prompts,
98 - sampling_params)
100 + deserialized_outputs = loaded_hf_model.generate(
101 + prompts, sampling_params)
102 # noqa: E501
103
104 assert deserialized_outputs
@@ -111,23 +114,21 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
114
115 outputs = vllm_model.generate(prompts, sampling_params)
116
114 - config_for_serializing = TensorizerConfig(
115 - tensorizer_uri=model_path,
116 - encryption_keyfile=key_path
117 - )
117 + config_for_serializing = TensorizerConfig(tensorizer_uri=model_path,
118 + encryption_keyfile=key_path)
119 serialize_vllm_model(get_torch_model(vllm_model),
120 config_for_serializing)
121
122 config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
123 encryption_keyfile=key_path)
124
124 - with vllm_runner(
125 - model_ref,
126 - load_format="tensorizer",
127 - model_loader_extra_config=config_for_deserializing) as loaded_vllm_model: # noqa: E501
125 + with vllm_runner(model_ref,
126 + load_format="tensorizer",
127 + model_loader_extra_config=config_for_deserializing
128 + ) as loaded_vllm_model: # noqa: E501
129
129 - deserialized_outputs = loaded_vllm_model.generate(prompts,
130 - sampling_params)
130 + deserialized_outputs = loaded_vllm_model.generate(
131 + prompts, sampling_params)
132 # noqa: E501
133
134 assert outputs == deserialized_outputs
@@ -156,14 +157,14 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
157
158
159 def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
159 - from huggingface_hub import snapshot_download
160 -
161 - from examples.multilora_inference import (create_test_prompts,
162 - process_requests)
160 + multilora_inference = import_from_path(
161 + "examples.multilora_inference",
162 + EXAMPLES_PATH / "multilora_inference.py",
163 + )
164
165 model_ref = "meta-llama/Llama-2-7b-hf"
166 lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
166 - test_prompts = create_test_prompts(lora_path)
167 + test_prompts = multilora_inference.create_test_prompts(lora_path)
168
169 # Serialize model before deserializing and binding LoRA adapters
170 with vllm_runner(model_ref, ) as vllm_model:
@@ -186,7 +187,8 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
187 max_num_seqs=50,
188 max_model_len=1000,
189 ) as loaded_vllm_model:
189 - process_requests(loaded_vllm_model.model.llm_engine, test_prompts)
190 + multilora_inference.process_requests(
191 + loaded_vllm_model.model.llm_engine, test_prompts)
192
193 assert loaded_vllm_model
194
@@ -217,8 +219,11 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
219
220 ## Start OpenAI API server
221 openai_args = [
220 - "--dtype", "float16", "--load-format",
221 - "tensorizer", "--model-loader-extra-config",
222 + "--dtype",
223 + "float16",
224 + "--load-format",
225 + "tensorizer",
226 + "--model-loader-extra-config",
227 json.dumps(model_loader_extra_config),
228 ]
229
@@ -251,8 +256,7 @@ def test_raise_value_error_on_invalid_load_format(vllm_runner):
256 torch.cuda.empty_cache()
257
258
254 -@pytest.mark.skipif(torch.cuda.device_count() < 2,
255 - reason="Requires 2 GPUs")
259 +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Requires 2 GPUs")
260 def test_tensorizer_with_tp_path_without_template(vllm_runner):
261 with pytest.raises(ValueError):
262 model_ref = "EleutherAI/pythia-1.4b"
@@ -271,10 +275,9 @@ def test_tensorizer_with_tp_path_without_template(vllm_runner):
275 )
276
277
274 -@pytest.mark.skipif(torch.cuda.device_count() < 2,
275 - reason="Requires 2 GPUs")
276 -def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
277 - tmp_path):
278 +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Requires 2 GPUs")
279 +def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(
280 + vllm_runner, tmp_path):
281 model_ref = "EleutherAI/pythia-1.4b"
282 # record outputs from un-sharded un-tensorized model
283 with vllm_runner(
@@ -313,13 +316,12 @@ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
316 disable_custom_all_reduce=True,
317 enforce_eager=True,
318 model_loader_extra_config=tensorizer_config) as loaded_vllm_model:
316 - deserialized_outputs = loaded_vllm_model.generate(prompts,
317 - sampling_params)
319 + deserialized_outputs = loaded_vllm_model.generate(
320 + prompts, sampling_params)
321
322 assert outputs == deserialized_outputs
323
324
322 -
325 @retry_until_skip(3)
326 def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
327 gc.collect()
@@ -337,8 +339,8 @@ def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
339 with vllm_runner(model_ref,
340 load_format="tensorizer",
341 model_loader_extra_config=config) as loaded_vllm_model:
340 - deserialized_outputs = loaded_vllm_model.generate(prompts,
341 - sampling_params)
342 + deserialized_outputs = loaded_vllm_model.generate(
343 + prompts, sampling_params)
344 # noqa: E501
345
346 assert outputs == deserialized_outputs
vllm/engine/llm_engine.py
-3
@@ -2002,9 +2002,6 @@ class LLMEngine:
2002 SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE,
2003 metrics.model_execute_time)
2004
2005 - def is_encoder_decoder_model(self):
2006 - return self.input_preprocessor.is_encoder_decoder_model()
2007 -
2005 def _validate_model_inputs(self, inputs: ProcessorInputs,
2006 lora_request: Optional[LoRARequest]):
2007 if is_encoder_decoder_inputs(inputs):
vllm/entrypoints/llm.py
-3
@@ -964,6 +964,3 @@ class LLM:
964 # This is necessary because some requests may be finished earlier than
965 # its previous requests.
966 return sorted(outputs, key=lambda x: int(x.request_id))
967 -
968 - def _is_encoder_decoder_model(self):
969 - return self.llm_engine.is_encoder_decoder_model()
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
+8 -9
@@ -1,5 +1,3 @@
1 -import importlib
2 -import importlib.util
1 import os
2 from functools import cached_property
3 from typing import Callable, Dict, List, Optional, Sequence, Type, Union
@@ -9,7 +7,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
7 ExtractedToolCallInformation)
8 from vllm.logger import init_logger
9 from vllm.transformers_utils.tokenizer import AnyTokenizer
12 -from vllm.utils import is_list_of
10 +from vllm.utils import import_from_path, is_list_of
11
12 logger = init_logger(__name__)
13
@@ -149,13 +147,14 @@ class ToolParserManager:
147 @classmethod
148 def import_tool_parser(cls, plugin_path: str) -> None:
149 """
152 - Import a user defined tool parser by the path of the tool parser define
150 + Import a user-defined tool parser by the path of the tool parser define
151 file.
152 """
153 module_name = os.path.splitext(os.path.basename(plugin_path))[0]
156 - spec = importlib.util.spec_from_file_location(module_name, plugin_path)
157 - if spec is None or spec.loader is None:
158 - logger.error("load %s from %s failed.", module_name, plugin_path)
154 +
155 + try:
156 + import_from_path(module_name, plugin_path)
157 + except Exception:
158 + logger.exception("Failed to load module '%s' from %s.",
159 + module_name, plugin_path)
160 return
160 - module = importlib.util.module_from_spec(spec)
161 - spec.loader.exec_module(module)
vllm/inputs/preprocess.py
+3 -6
@@ -67,7 +67,7 @@ class InputPreprocessor:
67 model config is unavailable.
68 '''
69
70 - if not self.is_encoder_decoder_model():
70 + if not self.model_config.is_encoder_decoder:
71 print_warning_once("Using None for decoder start token id because "
72 "this is not an encoder/decoder model.")
73 return None
@@ -632,7 +632,7 @@ class InputPreprocessor:
632 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
633 ) -> ProcessorInputs:
634 """Preprocess the input prompt."""
635 - if self.is_encoder_decoder_model():
635 + if self.model_config.is_encoder_decoder:
636 # Encoder-decoder model requires special mapping of
637 # input prompts to encoder & decoder
638 return self._process_encoder_decoder_prompt(
@@ -660,7 +660,7 @@ class InputPreprocessor:
660 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
661 ) -> ProcessorInputs:
662 """Async version of :meth:`preprocess`."""
663 - if self.is_encoder_decoder_model():
663 + if self.model_config.is_encoder_decoder:
664 # Encoder-decoder model requires special mapping of
665 # input prompts to encoder & decoder
666 return await self._process_encoder_decoder_prompt_async(
@@ -679,6 +679,3 @@ class InputPreprocessor:
679 lora_request=lora_request,
680 prompt_adapter_request=prompt_adapter_request,
681 )
682 -
683 - def is_encoder_decoder_model(self):
684 - return self.model_config.is_encoder_decoder
vllm/utils.py
+20
@@ -5,6 +5,7 @@ import datetime
5 import enum
6 import gc
7 import getpass
8 +import importlib.util
9 import inspect
10 import ipaddress
11 import os
@@ -1539,6 +1540,25 @@ def is_in_doc_build() -> bool:
1540 return False
1541
1542
1543 +def import_from_path(module_name: str, file_path: Union[str, os.PathLike]):
1544 + """
1545 + Import a Python file according to its file path.
1546 +
1547 + Based on the official recipe:
1548 + https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
1549 + """
1550 + spec = importlib.util.spec_from_file_location(module_name, file_path)
1551 + if spec is None:
1552 + raise ModuleNotFoundError(f"No module named '{module_name}'")
1553 +
1554 + assert spec.loader is not None
1555 +
1556 + module = importlib.util.module_from_spec(spec)
1557 + sys.modules[module_name] = module
1558 + spec.loader.exec_module(module)
1559 + return module
1560 +
1561 +
1562 # create a library to hold the custom op
1563 vllm_lib = Library("vllm", "FRAGMENT") # noqa
1564
vllm/v1/engine/llm_engine.py
-3
@@ -163,9 +163,6 @@ class LLMEngine:
163 def get_model_config(self):
164 pass
165
166 - def is_encoder_decoder_model(self):
167 - pass
168 -
166 def start_profile(self):
167 pass
168