A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Misc] Fix import error in tensorizer tests and cleanup some code (#10349)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 15, 2024 at 17:34 UTC
b311efd0bd84faffcb1fe47aaa27ffd8c53688be
7 files changed
+67
-58
tests/tensorizer_loader/test_tensorizer.py
+36
-34
@@ -8,10 +8,12 @@ from unittest.mock import MagicMock, patch
8
import openai
9
import pytest
10
import torch
11
+from huggingface_hub import snapshot_download
12
from tensorizer import EncryptionParams
13
14
from vllm import SamplingParams
15
from vllm.engine.arg_utils import EngineArgs
16
+# yapf conflicts with isort for this docstring
17
# yapf: disable
18
from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
19
TensorSerializer,
@@ -20,13 +22,14 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
22
open_stream,
23
serialize_vllm_model,
24
tensorize_vllm_model)
25
+# yapf: enable
26
+from vllm.utils import import_from_path
27
28
from ..conftest import VllmRunner
25
-from ..utils import RemoteOpenAIServer
29
+from ..utils import VLLM_PATH, RemoteOpenAIServer
30
from .conftest import retry_until_skip
31
28
-# yapf conflicts with isort for this docstring
29
-
32
+EXAMPLES_PATH = VLLM_PATH / "examples"
33
34
prompts = [
35
"Hello, my name is",
@@ -94,8 +97,8 @@ def test_can_deserialize_s3(vllm_runner):
97
num_readers=1,
98
s3_endpoint="object.ord1.coreweave.com",
99
)) as loaded_hf_model:
97
- deserialized_outputs = loaded_hf_model.generate(prompts,
98
- sampling_params)
100
+ deserialized_outputs = loaded_hf_model.generate(
101
+ prompts, sampling_params)
102
# noqa: E501
103
104
assert deserialized_outputs
@@ -111,23 +114,21 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
114
115
outputs = vllm_model.generate(prompts, sampling_params)
116
114
- config_for_serializing = TensorizerConfig(
115
- tensorizer_uri=model_path,
116
- encryption_keyfile=key_path
117
- )
117
+ config_for_serializing = TensorizerConfig(tensorizer_uri=model_path,
118
+ encryption_keyfile=key_path)
119
serialize_vllm_model(get_torch_model(vllm_model),
120
config_for_serializing)
121
122
config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
123
encryption_keyfile=key_path)
124
124
- with vllm_runner(
125
- model_ref,
126
- load_format="tensorizer",
127
- model_loader_extra_config=config_for_deserializing) as loaded_vllm_model: # noqa: E501
125
+ with vllm_runner(model_ref,
126
+ load_format="tensorizer",
127
+ model_loader_extra_config=config_for_deserializing
128
+ ) as loaded_vllm_model: # noqa: E501
129
129
- deserialized_outputs = loaded_vllm_model.generate(prompts,
130
- sampling_params)
130
+ deserialized_outputs = loaded_vllm_model.generate(
131
+ prompts, sampling_params)
132
# noqa: E501
133
134
assert outputs == deserialized_outputs
@@ -156,14 +157,14 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
157
158
159
def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
159
- from huggingface_hub import snapshot_download
160
-
161
- from examples.multilora_inference import (create_test_prompts,
162
- process_requests)
160
+ multilora_inference = import_from_path(
161
+ "examples.multilora_inference",
162
+ EXAMPLES_PATH / "multilora_inference.py",
163
+ )
164
165
model_ref = "meta-llama/Llama-2-7b-hf"
166
lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
166
- test_prompts = create_test_prompts(lora_path)
167
+ test_prompts = multilora_inference.create_test_prompts(lora_path)
168
169
# Serialize model before deserializing and binding LoRA adapters
170
with vllm_runner(model_ref, ) as vllm_model:
@@ -186,7 +187,8 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
187
max_num_seqs=50,
188
max_model_len=1000,
189
) as loaded_vllm_model:
189
- process_requests(loaded_vllm_model.model.llm_engine, test_prompts)
190
+ multilora_inference.process_requests(
191
+ loaded_vllm_model.model.llm_engine, test_prompts)
192
193
assert loaded_vllm_model
194
@@ -217,8 +219,11 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
219
220
## Start OpenAI API server
221
openai_args = [
220
- "--dtype", "float16", "--load-format",
221
- "tensorizer", "--model-loader-extra-config",
222
+ "--dtype",
223
+ "float16",
224
+ "--load-format",
225
+ "tensorizer",
226
+ "--model-loader-extra-config",
227
json.dumps(model_loader_extra_config),
228
]
229
@@ -251,8 +256,7 @@ def test_raise_value_error_on_invalid_load_format(vllm_runner):
256
torch.cuda.empty_cache()
257
258
254
-@pytest.mark.skipif(torch.cuda.device_count() < 2,
255
- reason="Requires 2 GPUs")
259
+@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Requires 2 GPUs")
260
def test_tensorizer_with_tp_path_without_template(vllm_runner):
261
with pytest.raises(ValueError):
262
model_ref = "EleutherAI/pythia-1.4b"
@@ -271,10 +275,9 @@ def test_tensorizer_with_tp_path_without_template(vllm_runner):
275
)
276
277
274
-@pytest.mark.skipif(torch.cuda.device_count() < 2,
275
- reason="Requires 2 GPUs")
276
-def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
277
- tmp_path):
278
+@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Requires 2 GPUs")
279
+def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(
280
+ vllm_runner, tmp_path):
281
model_ref = "EleutherAI/pythia-1.4b"
282
# record outputs from un-sharded un-tensorized model
283
with vllm_runner(
@@ -313,13 +316,12 @@ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
316
disable_custom_all_reduce=True,
317
enforce_eager=True,
318
model_loader_extra_config=tensorizer_config) as loaded_vllm_model:
316
- deserialized_outputs = loaded_vllm_model.generate(prompts,
317
- sampling_params)
319
+ deserialized_outputs = loaded_vllm_model.generate(
320
+ prompts, sampling_params)
321
322
assert outputs == deserialized_outputs
323
324
322
-
325
@retry_until_skip(3)
326
def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
327
gc.collect()
@@ -337,8 +339,8 @@ def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
339
with vllm_runner(model_ref,
340
load_format="tensorizer",
341
model_loader_extra_config=config) as loaded_vllm_model:
340
- deserialized_outputs = loaded_vllm_model.generate(prompts,
341
- sampling_params)
342
+ deserialized_outputs = loaded_vllm_model.generate(
343
+ prompts, sampling_params)
344
# noqa: E501
345
346
assert outputs == deserialized_outputs
vllm/engine/llm_engine.py
-3
@@ -2002,9 +2002,6 @@ class LLMEngine:
2002
SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE,
2003
metrics.model_execute_time)
2004
2005
- def is_encoder_decoder_model(self):
2006
- return self.input_preprocessor.is_encoder_decoder_model()
2007
-
2005
def _validate_model_inputs(self, inputs: ProcessorInputs,
2006
lora_request: Optional[LoRARequest]):
2007
if is_encoder_decoder_inputs(inputs):
vllm/entrypoints/llm.py
-3
@@ -964,6 +964,3 @@ class LLM:
964
# This is necessary because some requests may be finished earlier than
965
# its previous requests.
966
return sorted(outputs, key=lambda x: int(x.request_id))
967
-
968
- def _is_encoder_decoder_model(self):
969
- return self.llm_engine.is_encoder_decoder_model()
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
+8
-9
@@ -1,5 +1,3 @@
1
-import importlib
2
-import importlib.util
1
import os
2
from functools import cached_property
3
from typing import Callable, Dict, List, Optional, Sequence, Type, Union
@@ -9,7 +7,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
7
ExtractedToolCallInformation)
8
from vllm.logger import init_logger
9
from vllm.transformers_utils.tokenizer import AnyTokenizer
12
-from vllm.utils import is_list_of
10
+from vllm.utils import import_from_path, is_list_of
11
12
logger = init_logger(__name__)
13
@@ -149,13 +147,14 @@ class ToolParserManager:
147
@classmethod
148
def import_tool_parser(cls, plugin_path: str) -> None:
149
"""
152
- Import a user defined tool parser by the path of the tool parser define
150
+ Import a user-defined tool parser by the path of the tool parser define
151
file.
152
"""
153
module_name = os.path.splitext(os.path.basename(plugin_path))[0]
156
- spec = importlib.util.spec_from_file_location(module_name, plugin_path)
157
- if spec is None or spec.loader is None:
158
- logger.error("load %s from %s failed.", module_name, plugin_path)
154
+
155
+ try:
156
+ import_from_path(module_name, plugin_path)
157
+ except Exception:
158
+ logger.exception("Failed to load module '%s' from %s.",
159
+ module_name, plugin_path)
160
return
160
- module = importlib.util.module_from_spec(spec)
161
- spec.loader.exec_module(module)
vllm/inputs/preprocess.py
+3
-6
@@ -67,7 +67,7 @@ class InputPreprocessor:
67
model config is unavailable.
68
'''
69
70
- if not self.is_encoder_decoder_model():
70
+ if not self.model_config.is_encoder_decoder:
71
print_warning_once("Using None for decoder start token id because "
72
"this is not an encoder/decoder model.")
73
return None
@@ -632,7 +632,7 @@ class InputPreprocessor:
632
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
633
) -> ProcessorInputs:
634
"""Preprocess the input prompt."""
635
- if self.is_encoder_decoder_model():
635
+ if self.model_config.is_encoder_decoder:
636
# Encoder-decoder model requires special mapping of
637
# input prompts to encoder & decoder
638
return self._process_encoder_decoder_prompt(
@@ -660,7 +660,7 @@ class InputPreprocessor:
660
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
661
) -> ProcessorInputs:
662
"""Async version of :meth:`preprocess`."""
663
- if self.is_encoder_decoder_model():
663
+ if self.model_config.is_encoder_decoder:
664
# Encoder-decoder model requires special mapping of
665
# input prompts to encoder & decoder
666
return await self._process_encoder_decoder_prompt_async(
@@ -679,6 +679,3 @@ class InputPreprocessor:
679
lora_request=lora_request,
680
prompt_adapter_request=prompt_adapter_request,
681
)
682
-
683
- def is_encoder_decoder_model(self):
684
- return self.model_config.is_encoder_decoder
vllm/utils.py
+20
@@ -5,6 +5,7 @@ import datetime
5
import enum
6
import gc
7
import getpass
8
+import importlib.util
9
import inspect
10
import ipaddress
11
import os
@@ -1539,6 +1540,25 @@ def is_in_doc_build() -> bool:
1540
return False
1541
1542
1543
+def import_from_path(module_name: str, file_path: Union[str, os.PathLike]):
1544
+ """
1545
+ Import a Python file according to its file path.
1546
+
1547
+ Based on the official recipe:
1548
+ https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
1549
+ """
1550
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
1551
+ if spec is None:
1552
+ raise ModuleNotFoundError(f"No module named '{module_name}'")
1553
+
1554
+ assert spec.loader is not None
1555
+
1556
+ module = importlib.util.module_from_spec(spec)
1557
+ sys.modules[module_name] = module
1558
+ spec.loader.exec_module(module)
1559
+ return module
1560
+
1561
+
1562
# create a library to hold the custom op
1563
vllm_lib = Library("vllm", "FRAGMENT") # noqa
1564
vllm/v1/engine/llm_engine.py
-3
@@ -163,9 +163,6 @@ class LLMEngine:
163
def get_model_config(self):
164
pass
165
166
- def is_encoder_decoder_model(self):
167
- pass
168
-
166
def start_profile(self):
167
pass
168