A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Bugfix] [CI] Fix Tensorizer LoRA test (#20760)
Signed-off-by: Sanger Steel <sangersteel@gmail.com>
Sanger Steel committed
Jul 10, 2025 at 15:07 UTC
5e53c89a74ff98b7b0a9299f8854d17eb8142e9e
3 files changed
+14
-19
tests/lora/test_llama_tp.py
+3
-8
@@ -4,8 +4,6 @@ import subprocess
4
import sys
5
from typing import Union
6
7
-import pytest
8
-
7
import vllm
8
from vllm import LLM
9
from vllm.lora.request import LoRARequest
@@ -151,8 +149,6 @@ def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):
149
generate_and_test(llm, sql_lora_files)
150
151
154
-@pytest.mark.skip(reason=("Skipping this test as tensorizer is not "
155
- "working with LoRA as of #19619"))
152
@multi_gpu_test(num_gpus=2)
153
@create_new_process_for_each_test()
154
def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,
@@ -189,7 +185,6 @@ def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,
185
186
model_uri = tmp_path / "vllm" / model_ref / suffix / model_name
187
tensorizer_config = TensorizerConfig(tensorizer_uri=str(model_uri))
192
- tensorizer_config.lora_dir = tensorizer_config.tensorizer_dir
188
189
loaded_vllm_model = LLM(model=model_ref,
190
load_format="tensorizer",
@@ -200,16 +195,16 @@ def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,
195
tensor_parallel_size=2,
196
max_loras=2)
197
203
- tensorizer_config_dict = tensorizer_config.to_serializable()
198
+ tc_as_dict = tensorizer_config.to_serializable()
199
200
print("lora adapter created")
201
assert do_sample(loaded_vllm_model,
202
sql_lora_files,
208
- tensorizer_config_dict=tensorizer_config_dict,
203
+ tensorizer_config_dict=tc_as_dict,
204
lora_id=0) == EXPECTED_NO_LORA_OUTPUT
205
206
print("lora 1")
207
assert do_sample(loaded_vllm_model,
208
sql_lora_files,
214
- tensorizer_config_dict=tensorizer_config_dict,
209
+ tensorizer_config_dict=tc_as_dict,
210
lora_id=1) == EXPECTED_LORA_OUTPUT
vllm/lora/peft_helper.py
+2
-2
@@ -102,7 +102,7 @@ class PEFTHelper:
102
tensorizer_config = TensorizerConfig(**tensorizer_config_dict)
103
tensorizer_args = tensorizer_config._construct_tensorizer_args()
104
from tensorizer.stream_io import open_stream
105
- lora_config_path = os.path.join(tensorizer_config.lora_dir,
105
+ lora_config_path = os.path.join(tensorizer_config.tensorizer_dir,
106
"adapter_config.json")
107
with open_stream(lora_config_path,
108
mode="rb",
@@ -110,7 +110,7 @@ class PEFTHelper:
110
config = json.load(f)
111
112
logger.info("Successfully deserialized LoRA config from %s",
113
- tensorizer_config.lora_dir)
113
+ tensorizer_config.tensorizer_dir)
114
115
else:
116
with open(lora_config_path) as f:
vllm/model_executor/model_loader/tensorizer.py
+9
-9
@@ -222,17 +222,17 @@ class TensorizerConfig(MutableMapping):
222
self._is_sharded = isinstance(self.tensorizer_uri, str) \
223
and re.search(r'%0\dd', self.tensorizer_uri) is not None
224
225
+ if self.tensorizer_dir and self.lora_dir:
226
+ raise ValueError(
227
+ "Only one of tensorizer_dir or lora_dir may be specified. "
228
+ "Use lora_dir exclusively when serializing LoRA adapters, "
229
+ "and tensorizer_dir or tensorizer_uri otherwise.")
230
if self.tensorizer_dir and self.tensorizer_uri:
231
logger.warning_once(
232
"Provided both tensorizer_dir and tensorizer_uri. "
233
"Inferring tensorizer_dir from tensorizer_uri as the "
234
"latter takes precedence.")
235
self.tensorizer_dir = os.path.dirname(self.tensorizer_uri)
231
- if self.tensorizer_dir and self.lora_dir:
232
- raise ValueError(
233
- "Only one of tensorizer_dir or lora_dir may be specified. "
234
- "Use lora_dir exclusively when serializing LoRA adapters, "
235
- "and tensorizer_dir or tensorizer_uri otherwise.")
236
if not self.tensorizer_uri:
237
if self.lora_dir:
238
self.tensorizer_uri = f"{self.lora_dir}/adapter_model.tensors"
@@ -695,7 +695,7 @@ def tensorize_lora_adapter(lora_path: str,
695
needed to load a LoRA adapter are a safetensors-format file called
696
adapter_model.safetensors and a json config file called adapter_config.json.
697
698
- Serializes the files in the tensorizer_config.lora_dir
698
+ Serializes the files in the tensorizer_config.tensorizer_dir
699
"""
700
import safetensors
701
@@ -725,13 +725,13 @@ def tensorize_lora_adapter(lora_path: str,
725
726
tensorizer_args = tensorizer_config._construct_tensorizer_args()
727
728
- with open_stream(f"{tensorizer_config.lora_dir}/adapter_config.json",
728
+ with open_stream(f"{tensorizer_config.tensorizer_dir}/adapter_config.json",
729
mode="wb+",
730
**tensorizer_args.stream_kwargs) as f:
731
732
f.write(json.dumps(config).encode("utf-8"))
733
734
- lora_uri = (f"{tensorizer_config.lora_dir}"
734
+ lora_uri = (f"{tensorizer_config.tensorizer_dir}"
735
f"/adapter_model.tensors")
736
with open_stream(lora_uri, mode="wb+",
737
**tensorizer_args.stream_kwargs) as f:
@@ -740,4 +740,4 @@ def tensorize_lora_adapter(lora_path: str,
740
serializer.close()
741
742
logger.info("Successfully serialized LoRA files to %s",
743
- str(tensorizer_config.lora_dir))
743
+ str(tensorizer_config.tensorizer_dir))