A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix]: Fix Tensorizer test failures (#6835)
Sanger Steel committed
Jul 26, 2024 at 23:02 UTC
969d03226514d99f43cf7d17d1e336231d91751a
3 files changed
+47
-6
.buildkite/test-pipeline.yaml
-1
@@ -220,7 +220,6 @@ steps:
220
221
- label: Tensorizer Test
222
#mirror_hardwares: [amd]
223
- soft_fail: true
223
fast_check: true
224
commands:
225
- apt-get install -y curl libsodium23
tests/tensorizer_loader/conftest.py
new
+45
@@ -0,0 +1,45 @@
1
+# isort: skip_file
2
+
3
+import contextlib
4
+import gc
5
+
6
+import pytest
7
+import ray
8
+import torch
9
+
10
+from vllm.distributed import (destroy_distributed_environment,
11
+ destroy_model_parallel)
12
+from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
13
+
14
+
15
+def cleanup():
16
+ destroy_model_parallel()
17
+ destroy_distributed_environment()
18
+ with contextlib.suppress(AssertionError):
19
+ torch.distributed.destroy_process_group()
20
+ gc.collect()
21
+ torch.cuda.empty_cache()
22
+ ray.shutdown()
23
+
24
+
25
+@pytest.fixture()
26
+def should_do_global_cleanup_after_test(request) -> bool:
27
+ """Allow subdirectories to skip global cleanup by overriding this fixture.
28
+ This can provide a ~10x speedup for non-GPU unit tests since they don't need
29
+ to initialize torch.
30
+ """
31
+
32
+ return True
33
+
34
+
35
+@pytest.fixture(autouse=True)
36
+def cleanup_fixture(should_do_global_cleanup_after_test: bool):
37
+ yield
38
+ if should_do_global_cleanup_after_test:
39
+ cleanup()
40
+
41
+
42
+@pytest.fixture(autouse=True)
43
+def tensorizer_config():
44
+ config = TensorizerConfig(tensorizer_uri="vllm")
45
+ return config
\ No newline at end of file
tests/tensorizer_loader/test_tensorizer.py
+2
-5
@@ -40,7 +40,6 @@ model_ref = "facebook/opt-125m"
40
tensorize_model_for_testing_script = os.path.join(
41
os.path.dirname(__file__), "tensorize_vllm_model_for_testing.py")
42
43
-
43
def is_curl_installed():
44
try:
45
subprocess.check_call(['curl', '--version'])
@@ -63,10 +62,6 @@ def write_keyfile(keyfile_path: str):
62
with open(keyfile_path, 'wb') as f:
63
f.write(encryption_params.key)
64
66
-@pytest.fixture(autouse=True)
67
-def tensorizer_config():
68
- config = TensorizerConfig(tensorizer_uri="vllm")
69
- return config
65
66
67
@patch('vllm.model_executor.model_loader.tensorizer.TensorizerAgent')
@@ -105,6 +100,7 @@ def test_can_deserialize_s3(vllm_runner):
100
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
101
def test_deserialized_encrypted_vllm_model_has_same_outputs(
102
vllm_runner, tmp_path):
103
+ cleanup()
104
with vllm_runner(model_ref) as vllm_model:
105
model_path = tmp_path / (model_ref + ".tensors")
106
key_path = tmp_path / (model_ref + ".key")
@@ -316,6 +312,7 @@ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
312
313
314
def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
315
+ cleanup()
316
model_ref = "facebook/opt-125m"
317
model_path = tmp_path / (model_ref + ".tensors")
318
config = TensorizerConfig(tensorizer_uri=str(model_path))