@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 969d032265
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix]: Fix Tensorizer test failures (#6835)

Sanger Steel committed Jul 26, 2024 at 23:02 UTC 969d03226514d99f43cf7d17d1e336231d91751a
3 files changed +47 -6
.buildkite/test-pipeline.yaml
-1
@@ -220,7 +220,6 @@ steps:
220
221 - label: Tensorizer Test
222 #mirror_hardwares: [amd]
223 - soft_fail: true
223 fast_check: true
224 commands:
225 - apt-get install -y curl libsodium23
tests/tensorizer_loader/conftest.py new
+45
@@ -0,0 +1,45 @@
1 +# isort: skip_file
2 +
3 +import contextlib
4 +import gc
5 +
6 +import pytest
7 +import ray
8 +import torch
9 +
10 +from vllm.distributed import (destroy_distributed_environment,
11 + destroy_model_parallel)
12 +from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
13 +
14 +
15 +def cleanup():
16 + destroy_model_parallel()
17 + destroy_distributed_environment()
18 + with contextlib.suppress(AssertionError):
19 + torch.distributed.destroy_process_group()
20 + gc.collect()
21 + torch.cuda.empty_cache()
22 + ray.shutdown()
23 +
24 +
25 +@pytest.fixture()
26 +def should_do_global_cleanup_after_test(request) -> bool:
27 + """Allow subdirectories to skip global cleanup by overriding this fixture.
28 + This can provide a ~10x speedup for non-GPU unit tests since they don't need
29 + to initialize torch.
30 + """
31 +
32 + return True
33 +
34 +
35 +@pytest.fixture(autouse=True)
36 +def cleanup_fixture(should_do_global_cleanup_after_test: bool):
37 + yield
38 + if should_do_global_cleanup_after_test:
39 + cleanup()
40 +
41 +
42 +@pytest.fixture(autouse=True)
43 +def tensorizer_config():
44 + config = TensorizerConfig(tensorizer_uri="vllm")
45 + return config
\ No newline at end of file
tests/tensorizer_loader/test_tensorizer.py
+2 -5
@@ -40,7 +40,6 @@ model_ref = "facebook/opt-125m"
40 tensorize_model_for_testing_script = os.path.join(
41 os.path.dirname(__file__), "tensorize_vllm_model_for_testing.py")
42
43 -
43 def is_curl_installed():
44 try:
45 subprocess.check_call(['curl', '--version'])
@@ -63,10 +62,6 @@ def write_keyfile(keyfile_path: str):
62 with open(keyfile_path, 'wb') as f:
63 f.write(encryption_params.key)
64
66 -@pytest.fixture(autouse=True)
67 -def tensorizer_config():
68 - config = TensorizerConfig(tensorizer_uri="vllm")
69 - return config
65
66
67 @patch('vllm.model_executor.model_loader.tensorizer.TensorizerAgent')
@@ -105,6 +100,7 @@ def test_can_deserialize_s3(vllm_runner):
100 @pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
101 def test_deserialized_encrypted_vllm_model_has_same_outputs(
102 vllm_runner, tmp_path):
103 + cleanup()
104 with vllm_runner(model_ref) as vllm_model:
105 model_path = tmp_path / (model_ref + ".tensors")
106 key_path = tmp_path / (model_ref + ".key")
@@ -316,6 +312,7 @@ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
312
313
314 def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
315 + cleanup()
316 model_ref = "facebook/opt-125m"
317 model_path = tmp_path / (model_ref + ".tensors")
318 config = TensorizerConfig(tensorizer_uri=str(model_path))