A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Fix circular reference which leaked llm instance in local dev env (#4737)
Storing exception frame is extremely prone to circular refernece because it contains the reference to objects. When tensorizer is not installed, it leaks llm instance because error frame has references to various modules which cause circular reference problem. I also found spec decoding has a circular reference issue, and I solved it using weakref.proxy.
SangBin Cho committed
May 10, 2024 at 23:54 UTC
6a0f617210dfba76f3db4db1155d1f1489609133
4 files changed
+22
-7
tests/basic_correctness/test_basic_correctness.py
+13
@@ -3,9 +3,12 @@
3
Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4
"""
5
import os
6
+import weakref
7
8
import pytest
9
10
+from vllm import LLM
11
+
12
MODELS = [
13
"facebook/opt-125m",
14
"meta-llama/Llama-2-7b-hf",
@@ -13,6 +16,16 @@ MODELS = [
16
VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
17
18
19
+def test_vllm_gc_ed():
20
+ """Verify vllm instance is GC'ed when it is deleted"""
21
+ llm = LLM("facebook/opt-125m")
22
+ weak_llm = weakref.ref(llm)
23
+ del llm
24
+ # If there's any circular reference to vllm, this fails
25
+ # because llm instance is not GC'ed.
26
+ assert weak_llm() is None
27
+
28
+
29
@pytest.mark.parametrize("model", MODELS)
30
@pytest.mark.parametrize("dtype", ["half"])
31
@pytest.mark.parametrize("max_tokens", [5])
vllm/model_executor/model_loader/tensorizer.py
+5
-5
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization.base_config import (
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
VocabParallelEmbedding)
21
22
-tensorizer_load_fail = None
22
+tensorizer_error_msg = None
23
24
try:
25
from tensorizer import (DecryptionParams, EncryptionParams,
@@ -28,7 +28,7 @@ try:
28
from tensorizer.utils import (convert_bytes, get_mem_usage,
29
no_init_or_tensor)
30
except ImportError as e:
31
- tensorizer_load_fail = e
31
+ tensorizer_error_msg = str(e)
32
33
__all__ = [
34
'EncryptionParams', 'DecryptionParams', 'TensorDeserializer',
@@ -254,11 +254,11 @@ class TensorizerAgent:
254
255
def __init__(self, tensorizer_config: TensorizerConfig,
256
quant_config: QuantizationConfig, **extra_kwargs):
257
- if tensorizer_load_fail is not None:
257
+ if tensorizer_error_msg is not None:
258
raise ImportError(
259
"Tensorizer is not installed. Please install tensorizer "
260
- "to use this feature with `pip install vllm[tensorizer]`."
261
- ) from tensorizer_load_fail
260
+ "to use this feature with `pip install vllm[tensorizer]`. "
261
+ "Error message: {}".format(tensorizer_error_msg))
262
263
self.tensorizer_config = tensorizer_config
264
self.tensorizer_args = (
vllm/spec_decode/multi_step_worker.py
+2
-1
@@ -1,4 +1,5 @@
1
import copy
2
+import weakref
3
from typing import List, Tuple
4
5
import torch
@@ -32,7 +33,7 @@ class MultiStepWorker(Worker):
33
super().init_device()
34
35
self._proposer = Top1Proposer(
35
- self,
36
+ weakref.proxy(self),
37
self.device,
38
self.vocab_size,
39
max_proposal_len=self.max_model_len,
vllm/spec_decode/ngram_worker.py
+2
-1
@@ -1,3 +1,4 @@
1
+import weakref
2
from typing import List, Optional, Tuple
3
4
import torch
@@ -37,7 +38,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
38
39
# Current only support Top1Proposer
40
self._proposer = Top1Proposer(
40
- self,
41
+ weakref.proxy(self),
42
device=self.device,
43
vocab_size=self.vocab_size,
44
)