A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[V1][Structured Output] Clear xgrammar compiler object when engine core shut down to avoid nanobind leaked warning (#16954)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
Apr 24, 2025 at 21:15 UTC
b724afe343b788d61ca0425892f2631669f97f45
5 files changed
+17
vllm/v1/engine/core.py
+1
@@ -253,6 +253,7 @@ class EngineCore:
253
return engine_core_outputs
254
255
def shutdown(self):
256
+ self.structured_output_manager.clear_backend()
257
if self.model_executor:
258
self.model_executor.shutdown()
259
vllm/v1/structured_output/__init__.py
+4
@@ -107,3 +107,7 @@ class StructuredOutputManager:
107
# np.ndarray, because that is much more efficient for serialization
108
# and deserialization when sending this to the GPU workers.
109
return bitmask_tensor.numpy()
110
+
111
+ def clear_backend(self) -> None:
112
+ if self.backend is not None:
113
+ self.backend.destroy()
vllm/v1/structured_output/backend_guidance.py
+3
@@ -108,6 +108,9 @@ class GuidanceBackend(StructuredOutputBackend):
108
return llguidance_torch.allocate_token_bitmask(
109
max_num_seqs, self.ll_tokenizer.vocab_size)
110
111
+ def destroy(self):
112
+ pass
113
+
114
115
@dataclass
116
class GuidanceGrammar(StructuredOutputGrammar):
vllm/v1/structured_output/backend_types.py
+6
@@ -87,3 +87,9 @@ class StructuredOutputBackend(ABC):
87
max_num_seqs (int): The maximum number of sequences for which
88
to allocate the bitmask.
89
"""
90
+
91
+ @abstractmethod
92
+ def destroy(self):
93
+ """
94
+ Backend-specific cleanup.
95
+ """
vllm/v1/structured_output/backend_xgrammar.py
+3
@@ -124,6 +124,9 @@ class XgrammarBackend(StructuredOutputBackend):
124
def allocate_token_bitmask(self, max_num_seqs: int):
125
return xgr.allocate_token_bitmask(max_num_seqs, self.vocab_size)
126
127
+ def destroy(self):
128
+ del self.compiler
129
+
130
131
@dataclass
132
class XgrammarGrammar(StructuredOutputGrammar):