A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[misc] avoid circular import (#9765)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 28, 2024 at 13:51 UTC
97b61bfae63636e4916b49c3a2ff20353cb86db7
1 file changed
+2
-1
vllm/sequence.py
+2
-1
@@ -18,7 +18,6 @@ from vllm.lora.request import LoRARequest
18
from vllm.pooling_params import PoolingParams
19
from vllm.prompt_adapter.request import PromptAdapterRequest
20
from vllm.sampling_params import RequestOutputKind, SamplingParams
21
-from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22
if TYPE_CHECKING:
23
from vllm.inputs import SingletonInputs
@@ -1132,6 +1131,8 @@ class PoolerOutput(
1131
"""The output from a pooling operation in the embedding model."""
1132
outputs: List[EmbeddingSequenceGroupOutput]
1133
1134
+ # lazy import to avoid circular import
1135
+ from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
1136
spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1137
1138
def __getitem__(self, idx: int) -> EmbeddingSequenceGroupOutput: