@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 97b61bfae6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[misc] avoid circular import (#9765)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 28, 2024 at 13:51 UTC 97b61bfae63636e4916b49c3a2ff20353cb86db7
1 file changed +2 -1
vllm/sequence.py
+2 -1
@@ -18,7 +18,6 @@ from vllm.lora.request import LoRARequest
18 from vllm.pooling_params import PoolingParams
19 from vllm.prompt_adapter.request import PromptAdapterRequest
20 from vllm.sampling_params import RequestOutputKind, SamplingParams
21 -from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22 if TYPE_CHECKING:
23 from vllm.inputs import SingletonInputs
@@ -1132,6 +1131,8 @@ class PoolerOutput(
1131 """The output from a pooling operation in the embedding model."""
1132 outputs: List[EmbeddingSequenceGroupOutput]
1133
1134 + # lazy import to avoid circular import
1135 + from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
1136 spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1137
1138 def __getitem__(self, idx: int) -> EmbeddingSequenceGroupOutput: