A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Misc] SpecDecodeWorker supports profiling (#9719)
Signed-off-by: Abatom <abatom@163.com>
科英 committed
Oct 27, 2024 at 12:18 UTC
67a6882da474a45dde0d35b3789e096e7bd0fd4e
1 file changed
+8
vllm/spec_decode/spec_decode_worker.py
+8
@@ -1038,6 +1038,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
1038
"""
1039
raise NotImplementedError
1040
1041
+ def start_profile(self):
1042
+ if isinstance(self.scorer_worker, Worker):
1043
+ self.scorer_worker.start_profile()
1044
+
1045
+ def stop_profile(self):
1046
+ if isinstance(self.scorer_worker, Worker):
1047
+ self.scorer_worker.stop_profile()
1048
+
1049
1050
def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
1051
proposer_cache_block_size_bytes: int,