@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 67a6882da4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc] SpecDecodeWorker supports profiling (#9719)

Signed-off-by: Abatom <abatom@163.com>

科英 committed Oct 27, 2024 at 12:18 UTC 67a6882da474a45dde0d35b3789e096e7bd0fd4e
1 file changed +8
vllm/spec_decode/spec_decode_worker.py
+8
@@ -1038,6 +1038,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
1038 """
1039 raise NotImplementedError
1040
1041 + def start_profile(self):
1042 + if isinstance(self.scorer_worker, Worker):
1043 + self.scorer_worker.start_profile()
1044 +
1045 + def stop_profile(self):
1046 + if isinstance(self.scorer_worker, Worker):
1047 + self.scorer_worker.stop_profile()
1048 +
1049
1050 def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
1051 proposer_cache_block_size_bytes: int,