@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / faeddb565d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[misc] Add Torch profiler support for CPU-only devices (#7806)

Jie Fu (傅杰) committed Aug 23, 2024 at 13:46 UTC faeddb565d6a528d1cbd169e90bc538178fc2828
1 file changed +26
vllm/worker/cpu_worker.py
+26
@@ -179,6 +179,32 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
179 self.cache_engine: List[CPUCacheEngine]
180 self.cpu_cache: List[List[torch.Tensor]]
181
182 + # Torch profiler. Enabled and configured through env vars:
183 + # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
184 + if envs.VLLM_TORCH_PROFILER_DIR:
185 + torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR
186 + logger.info("Profiling enabled. Traces will be saved to: %s",
187 + torch_profiler_trace_dir)
188 + self.profiler = torch.profiler.profile(
189 + activities=[
190 + torch.profiler.ProfilerActivity.CPU,
191 + ],
192 + with_stack=True,
193 + on_trace_ready=torch.profiler.tensorboard_trace_handler(
194 + torch_profiler_trace_dir, use_gzip=True))
195 + else:
196 + self.profiler = None
197 +
198 + def start_profile(self):
199 + if self.profiler is None:
200 + raise RuntimeError("Profiler is not enabled.")
201 + self.profiler.start()
202 +
203 + def stop_profile(self):
204 + if self.profiler is None:
205 + raise RuntimeError("Profiler is not enabled.")
206 + self.profiler.stop()
207 +
208 def init_device(self) -> None:
209 if self.local_omp_cpuid != "all":
210 torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)