A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[misc] Add Torch profiler support for CPU-only devices (#7806)
Jie Fu (傅杰) committed
Aug 23, 2024 at 13:46 UTC
faeddb565d6a528d1cbd169e90bc538178fc2828
1 file changed
+26
vllm/worker/cpu_worker.py
+26
@@ -179,6 +179,32 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
179
self.cache_engine: List[CPUCacheEngine]
180
self.cpu_cache: List[List[torch.Tensor]]
181
182
+ # Torch profiler. Enabled and configured through env vars:
183
+ # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
184
+ if envs.VLLM_TORCH_PROFILER_DIR:
185
+ torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR
186
+ logger.info("Profiling enabled. Traces will be saved to: %s",
187
+ torch_profiler_trace_dir)
188
+ self.profiler = torch.profiler.profile(
189
+ activities=[
190
+ torch.profiler.ProfilerActivity.CPU,
191
+ ],
192
+ with_stack=True,
193
+ on_trace_ready=torch.profiler.tensorboard_trace_handler(
194
+ torch_profiler_trace_dir, use_gzip=True))
195
+ else:
196
+ self.profiler = None
197
+
198
+ def start_profile(self):
199
+ if self.profiler is None:
200
+ raise RuntimeError("Profiler is not enabled.")
201
+ self.profiler.start()
202
+
203
+ def stop_profile(self):
204
+ if self.profiler is None:
205
+ raise RuntimeError("Profiler is not enabled.")
206
+ self.profiler.stop()
207
+
208
def init_device(self) -> None:
209
if self.local_omp_cpuid != "all":
210
torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)