@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4695397dcf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix ray workers profiling with nsight (#4095)

Ricky Xu committed Apr 15, 2024 at 14:24 UTC 4695397dcfef693a0a10f1eb8bf77ea905c54829
1 file changed +19
vllm/executor/ray_gpu_executor.py
+19
@@ -48,6 +48,21 @@ class RayGPUExecutor(ExecutorBase):
48 if USE_RAY_COMPILED_DAG:
49 self.forward_dag = self._compiled_ray_dag()
50
51 + def _configure_ray_workers_use_nsight(self,
52 + ray_remote_kwargs) -> Dict[str, Any]:
53 + # If nsight profiling is enabled, we need to set the profiling
54 + # configuration for the ray workers as runtime env.
55 + runtime_env = ray_remote_kwargs.setdefault("runtime_env", {})
56 + runtime_env.update({
57 + "nsight": {
58 + "t": "cuda,cudnn,cublas",
59 + "o": "'worker_process_%p'",
60 + "cuda-graph-trace": "node",
61 + }
62 + })
63 +
64 + return ray_remote_kwargs
65 +
66 def _init_workers_ray(self, placement_group: "PlacementGroup",
67 **ray_remote_kwargs):
68 if self.parallel_config.tensor_parallel_size == 1:
@@ -63,6 +78,10 @@ class RayGPUExecutor(ExecutorBase):
78 # The remaining workers are the actual ray actors.
79 self.workers: List[RayWorkerVllm] = []
80
81 + if self.parallel_config.ray_workers_use_nsight:
82 + ray_remote_kwargs = self._configure_ray_workers_use_nsight(
83 + ray_remote_kwargs)
84 +
85 # Create the workers.
86 driver_ip = get_ip()
87 for bundle_id, bundle in enumerate(placement_group.bundle_specs):