A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix ray workers profiling with nsight (#4095)
Ricky Xu committed
Apr 15, 2024 at 14:24 UTC
4695397dcfef693a0a10f1eb8bf77ea905c54829
1 file changed
+19
vllm/executor/ray_gpu_executor.py
+19
@@ -48,6 +48,21 @@ class RayGPUExecutor(ExecutorBase):
48
if USE_RAY_COMPILED_DAG:
49
self.forward_dag = self._compiled_ray_dag()
50
51
+ def _configure_ray_workers_use_nsight(self,
52
+ ray_remote_kwargs) -> Dict[str, Any]:
53
+ # If nsight profiling is enabled, we need to set the profiling
54
+ # configuration for the ray workers as runtime env.
55
+ runtime_env = ray_remote_kwargs.setdefault("runtime_env", {})
56
+ runtime_env.update({
57
+ "nsight": {
58
+ "t": "cuda,cudnn,cublas",
59
+ "o": "'worker_process_%p'",
60
+ "cuda-graph-trace": "node",
61
+ }
62
+ })
63
+
64
+ return ray_remote_kwargs
65
+
66
def _init_workers_ray(self, placement_group: "PlacementGroup",
67
**ray_remote_kwargs):
68
if self.parallel_config.tensor_parallel_size == 1:
@@ -63,6 +78,10 @@ class RayGPUExecutor(ExecutorBase):
78
# The remaining workers are the actual ray actors.
79
self.workers: List[RayWorkerVllm] = []
80
81
+ if self.parallel_config.ray_workers_use_nsight:
82
+ ray_remote_kwargs = self._configure_ray_workers_use_nsight(
83
+ ray_remote_kwargs)
84
+
85
# Create the workers.
86
driver_ip = get_ip()
87
for bundle_id, bundle in enumerate(placement_group.bundle_specs):