[Profiler] Fix start_profile permanently no-op after max_iterations auto-stop (#51839)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Elvir Crnčević committed
Aug 25, 2026 at 14:13 UTC
afc91fa0d2eb8bc050d80e9bcd74c8b1d72b4191
2 files changed
+34
-4
tests/v1/worker/test_gpu_profiler.py
+28
@@ -112,6 +112,34 @@ def test_max_iterations(default_profiler_config):
112
# Should have stopped now
113
assert profiler._running is False
114
assert profiler.stop_call_count == 1
115
+ # And fully reset, not just paused -- a later start_profile must not be a
116
+ # permanent no-op just because max_iterations already fired once.
117
+ assert profiler._active is False
118
+ assert profiler._active_iteration_count == 0
119
+ assert profiler._profiling_for_iters == 0
120
+
121
+
122
+def test_restart_after_max_iterations(default_profiler_config):
123
+ """A start_profile after an auto-stop must actually restart, not be
124
+ silently ignored (regression test: auto-stop used to leave _active
125
+ True forever, so start() always bailed out early)."""
126
+ default_profiler_config.max_iterations = 2
127
+ profiler = ConcreteWorkerProfiler(default_profiler_config)
128
+
129
+ profiler.start()
130
+ profiler.step()
131
+ profiler.step()
132
+ profiler.step() # exceeds max, auto-stops
133
+ assert profiler._running is False
134
+ assert profiler.start_call_count == 1
135
+
136
+ profiler.start()
137
+ assert profiler._active is True
138
+ assert profiler._running is True
139
+ assert profiler.start_call_count == 2
140
+
141
+ profiler.step()
142
+ assert profiler._running is True
143
144
145
def test_delayed_start_and_max_iters(default_profiler_config):
vllm/profiler/wrapper.py
+6
-4
@@ -120,11 +120,13 @@ class WorkerProfiler(ABC):
120
and self._running
121
and self._profiling_for_iters > self._max_iters
122
):
123
- # Automatically stop the profiler after max iters
124
- # will be marked as not running, but leave as active so that stop
125
- # can clean up properly
123
+ # Automatically stop the profiler after max iters. Go through the
124
+ # public stop() (not _call_stop() directly) so _active and the
125
+ # iteration counters reset too -- otherwise a later start_profile
126
+ # is silently ignored forever, since start() bails out early
127
+ # whenever _active is still True.
128
logger.info_once("Max profiling iterations reached. Stopping profiler...")
127
- self._call_stop()
129
+ self.stop()
130
return
131
132
def _profiler_step(self) -> bool: