A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] [SpecDecode] AsyncMetricsCollector: update time since last collection (#6578)
Thomas Parnell committed
Jul 19, 2024 at 23:01 UTC
f0bbfaf917edef99fe7817d607d56803695610ca
2 files changed
+47
tests/spec_decode/test_metrics.py
+43
@@ -105,6 +105,49 @@ def test_noop_until_time():
105
assert metrics is not None
106
107
108
+def test_timer_is_reset():
109
+ """Verify that the internal timer inside AsyncMetricsCollector
110
+ is reset after collection.
111
+ """
112
+ spec_decode_sampler = MagicMock()
113
+ spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
114
+ dtype=torch.long,
115
+ device='cuda')
116
+ spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
117
+ dtype=torch.long,
118
+ device='cuda')
119
+ spec_decode_sampler.num_draft_tokens = 0
120
+
121
+ collect_interval_s = 5.0
122
+ timer = MagicMock()
123
+ timer.side_effect = [
124
+ 0.0,
125
+ collect_interval_s + 0.1,
126
+ collect_interval_s + 0.1,
127
+ collect_interval_s + 0.2,
128
+ collect_interval_s + 0.2,
129
+ 2 * collect_interval_s + 0.1,
130
+ 2 * collect_interval_s + 0.1,
131
+ ]
132
+
133
+ collector = AsyncMetricsCollector(spec_decode_sampler=spec_decode_sampler,
134
+ timer=timer,
135
+ collect_interval_s=collect_interval_s)
136
+ collector.init_gpu_tensors(rank=0)
137
+
138
+ _ = collector.maybe_collect_rejsample_metrics(k=5)
139
+ metrics = collector.maybe_collect_rejsample_metrics(k=5)
140
+ assert metrics is not None
141
+
142
+ _ = collector.maybe_collect_rejsample_metrics(k=5)
143
+ metrics = collector.maybe_collect_rejsample_metrics(k=5)
144
+ assert metrics is None
145
+
146
+ _ = collector.maybe_collect_rejsample_metrics(k=5)
147
+ metrics = collector.maybe_collect_rejsample_metrics(k=5)
148
+ assert metrics is not None
149
+
150
+
151
@pytest.mark.parametrize("has_data", [True, False])
152
def test_initial_metrics_has_correct_values(has_data: bool):
153
"""Test correctness of metrics data.
vllm/spec_decode/metrics.py
+4
@@ -145,6 +145,10 @@ class AsyncMetricsCollector:
145
"""
146
147
ready_event.synchronize()
148
+
149
+ # update time of last collection
150
+ self._last_metrics_collect_time = self._timer()
151
+
152
accepted_tokens = self._aggregate_num_accepted_tokens.item()
153
emitted_tokens = self._aggregate_num_emitted_tokens.item()
154
draft_tokens = self._aggregate_num_draft_tokens