@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f0bbfaf917
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] [SpecDecode] AsyncMetricsCollector: update time since last collection (#6578)

Thomas Parnell committed Jul 19, 2024 at 23:01 UTC f0bbfaf917edef99fe7817d607d56803695610ca
2 files changed +47
tests/spec_decode/test_metrics.py
+43
@@ -105,6 +105,49 @@ def test_noop_until_time():
105 assert metrics is not None
106
107
108 +def test_timer_is_reset():
109 + """Verify that the internal timer inside AsyncMetricsCollector
110 + is reset after collection.
111 + """
112 + spec_decode_sampler = MagicMock()
113 + spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
114 + dtype=torch.long,
115 + device='cuda')
116 + spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
117 + dtype=torch.long,
118 + device='cuda')
119 + spec_decode_sampler.num_draft_tokens = 0
120 +
121 + collect_interval_s = 5.0
122 + timer = MagicMock()
123 + timer.side_effect = [
124 + 0.0,
125 + collect_interval_s + 0.1,
126 + collect_interval_s + 0.1,
127 + collect_interval_s + 0.2,
128 + collect_interval_s + 0.2,
129 + 2 * collect_interval_s + 0.1,
130 + 2 * collect_interval_s + 0.1,
131 + ]
132 +
133 + collector = AsyncMetricsCollector(spec_decode_sampler=spec_decode_sampler,
134 + timer=timer,
135 + collect_interval_s=collect_interval_s)
136 + collector.init_gpu_tensors(rank=0)
137 +
138 + _ = collector.maybe_collect_rejsample_metrics(k=5)
139 + metrics = collector.maybe_collect_rejsample_metrics(k=5)
140 + assert metrics is not None
141 +
142 + _ = collector.maybe_collect_rejsample_metrics(k=5)
143 + metrics = collector.maybe_collect_rejsample_metrics(k=5)
144 + assert metrics is None
145 +
146 + _ = collector.maybe_collect_rejsample_metrics(k=5)
147 + metrics = collector.maybe_collect_rejsample_metrics(k=5)
148 + assert metrics is not None
149 +
150 +
151 @pytest.mark.parametrize("has_data", [True, False])
152 def test_initial_metrics_has_correct_values(has_data: bool):
153 """Test correctness of metrics data.
vllm/spec_decode/metrics.py
+4
@@ -145,6 +145,10 @@ class AsyncMetricsCollector:
145 """
146
147 ready_event.synchronize()
148 +
149 + # update time of last collection
150 + self._last_metrics_collect_time = self._timer()
151 +
152 accepted_tokens = self._aggregate_num_accepted_tokens.item()
153 emitted_tokens = self._aggregate_num_emitted_tokens.item()
154 draft_tokens = self._aggregate_num_draft_tokens