@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6f7de33bed

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 13:15 UTC

[Metrics] Refactor LoRA state tracking (#26801)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Nov 10, 2025 at 08:34 UTC 6f7de33bed412869bec4631add885e5ff88c22cf

7 files changed +268 -106

tests/v1/engine/test_output_processor.py

+173 -2

@@ -15,12 +15,19 @@ from tests.v1.engine.utils import (

15 )

16 from vllm import PoolingParams

17 from vllm.logprobs import PromptLogprobs, SampleLogprobs

18 +from vllm.lora.request import LoRARequest

19 from vllm.outputs import CompletionOutput, RequestOutput

20 from vllm.sampling_params import RequestOutputKind, SamplingParams

21 from vllm.transformers_utils.tokenizer import AnyTokenizer

21 -from vllm.v1.engine import EngineCoreRequest

22 +from vllm.v1.engine import (

23 + EngineCoreEvent,

24 + EngineCoreEventType,

25 + EngineCoreOutputs,

26 + EngineCoreRequest,

27 + FinishReason,

28 +)

29 from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector

23 -from vllm.v1.metrics.stats import IterationStats

30 +from vllm.v1.metrics.stats import IterationStats, SchedulerStats

33 def _ref_convert_id_to_token(

@@ -895,6 +902,170 @@ def test_iteration_stats(dummy_test_vectors):

902 assert iteration_stats.num_generation_tokens == num_active

903

904

905 +@pytest.mark.parametrize("log_stats", [True, False])

906 +def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):

907 + """Test LoRA request lifecycle tracking through waiting -> running -> finished."""

908 + output_processor = OutputProcessor(

909 + dummy_test_vectors.tokenizer, log_stats=log_stats

910 + )

911 + engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)

912 + engine_core_timestamp = time.monotonic()

913 +

914 + # Create LoRA requests

915 + lora1 = LoRARequest(lora_name="lora-1", lora_int_id=1, lora_path="/path/to/lora1")

916 + lora2 = LoRARequest(lora_name="lora-2", lora_int_id=2, lora_path="/path/to/lora2")

917 +

918 + # Create requests with different LoRA adapters:

919 + # - request-0: lora-1

920 + # - request-1: lora-2

921 + # - request-2: None (no LoRA)

922 + lora_assignments = [lora1, lora2, None]

923 + requests = [

924 + EngineCoreRequest(

925 + request_id=f"request-{idx}",

926 + prompt_token_ids=prompt_tokens,

927 + mm_features=None,

928 + eos_token_id=None,

929 + arrival_time=0,

930 + lora_request=lora_assignments[idx],

931 + cache_salt=None,

932 + data_parallel_rank=None,

933 + sampling_params=SamplingParams(),

934 + pooling_params=None,

935 + )

936 + for idx, prompt_tokens in enumerate(dummy_test_vectors.prompt_tokens)

937 + ]

938 +

939 + # Add all requests to the OutputProcessor

940 + for request in requests:

941 + output_processor.add_request(request, None)

942 +

943 + # First iteration: process outputs with QUEUED events

944 + outputs = EngineCoreOutputs(

945 + outputs=engine_core.get_outputs(), scheduler_stats=SchedulerStats()

946 + )

947 + for output in outputs.outputs:

948 + output.events = [

949 + EngineCoreEvent.new_event(EngineCoreEventType.QUEUED, engine_core_timestamp)

950 + ]

951 +

952 + iteration_stats = IterationStats() if log_stats else None

953 + output_processor.process_outputs(

954 + outputs.outputs, engine_core_timestamp, iteration_stats

955 + )

956 + output_processor.update_scheduler_stats(outputs.scheduler_stats)

957 +

958 + if log_stats:

959 + # Verify waiting counts

960 + assert outputs.scheduler_stats.waiting_lora_adapters.get("lora-1") == 1

961 + assert outputs.scheduler_stats.waiting_lora_adapters.get("lora-2") == 1

962 + assert outputs.scheduler_stats.running_lora_adapters.get("lora-1") == 0

963 + assert outputs.scheduler_stats.running_lora_adapters.get("lora-2") == 0

964 + # Verify internal state

965 + assert len(output_processor.lora_states.requests) == 2

966 + assert "lora-1" in output_processor.lora_states.requests

967 + assert "lora-2" in output_processor.lora_states.requests

968 + else:

969 + # When log_stats=False, no tracking should occur

970 + assert iteration_stats is None

971 + assert len(output_processor.lora_states.requests) == 0

972 +

973 + # Second iteration: process outputs with SCHEDULED events

974 + outputs = EngineCoreOutputs(

975 + outputs=engine_core.get_outputs(), scheduler_stats=SchedulerStats()

976 + )

977 + for output in outputs.outputs:

978 + output.events = [

979 + EngineCoreEvent.new_event(

980 + EngineCoreEventType.SCHEDULED, engine_core_timestamp

981 + )

982 + ]

983 +

984 + iteration_stats = IterationStats() if log_stats else None

985 + output_processor.process_outputs(

986 + outputs.outputs, engine_core_timestamp, iteration_stats

987 + )

988 + output_processor.update_scheduler_stats(outputs.scheduler_stats)

989 +

990 + if log_stats:

991 + # Verify running counts

992 + assert outputs.scheduler_stats.waiting_lora_adapters.get("lora-1") == 0

993 + assert outputs.scheduler_stats.waiting_lora_adapters.get("lora-2") == 0

994 + assert outputs.scheduler_stats.running_lora_adapters.get("lora-1") == 1

995 + assert outputs.scheduler_stats.running_lora_adapters.get("lora-2") == 1

996 + else:

997 + assert iteration_stats is None

998 + assert len(output_processor.lora_states.requests) == 0

999 +

1000 + # Third iteration: finish request-0 (lora-1)

1001 + outputs = EngineCoreOutputs(

1002 + outputs=engine_core.get_outputs(), scheduler_stats=SchedulerStats()

1003 + )

1004 + # Find and mark request-0 as finished (it uses lora-1)

1005 + for output in outputs.outputs:

1006 + if output.request_id == "request-0":

1007 + output.finish_reason = FinishReason.LENGTH

1008 + break

1009 +

1010 + iteration_stats = IterationStats() if log_stats else None

1011 + output_processor.process_outputs(

1012 + outputs.outputs, engine_core_timestamp, iteration_stats

1013 + )

1014 + output_processor.update_scheduler_stats(outputs.scheduler_stats)

1015 +

1016 + if log_stats:

1017 + # lora-1 should be removed since no requests remain

1018 + assert "lora-1" not in output_processor.lora_states.requests

1019 + # lora-2 should still be running

1020 + assert outputs.scheduler_stats.running_lora_adapters.get("lora-2") == 1

1021 + assert len(output_processor.lora_states.requests) == 1

1022 + else:

1023 + assert len(output_processor.lora_states.requests) == 0

1024 +

1025 + # Fourth iteration: finish request-1 (lora-2)

1026 + outputs = EngineCoreOutputs(

1027 + outputs=engine_core.get_outputs(), scheduler_stats=SchedulerStats()

1028 + )

1029 + # Find and mark request-1 as finished (it uses lora-2)

1030 + for output in outputs.outputs:

1031 + if output.request_id == "request-1":

1032 + output.finish_reason = FinishReason.LENGTH

1033 + break

1034 +

1035 + iteration_stats = IterationStats() if log_stats else None

1036 + output_processor.process_outputs(

1037 + outputs.outputs, engine_core_timestamp, iteration_stats

1038 + )

1039 + output_processor.update_scheduler_stats(outputs.scheduler_stats)

1040 +

1041 + if log_stats:

1042 + # lora-2 should be removed since no requests remain

1043 + assert "lora-2" not in output_processor.lora_states.requests

1044 + assert len(outputs.scheduler_stats.running_lora_adapters) == 0

1045 + assert len(output_processor.lora_states.requests) == 0

1046 + else:

1047 + assert len(output_processor.lora_states.requests) == 0

1048 +

1049 + # Finish the last request (no LoRA)

1050 + outputs = EngineCoreOutputs(

1051 + outputs=engine_core.get_outputs(), scheduler_stats=SchedulerStats()

1052 + )

1053 + # Find and mark request-2 as finished (it has no LoRA)

1054 + for output in outputs.outputs:

1055 + if output.request_id == "request-2":

1056 + output.finish_reason = FinishReason.LENGTH

1057 + break

1058 +

1059 + iteration_stats = IterationStats() if log_stats else None

1060 + output_processor.process_outputs(

1061 + outputs.outputs, engine_core_timestamp, iteration_stats

1062 + )

1063 + output_processor.update_scheduler_stats(outputs.scheduler_stats)

1064 +

1065 + # Verify all requests are finished

1066 + assert output_processor.get_num_unfinished_requests() == 0

1067 +

1068 +

1069 @pytest.mark.asyncio

1070 async def test_request_output_collector():

1071 NUM_REQS = 3

tests/v1/metrics/test_stats.py

+1 -17

@@ -5,20 +5,4 @@ from vllm.v1.metrics.stats import IterationStats

6 def test_iteration_stats_repr():

7 iteration_stats = IterationStats()

8 - iteration_stats.iteration_timestamp = 0

9 - expected_repr = (

10 - "IterationStats("

11 - "iteration_timestamp=0, "

12 - "num_generation_tokens=0, "

13 - "num_prompt_tokens=0, "

14 - "num_preempted_reqs=0, "

15 - "finished_requests=[], "

16 - "max_num_generation_tokens_iter=[], "

17 - "n_params_iter=[], "

18 - "time_to_first_tokens_iter=[], "

19 - "inter_token_latencies_iter=[], "

20 - "waiting_lora_adapters={}, "

21 - "running_lora_adapters={}, "

22 - "num_corrupted_reqs=0)"

23 - )

24 - assert repr(iteration_stats) == expected_repr

8 + assert repr(iteration_stats).startswith("IterationStats(")

vllm/v1/engine/async_llm.py

@@ -508,6 +508,8 @@ class AsyncLLM(EngineClient):

508 processed_outputs.reqs_to_abort

509 )

510

511 + output_processor.update_scheduler_stats(outputs.scheduler_stats)

512 +

513 # 4) Logging.

514 # TODO(rob): make into a coroutine and launch it in

515 # background thread once Prometheus overhead is non-trivial.

vllm/v1/engine/llm_engine.py

@@ -289,6 +289,7 @@ class LLMEngine:

289 engine_core_timestamp=outputs.timestamp,

290 iteration_stats=iteration_stats,

291 )

292 + self.output_processor.update_scheduler_stats(outputs.scheduler_stats)

293

294 # 3) Abort any reqs that finished due to stop strings.

295 self.engine_core.abort_requests(processed_outputs.reqs_to_abort)

vllm/v1/engine/output_processor.py

+14 -9

@@ -22,7 +22,12 @@ from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason

22 from vllm.v1.engine.detokenizer import IncrementalDetokenizer

23 from vllm.v1.engine.logprobs import LogprobsProcessor

24 from vllm.v1.engine.parallel_sampling import ParentRequest

25 -from vllm.v1.metrics.stats import IterationStats, LoRARequestStates, RequestStateStats

25 +from vllm.v1.metrics.stats import (

26 + IterationStats,

27 + LoRARequestStates,

28 + RequestStateStats,

29 + SchedulerStats,

30 +)

33 class RequestOutputCollector:

@@ -310,7 +315,7 @@ class OutputProcessor:

315 self.tokenizer = tokenizer

316 self.request_states: dict[str, RequestState] = {}

317 self.parent_requests: dict[str, ParentRequest] = {}

313 - self.lora_states = LoRARequestStates()

318 + self.lora_states = LoRARequestStates(log_stats)

319 self.tracer: Tracer | None = None

320

321 def get_num_unfinished_requests(self):

@@ -334,7 +339,7 @@ class OutputProcessor:

339 for request_id in request_ids:

340 req_state = self.request_states.pop(request_id, None)

341 if req_state is not None:

337 - self.lora_states.abort_request(req_state)

342 + self.lora_states.request_finished(request_id, req_state.lora_name)

343 request_ids_to_abort.append(request_id)

344 # Produce final abort output.

345 if req_state.queue is not None and (

@@ -382,7 +387,6 @@ class OutputProcessor:

387 log_stats=self.log_stats,

388 )

389 self.request_states[request_id] = req_state

385 - self.lora_states.add_request(req_state)

390 if parent_req:

391 self.parent_requests[parent_req.request_id] = parent_req

392

@@ -484,13 +488,15 @@ class OutputProcessor:

488 )

489 if self.tracer:

490 self.do_tracing(engine_core_output, req_state, iteration_stats)

487 - self.lora_states.update_iteration_stats(iteration_stats)

491

492 return OutputProcessorOutput(

493 request_outputs=request_outputs,

494 reqs_to_abort=reqs_to_abort,

495 )

496

497 + def update_scheduler_stats(self, scheduler_stats: SchedulerStats | None):

498 + self.lora_states.update_scheduler_stats(scheduler_stats)

499 +

500 def do_tracing(

501 self,

502 engine_core_output: EngineCoreOutput,

@@ -564,8 +570,6 @@ class OutputProcessor:

570 if iteration_stats is None:

571 return

572

567 - lora_stats = self.lora_states.get_stats(req_state)

568 -

573 assert engine_core_timestamp is not None

574 assert req_state.stats is not None

575 iteration_stats.update_from_output(

@@ -574,7 +578,8 @@ class OutputProcessor:

578 req_state.is_prefilling,

579 req_state.prompt_len,

580 req_state.stats,

577 - lora_stats,

581 + self.lora_states,

582 + req_state.lora_name,

583 )

584

585 def _update_stats_from_finished(

@@ -596,7 +601,7 @@ class OutputProcessor:

601 max_tokens_param=req_state.max_tokens_param,

602 req_stats=req_state.stats,

603 )

599 - self.lora_states.finish_request(req_state)

604 + self.lora_states.request_finished(req_state.request_id, req_state.lora_name)

605

606 ParentRequest.observe_finished_request(

607 req_state.parent_req, iteration_stats, req_state.stats.num_generation_tokens

vllm/v1/metrics/loggers.py

+14 -14

@@ -989,6 +989,20 @@ class PrometheusStatLogger(AggregateStatLoggerBase):

989 scheduler_stats.kv_connector_stats, engine_idx

990 )

991

992 + if self.gauge_lora_info is not None:

993 + running_lora_adapters = ",".join(

994 + scheduler_stats.running_lora_adapters.keys()

995 + )

996 + waiting_lora_adapters = ",".join(

997 + scheduler_stats.waiting_lora_adapters.keys()

998 + )

999 + lora_info_labels = {

1000 + self.labelname_running_lora_adapters: running_lora_adapters,

1001 + self.labelname_waiting_lora_adapters: waiting_lora_adapters,

1002 + self.labelname_max_lora: self.max_lora,

1003 + }

1004 + self.gauge_lora_info.labels(**lora_info_labels).set_to_current_time()

1005 +

1006 if mm_cache_stats is not None:

1007 self.counter_mm_cache_queries[engine_idx].inc(mm_cache_stats.queries)

1008 self.counter_mm_cache_hits[engine_idx].inc(mm_cache_stats.hits)

@@ -1055,20 +1069,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase):

1069 finished_request.max_tokens_param

1070 )

1071

1058 - if self.gauge_lora_info is not None:

1059 - running_lora_adapters = ",".join(

1060 - iteration_stats.running_lora_adapters.keys()

1061 - )

1062 - waiting_lora_adapters = ",".join(

1063 - iteration_stats.waiting_lora_adapters.keys()

1064 - )

1065 - lora_info_labels = {

1066 - self.labelname_running_lora_adapters: running_lora_adapters,

1067 - self.labelname_waiting_lora_adapters: waiting_lora_adapters,

1068 - self.labelname_max_lora: self.max_lora,

1069 - }

1070 - self.gauge_lora_info.labels(**lora_info_labels).set_to_current_time()

1071 -

1072 def record_sleep_state(self, sleep: int = 0, level: int = 0):

1073 awake = 1

1074 discard_all = 0

vllm/v1/metrics/stats.py

+63 -64

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import time

5 -from collections import deque

5 +from collections import defaultdict, deque

6 from dataclasses import dataclass, field

7 from typing import TYPE_CHECKING, Any

@@ -11,7 +11,6 @@ from vllm.v1.spec_decode.metrics import SpecDecodingStats

12 if TYPE_CHECKING:

13 from vllm.v1.engine import EngineCoreEvent, EngineCoreOutput, FinishReason

14 - from vllm.v1.engine.output_processor import RequestState

16 @dataclass

@@ -170,11 +169,8 @@ class SchedulerStats:

169 spec_decoding_stats: SpecDecodingStats | None = None

170 kv_connector_stats: dict[str, Any] | None = None

171

173 -

174 -@dataclass

175 -class LoRAStats:

176 - waiting_requests: set[str] = field(default_factory=set)

177 - running_requests: set[str] = field(default_factory=set)

172 + waiting_lora_adapters: dict[str, int] = field(default_factory=dict)

173 + running_lora_adapters: dict[str, int] = field(default_factory=dict)

174

175

176 @dataclass

@@ -229,8 +225,6 @@ class IterationStats:

225 self.n_params_iter: list[int] = []

226 self.time_to_first_tokens_iter: list[float] = []

227 self.inter_token_latencies_iter: list[float] = []

232 - self.waiting_lora_adapters: dict[str, int] = {}

233 - self.running_lora_adapters: dict[str, int] = {}

228 self.num_corrupted_reqs: int = 0

229

230 def __repr__(self) -> str:

@@ -248,7 +242,8 @@ class IterationStats:

242 is_prefilling: bool,

243 prompt_len: int,

244 req_stats: RequestStateStats,

251 - lora_stats: LoRAStats | None,

245 + lora_states: "LoRARequestStates",

246 + lora_name: str | None,

247 ):

248 num_new_generation_tokens = len(output.new_token_ids)

249

@@ -274,7 +269,12 @@ class IterationStats:

269 # Process request-level engine core events

270 if output.events is not None:

271 self.update_from_events(

277 - output.request_id, output.events, is_prefilling, req_stats, lora_stats

272 + output.request_id,

273 + output.events,

274 + is_prefilling,

275 + req_stats,

276 + lora_states,

277 + lora_name,

278 )

279

280 # Process the batch-level "new tokens" engine core event

@@ -292,7 +292,8 @@ class IterationStats:

292 events: list["EngineCoreEvent"],

293 is_prefilling: bool,

294 req_stats: RequestStateStats,

295 - lora_stats: LoRAStats | None,

295 + lora_states: "LoRARequestStates",

296 + lora_name: str | None,

297 ):

298 # Avoid circular dependency

299 from vllm.v1.engine import EngineCoreEventType

@@ -300,15 +301,14 @@ class IterationStats:

301 for event in events:

302 if event.type == EngineCoreEventType.QUEUED:

303 req_stats.queued_ts = event.timestamp

303 - if lora_stats is not None:

304 - lora_stats.waiting_requests.add(req_id)

304 + lora_states.request_waiting(req_id, lora_name)

305 elif event.type == EngineCoreEventType.SCHEDULED:

306 if req_stats.scheduled_ts == 0.0: # ignore preemptions

307 req_stats.scheduled_ts = event.timestamp

308 - LoRARequestStates.scheduled_request(lora_stats, req_id)

308 + lora_states.request_running(req_id, lora_name)

309 elif event.type == EngineCoreEventType.PREEMPTED:

310 self.num_preempted_reqs += 1

311 - LoRARequestStates.preempted_request(lora_stats, req_id)

311 + lora_states.request_waiting(req_id, lora_name)

312

313 def update_from_finished_request(

314 self,

@@ -361,61 +361,60 @@ class IterationStats:

361 self.num_corrupted_reqs += 1

362

363

364 -class LoRARequestStates:

365 - """Per-LoRA request state stats."""

364 +class LoRAStats:

365 + """Tracks waiting and running request IDs for a single LoRA."""

366

367 def __init__(self):

368 - self.lora_name_to_stats: dict[str, LoRAStats] = {}

368 + self.waiting: set[str] = set()

369 + self.running: set[str] = set()

370

370 - def get_stats(self, req_state: "RequestState") -> LoRAStats | None:

371 - if req_state.lora_name is None:

372 - return None

373 - if req_state.lora_name not in self.lora_name_to_stats:

374 - self.lora_name_to_stats[req_state.lora_name] = LoRAStats()

375 - return self.lora_name_to_stats[req_state.lora_name]

371 + def update(self, req_id: str, waiting: bool, running: bool):

372 + assert not (waiting and running)

373 + if waiting:

374 + self.waiting.add(req_id)

375 + else:

376 + self.waiting.discard(req_id)

377

377 - def add_request(self, req_state: "RequestState"):

378 - if (lora_stats := self.get_stats(req_state)) is not None:

379 - lora_stats.waiting_requests.add(req_state.request_id)

378 + if running:

379 + self.running.add(req_id)

380 + else:

381 + self.running.discard(req_id)

382

381 - def finish_request(self, req_state: "RequestState"):

382 - if req_state.lora_name is None:

383 - return

384 - lora_stats = self.lora_name_to_stats[req_state.lora_name]

385 - lora_stats.running_requests.remove(req_state.request_id)

383 + @property

384 + def empty(self) -> bool:

385 + return not (self.waiting or self.running)

386

387 - def abort_request(self, req_state: "RequestState"):

388 - if req_state.lora_name is None:

389 - return

390 - lora_stats = self.lora_name_to_stats[req_state.lora_name]

391 - lora_stats.waiting_requests.discard(req_state.request_id)

392 - lora_stats.running_requests.discard(req_state.request_id)

393 -

394 - # Break the pattern for this lifecycle methods so we can

395 - # call this from IterationStats.update_from_events()

396 - @staticmethod

397 - def scheduled_request(lora_stats: LoRAStats | None, request_id: str):

398 - if lora_stats is None:

399 - return

400 - lora_stats.waiting_requests.remove(request_id)

401 - lora_stats.running_requests.add(request_id)

387

403 - @staticmethod

404 - def preempted_request(lora_stats: LoRAStats | None, request_id: str):

405 - if lora_stats is None:

388 +class LoRARequestStates:

389 + """A per-LoRA count of running and waiting requests."""

390 +

391 + def __init__(self, log_stats: bool = False):

392 + self.log_stats = log_stats

393 + self.requests: defaultdict[str, LoRAStats] = defaultdict(LoRAStats)

394 +

395 + def _request_update(

396 + self, req_id: str, lora_name: str | None, waiting: bool, running: bool

397 + ):

398 + if not self.log_stats or lora_name is None:

399 return

407 - lora_stats.running_requests.remove(request_id)

408 - lora_stats.waiting_requests.add(request_id)

400

410 - def update_iteration_stats(self, iteration_stats: IterationStats | None):

411 - if iteration_stats is None:

401 + lora_stats = self.requests[lora_name]

402 + lora_stats.update(req_id, waiting, running)

403 + if lora_stats.empty:

404 + del self.requests[lora_name]

405 +

406 + def request_waiting(self, req_id: str, lora_name: str | None):

407 + self._request_update(req_id, lora_name, waiting=True, running=False)

408 +

409 + def request_running(self, req_id: str, lora_name: str | None):

410 + self._request_update(req_id, lora_name, waiting=False, running=True)

411 +

412 + def request_finished(self, req_id: str, lora_name: str | None):

413 + self._request_update(req_id, lora_name, waiting=False, running=False)

414 +

415 + def update_scheduler_stats(self, scheduler_stats: SchedulerStats | None):

416 + if not self.log_stats or scheduler_stats is None:

417 return

413 - for lora_name, stats in self.lora_name_to_stats.items():

414 - if stats.waiting_requests:

415 - iteration_stats.waiting_lora_adapters[lora_name] = len(

416 - stats.waiting_requests

417 - )

418 - if stats.running_requests:

419 - iteration_stats.running_lora_adapters[lora_name] = len(

420 - stats.running_requests

421 - )

418 + for lora_name, stats in self.requests.items():

419 + scheduler_stats.waiting_lora_adapters[lora_name] = len(stats.waiting)

420 + scheduler_stats.running_lora_adapters[lora_name] = len(stats.running)