A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Multiprocessing Tensor Parallel Support for v1 (#9856)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Dec 10, 2024 at 01:28 UTC
28b3a1c7e596c08efac0fcfa59a629d16197be30
21 files changed
+733
-146
tests/basic_correctness/test_basic_correctness.py
+16
@@ -26,6 +26,14 @@ MODELS = [
26
TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
27
28
29
+@pytest.fixture(autouse=True)
30
+def v1(run_with_both_engines):
31
+ # Simple autouse wrapper to run both engines for each test
32
+ # This can be promoted up to conftest.py to run for every
33
+ # test in a package
34
+ pass
35
+
36
+
37
def test_vllm_gc_ed():
38
"""Verify vllm instance is GC'ed when it is deleted"""
39
llm = LLM("facebook/opt-125m")
@@ -36,6 +44,7 @@ def test_vllm_gc_ed():
44
assert weak_llm() is None
45
46
47
+@pytest.mark.skip_v1
48
@pytest.mark.parametrize("model", MODELS)
49
@pytest.mark.parametrize("backend", ["FLASH_ATTN", "XFORMERS", "FLASHINFER"])
50
@pytest.mark.parametrize("dtype", ["half"])
@@ -118,6 +127,11 @@ def test_models_distributed(
127
if attention_backend:
128
os.environ["VLLM_ATTENTION_BACKEND"] = attention_backend
129
130
+ # Import VLLM_USE_V1 dynamically to handle patching
131
+ from vllm.envs import VLLM_USE_V1
132
+ if VLLM_USE_V1 and distributed_executor_backend != "mp":
133
+ pytest.skip(f"Skip {distributed_executor_backend} for V1")
134
+
135
dtype = "half"
136
max_tokens = 5
137
@@ -143,6 +157,7 @@ def test_models_distributed(
157
)
158
159
160
+@pytest.mark.skip_v1
161
def test_model_with_failure(vllm_runner) -> None:
162
try:
163
with patch("vllm.model_executor.models.opt.OPTForCausalLM.forward",
@@ -169,6 +184,7 @@ def test_model_with_failure(vllm_runner) -> None:
184
os.remove(filename)
185
186
187
+@pytest.mark.skip_v1
188
def test_failure_with_async_out_proc(vllm_runner) -> None:
189
190
filename = None
tests/conftest.py
+5
-6
@@ -5,7 +5,6 @@ from collections import UserList
5
from enum import Enum
6
from typing import (Any, Callable, Dict, List, Optional, Tuple, Type,
7
TypedDict, TypeVar, Union)
8
-from unittest.mock import patch
8
9
import numpy as np
10
import pytest
@@ -110,7 +109,7 @@ VIDEO_ASSETS = _VideoAssets()
109
110
111
@pytest.fixture(params=[True, False])
113
-def run_with_both_engines(request):
112
+def run_with_both_engines(request, monkeypatch):
113
# Automatically runs tests twice, once with V1 and once without
114
use_v1 = request.param
115
# Tests decorated with `@skip_v1` are only run without v1
@@ -119,11 +118,11 @@ def run_with_both_engines(request):
118
if use_v1:
119
if skip_v1:
120
pytest.skip("Skipping test on vllm V1")
122
- with patch('vllm.envs.VLLM_USE_V1', True):
123
- yield
121
+ monkeypatch.setenv('VLLM_USE_V1', '1')
122
else:
125
- with patch('vllm.envs.VLLM_USE_V1', False):
126
- yield
123
+ monkeypatch.setenv('VLLM_USE_V1', '0')
124
+
125
+ yield
126
127
128
@pytest.fixture(autouse=True)
vllm/distributed/device_communicators/shm_broadcast.py
+52
-24
@@ -1,10 +1,11 @@
1
import os
2
import pickle
3
+import sys
4
import time
5
from contextlib import contextmanager
6
from dataclasses import dataclass, field
7
from multiprocessing import shared_memory
7
-from typing import List, Optional
8
+from typing import List, Optional, Tuple
9
from unittest.mock import patch
10
11
import torch
@@ -21,6 +22,20 @@ VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
22
23
logger = init_logger(__name__)
24
25
+# We prefer to use os.sched_yield as it results in tighter polling loops,
26
+# measured to be around 3e-7 seconds. However on earlier versions of Python
27
+# os.sched_yield() does not release the GIL, so we fall back to time.sleep(0)
28
+USE_SCHED_YIELD = ((sys.version_info[:3] >= (3, 11, 1))
29
+ or (sys.version_info[:2] == (3, 10)
30
+ and sys.version_info[2] >= 8))
31
+
32
+
33
+def sched_yield():
34
+ if USE_SCHED_YIELD:
35
+ os.sched_yield()
36
+ else:
37
+ time.sleep(0)
38
+
39
40
class ShmRingBuffer:
41
@@ -114,11 +129,14 @@ class ShmRingBuffer:
129
# and we should suppress the error
130
pass
131
132
+ def handle(self):
133
+ return (self.n_reader, self.max_chunk_bytes, self.max_chunks,
134
+ self.shared_memory.name)
135
+
136
def __reduce__(self):
137
return (
138
self.__class__,
120
- (self.n_reader, self.max_chunk_bytes, self.max_chunks,
121
- self.shared_memory.name),
139
+ self.handle(),
140
)
141
142
def __del__(self):
@@ -147,7 +165,7 @@ class Handle:
165
connect_ip: str
166
local_reader_ranks: List[int] = field(default_factory=list)
167
150
- buffer: Optional[ShmRingBuffer] = None
168
+ buffer_handle: Optional[Tuple[int, int, int, str]] = None
169
local_subscribe_port: Optional[int] = None
170
remote_subscribe_port: Optional[int] = None
171
@@ -228,7 +246,7 @@ class MessageQueue:
246
self.handle = Handle(
247
connect_ip=connect_ip,
248
local_reader_ranks=local_reader_ranks,
231
- buffer=self.buffer,
249
+ buffer_handle=self.buffer.handle(),
250
local_subscribe_port=local_subscribe_port,
251
remote_subscribe_port=remote_subscribe_port,
252
)
@@ -247,8 +265,8 @@ class MessageQueue:
265
context = Context()
266
267
if rank in handle.local_reader_ranks:
250
- assert handle.buffer is not None
251
- self.buffer = handle.buffer
268
+ assert handle.buffer_handle is not None
269
+ self.buffer = ShmRingBuffer(*handle.buffer_handle)
270
self.current_idx = 0
271
self.local_reader_rank = handle.local_reader_ranks.index(rank)
272
self._is_local_reader = True
@@ -314,7 +332,7 @@ class MessageQueue:
332
assert recv == b"READY"
333
334
@contextmanager
317
- def acquire_write(self):
335
+ def acquire_write(self, timeout: Optional[float] = None):
336
assert self._is_writer, "Only writers can acquire write"
337
start_time = time.monotonic()
338
n_warning = 1
@@ -329,16 +347,20 @@ class MessageQueue:
347
# we need to wait until it is read by all readers
348
349
# Release the processor to other threads
332
- os.sched_yield()
350
+ sched_yield()
351
334
- # if we wait for a long time, we should warn the user
352
+ # if we wait for a long time, log a message
353
if (time.monotonic() - start_time >
354
VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
337
- logger.warning(
338
- "No available block found in %s second. ",
339
- VLLM_RINGBUFFER_WARNING_INTERVAL)
355
+ logger.debug("No available block found in %s second. ",
356
+ VLLM_RINGBUFFER_WARNING_INTERVAL)
357
n_warning += 1
358
359
+ # if we time out, raise an exception
360
+ if (timeout is not None
361
+ and time.monotonic() - start_time > timeout):
362
+ raise TimeoutError
363
+
364
continue
365
# found a block that is either
366
# (1) not written
@@ -365,7 +387,7 @@ class MessageQueue:
387
break
388
389
@contextmanager
368
- def acquire_read(self):
390
+ def acquire_read(self, timeout: Optional[float] = None):
391
assert self._is_local_reader, "Only readers can acquire read"
392
start_time = time.monotonic()
393
n_warning = 1
@@ -383,16 +405,20 @@ class MessageQueue:
405
# we need to wait until it is written
406
407
# Release the processor to other threads
386
- os.sched_yield()
408
+ sched_yield()
409
388
- # if we wait for a long time, we should warn the user
410
+ # if we wait for a long time, log a message
411
if (time.monotonic() - start_time >
412
VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
391
- logger.warning(
392
- "No available block found in %s second. ",
393
- VLLM_RINGBUFFER_WARNING_INTERVAL)
413
+ logger.debug("No available block found in %s second. ",
414
+ VLLM_RINGBUFFER_WARNING_INTERVAL)
415
n_warning += 1
416
417
+ # if we time out, raise an exception
418
+ if (timeout is not None
419
+ and time.monotonic() - start_time > timeout):
420
+ raise TimeoutError
421
+
422
continue
423
# found a block that is not read by this reader
424
# let caller read from the buffer
@@ -406,24 +432,26 @@ class MessageQueue:
432
1) % self.buffer.max_chunks
433
break
434
409
- def enqueue(self, obj):
435
+ def enqueue(self, obj, timeout: Optional[float] = None):
436
+ """ Write to message queue with optional timeout (in seconds) """
437
assert self._is_writer, "Only writers can enqueue"
438
serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
439
if self.n_local_reader > 0:
440
if len(serialized_obj) >= self.buffer.max_chunk_bytes:
414
- with self.acquire_write() as buf:
441
+ with self.acquire_write(timeout) as buf:
442
buf[0] = 1 # overflow
443
self.local_socket.send(serialized_obj)
444
else:
418
- with self.acquire_write() as buf:
445
+ with self.acquire_write(timeout) as buf:
446
buf[0] = 0 # not overflow
447
buf[1:len(serialized_obj) + 1] = serialized_obj
448
if self.n_remote_reader > 0:
449
self.remote_socket.send(serialized_obj)
450
424
- def dequeue(self):
451
+ def dequeue(self, timeout: Optional[float] = None):
452
+ """ Read from message queue with optional timeout (in seconds) """
453
if self._is_local_reader:
426
- with self.acquire_read() as buf:
454
+ with self.acquire_read(timeout) as buf:
455
overflow = buf[0] == 1
456
if not overflow:
457
# no need to know the size of serialized object
vllm/executor/multiproc_gpu_executor.py
+6
-41
@@ -3,25 +3,19 @@ import os
3
from functools import partial
4
from typing import Any, List, Optional
5
6
-import torch
7
-
6
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
7
DistributedGPUExecutor, DistributedGPUExecutorAsync)
8
from vllm.executor.gpu_executor import create_worker
11
-from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
12
- ResultHandler, WorkerMonitor)
9
+from vllm.executor.multiproc_worker_utils import (
10
+ ProcessWorkerWrapper, ResultHandler, WorkerMonitor,
11
+ set_multiprocessing_worker_envs)
12
from vllm.logger import init_logger
13
from vllm.model_executor.layers.sampler import SamplerOutput
14
from vllm.sequence import ExecuteModelRequest
16
-from vllm.triton_utils.importing import HAS_TRITON
15
from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
18
- cuda_is_initialized, get_distributed_init_method,
19
- get_open_port, make_async,
16
+ get_distributed_init_method, get_open_port, make_async,
17
update_environment_variables)
18
22
-if HAS_TRITON:
23
- from vllm.triton_utils import maybe_set_triton_cache_manager
24
-
19
logger = init_logger(__name__)
20
21
@@ -37,30 +31,8 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
31
world_size = self.parallel_config.world_size
32
tensor_parallel_size = self.parallel_config.tensor_parallel_size
33
40
- # Disable torch async compiling which won't work with daemonic processes
41
- os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
42
-
43
- # Configure thread parallelism if OMP_NUM_THREADS isn't set
44
- #
45
- # Helps to avoid CPU contention. The default of spawning a thread per
46
- # core combined with multiprocessing for each GPU can have a negative
47
- # impact on performance. The contention is amplified when running in a
48
- # container where CPU limits can cause throttling.
49
- default_omp_num_threads = 1
50
- if "OMP_NUM_THREADS" not in os.environ and (
51
- current_parallelism :=
52
- torch.get_num_threads()) > default_omp_num_threads:
53
- logger.warning(
54
- "Reducing Torch parallelism from %d threads to %d to avoid "
55
- "unnecessary CPU contention. Set OMP_NUM_THREADS in the "
56
- "external environment to tune this value as needed.",
57
- current_parallelism, default_omp_num_threads)
58
- os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)
59
- torch.set_num_threads(default_omp_num_threads)
60
-
61
- # workaround for https://github.com/vllm-project/vllm/issues/6103
62
- if HAS_TRITON and world_size > 1:
63
- maybe_set_triton_cache_manager()
34
+ # Set multiprocessing envs that are common to V0 and V1
35
+ set_multiprocessing_worker_envs(self.parallel_config)
36
37
# Multiprocessing-based executor does not support multi-node setting.
38
# Since it only works for single node, we can use the loopback address
@@ -122,13 +94,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
94
"CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
95
})
96
125
- if (cuda_is_initialized()
126
- and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
127
- logger.warning("CUDA was previously initialized. We must use "
128
- "the `spawn` multiprocessing start method. Setting "
129
- "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'.")
130
- os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
131
-
97
cuda_device_count = cuda_device_count_stateless()
98
# Use confusing message for more common TP-only case.
99
assert tensor_parallel_size <= cuda_device_count, (
vllm/executor/multiproc_worker_utils.py
+42
@@ -11,8 +11,15 @@ from multiprocessing.process import BaseProcess
11
from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
12
TypeVar, Union)
13
14
+import torch
15
+
16
import vllm.envs as envs
17
from vllm.logger import init_logger
18
+from vllm.triton_utils.importing import HAS_TRITON
19
+from vllm.utils import cuda_is_initialized
20
+
21
+if HAS_TRITON:
22
+ from vllm.triton_utils import maybe_set_triton_cache_manager
23
24
logger = init_logger(__name__)
25
@@ -270,3 +277,38 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
277
def get_mp_context():
278
mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
279
return multiprocessing.get_context(mp_method)
280
+
281
+
282
+def set_multiprocessing_worker_envs(parallel_config):
283
+ """ Set up environment variables that should be used when there are workers
284
+ in a multiprocessing environment. This should be called by the parent
285
+ process before worker processes are created"""
286
+
287
+ if (cuda_is_initialized()
288
+ and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
289
+ logger.warning("CUDA was previously initialized. We must use "
290
+ "the `spawn` multiprocessing start method. Setting "
291
+ "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'.")
292
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
293
+
294
+ # Configure thread parallelism if OMP_NUM_THREADS isn't set
295
+ #
296
+ # Helps to avoid CPU contention. The default of spawning a thread per
297
+ # core combined with multiprocessing for each GPU can have a negative
298
+ # impact on performance. The contention is amplified when running in a
299
+ # container where CPU limits can cause throttling.
300
+ default_omp_num_threads = 1
301
+ if "OMP_NUM_THREADS" not in os.environ and (
302
+ current_parallelism :=
303
+ torch.get_num_threads()) > default_omp_num_threads:
304
+ logger.warning(
305
+ "Reducing Torch parallelism from %d threads to %d to avoid "
306
+ "unnecessary CPU contention. Set OMP_NUM_THREADS in the "
307
+ "external environment to tune this value as needed.",
308
+ current_parallelism, default_omp_num_threads)
309
+ os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)
310
+ torch.set_num_threads(default_omp_num_threads)
311
+
312
+ # workaround for https://github.com/vllm-project/vllm/issues/6103
313
+ if HAS_TRITON and parallel_config.world_size > 1:
314
+ maybe_set_triton_cache_manager()
vllm/model_executor/layers/logits_processor.py
+4
-1
@@ -5,6 +5,7 @@ from typing import Optional
5
import torch
6
import torch.nn as nn
7
8
+import vllm.envs as envs
9
from vllm.distributed import (tensor_model_parallel_all_gather,
10
tensor_model_parallel_gather)
11
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -42,7 +43,9 @@ class LogitsProcessor(nn.Module):
43
# Soft cap the logits. Used in Gemma 2.
44
self.soft_cap = soft_cap
45
# Whether to use gather or all-gather to gather the logits.
45
- self.use_gather = not current_platform.is_tpu()
46
+
47
+ self.use_gather = not current_platform.is_tpu(
48
+ ) and not envs.VLLM_USE_V1
49
50
def forward(
51
self,
vllm/platforms/cuda.py
+20
-8
@@ -12,6 +12,7 @@ from typing_extensions import ParamSpec
12
13
# import custom ops, trigger op registration
14
import vllm._C # noqa
15
+import vllm.envs as envs
16
from vllm.logger import init_logger
17
18
from .interface import DeviceCapability, Platform, PlatformEnum
@@ -110,17 +111,28 @@ class CudaPlatformBase(Platform):
111
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
112
parallel_config = vllm_config.parallel_config
113
scheduler_config = vllm_config.scheduler_config
114
+
115
if parallel_config.worker_cls == "auto":
116
if scheduler_config.is_multi_step:
115
- parallel_config.worker_cls = \
116
- "vllm.worker.multi_step_worker.MultiStepWorker"
117
+ if envs.VLLM_USE_V1:
118
+ raise NotImplementedError
119
+ else:
120
+ parallel_config.worker_cls = \
121
+ "vllm.worker.multi_step_worker.MultiStepWorker"
122
elif vllm_config.speculative_config:
118
- parallel_config.worker_cls = \
119
- "vllm.spec_decode.spec_decode_worker.create_spec_worker"
120
- parallel_config.sd_worker_cls = \
121
- "vllm.worker.worker.Worker"
123
+ if envs.VLLM_USE_V1:
124
+ raise NotImplementedError
125
+ else:
126
+ parallel_config.worker_cls = \
127
+ "vllm.spec_decode.spec_decode_worker.create_spec_worker"
128
+ parallel_config.sd_worker_cls = \
129
+ "vllm.worker.worker.Worker"
130
else:
123
- parallel_config.worker_cls = "vllm.worker.worker.Worker"
131
+ if envs.VLLM_USE_V1:
132
+ parallel_config.worker_cls = \
133
+ "vllm.v1.worker.gpu_worker.Worker"
134
+ else:
135
+ parallel_config.worker_cls = "vllm.worker.worker.Worker"
136
137
138
# NVML utils
@@ -249,4 +261,4 @@ try:
261
if not isinstance(pynvml, _MockModule):
262
CudaPlatform.log_warnings()
263
except ModuleNotFoundError:
252
- CudaPlatform.log_warnings()
\ No newline at end of file
264
+ CudaPlatform.log_warnings()
vllm/utils.py
+26
@@ -10,6 +10,7 @@ import importlib.util
10
import inspect
11
import ipaddress
12
import os
13
+import signal
14
import socket
15
import subprocess
16
import sys
@@ -1652,3 +1653,28 @@ def resolve_obj_by_qualname(qualname: str) -> Any:
1653
module_name, obj_name = qualname.rsplit(".", 1)
1654
module = importlib.import_module(module_name)
1655
return getattr(module, obj_name)
1656
+
1657
+
1658
+def kill_process_tree(pid: int):
1659
+ """
1660
+ Kills all descendant processes of the given pid by sending SIGKILL.
1661
+
1662
+ Args:
1663
+ pid (int): Process ID of the parent process
1664
+ """
1665
+ try:
1666
+ parent = psutil.Process(pid)
1667
+ except psutil.NoSuchProcess:
1668
+ return
1669
+
1670
+ # Get all children recursively
1671
+ children = parent.children(recursive=True)
1672
+
1673
+ # Send SIGKILL to all children first
1674
+ for child in children:
1675
+ with contextlib.suppress(ProcessLookupError):
1676
+ os.kill(child.pid, signal.SIGKILL)
1677
+
1678
+ # Finally kill the parent
1679
+ with contextlib.suppress(ProcessLookupError):
1680
+ os.kill(pid, signal.SIGKILL)
vllm/v1/core/scheduler.py
+3
-1
@@ -5,6 +5,8 @@ from typing import (TYPE_CHECKING, Deque, Dict, Iterable, List, Optional, Set,
5
6
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7
from vllm.logger import init_logger
8
+from vllm.multimodal import MultiModalKwargs
9
+from vllm.multimodal.base import PlaceholderRange
10
from vllm.sampling_params import SamplingParams
11
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
12
from vllm.v1.core.kv_cache_manager import KVCacheManager
@@ -383,7 +385,7 @@ class Scheduler:
385
model_runner_output: "ModelRunnerOutput",
386
) -> List[EngineCoreOutput]:
387
# NOTE(woosuk): This method doesn't consider speculative decoding.
386
- sampled_token_ids = model_runner_output.sampled_token_ids_cpu.tolist()
388
+ sampled_token_ids = model_runner_output.sampled_token_ids
389
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
390
new_running: List[Request] = []
391
engine_core_outputs: List[EngineCoreOutput] = []
vllm/v1/engine/async_llm.py
+14
-4
@@ -20,7 +20,7 @@ from vllm.v1.engine.async_stream import AsyncStream
20
from vllm.v1.engine.core_client import EngineCoreClient
21
from vllm.v1.engine.detokenizer import Detokenizer
22
from vllm.v1.engine.processor import Processor
23
-from vllm.v1.executor.gpu_executor import GPUExecutor
23
+from vllm.v1.executor.abstract import Executor
24
25
logger = init_logger(__name__)
26
@@ -30,7 +30,7 @@ class AsyncLLM(EngineClient):
30
def __init__(
31
self,
32
vllm_config: VllmConfig,
33
- executor_class: Type[GPUExecutor],
33
+ executor_class: Type[Executor],
34
log_stats: bool,
35
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
36
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
@@ -119,14 +119,24 @@ class AsyncLLM(EngineClient):
119
def shutdown(self):
120
"""Shutdown, cleaning up the background proc and IPC."""
121
122
- self.engine_core.shutdown()
122
+ if engine_core := getattr(self, "engine_core", None):
123
+ engine_core.shutdown()
124
125
if handler := getattr(self, "output_handler", None):
126
handler.cancel()
127
128
@classmethod
129
def _get_executor_cls(cls, vllm_config: VllmConfig):
129
- return GPUExecutor
130
+ distributed_executor_backend = (
131
+ vllm_config.parallel_config.distributed_executor_backend)
132
+ if distributed_executor_backend == "mp":
133
+ from vllm.v1.executor.multiproc_executor import MultiprocExecutor
134
+ executor_class = MultiprocExecutor
135
+ else:
136
+ assert (distributed_executor_backend is None)
137
+ from vllm.v1.executor.uniproc_executor import UniprocExecutor
138
+ executor_class = UniprocExecutor
139
+ return executor_class
140
141
async def add_request(
142
self,
vllm/v1/engine/core.py
+39
-35
@@ -1,12 +1,12 @@
1
import multiprocessing
2
import pickle
3
import queue
4
+import signal
5
import threading
6
import time
6
-from contextlib import contextmanager
7
from multiprocessing.process import BaseProcess
8
from multiprocessing.sharedctypes import Synchronized
9
-from typing import Any, Iterator, List, Tuple, Type, Union
9
+from typing import List, Tuple, Type, Union
10
11
import zmq
12
import zmq.asyncio
@@ -20,9 +20,10 @@ from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
20
EngineCoreProfile, EngineCoreRequest,
21
EngineCoreRequestType)
22
from vllm.v1.engine.mm_input_mapper import MMInputMapper
23
-from vllm.v1.executor.gpu_executor import GPUExecutor
23
+from vllm.v1.executor.abstract import Executor
24
from vllm.v1.request import Request, RequestStatus
25
from vllm.v1.serial_utils import PickleEncoder
26
+from vllm.v1.utils import make_zmq_socket
27
from vllm.version import __version__ as VLLM_VERSION
28
29
logger = init_logger(__name__)
@@ -38,7 +39,7 @@ class EngineCore:
39
def __init__(
40
self,
41
vllm_config: VllmConfig,
41
- executor_class: Type[GPUExecutor],
42
+ executor_class: Type[Executor],
43
usage_context: UsageContext,
44
):
45
assert vllm_config.model_config.task != "embedding"
@@ -80,7 +81,7 @@ class EngineCore:
81
num_gpu_blocks = num_gpu_blocks_override
82
83
num_cpu_blocks = 0
83
- self.model_executor.initialize_cache(num_gpu_blocks)
84
+ self.model_executor.initialize(num_gpu_blocks)
85
elapsed = time.time() - start
86
logger.info(("init engine (profile, create kv cache, "
87
"warmup model) took %.2f seconds"), elapsed)
@@ -112,8 +113,11 @@ class EngineCore:
113
scheduler_output, output)
114
return engine_core_outputs
115
116
+ def shutdown(self):
117
+ self.model_executor.shutdown()
118
+
119
def profile(self, is_start=True):
116
- self.model_executor.worker.profile(is_start)
120
+ self.model_executor.profile(is_start)
121
122
123
class EngineCoreProc(EngineCore):
@@ -124,7 +128,7 @@ class EngineCoreProc(EngineCore):
128
def __init__(
129
self,
130
vllm_config: VllmConfig,
127
- executor_class: Type[GPUExecutor],
131
+ executor_class: Type[Executor],
132
usage_context: UsageContext,
133
input_path: str,
134
output_path: str,
@@ -151,32 +155,9 @@ class EngineCoreProc(EngineCore):
155
daemon=True).start()
156
157
# Send Readiness signal to EngineClient.
154
- with self.make_socket(ready_path, zmq.constants.PUSH) as ready_socket:
158
+ with make_zmq_socket(ready_path, zmq.constants.PUSH) as ready_socket:
159
ready_socket.send_string(EngineCoreProc.READY_STR)
160
157
- @contextmanager
158
- def make_socket(self, path: str, type: Any) -> Iterator[zmq.Socket]:
159
- """Context manager for use """
160
-
161
- ctx = zmq.Context()
162
- try:
163
- socket = ctx.socket(type)
164
-
165
- if type == zmq.constants.PULL:
166
- socket.connect(path)
167
- elif type == zmq.constants.PUSH:
168
- socket.bind(path)
169
- else:
170
- raise ValueError(f"Unknown Socket Type: {type}")
171
-
172
- yield socket
173
-
174
- except KeyboardInterrupt:
175
- logger.debug("EngineCore had Keyboard Interrupt.")
176
-
177
- finally:
178
- ctx.destroy(linger=0)
179
-
161
@staticmethod
162
def wait_for_startup(
163
proc: BaseProcess,
@@ -209,7 +190,7 @@ class EngineCoreProc(EngineCore):
190
@staticmethod
191
def make_engine_core_process(
192
vllm_config: VllmConfig,
212
- executor_class: Type[GPUExecutor],
193
+ executor_class: Type[Executor],
194
usage_context: UsageContext,
195
input_path: str,
196
output_path: str,
@@ -244,17 +225,38 @@ class EngineCoreProc(EngineCore):
225
def run_engine_core(*args, **kwargs):
226
"""Launch EngineCore busy loop in background process."""
227
228
+ # Signal handler used for graceful termination.
229
+ # SystemExit exception is only raised once to allow this and worker
230
+ # processes to terminate without error
231
+ shutdown_requested = False
232
+
233
+ def signal_handler(signum, frame):
234
+ nonlocal shutdown_requested
235
+ if not shutdown_requested:
236
+ shutdown_requested = True
237
+ raise SystemExit()
238
+
239
+ # Either SIGTERM or SIGINT will terminate the engine_core
240
+ signal.signal(signal.SIGTERM, signal_handler)
241
+ signal.signal(signal.SIGINT, signal_handler)
242
+
243
+ engine_core = None
244
try:
245
engine_core = EngineCoreProc(*args, **kwargs)
246
engine_core.run_busy_loop()
247
251
- except KeyboardInterrupt:
248
+ except SystemExit:
249
logger.debug("EngineCore interrupted.")
250
251
except BaseException as e:
252
logger.exception(e)
253
raise e
254
255
+ finally:
256
+ if engine_core is not None:
257
+ engine_core.shutdown()
258
+ engine_core = None
259
+
260
def run_busy_loop(self):
261
"""Core busy loop of the EngineCore."""
262
@@ -272,6 +274,8 @@ class EngineCoreProc(EngineCore):
274
logger.debug("EngineCore busy loop waiting.")
275
if self.should_shutdown:
276
return
277
+ except BaseException:
278
+ raise
279
280
# 2) Handle any new client requests (Abort or Add).
281
while not self.input_queue.empty():
@@ -321,7 +325,7 @@ class EngineCoreProc(EngineCore):
325
decoder_add_req = PickleEncoder()
326
decoder_abort_req = PickleEncoder()
327
324
- with self.make_socket(input_path, zmq.constants.PULL) as socket:
328
+ with make_zmq_socket(input_path, zmq.constants.PULL) as socket:
329
while True:
330
# (RequestType, RequestData)
331
type_frame, data_frame = socket.recv_multipart(copy=False)
@@ -349,7 +353,7 @@ class EngineCoreProc(EngineCore):
353
# Reuse send buffer.
354
buffer = bytearray()
355
352
- with self.make_socket(output_path, zmq.constants.PUSH) as socket:
356
+ with make_zmq_socket(output_path, zmq.constants.PUSH) as socket:
357
while True:
358
engine_core_outputs = self.output_queue.get()
359
outputs = EngineCoreOutputs(outputs=engine_core_outputs)
vllm/v1/engine/core_client.py
+9
-4
@@ -1,5 +1,4 @@
1
import multiprocessing
2
-import time
2
from typing import List, Union
3
4
import msgspec
@@ -7,7 +6,7 @@ import zmq
6
import zmq.asyncio
7
8
from vllm.logger import init_logger
10
-from vllm.utils import get_open_zmq_ipc_path
9
+from vllm.utils import get_open_zmq_ipc_path, kill_process_tree
10
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
11
EngineCoreProfile, EngineCoreRequest,
12
EngineCoreRequestType)
@@ -99,6 +98,12 @@ class InprocClient(EngineCoreClient):
98
def abort_requests(self, request_ids: List[str]) -> None:
99
self.engine_core.abort_requests(request_ids)
100
101
+ def shutdown(self):
102
+ self.engine_core.shutdown()
103
+
104
+ def __del__(self):
105
+ self.shutdown()
106
+
107
async def profile(self, is_start=True) -> None:
108
self.engine_core.profile(is_start)
109
@@ -163,10 +168,10 @@ class MPClient(EngineCoreClient):
168
# Shutdown the process if needed.
169
if hasattr(self, "proc") and self.proc.is_alive():
170
self.proc.terminate()
171
+ self.proc.join(5)
172
167
- time.sleep(5)
173
if self.proc.is_alive():
169
- self.proc.kill()
174
+ kill_process_tree(self.proc.pid)
175
176
def __del__(self):
177
self.shutdown()
vllm/v1/engine/llm_engine.py
+13
-6
@@ -20,7 +20,7 @@ from vllm.usage.usage_lib import UsageContext
20
from vllm.v1.engine.core_client import EngineCoreClient
21
from vllm.v1.engine.detokenizer import Detokenizer
22
from vllm.v1.engine.processor import Processor
23
-from vllm.v1.executor.gpu_executor import GPUExecutor
23
+from vllm.v1.executor.abstract import Executor
24
25
logger = init_logger(__name__)
26
@@ -33,7 +33,7 @@ class LLMEngine:
33
def __init__(
34
self,
35
vllm_config: VllmConfig,
36
- executor_class: Type[GPUExecutor],
36
+ executor_class: Type[Executor],
37
log_stats: bool,
38
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
39
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
@@ -104,10 +104,17 @@ class LLMEngine:
104
105
@classmethod
106
def _get_executor_cls(cls, vllm_config: VllmConfig):
107
- return GPUExecutor
108
-
109
- def stop_remote_worker_execution_loop(self) -> None:
110
- raise NotImplementedError("TP not implemented yet.")
107
+ distributed_executor_backend = (
108
+ vllm_config.parallel_config.distributed_executor_backend)
109
+ if distributed_executor_backend == "mp":
110
+ from vllm.v1.executor.multiproc_executor import MultiprocExecutor
111
+ executor_class = MultiprocExecutor
112
+ else:
113
+ assert (distributed_executor_backend is None)
114
+ from vllm.v1.executor.uniproc_executor import UniprocExecutor
115
+ executor_class = UniprocExecutor
116
+
117
+ return executor_class
118
119
def get_num_unfinished_requests(self) -> int:
120
return self.detokenizer.get_num_unfinished_requests()
vllm/v1/executor/abstract.py
new
+48
@@ -0,0 +1,48 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Dict, Optional, Tuple
3
+
4
+from vllm.config import VllmConfig
5
+from vllm.v1.outputs import ModelRunnerOutput
6
+
7
+
8
+class Executor(ABC):
9
+ """Abstract class for executors."""
10
+
11
+ @abstractmethod
12
+ def __init__(self, vllm_config: VllmConfig) -> None:
13
+ raise NotImplementedError
14
+
15
+ @abstractmethod
16
+ def initialize(self, num_gpu_blocks: int) -> None:
17
+ raise NotImplementedError
18
+
19
+ @abstractmethod
20
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
21
+ raise NotImplementedError
22
+
23
+ @abstractmethod
24
+ def execute_model(
25
+ self,
26
+ scheduler_output,
27
+ ) -> ModelRunnerOutput:
28
+ raise NotImplementedError
29
+
30
+ @abstractmethod
31
+ def profile(self, is_start=True):
32
+ raise NotImplementedError
33
+
34
+ @abstractmethod
35
+ def shutdown(self):
36
+ pass
37
+
38
+ @abstractmethod
39
+ def check_health(self) -> None:
40
+ raise NotImplementedError
41
+
42
+ @abstractmethod
43
+ def collective_rpc(self,
44
+ method: str,
45
+ timeout: Optional[float] = None,
46
+ args: Tuple = (),
47
+ kwargs: Optional[Dict] = None) -> []:
48
+ raise NotImplementedError
vllm/v1/executor/multiproc_executor.py
new
+375
@@ -0,0 +1,375 @@
1
+import atexit
2
+import os
3
+import pickle
4
+import signal
5
+import sys
6
+import time
7
+from dataclasses import dataclass
8
+from enum import Enum, auto
9
+from multiprocessing.process import BaseProcess
10
+from typing import Dict, List, Optional, Tuple
11
+
12
+import zmq
13
+
14
+from vllm.config import VllmConfig
15
+from vllm.distributed import (destroy_distributed_environment,
16
+ destroy_model_parallel)
17
+from vllm.distributed.device_communicators.shm_broadcast import (Handle,
18
+ MessageQueue)
19
+from vllm.executor.multiproc_worker_utils import (
20
+ _add_prefix, get_mp_context, set_multiprocessing_worker_envs)
21
+from vllm.logger import init_logger
22
+from vllm.utils import (get_distributed_init_method, get_open_port,
23
+ get_open_zmq_ipc_path)
24
+from vllm.v1.outputs import ModelRunnerOutput
25
+from vllm.v1.utils import make_zmq_socket
26
+from vllm.worker.worker_base import WorkerWrapperBase
27
+
28
+logger = init_logger(__name__)
29
+
30
+POLLING_TIMEOUT_MS = 5000
31
+POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
32
+
33
+
34
+class MultiprocExecutor:
35
+
36
+ def __init__(self, vllm_config: VllmConfig) -> None:
37
+ # Call self.shutdown at exit to clean up
38
+ # and ensure workers will be terminated.
39
+ atexit.register(self.shutdown)
40
+
41
+ self.vllm_config = vllm_config
42
+ self.parallel_config = vllm_config.parallel_config
43
+
44
+ self.world_size = self.parallel_config.world_size
45
+ tensor_parallel_size = self.parallel_config.tensor_parallel_size
46
+ assert self.world_size == tensor_parallel_size, (
47
+ f"world_size ({self.world_size}) must be equal to the "
48
+ f"tensor_parallel_size ({tensor_parallel_size}). "
49
+ f"Pipeline parallelism is not yet implemented in v1")
50
+
51
+ # Set multiprocessing envs that are common to V0 and V1
52
+ set_multiprocessing_worker_envs(self.parallel_config)
53
+
54
+ # Multiprocessing-based executor does not support multi-node setting.
55
+ # Since it only works for single node, we can use the loopback address
56
+ # 127.0.0.1 for communication.
57
+ distributed_init_method = get_distributed_init_method(
58
+ "127.0.0.1", get_open_port())
59
+
60
+ # Initialize worker and set up message queues for SchedulerOutputs
61
+ # and ModelRunnerOutputs
62
+ self.rpc_broadcast_mq = MessageQueue(self.world_size, self.world_size)
63
+ scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
64
+
65
+ # Create workers
66
+ self.workers: List[WorkerProcHandle] = []
67
+ for rank in range(self.world_size):
68
+ worker = WorkerProc.make_worker_process(vllm_config, rank, rank,
69
+ distributed_init_method,
70
+ scheduler_output_handle)
71
+ self.workers.append(worker)
72
+
73
+ # Ensure message queues are ready. Will deadlock if re-ordered
74
+ # Must be kept consistent with the WorkerProc
75
+ self.rpc_broadcast_mq.wait_until_ready()
76
+ for w in self.workers:
77
+ w.worker_response_mq.wait_until_ready()
78
+
79
+ def initialize(self, num_gpu_blocks: int) -> None:
80
+ """
81
+ Initialize the KV caches and begin the model execution loop of the
82
+ underlying workers.
83
+ """
84
+ self.collective_rpc("initialize_cache", args=(num_gpu_blocks, ))
85
+ self.collective_rpc("compile_or_warm_up_model")
86
+
87
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
88
+ """
89
+ Determine the number of available KV blocks by invoking the
90
+ underlying worker.
91
+ """
92
+ num_blocks = self.collective_rpc("determine_num_available_blocks")
93
+
94
+ # Since we use a shared centralized controller, we take the minimum
95
+ # number of blocks across all workers to make sure all the memory
96
+ # operators can be applied to all workers.
97
+ num_gpu_blocks = min(b[0] for b in num_blocks)
98
+ num_cpu_blocks = min(b[1] for b in num_blocks)
99
+
100
+ return num_gpu_blocks, num_cpu_blocks
101
+
102
+ def collective_rpc(self,
103
+ method: str,
104
+ timeout: Optional[float] = None,
105
+ args: Tuple = (),
106
+ kwargs: Optional[Dict] = None) -> []:
107
+ """
108
+ Execute an RPC call on workers.
109
+
110
+ Args:
111
+ method: Name of the worker method to execute
112
+ timeout: Maximum time in seconds to wait for execution. Rases a
113
+ TimeoutError on timeout. None means wait indefinitely.
114
+ args: Positional arguments to pass to the worker method
115
+ kwargs: Keyword arguments to pass to the worker method
116
+
117
+ Returns:
118
+ List of results from each worker
119
+ """
120
+ start_time = time.monotonic()
121
+ kwargs = kwargs or {}
122
+
123
+ try:
124
+ self.rpc_broadcast_mq.enqueue((method, args, kwargs))
125
+
126
+ responses = [None] * self.world_size
127
+ for w in self.workers:
128
+ dequeue_timeout = timeout - (time.monotonic() - start_time()
129
+ ) if timeout is not None else None
130
+ status, result = w.worker_response_mq.dequeue(
131
+ timeout=dequeue_timeout)
132
+
133
+ if status != WorkerProc.ResponseStatus.SUCCESS:
134
+ if isinstance(result, Exception):
135
+ raise result
136
+ else:
137
+ raise RuntimeError("Worker failed")
138
+
139
+ responses[w.rank] = result
140
+
141
+ return responses
142
+ except TimeoutError as e:
143
+ raise TimeoutError(f"RPC call to {method} timed out.") from e
144
+ except Exception as e:
145
+ # Re-raise any other exceptions
146
+ raise e
147
+
148
+ def execute_model(
149
+ self,
150
+ scheduler_output,
151
+ ) -> ModelRunnerOutput:
152
+ model_output = self.collective_rpc("execute_model",
153
+ args=(scheduler_output, ))[0]
154
+ return model_output
155
+
156
+ def profile(self, is_start=True):
157
+ self.collective_rpc("profile", args=(is_start, ))
158
+ return
159
+
160
+ def _ensure_worker_termination(self):
161
+ """Ensure that all worker processes are terminated. Assumes workers have
162
+ received termination requests. Waits for processing, then sends
163
+ termination and kill signals if needed."""
164
+
165
+ def wait_for_termination(procs, timeout):
166
+ start_time = time.time()
167
+ while time.time() - start_time < timeout:
168
+ if all(not proc.is_alive() for proc in procs):
169
+ return True
170
+ time.sleep(0.1)
171
+ return False
172
+
173
+ # Send SIGTERM if still running
174
+ active_procs = [w.proc for w in self.workers if w.proc.is_alive()]
175
+ self.workers = None
176
+ for p in active_procs:
177
+ p.terminate()
178
+ if wait_for_termination(active_procs, 4):
179
+ return
180
+
181
+ # Send SIGKILL if still running
182
+ active_procs = [p for p in active_procs if p.is_alive()]
183
+ for p in active_procs:
184
+ p.kill()
185
+
186
+ def shutdown(self):
187
+ """Properly shut down the executor and its workers"""
188
+ if (hasattr(self, 'workers') and self.workers is not None):
189
+ for w in self.workers: #TODO: not sure if needed
190
+ w.worker_response_mq = None
191
+ self._ensure_worker_termination()
192
+
193
+ self.rpc_broadcast_mq = None
194
+
195
+ def check_health(self) -> None:
196
+ self.collective_rpc("check_health", timeout=10)
197
+ return
198
+
199
+
200
+@dataclass
201
+class WorkerProcHandle:
202
+ proc: BaseProcess
203
+ rank: int
204
+ ready_path: str
205
+ worker_response_mq: MessageQueue # The worker process writes to this MQ
206
+
207
+
208
+class WorkerProc:
209
+ """Wrapper that runs one Worker in a separate process."""
210
+
211
+ READY_STR = "READY"
212
+
213
+ def __init__(
214
+ self,
215
+ vllm_config: VllmConfig,
216
+ local_rank: int,
217
+ rank: int,
218
+ distributed_init_method: str,
219
+ input_shm_handle: Handle,
220
+ ready_path: str,
221
+ ):
222
+ self.rank = rank
223
+ wrapper = WorkerWrapperBase(vllm_config=vllm_config)
224
+ wrapper.init_worker(vllm_config, local_rank, rank,
225
+ distributed_init_method)
226
+ self.worker = wrapper.worker
227
+
228
+ pid = os.getpid()
229
+ _add_prefix(sys.stdout, f"VllmWorker rank={rank}", pid)
230
+ _add_prefix(sys.stderr, f"VllmWorker rank={rank}", pid)
231
+
232
+ # Initialize MessageQueue for receiving SchedulerOutput
233
+ self.rpc_broadcast_mq = MessageQueue.create_from_handle(
234
+ input_shm_handle, self.worker.rank)
235
+
236
+ # Initializes a message queue for sending the model output
237
+ self.worker_response_mq = MessageQueue(1, 1)
238
+ worker_response_mq_handle = self.worker_response_mq.export_handle()
239
+
240
+ # Send Readiness signal to EngineCore process.
241
+ with make_zmq_socket(ready_path, zmq.constants.PUSH) as ready_socket:
242
+ payload = pickle.dumps(worker_response_mq_handle,
243
+ protocol=pickle.HIGHEST_PROTOCOL)
244
+ ready_socket.send_string(WorkerProc.READY_STR)
245
+ ready_socket.send(payload)
246
+
247
+ self.worker.initialize()
248
+ self.worker.load_model()
249
+
250
+ @staticmethod
251
+ def make_worker_process(
252
+ vllm_config: VllmConfig,
253
+ local_rank: int,
254
+ rank: int,
255
+ distributed_init_method: str,
256
+ input_shm_handle, # Receive SchedulerOutput
257
+ ) -> WorkerProcHandle:
258
+ context = get_mp_context()
259
+
260
+ # ZMQ path for worker to send ready message and shm_broadcast handle
261
+ # back to core process.
262
+ ready_path = get_open_zmq_ipc_path()
263
+
264
+ process_kwargs = {
265
+ "vllm_config": vllm_config,
266
+ "local_rank": local_rank,
267
+ "rank": rank,
268
+ "distributed_init_method": distributed_init_method,
269
+ "input_shm_handle": input_shm_handle,
270
+ "ready_path": ready_path,
271
+ }
272
+ # Run EngineCore busy loop in background process.
273
+ proc = context.Process(target=WorkerProc.worker_main,
274
+ kwargs=process_kwargs,
275
+ daemon=True)
276
+ proc.start()
277
+
278
+ # Wait for startup
279
+ worker_response_mq_handle = WorkerProc.wait_for_startup(
280
+ proc, ready_path)
281
+
282
+ worker_response_mq = MessageQueue.create_from_handle(
283
+ worker_response_mq_handle, 0)
284
+
285
+ return WorkerProcHandle(proc, rank, ready_path, worker_response_mq)
286
+
287
+ def shutdown(self):
288
+ self.rpc_broadcast_mq = None
289
+ self.worker_response_mq = None
290
+ destroy_model_parallel()
291
+ destroy_distributed_environment()
292
+
293
+ @staticmethod
294
+ def worker_main(*args, **kwargs):
295
+ """ Worker initialization and execution loops.
296
+ This runs a background process """
297
+
298
+ # Signal handler used for graceful termination.
299
+ # SystemExit exception is only raised once to allow this and worker
300
+ # processes to terminate without error
301
+ shutdown_requested = False
302
+
303
+ def signal_handler(signum, frame):
304
+ nonlocal shutdown_requested
305
+ if not shutdown_requested:
306
+ shutdown_requested = True
307
+ raise SystemExit()
308
+
309
+ # Either SIGTERM or SIGINT will terminate the worker
310
+ signal.signal(signal.SIGTERM, signal_handler)
311
+ signal.signal(signal.SIGINT, signal_handler)
312
+
313
+ worker = None
314
+ try:
315
+ worker = WorkerProc(*args, **kwargs)
316
+
317
+ # Ensure message queues are ready. Will deadlock if re-ordered.
318
+ # Must be kept consistent with the Executor
319
+ worker.rpc_broadcast_mq.wait_until_ready()
320
+ worker.worker_response_mq.wait_until_ready()
321
+
322
+ worker.worker_busy_loop()
323
+
324
+ except SystemExit:
325
+ logger.debug("Worker interrupted.")
326
+
327
+ except BaseException as e:
328
+ logger.exception(e)
329
+ raise
330
+
331
+ finally:
332
+ # Clean up once worker exits busy loop
333
+ if worker is not None:
334
+ worker.shutdown()
335
+ worker = None
336
+
337
+ @staticmethod
338
+ def wait_for_startup(
339
+ proc: BaseProcess,
340
+ ready_path: str,
341
+ ) -> Optional[Handle]:
342
+ """Wait until the Worker is ready."""
343
+ with make_zmq_socket(ready_path, zmq.constants.PULL) as socket:
344
+
345
+ # Wait for Worker to send READY.
346
+ while socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
347
+ logger.debug("Waiting for WorkerProc to startup.")
348
+
349
+ if not proc.is_alive():
350
+ raise RuntimeError("WorkerProc failed to start.")
351
+
352
+ message = socket.recv_string()
353
+ assert message == WorkerProc.READY_STR
354
+ handle_frame = socket.recv(copy=False)
355
+ handle = pickle.loads(handle_frame.buffer)
356
+ return handle
357
+
358
+ class ResponseStatus(Enum):
359
+ SUCCESS = auto()
360
+ FAILURE = auto()
361
+
362
+ def worker_busy_loop(self):
363
+ """Main busy loop for Multiprocessing Workers"""
364
+ while True:
365
+ method, args, kwargs = self.rpc_broadcast_mq.dequeue()
366
+
367
+ try:
368
+ output = getattr(self.worker, method)(*args, **kwargs)
369
+ except BaseException as e:
370
+ self.worker_response_mq.enqueue(
371
+ (WorkerProc.ResponseStatus.FAILURE, e))
372
+ continue
373
+
374
+ self.worker_response_mq.enqueue(
375
+ (WorkerProc.ResponseStatus.SUCCESS, output))
vllm/v1/executor/uniproc_executor.py
renamed
+9
-3
@@ -10,7 +10,7 @@ from vllm.v1.worker.gpu_worker import Worker
10
logger = init_logger(__name__)
11
12
13
-class GPUExecutor:
13
+class UniprocExecutor:
14
15
def __init__(self, vllm_config: VllmConfig) -> None:
16
self.vllm_config = vllm_config
@@ -54,7 +54,7 @@ class GPUExecutor:
54
"""
55
return self.worker.determine_num_available_blocks()
56
57
- def initialize_cache(self, num_gpu_blocks: int) -> None:
57
+ def initialize(self, num_gpu_blocks: int) -> None:
58
"""Initialize the KV cache by invoking the underlying worker.
59
"""
60
# NOTE: This is logged in the executor because there can be >1 worker
@@ -71,7 +71,13 @@ class GPUExecutor:
71
output = self.worker.execute_model(scheduler_output)
72
return output
73
74
+ def profile(self, is_start: bool = True):
75
+ self.worker.profile(is_start)
76
+
77
+ def shutdown(self):
78
+ self.worker = None
79
+
80
def check_health(self) -> None:
75
- # GPUExecutor will always be healthy as long as
81
+ # UniprocExecutor will always be healthy as long as
82
# it's running.
83
return
vllm/v1/outputs.py
+4
-2
@@ -8,7 +8,7 @@ import torch
8
class SamplerOutput:
9
10
# [num_reqs]
11
- sampled_token_ids: torch.Tensor
11
+ sampled_token_ids: List[int]
12
13
# [num_reqs, max_num_logprobs + 1]
14
logprob_token_ids: Optional[torch.Tensor]
@@ -20,6 +20,8 @@ class SamplerOutput:
20
prompt_logprobs: Optional[torch.Tensor]
21
22
23
+# ModelRunnerOutput is serialized and sent to the scheduler process.
24
+# This is expensive for torch.Tensor so prefer to use List instead.
25
@dataclass
26
class ModelRunnerOutput:
27
@@ -29,7 +31,7 @@ class ModelRunnerOutput:
31
req_id_to_index: Dict[str, int]
32
33
# [num_reqs]
32
- sampled_token_ids_cpu: torch.Tensor
34
+ sampled_token_ids: List[int]
35
36
# [num_reqs, max_num_logprobs + 1]
37
logprob_token_ids_cpu: Optional[torch.Tensor]
vllm/v1/sample/sampler.py
+2
-1
@@ -37,8 +37,9 @@ class Sampler(nn.Module):
37
topk_logprobs = None
38
topk_indices = None
39
40
+ # NOTE: CPU-GPU synchronization happens here.
41
sampler_output = SamplerOutput(
41
- sampled_token_ids=sampled,
42
+ sampled_token_ids=sampled.tolist(),
43
logprob_token_ids=topk_indices,
44
logprobs=topk_logprobs,
45
prompt_logprob_token_ids=None,
vllm/v1/utils.py
+32
-1
@@ -1,4 +1,11 @@
1
-from typing import Generic, List, TypeVar, overload
1
+from contextlib import contextmanager
2
+from typing import Any, Generic, Iterator, List, TypeVar, overload
3
+
4
+import zmq
5
+
6
+from vllm.logger import init_logger
7
+
8
+logger = init_logger(__name__)
9
10
T = TypeVar("T")
11
@@ -62,3 +69,27 @@ class ConstantList(Generic[T]):
69
70
def __len__(self):
71
return len(self._x)
72
+
73
+
74
+@contextmanager
75
+def make_zmq_socket(path: str, type: Any) -> Iterator[zmq.Socket]:
76
+ """Context manager for a ZMQ socket"""
77
+
78
+ ctx = zmq.Context()
79
+ try:
80
+ socket = ctx.socket(type)
81
+
82
+ if type == zmq.constants.PULL:
83
+ socket.connect(path)
84
+ elif type == zmq.constants.PUSH:
85
+ socket.bind(path)
86
+ else:
87
+ raise ValueError(f"Unknown Socket Type: {type}")
88
+
89
+ yield socket
90
+
91
+ except KeyboardInterrupt:
92
+ logger.debug("Worker had Keyboard Interrupt.")
93
+
94
+ finally:
95
+ ctx.destroy(linger=0)
vllm/v1/worker/gpu_model_runner.py
+5
-7
@@ -34,6 +34,7 @@ class GPUModelRunner:
34
def __init__(
35
self,
36
vllm_config: VllmConfig,
37
+ device: torch.device,
38
input_registry: InputRegistry = INPUT_REGISTRY,
39
):
40
self.vllm_config = vllm_config
@@ -43,7 +44,6 @@ class GPUModelRunner:
44
self.load_config = vllm_config.load_config
45
self.parallel_config = vllm_config.parallel_config
46
self.scheduler_config = vllm_config.scheduler_config
46
- self.device_config = vllm_config.device_config
47
self.speculative_config = vllm_config.speculative_config
48
self.prompt_adapter_config = vllm_config.prompt_adapter_config
49
self.observability_config = vllm_config.observability_config
@@ -52,7 +52,7 @@ class GPUModelRunner:
52
cache_config = self.cache_config
53
scheduler_config = self.scheduler_config
54
parallel_config = self.parallel_config
55
- self.device = self.device_config.device
55
+ self.device = device
56
self.pin_memory = is_pin_memory_available()
57
self.dtype = self.model_config.dtype
58
if cache_config.cache_dtype == "auto":
@@ -477,9 +477,7 @@ class GPUModelRunner:
477
sampling_metadata=sampling_metadata,
478
)
479
480
- # NOTE: CPU-GPU synchronization happens here.
481
- sampled_token_ids = sampler_output.sampled_token_ids.cpu()
482
- sampled_token_ids_list = sampled_token_ids.tolist()
480
+ sampled_token_ids = sampler_output.sampled_token_ids
481
# TODO(woosuk): The following loop can be slow since it iterates over
482
# the requests one by one. Optimize.
483
num_reqs = self.input_batch.num_reqs
@@ -490,7 +488,7 @@ class GPUModelRunner:
488
assert seq_len <= req_state.num_tokens
489
if seq_len == req_state.num_tokens:
490
# Append the sampled token to the output token ids.
493
- token_id = sampled_token_ids_list[i]
491
+ token_id = sampled_token_ids[i]
492
self.input_batch.token_ids_cpu[i, seq_len] = token_id
493
req_state.output_token_ids.append(token_id)
494
else:
@@ -512,7 +510,7 @@ class GPUModelRunner:
510
model_runner_output = ModelRunnerOutput(
511
req_ids=self.input_batch.req_ids[:num_reqs],
512
req_id_to_index=self.input_batch.req_id_to_index,
515
- sampled_token_ids_cpu=sampled_token_ids,
513
+ sampled_token_ids=sampled_token_ids,
514
logprob_token_ids_cpu=logprob_token_ids,
515
logprobs_cpu=logprobs,
516
)
vllm/v1/worker/gpu_worker.py
+9
-2
@@ -15,6 +15,7 @@ from vllm.logger import init_logger
15
from vllm.model_executor import set_random_seed
16
from vllm.platforms import current_platform
17
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
18
+from vllm.v1.core.scheduler import SchedulerOutput
19
from vllm.v1.outputs import ModelRunnerOutput
20
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
21
@@ -56,7 +57,6 @@ class Worker:
57
from vllm.utils import init_cached_hf_modules
58
init_cached_hf_modules()
59
59
- self.model_runner = GPUModelRunner(vllm_config)
60
# Torch profiler. Enabled and configured through env vars:
61
# VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
62
if envs.VLLM_TORCH_PROFILER_DIR:
@@ -103,6 +103,9 @@ class Worker:
103
# Set random seed.
104
set_random_seed(self.model_config.seed)
105
106
+ # Construct the model runner
107
+ self.model_runner = GPUModelRunner(self.vllm_config, self.device)
108
+
109
def load_model(self) -> None:
110
self.model_runner.load_model()
111
@@ -198,7 +201,7 @@ class Worker:
201
scheduler_output: "SchedulerOutput",
202
) -> ModelRunnerOutput:
203
output = self.model_runner.execute_model(scheduler_output)
201
- # TODO(woosuk): Send the output to the engine process.
204
+ return output if self.rank == 0 else None
205
return output
206
207
def profile(self, is_start=True):
@@ -209,6 +212,10 @@ class Worker:
212
else:
213
self.profiler.stop()
214
215
+ def check_health(self) -> None:
216
+ # worker will always be healthy as long as it's running.
217
+ return
218
+
219
220
def init_worker_distributed_environment(
221
parallel_config: ParallelConfig,