@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 28b3a1c7e5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Multiprocessing Tensor Parallel Support for v1 (#9856)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Dec 10, 2024 at 01:28 UTC 28b3a1c7e596c08efac0fcfa59a629d16197be30
21 files changed +733 -146
tests/basic_correctness/test_basic_correctness.py
+16
@@ -26,6 +26,14 @@ MODELS = [
26 TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
27
28
29 +@pytest.fixture(autouse=True)
30 +def v1(run_with_both_engines):
31 + # Simple autouse wrapper to run both engines for each test
32 + # This can be promoted up to conftest.py to run for every
33 + # test in a package
34 + pass
35 +
36 +
37 def test_vllm_gc_ed():
38 """Verify vllm instance is GC'ed when it is deleted"""
39 llm = LLM("facebook/opt-125m")
@@ -36,6 +44,7 @@ def test_vllm_gc_ed():
44 assert weak_llm() is None
45
46
47 +@pytest.mark.skip_v1
48 @pytest.mark.parametrize("model", MODELS)
49 @pytest.mark.parametrize("backend", ["FLASH_ATTN", "XFORMERS", "FLASHINFER"])
50 @pytest.mark.parametrize("dtype", ["half"])
@@ -118,6 +127,11 @@ def test_models_distributed(
127 if attention_backend:
128 os.environ["VLLM_ATTENTION_BACKEND"] = attention_backend
129
130 + # Import VLLM_USE_V1 dynamically to handle patching
131 + from vllm.envs import VLLM_USE_V1
132 + if VLLM_USE_V1 and distributed_executor_backend != "mp":
133 + pytest.skip(f"Skip {distributed_executor_backend} for V1")
134 +
135 dtype = "half"
136 max_tokens = 5
137
@@ -143,6 +157,7 @@ def test_models_distributed(
157 )
158
159
160 +@pytest.mark.skip_v1
161 def test_model_with_failure(vllm_runner) -> None:
162 try:
163 with patch("vllm.model_executor.models.opt.OPTForCausalLM.forward",
@@ -169,6 +184,7 @@ def test_model_with_failure(vllm_runner) -> None:
184 os.remove(filename)
185
186
187 +@pytest.mark.skip_v1
188 def test_failure_with_async_out_proc(vllm_runner) -> None:
189
190 filename = None
tests/conftest.py
+5 -6
@@ -5,7 +5,6 @@ from collections import UserList
5 from enum import Enum
6 from typing import (Any, Callable, Dict, List, Optional, Tuple, Type,
7 TypedDict, TypeVar, Union)
8 -from unittest.mock import patch
8
9 import numpy as np
10 import pytest
@@ -110,7 +109,7 @@ VIDEO_ASSETS = _VideoAssets()
109
110
111 @pytest.fixture(params=[True, False])
113 -def run_with_both_engines(request):
112 +def run_with_both_engines(request, monkeypatch):
113 # Automatically runs tests twice, once with V1 and once without
114 use_v1 = request.param
115 # Tests decorated with `@skip_v1` are only run without v1
@@ -119,11 +118,11 @@ def run_with_both_engines(request):
118 if use_v1:
119 if skip_v1:
120 pytest.skip("Skipping test on vllm V1")
122 - with patch('vllm.envs.VLLM_USE_V1', True):
123 - yield
121 + monkeypatch.setenv('VLLM_USE_V1', '1')
122 else:
125 - with patch('vllm.envs.VLLM_USE_V1', False):
126 - yield
123 + monkeypatch.setenv('VLLM_USE_V1', '0')
124 +
125 + yield
126
127
128 @pytest.fixture(autouse=True)
vllm/distributed/device_communicators/shm_broadcast.py
+52 -24
@@ -1,10 +1,11 @@
1 import os
2 import pickle
3 +import sys
4 import time
5 from contextlib import contextmanager
6 from dataclasses import dataclass, field
7 from multiprocessing import shared_memory
7 -from typing import List, Optional
8 +from typing import List, Optional, Tuple
9 from unittest.mock import patch
10
11 import torch
@@ -21,6 +22,20 @@ VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
22
23 logger = init_logger(__name__)
24
25 +# We prefer to use os.sched_yield as it results in tighter polling loops,
26 +# measured to be around 3e-7 seconds. However on earlier versions of Python
27 +# os.sched_yield() does not release the GIL, so we fall back to time.sleep(0)
28 +USE_SCHED_YIELD = ((sys.version_info[:3] >= (3, 11, 1))
29 + or (sys.version_info[:2] == (3, 10)
30 + and sys.version_info[2] >= 8))
31 +
32 +
33 +def sched_yield():
34 + if USE_SCHED_YIELD:
35 + os.sched_yield()
36 + else:
37 + time.sleep(0)
38 +
39
40 class ShmRingBuffer:
41
@@ -114,11 +129,14 @@ class ShmRingBuffer:
129 # and we should suppress the error
130 pass
131
132 + def handle(self):
133 + return (self.n_reader, self.max_chunk_bytes, self.max_chunks,
134 + self.shared_memory.name)
135 +
136 def __reduce__(self):
137 return (
138 self.__class__,
120 - (self.n_reader, self.max_chunk_bytes, self.max_chunks,
121 - self.shared_memory.name),
139 + self.handle(),
140 )
141
142 def __del__(self):
@@ -147,7 +165,7 @@ class Handle:
165 connect_ip: str
166 local_reader_ranks: List[int] = field(default_factory=list)
167
150 - buffer: Optional[ShmRingBuffer] = None
168 + buffer_handle: Optional[Tuple[int, int, int, str]] = None
169 local_subscribe_port: Optional[int] = None
170 remote_subscribe_port: Optional[int] = None
171
@@ -228,7 +246,7 @@ class MessageQueue:
246 self.handle = Handle(
247 connect_ip=connect_ip,
248 local_reader_ranks=local_reader_ranks,
231 - buffer=self.buffer,
249 + buffer_handle=self.buffer.handle(),
250 local_subscribe_port=local_subscribe_port,
251 remote_subscribe_port=remote_subscribe_port,
252 )
@@ -247,8 +265,8 @@ class MessageQueue:
265 context = Context()
266
267 if rank in handle.local_reader_ranks:
250 - assert handle.buffer is not None
251 - self.buffer = handle.buffer
268 + assert handle.buffer_handle is not None
269 + self.buffer = ShmRingBuffer(*handle.buffer_handle)
270 self.current_idx = 0
271 self.local_reader_rank = handle.local_reader_ranks.index(rank)
272 self._is_local_reader = True
@@ -314,7 +332,7 @@ class MessageQueue:
332 assert recv == b"READY"
333
334 @contextmanager
317 - def acquire_write(self):
335 + def acquire_write(self, timeout: Optional[float] = None):
336 assert self._is_writer, "Only writers can acquire write"
337 start_time = time.monotonic()
338 n_warning = 1
@@ -329,16 +347,20 @@ class MessageQueue:
347 # we need to wait until it is read by all readers
348
349 # Release the processor to other threads
332 - os.sched_yield()
350 + sched_yield()
351
334 - # if we wait for a long time, we should warn the user
352 + # if we wait for a long time, log a message
353 if (time.monotonic() - start_time >
354 VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
337 - logger.warning(
338 - "No available block found in %s second. ",
339 - VLLM_RINGBUFFER_WARNING_INTERVAL)
355 + logger.debug("No available block found in %s second. ",
356 + VLLM_RINGBUFFER_WARNING_INTERVAL)
357 n_warning += 1
358
359 + # if we time out, raise an exception
360 + if (timeout is not None
361 + and time.monotonic() - start_time > timeout):
362 + raise TimeoutError
363 +
364 continue
365 # found a block that is either
366 # (1) not written
@@ -365,7 +387,7 @@ class MessageQueue:
387 break
388
389 @contextmanager
368 - def acquire_read(self):
390 + def acquire_read(self, timeout: Optional[float] = None):
391 assert self._is_local_reader, "Only readers can acquire read"
392 start_time = time.monotonic()
393 n_warning = 1
@@ -383,16 +405,20 @@ class MessageQueue:
405 # we need to wait until it is written
406
407 # Release the processor to other threads
386 - os.sched_yield()
408 + sched_yield()
409
388 - # if we wait for a long time, we should warn the user
410 + # if we wait for a long time, log a message
411 if (time.monotonic() - start_time >
412 VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
391 - logger.warning(
392 - "No available block found in %s second. ",
393 - VLLM_RINGBUFFER_WARNING_INTERVAL)
413 + logger.debug("No available block found in %s second. ",
414 + VLLM_RINGBUFFER_WARNING_INTERVAL)
415 n_warning += 1
416
417 + # if we time out, raise an exception
418 + if (timeout is not None
419 + and time.monotonic() - start_time > timeout):
420 + raise TimeoutError
421 +
422 continue
423 # found a block that is not read by this reader
424 # let caller read from the buffer
@@ -406,24 +432,26 @@ class MessageQueue:
432 1) % self.buffer.max_chunks
433 break
434
409 - def enqueue(self, obj):
435 + def enqueue(self, obj, timeout: Optional[float] = None):
436 + """ Write to message queue with optional timeout (in seconds) """
437 assert self._is_writer, "Only writers can enqueue"
438 serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
439 if self.n_local_reader > 0:
440 if len(serialized_obj) >= self.buffer.max_chunk_bytes:
414 - with self.acquire_write() as buf:
441 + with self.acquire_write(timeout) as buf:
442 buf[0] = 1 # overflow
443 self.local_socket.send(serialized_obj)
444 else:
418 - with self.acquire_write() as buf:
445 + with self.acquire_write(timeout) as buf:
446 buf[0] = 0 # not overflow
447 buf[1:len(serialized_obj) + 1] = serialized_obj
448 if self.n_remote_reader > 0:
449 self.remote_socket.send(serialized_obj)
450
424 - def dequeue(self):
451 + def dequeue(self, timeout: Optional[float] = None):
452 + """ Read from message queue with optional timeout (in seconds) """
453 if self._is_local_reader:
426 - with self.acquire_read() as buf:
454 + with self.acquire_read(timeout) as buf:
455 overflow = buf[0] == 1
456 if not overflow:
457 # no need to know the size of serialized object
vllm/executor/multiproc_gpu_executor.py
+6 -41
@@ -3,25 +3,19 @@ import os
3 from functools import partial
4 from typing import Any, List, Optional
5
6 -import torch
7 -
6 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
7 DistributedGPUExecutor, DistributedGPUExecutorAsync)
8 from vllm.executor.gpu_executor import create_worker
11 -from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
12 - ResultHandler, WorkerMonitor)
9 +from vllm.executor.multiproc_worker_utils import (
10 + ProcessWorkerWrapper, ResultHandler, WorkerMonitor,
11 + set_multiprocessing_worker_envs)
12 from vllm.logger import init_logger
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 from vllm.sequence import ExecuteModelRequest
16 -from vllm.triton_utils.importing import HAS_TRITON
15 from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
18 - cuda_is_initialized, get_distributed_init_method,
19 - get_open_port, make_async,
16 + get_distributed_init_method, get_open_port, make_async,
17 update_environment_variables)
18
22 -if HAS_TRITON:
23 - from vllm.triton_utils import maybe_set_triton_cache_manager
24 -
19 logger = init_logger(__name__)
20
21
@@ -37,30 +31,8 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
31 world_size = self.parallel_config.world_size
32 tensor_parallel_size = self.parallel_config.tensor_parallel_size
33
40 - # Disable torch async compiling which won't work with daemonic processes
41 - os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
42 -
43 - # Configure thread parallelism if OMP_NUM_THREADS isn't set
44 - #
45 - # Helps to avoid CPU contention. The default of spawning a thread per
46 - # core combined with multiprocessing for each GPU can have a negative
47 - # impact on performance. The contention is amplified when running in a
48 - # container where CPU limits can cause throttling.
49 - default_omp_num_threads = 1
50 - if "OMP_NUM_THREADS" not in os.environ and (
51 - current_parallelism :=
52 - torch.get_num_threads()) > default_omp_num_threads:
53 - logger.warning(
54 - "Reducing Torch parallelism from %d threads to %d to avoid "
55 - "unnecessary CPU contention. Set OMP_NUM_THREADS in the "
56 - "external environment to tune this value as needed.",
57 - current_parallelism, default_omp_num_threads)
58 - os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)
59 - torch.set_num_threads(default_omp_num_threads)
60 -
61 - # workaround for https://github.com/vllm-project/vllm/issues/6103
62 - if HAS_TRITON and world_size > 1:
63 - maybe_set_triton_cache_manager()
34 + # Set multiprocessing envs that are common to V0 and V1
35 + set_multiprocessing_worker_envs(self.parallel_config)
36
37 # Multiprocessing-based executor does not support multi-node setting.
38 # Since it only works for single node, we can use the loopback address
@@ -122,13 +94,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
94 "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
95 })
96
125 - if (cuda_is_initialized()
126 - and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
127 - logger.warning("CUDA was previously initialized. We must use "
128 - "the `spawn` multiprocessing start method. Setting "
129 - "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'.")
130 - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
131 -
97 cuda_device_count = cuda_device_count_stateless()
98 # Use confusing message for more common TP-only case.
99 assert tensor_parallel_size <= cuda_device_count, (
vllm/executor/multiproc_worker_utils.py
+42
@@ -11,8 +11,15 @@ from multiprocessing.process import BaseProcess
11 from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
12 TypeVar, Union)
13
14 +import torch
15 +
16 import vllm.envs as envs
17 from vllm.logger import init_logger
18 +from vllm.triton_utils.importing import HAS_TRITON
19 +from vllm.utils import cuda_is_initialized
20 +
21 +if HAS_TRITON:
22 + from vllm.triton_utils import maybe_set_triton_cache_manager
23
24 logger = init_logger(__name__)
25
@@ -270,3 +277,38 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
277 def get_mp_context():
278 mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
279 return multiprocessing.get_context(mp_method)
280 +
281 +
282 +def set_multiprocessing_worker_envs(parallel_config):
283 + """ Set up environment variables that should be used when there are workers
284 + in a multiprocessing environment. This should be called by the parent
285 + process before worker processes are created"""
286 +
287 + if (cuda_is_initialized()
288 + and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
289 + logger.warning("CUDA was previously initialized. We must use "
290 + "the `spawn` multiprocessing start method. Setting "
291 + "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'.")
292 + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
293 +
294 + # Configure thread parallelism if OMP_NUM_THREADS isn't set
295 + #
296 + # Helps to avoid CPU contention. The default of spawning a thread per
297 + # core combined with multiprocessing for each GPU can have a negative
298 + # impact on performance. The contention is amplified when running in a
299 + # container where CPU limits can cause throttling.
300 + default_omp_num_threads = 1
301 + if "OMP_NUM_THREADS" not in os.environ and (
302 + current_parallelism :=
303 + torch.get_num_threads()) > default_omp_num_threads:
304 + logger.warning(
305 + "Reducing Torch parallelism from %d threads to %d to avoid "
306 + "unnecessary CPU contention. Set OMP_NUM_THREADS in the "
307 + "external environment to tune this value as needed.",
308 + current_parallelism, default_omp_num_threads)
309 + os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)
310 + torch.set_num_threads(default_omp_num_threads)
311 +
312 + # workaround for https://github.com/vllm-project/vllm/issues/6103
313 + if HAS_TRITON and parallel_config.world_size > 1:
314 + maybe_set_triton_cache_manager()
vllm/model_executor/layers/logits_processor.py
+4 -1
@@ -5,6 +5,7 @@ from typing import Optional
5 import torch
6 import torch.nn as nn
7
8 +import vllm.envs as envs
9 from vllm.distributed import (tensor_model_parallel_all_gather,
10 tensor_model_parallel_gather)
11 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -42,7 +43,9 @@ class LogitsProcessor(nn.Module):
43 # Soft cap the logits. Used in Gemma 2.
44 self.soft_cap = soft_cap
45 # Whether to use gather or all-gather to gather the logits.
45 - self.use_gather = not current_platform.is_tpu()
46 +
47 + self.use_gather = not current_platform.is_tpu(
48 + ) and not envs.VLLM_USE_V1
49
50 def forward(
51 self,
vllm/platforms/cuda.py
+20 -8
@@ -12,6 +12,7 @@ from typing_extensions import ParamSpec
12
13 # import custom ops, trigger op registration
14 import vllm._C # noqa
15 +import vllm.envs as envs
16 from vllm.logger import init_logger
17
18 from .interface import DeviceCapability, Platform, PlatformEnum
@@ -110,17 +111,28 @@ class CudaPlatformBase(Platform):
111 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
112 parallel_config = vllm_config.parallel_config
113 scheduler_config = vllm_config.scheduler_config
114 +
115 if parallel_config.worker_cls == "auto":
116 if scheduler_config.is_multi_step:
115 - parallel_config.worker_cls = \
116 - "vllm.worker.multi_step_worker.MultiStepWorker"
117 + if envs.VLLM_USE_V1:
118 + raise NotImplementedError
119 + else:
120 + parallel_config.worker_cls = \
121 + "vllm.worker.multi_step_worker.MultiStepWorker"
122 elif vllm_config.speculative_config:
118 - parallel_config.worker_cls = \
119 - "vllm.spec_decode.spec_decode_worker.create_spec_worker"
120 - parallel_config.sd_worker_cls = \
121 - "vllm.worker.worker.Worker"
123 + if envs.VLLM_USE_V1:
124 + raise NotImplementedError
125 + else:
126 + parallel_config.worker_cls = \
127 + "vllm.spec_decode.spec_decode_worker.create_spec_worker"
128 + parallel_config.sd_worker_cls = \
129 + "vllm.worker.worker.Worker"
130 else:
123 - parallel_config.worker_cls = "vllm.worker.worker.Worker"
131 + if envs.VLLM_USE_V1:
132 + parallel_config.worker_cls = \
133 + "vllm.v1.worker.gpu_worker.Worker"
134 + else:
135 + parallel_config.worker_cls = "vllm.worker.worker.Worker"
136
137
138 # NVML utils
@@ -249,4 +261,4 @@ try:
261 if not isinstance(pynvml, _MockModule):
262 CudaPlatform.log_warnings()
263 except ModuleNotFoundError:
252 - CudaPlatform.log_warnings()
\ No newline at end of file
264 + CudaPlatform.log_warnings()
vllm/utils.py
+26
@@ -10,6 +10,7 @@ import importlib.util
10 import inspect
11 import ipaddress
12 import os
13 +import signal
14 import socket
15 import subprocess
16 import sys
@@ -1652,3 +1653,28 @@ def resolve_obj_by_qualname(qualname: str) -> Any:
1653 module_name, obj_name = qualname.rsplit(".", 1)
1654 module = importlib.import_module(module_name)
1655 return getattr(module, obj_name)
1656 +
1657 +
1658 +def kill_process_tree(pid: int):
1659 + """
1660 + Kills all descendant processes of the given pid by sending SIGKILL.
1661 +
1662 + Args:
1663 + pid (int): Process ID of the parent process
1664 + """
1665 + try:
1666 + parent = psutil.Process(pid)
1667 + except psutil.NoSuchProcess:
1668 + return
1669 +
1670 + # Get all children recursively
1671 + children = parent.children(recursive=True)
1672 +
1673 + # Send SIGKILL to all children first
1674 + for child in children:
1675 + with contextlib.suppress(ProcessLookupError):
1676 + os.kill(child.pid, signal.SIGKILL)
1677 +
1678 + # Finally kill the parent
1679 + with contextlib.suppress(ProcessLookupError):
1680 + os.kill(pid, signal.SIGKILL)
vllm/v1/core/scheduler.py
+3 -1
@@ -5,6 +5,8 @@ from typing import (TYPE_CHECKING, Deque, Dict, Iterable, List, Optional, Set,
5
6 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7 from vllm.logger import init_logger
8 +from vllm.multimodal import MultiModalKwargs
9 +from vllm.multimodal.base import PlaceholderRange
10 from vllm.sampling_params import SamplingParams
11 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
12 from vllm.v1.core.kv_cache_manager import KVCacheManager
@@ -383,7 +385,7 @@ class Scheduler:
385 model_runner_output: "ModelRunnerOutput",
386 ) -> List[EngineCoreOutput]:
387 # NOTE(woosuk): This method doesn't consider speculative decoding.
386 - sampled_token_ids = model_runner_output.sampled_token_ids_cpu.tolist()
388 + sampled_token_ids = model_runner_output.sampled_token_ids
389 num_scheduled_tokens = scheduler_output.num_scheduled_tokens
390 new_running: List[Request] = []
391 engine_core_outputs: List[EngineCoreOutput] = []
vllm/v1/engine/async_llm.py
+14 -4
@@ -20,7 +20,7 @@ from vllm.v1.engine.async_stream import AsyncStream
20 from vllm.v1.engine.core_client import EngineCoreClient
21 from vllm.v1.engine.detokenizer import Detokenizer
22 from vllm.v1.engine.processor import Processor
23 -from vllm.v1.executor.gpu_executor import GPUExecutor
23 +from vllm.v1.executor.abstract import Executor
24
25 logger = init_logger(__name__)
26
@@ -30,7 +30,7 @@ class AsyncLLM(EngineClient):
30 def __init__(
31 self,
32 vllm_config: VllmConfig,
33 - executor_class: Type[GPUExecutor],
33 + executor_class: Type[Executor],
34 log_stats: bool,
35 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
36 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
@@ -119,14 +119,24 @@ class AsyncLLM(EngineClient):
119 def shutdown(self):
120 """Shutdown, cleaning up the background proc and IPC."""
121
122 - self.engine_core.shutdown()
122 + if engine_core := getattr(self, "engine_core", None):
123 + engine_core.shutdown()
124
125 if handler := getattr(self, "output_handler", None):
126 handler.cancel()
127
128 @classmethod
129 def _get_executor_cls(cls, vllm_config: VllmConfig):
129 - return GPUExecutor
130 + distributed_executor_backend = (
131 + vllm_config.parallel_config.distributed_executor_backend)
132 + if distributed_executor_backend == "mp":
133 + from vllm.v1.executor.multiproc_executor import MultiprocExecutor
134 + executor_class = MultiprocExecutor
135 + else:
136 + assert (distributed_executor_backend is None)
137 + from vllm.v1.executor.uniproc_executor import UniprocExecutor
138 + executor_class = UniprocExecutor
139 + return executor_class
140
141 async def add_request(
142 self,
vllm/v1/engine/core.py
+39 -35
@@ -1,12 +1,12 @@
1 import multiprocessing
2 import pickle
3 import queue
4 +import signal
5 import threading
6 import time
6 -from contextlib import contextmanager
7 from multiprocessing.process import BaseProcess
8 from multiprocessing.sharedctypes import Synchronized
9 -from typing import Any, Iterator, List, Tuple, Type, Union
9 +from typing import List, Tuple, Type, Union
10
11 import zmq
12 import zmq.asyncio
@@ -20,9 +20,10 @@ from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
20 EngineCoreProfile, EngineCoreRequest,
21 EngineCoreRequestType)
22 from vllm.v1.engine.mm_input_mapper import MMInputMapper
23 -from vllm.v1.executor.gpu_executor import GPUExecutor
23 +from vllm.v1.executor.abstract import Executor
24 from vllm.v1.request import Request, RequestStatus
25 from vllm.v1.serial_utils import PickleEncoder
26 +from vllm.v1.utils import make_zmq_socket
27 from vllm.version import __version__ as VLLM_VERSION
28
29 logger = init_logger(__name__)
@@ -38,7 +39,7 @@ class EngineCore:
39 def __init__(
40 self,
41 vllm_config: VllmConfig,
41 - executor_class: Type[GPUExecutor],
42 + executor_class: Type[Executor],
43 usage_context: UsageContext,
44 ):
45 assert vllm_config.model_config.task != "embedding"
@@ -80,7 +81,7 @@ class EngineCore:
81 num_gpu_blocks = num_gpu_blocks_override
82
83 num_cpu_blocks = 0
83 - self.model_executor.initialize_cache(num_gpu_blocks)
84 + self.model_executor.initialize(num_gpu_blocks)
85 elapsed = time.time() - start
86 logger.info(("init engine (profile, create kv cache, "
87 "warmup model) took %.2f seconds"), elapsed)
@@ -112,8 +113,11 @@ class EngineCore:
113 scheduler_output, output)
114 return engine_core_outputs
115
116 + def shutdown(self):
117 + self.model_executor.shutdown()
118 +
119 def profile(self, is_start=True):
116 - self.model_executor.worker.profile(is_start)
120 + self.model_executor.profile(is_start)
121
122
123 class EngineCoreProc(EngineCore):
@@ -124,7 +128,7 @@ class EngineCoreProc(EngineCore):
128 def __init__(
129 self,
130 vllm_config: VllmConfig,
127 - executor_class: Type[GPUExecutor],
131 + executor_class: Type[Executor],
132 usage_context: UsageContext,
133 input_path: str,
134 output_path: str,
@@ -151,32 +155,9 @@ class EngineCoreProc(EngineCore):
155 daemon=True).start()
156
157 # Send Readiness signal to EngineClient.
154 - with self.make_socket(ready_path, zmq.constants.PUSH) as ready_socket:
158 + with make_zmq_socket(ready_path, zmq.constants.PUSH) as ready_socket:
159 ready_socket.send_string(EngineCoreProc.READY_STR)
160
157 - @contextmanager
158 - def make_socket(self, path: str, type: Any) -> Iterator[zmq.Socket]:
159 - """Context manager for use """
160 -
161 - ctx = zmq.Context()
162 - try:
163 - socket = ctx.socket(type)
164 -
165 - if type == zmq.constants.PULL:
166 - socket.connect(path)
167 - elif type == zmq.constants.PUSH:
168 - socket.bind(path)
169 - else:
170 - raise ValueError(f"Unknown Socket Type: {type}")
171 -
172 - yield socket
173 -
174 - except KeyboardInterrupt:
175 - logger.debug("EngineCore had Keyboard Interrupt.")
176 -
177 - finally:
178 - ctx.destroy(linger=0)
179 -
161 @staticmethod
162 def wait_for_startup(
163 proc: BaseProcess,
@@ -209,7 +190,7 @@ class EngineCoreProc(EngineCore):
190 @staticmethod
191 def make_engine_core_process(
192 vllm_config: VllmConfig,
212 - executor_class: Type[GPUExecutor],
193 + executor_class: Type[Executor],
194 usage_context: UsageContext,
195 input_path: str,
196 output_path: str,
@@ -244,17 +225,38 @@ class EngineCoreProc(EngineCore):
225 def run_engine_core(*args, **kwargs):
226 """Launch EngineCore busy loop in background process."""
227
228 + # Signal handler used for graceful termination.
229 + # SystemExit exception is only raised once to allow this and worker
230 + # processes to terminate without error
231 + shutdown_requested = False
232 +
233 + def signal_handler(signum, frame):
234 + nonlocal shutdown_requested
235 + if not shutdown_requested:
236 + shutdown_requested = True
237 + raise SystemExit()
238 +
239 + # Either SIGTERM or SIGINT will terminate the engine_core
240 + signal.signal(signal.SIGTERM, signal_handler)
241 + signal.signal(signal.SIGINT, signal_handler)
242 +
243 + engine_core = None
244 try:
245 engine_core = EngineCoreProc(*args, **kwargs)
246 engine_core.run_busy_loop()
247
251 - except KeyboardInterrupt:
248 + except SystemExit:
249 logger.debug("EngineCore interrupted.")
250
251 except BaseException as e:
252 logger.exception(e)
253 raise e
254
255 + finally:
256 + if engine_core is not None:
257 + engine_core.shutdown()
258 + engine_core = None
259 +
260 def run_busy_loop(self):
261 """Core busy loop of the EngineCore."""
262
@@ -272,6 +274,8 @@ class EngineCoreProc(EngineCore):
274 logger.debug("EngineCore busy loop waiting.")
275 if self.should_shutdown:
276 return
277 + except BaseException:
278 + raise
279
280 # 2) Handle any new client requests (Abort or Add).
281 while not self.input_queue.empty():
@@ -321,7 +325,7 @@ class EngineCoreProc(EngineCore):
325 decoder_add_req = PickleEncoder()
326 decoder_abort_req = PickleEncoder()
327
324 - with self.make_socket(input_path, zmq.constants.PULL) as socket:
328 + with make_zmq_socket(input_path, zmq.constants.PULL) as socket:
329 while True:
330 # (RequestType, RequestData)
331 type_frame, data_frame = socket.recv_multipart(copy=False)
@@ -349,7 +353,7 @@ class EngineCoreProc(EngineCore):
353 # Reuse send buffer.
354 buffer = bytearray()
355
352 - with self.make_socket(output_path, zmq.constants.PUSH) as socket:
356 + with make_zmq_socket(output_path, zmq.constants.PUSH) as socket:
357 while True:
358 engine_core_outputs = self.output_queue.get()
359 outputs = EngineCoreOutputs(outputs=engine_core_outputs)
vllm/v1/engine/core_client.py
+9 -4
@@ -1,5 +1,4 @@
1 import multiprocessing
2 -import time
2 from typing import List, Union
3
4 import msgspec
@@ -7,7 +6,7 @@ import zmq
6 import zmq.asyncio
7
8 from vllm.logger import init_logger
10 -from vllm.utils import get_open_zmq_ipc_path
9 +from vllm.utils import get_open_zmq_ipc_path, kill_process_tree
10 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
11 EngineCoreProfile, EngineCoreRequest,
12 EngineCoreRequestType)
@@ -99,6 +98,12 @@ class InprocClient(EngineCoreClient):
98 def abort_requests(self, request_ids: List[str]) -> None:
99 self.engine_core.abort_requests(request_ids)
100
101 + def shutdown(self):
102 + self.engine_core.shutdown()
103 +
104 + def __del__(self):
105 + self.shutdown()
106 +
107 async def profile(self, is_start=True) -> None:
108 self.engine_core.profile(is_start)
109
@@ -163,10 +168,10 @@ class MPClient(EngineCoreClient):
168 # Shutdown the process if needed.
169 if hasattr(self, "proc") and self.proc.is_alive():
170 self.proc.terminate()
171 + self.proc.join(5)
172
167 - time.sleep(5)
173 if self.proc.is_alive():
169 - self.proc.kill()
174 + kill_process_tree(self.proc.pid)
175
176 def __del__(self):
177 self.shutdown()
vllm/v1/engine/llm_engine.py
+13 -6
@@ -20,7 +20,7 @@ from vllm.usage.usage_lib import UsageContext
20 from vllm.v1.engine.core_client import EngineCoreClient
21 from vllm.v1.engine.detokenizer import Detokenizer
22 from vllm.v1.engine.processor import Processor
23 -from vllm.v1.executor.gpu_executor import GPUExecutor
23 +from vllm.v1.executor.abstract import Executor
24
25 logger = init_logger(__name__)
26
@@ -33,7 +33,7 @@ class LLMEngine:
33 def __init__(
34 self,
35 vllm_config: VllmConfig,
36 - executor_class: Type[GPUExecutor],
36 + executor_class: Type[Executor],
37 log_stats: bool,
38 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
39 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
@@ -104,10 +104,17 @@ class LLMEngine:
104
105 @classmethod
106 def _get_executor_cls(cls, vllm_config: VllmConfig):
107 - return GPUExecutor
108 -
109 - def stop_remote_worker_execution_loop(self) -> None:
110 - raise NotImplementedError("TP not implemented yet.")
107 + distributed_executor_backend = (
108 + vllm_config.parallel_config.distributed_executor_backend)
109 + if distributed_executor_backend == "mp":
110 + from vllm.v1.executor.multiproc_executor import MultiprocExecutor
111 + executor_class = MultiprocExecutor
112 + else:
113 + assert (distributed_executor_backend is None)
114 + from vllm.v1.executor.uniproc_executor import UniprocExecutor
115 + executor_class = UniprocExecutor
116 +
117 + return executor_class
118
119 def get_num_unfinished_requests(self) -> int:
120 return self.detokenizer.get_num_unfinished_requests()
vllm/v1/executor/abstract.py new
+48
@@ -0,0 +1,48 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Dict, Optional, Tuple
3 +
4 +from vllm.config import VllmConfig
5 +from vllm.v1.outputs import ModelRunnerOutput
6 +
7 +
8 +class Executor(ABC):
9 + """Abstract class for executors."""
10 +
11 + @abstractmethod
12 + def __init__(self, vllm_config: VllmConfig) -> None:
13 + raise NotImplementedError
14 +
15 + @abstractmethod
16 + def initialize(self, num_gpu_blocks: int) -> None:
17 + raise NotImplementedError
18 +
19 + @abstractmethod
20 + def determine_num_available_blocks(self) -> Tuple[int, int]:
21 + raise NotImplementedError
22 +
23 + @abstractmethod
24 + def execute_model(
25 + self,
26 + scheduler_output,
27 + ) -> ModelRunnerOutput:
28 + raise NotImplementedError
29 +
30 + @abstractmethod
31 + def profile(self, is_start=True):
32 + raise NotImplementedError
33 +
34 + @abstractmethod
35 + def shutdown(self):
36 + pass
37 +
38 + @abstractmethod
39 + def check_health(self) -> None:
40 + raise NotImplementedError
41 +
42 + @abstractmethod
43 + def collective_rpc(self,
44 + method: str,
45 + timeout: Optional[float] = None,
46 + args: Tuple = (),
47 + kwargs: Optional[Dict] = None) -> []:
48 + raise NotImplementedError
vllm/v1/executor/multiproc_executor.py new
+375
@@ -0,0 +1,375 @@
1 +import atexit
2 +import os
3 +import pickle
4 +import signal
5 +import sys
6 +import time
7 +from dataclasses import dataclass
8 +from enum import Enum, auto
9 +from multiprocessing.process import BaseProcess
10 +from typing import Dict, List, Optional, Tuple
11 +
12 +import zmq
13 +
14 +from vllm.config import VllmConfig
15 +from vllm.distributed import (destroy_distributed_environment,
16 + destroy_model_parallel)
17 +from vllm.distributed.device_communicators.shm_broadcast import (Handle,
18 + MessageQueue)
19 +from vllm.executor.multiproc_worker_utils import (
20 + _add_prefix, get_mp_context, set_multiprocessing_worker_envs)
21 +from vllm.logger import init_logger
22 +from vllm.utils import (get_distributed_init_method, get_open_port,
23 + get_open_zmq_ipc_path)
24 +from vllm.v1.outputs import ModelRunnerOutput
25 +from vllm.v1.utils import make_zmq_socket
26 +from vllm.worker.worker_base import WorkerWrapperBase
27 +
28 +logger = init_logger(__name__)
29 +
30 +POLLING_TIMEOUT_MS = 5000
31 +POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
32 +
33 +
34 +class MultiprocExecutor:
35 +
36 + def __init__(self, vllm_config: VllmConfig) -> None:
37 + # Call self.shutdown at exit to clean up
38 + # and ensure workers will be terminated.
39 + atexit.register(self.shutdown)
40 +
41 + self.vllm_config = vllm_config
42 + self.parallel_config = vllm_config.parallel_config
43 +
44 + self.world_size = self.parallel_config.world_size
45 + tensor_parallel_size = self.parallel_config.tensor_parallel_size
46 + assert self.world_size == tensor_parallel_size, (
47 + f"world_size ({self.world_size}) must be equal to the "
48 + f"tensor_parallel_size ({tensor_parallel_size}). "
49 + f"Pipeline parallelism is not yet implemented in v1")
50 +
51 + # Set multiprocessing envs that are common to V0 and V1
52 + set_multiprocessing_worker_envs(self.parallel_config)
53 +
54 + # Multiprocessing-based executor does not support multi-node setting.
55 + # Since it only works for single node, we can use the loopback address
56 + # 127.0.0.1 for communication.
57 + distributed_init_method = get_distributed_init_method(
58 + "127.0.0.1", get_open_port())
59 +
60 + # Initialize worker and set up message queues for SchedulerOutputs
61 + # and ModelRunnerOutputs
62 + self.rpc_broadcast_mq = MessageQueue(self.world_size, self.world_size)
63 + scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
64 +
65 + # Create workers
66 + self.workers: List[WorkerProcHandle] = []
67 + for rank in range(self.world_size):
68 + worker = WorkerProc.make_worker_process(vllm_config, rank, rank,
69 + distributed_init_method,
70 + scheduler_output_handle)
71 + self.workers.append(worker)
72 +
73 + # Ensure message queues are ready. Will deadlock if re-ordered
74 + # Must be kept consistent with the WorkerProc
75 + self.rpc_broadcast_mq.wait_until_ready()
76 + for w in self.workers:
77 + w.worker_response_mq.wait_until_ready()
78 +
79 + def initialize(self, num_gpu_blocks: int) -> None:
80 + """
81 + Initialize the KV caches and begin the model execution loop of the
82 + underlying workers.
83 + """
84 + self.collective_rpc("initialize_cache", args=(num_gpu_blocks, ))
85 + self.collective_rpc("compile_or_warm_up_model")
86 +
87 + def determine_num_available_blocks(self) -> Tuple[int, int]:
88 + """
89 + Determine the number of available KV blocks by invoking the
90 + underlying worker.
91 + """
92 + num_blocks = self.collective_rpc("determine_num_available_blocks")
93 +
94 + # Since we use a shared centralized controller, we take the minimum
95 + # number of blocks across all workers to make sure all the memory
96 + # operators can be applied to all workers.
97 + num_gpu_blocks = min(b[0] for b in num_blocks)
98 + num_cpu_blocks = min(b[1] for b in num_blocks)
99 +
100 + return num_gpu_blocks, num_cpu_blocks
101 +
102 + def collective_rpc(self,
103 + method: str,
104 + timeout: Optional[float] = None,
105 + args: Tuple = (),
106 + kwargs: Optional[Dict] = None) -> []:
107 + """
108 + Execute an RPC call on workers.
109 +
110 + Args:
111 + method: Name of the worker method to execute
112 + timeout: Maximum time in seconds to wait for execution. Rases a
113 + TimeoutError on timeout. None means wait indefinitely.
114 + args: Positional arguments to pass to the worker method
115 + kwargs: Keyword arguments to pass to the worker method
116 +
117 + Returns:
118 + List of results from each worker
119 + """
120 + start_time = time.monotonic()
121 + kwargs = kwargs or {}
122 +
123 + try:
124 + self.rpc_broadcast_mq.enqueue((method, args, kwargs))
125 +
126 + responses = [None] * self.world_size
127 + for w in self.workers:
128 + dequeue_timeout = timeout - (time.monotonic() - start_time()
129 + ) if timeout is not None else None
130 + status, result = w.worker_response_mq.dequeue(
131 + timeout=dequeue_timeout)
132 +
133 + if status != WorkerProc.ResponseStatus.SUCCESS:
134 + if isinstance(result, Exception):
135 + raise result
136 + else:
137 + raise RuntimeError("Worker failed")
138 +
139 + responses[w.rank] = result
140 +
141 + return responses
142 + except TimeoutError as e:
143 + raise TimeoutError(f"RPC call to {method} timed out.") from e
144 + except Exception as e:
145 + # Re-raise any other exceptions
146 + raise e
147 +
148 + def execute_model(
149 + self,
150 + scheduler_output,
151 + ) -> ModelRunnerOutput:
152 + model_output = self.collective_rpc("execute_model",
153 + args=(scheduler_output, ))[0]
154 + return model_output
155 +
156 + def profile(self, is_start=True):
157 + self.collective_rpc("profile", args=(is_start, ))
158 + return
159 +
160 + def _ensure_worker_termination(self):
161 + """Ensure that all worker processes are terminated. Assumes workers have
162 + received termination requests. Waits for processing, then sends
163 + termination and kill signals if needed."""
164 +
165 + def wait_for_termination(procs, timeout):
166 + start_time = time.time()
167 + while time.time() - start_time < timeout:
168 + if all(not proc.is_alive() for proc in procs):
169 + return True
170 + time.sleep(0.1)
171 + return False
172 +
173 + # Send SIGTERM if still running
174 + active_procs = [w.proc for w in self.workers if w.proc.is_alive()]
175 + self.workers = None
176 + for p in active_procs:
177 + p.terminate()
178 + if wait_for_termination(active_procs, 4):
179 + return
180 +
181 + # Send SIGKILL if still running
182 + active_procs = [p for p in active_procs if p.is_alive()]
183 + for p in active_procs:
184 + p.kill()
185 +
186 + def shutdown(self):
187 + """Properly shut down the executor and its workers"""
188 + if (hasattr(self, 'workers') and self.workers is not None):
189 + for w in self.workers: #TODO: not sure if needed
190 + w.worker_response_mq = None
191 + self._ensure_worker_termination()
192 +
193 + self.rpc_broadcast_mq = None
194 +
195 + def check_health(self) -> None:
196 + self.collective_rpc("check_health", timeout=10)
197 + return
198 +
199 +
200 +@dataclass
201 +class WorkerProcHandle:
202 + proc: BaseProcess
203 + rank: int
204 + ready_path: str
205 + worker_response_mq: MessageQueue # The worker process writes to this MQ
206 +
207 +
208 +class WorkerProc:
209 + """Wrapper that runs one Worker in a separate process."""
210 +
211 + READY_STR = "READY"
212 +
213 + def __init__(
214 + self,
215 + vllm_config: VllmConfig,
216 + local_rank: int,
217 + rank: int,
218 + distributed_init_method: str,
219 + input_shm_handle: Handle,
220 + ready_path: str,
221 + ):
222 + self.rank = rank
223 + wrapper = WorkerWrapperBase(vllm_config=vllm_config)
224 + wrapper.init_worker(vllm_config, local_rank, rank,
225 + distributed_init_method)
226 + self.worker = wrapper.worker
227 +
228 + pid = os.getpid()
229 + _add_prefix(sys.stdout, f"VllmWorker rank={rank}", pid)
230 + _add_prefix(sys.stderr, f"VllmWorker rank={rank}", pid)
231 +
232 + # Initialize MessageQueue for receiving SchedulerOutput
233 + self.rpc_broadcast_mq = MessageQueue.create_from_handle(
234 + input_shm_handle, self.worker.rank)
235 +
236 + # Initializes a message queue for sending the model output
237 + self.worker_response_mq = MessageQueue(1, 1)
238 + worker_response_mq_handle = self.worker_response_mq.export_handle()
239 +
240 + # Send Readiness signal to EngineCore process.
241 + with make_zmq_socket(ready_path, zmq.constants.PUSH) as ready_socket:
242 + payload = pickle.dumps(worker_response_mq_handle,
243 + protocol=pickle.HIGHEST_PROTOCOL)
244 + ready_socket.send_string(WorkerProc.READY_STR)
245 + ready_socket.send(payload)
246 +
247 + self.worker.initialize()
248 + self.worker.load_model()
249 +
250 + @staticmethod
251 + def make_worker_process(
252 + vllm_config: VllmConfig,
253 + local_rank: int,
254 + rank: int,
255 + distributed_init_method: str,
256 + input_shm_handle, # Receive SchedulerOutput
257 + ) -> WorkerProcHandle:
258 + context = get_mp_context()
259 +
260 + # ZMQ path for worker to send ready message and shm_broadcast handle
261 + # back to core process.
262 + ready_path = get_open_zmq_ipc_path()
263 +
264 + process_kwargs = {
265 + "vllm_config": vllm_config,
266 + "local_rank": local_rank,
267 + "rank": rank,
268 + "distributed_init_method": distributed_init_method,
269 + "input_shm_handle": input_shm_handle,
270 + "ready_path": ready_path,
271 + }
272 + # Run EngineCore busy loop in background process.
273 + proc = context.Process(target=WorkerProc.worker_main,
274 + kwargs=process_kwargs,
275 + daemon=True)
276 + proc.start()
277 +
278 + # Wait for startup
279 + worker_response_mq_handle = WorkerProc.wait_for_startup(
280 + proc, ready_path)
281 +
282 + worker_response_mq = MessageQueue.create_from_handle(
283 + worker_response_mq_handle, 0)
284 +
285 + return WorkerProcHandle(proc, rank, ready_path, worker_response_mq)
286 +
287 + def shutdown(self):
288 + self.rpc_broadcast_mq = None
289 + self.worker_response_mq = None
290 + destroy_model_parallel()
291 + destroy_distributed_environment()
292 +
293 + @staticmethod
294 + def worker_main(*args, **kwargs):
295 + """ Worker initialization and execution loops.
296 + This runs a background process """
297 +
298 + # Signal handler used for graceful termination.
299 + # SystemExit exception is only raised once to allow this and worker
300 + # processes to terminate without error
301 + shutdown_requested = False
302 +
303 + def signal_handler(signum, frame):
304 + nonlocal shutdown_requested
305 + if not shutdown_requested:
306 + shutdown_requested = True
307 + raise SystemExit()
308 +
309 + # Either SIGTERM or SIGINT will terminate the worker
310 + signal.signal(signal.SIGTERM, signal_handler)
311 + signal.signal(signal.SIGINT, signal_handler)
312 +
313 + worker = None
314 + try:
315 + worker = WorkerProc(*args, **kwargs)
316 +
317 + # Ensure message queues are ready. Will deadlock if re-ordered.
318 + # Must be kept consistent with the Executor
319 + worker.rpc_broadcast_mq.wait_until_ready()
320 + worker.worker_response_mq.wait_until_ready()
321 +
322 + worker.worker_busy_loop()
323 +
324 + except SystemExit:
325 + logger.debug("Worker interrupted.")
326 +
327 + except BaseException as e:
328 + logger.exception(e)
329 + raise
330 +
331 + finally:
332 + # Clean up once worker exits busy loop
333 + if worker is not None:
334 + worker.shutdown()
335 + worker = None
336 +
337 + @staticmethod
338 + def wait_for_startup(
339 + proc: BaseProcess,
340 + ready_path: str,
341 + ) -> Optional[Handle]:
342 + """Wait until the Worker is ready."""
343 + with make_zmq_socket(ready_path, zmq.constants.PULL) as socket:
344 +
345 + # Wait for Worker to send READY.
346 + while socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
347 + logger.debug("Waiting for WorkerProc to startup.")
348 +
349 + if not proc.is_alive():
350 + raise RuntimeError("WorkerProc failed to start.")
351 +
352 + message = socket.recv_string()
353 + assert message == WorkerProc.READY_STR
354 + handle_frame = socket.recv(copy=False)
355 + handle = pickle.loads(handle_frame.buffer)
356 + return handle
357 +
358 + class ResponseStatus(Enum):
359 + SUCCESS = auto()
360 + FAILURE = auto()
361 +
362 + def worker_busy_loop(self):
363 + """Main busy loop for Multiprocessing Workers"""
364 + while True:
365 + method, args, kwargs = self.rpc_broadcast_mq.dequeue()
366 +
367 + try:
368 + output = getattr(self.worker, method)(*args, **kwargs)
369 + except BaseException as e:
370 + self.worker_response_mq.enqueue(
371 + (WorkerProc.ResponseStatus.FAILURE, e))
372 + continue
373 +
374 + self.worker_response_mq.enqueue(
375 + (WorkerProc.ResponseStatus.SUCCESS, output))
vllm/v1/executor/uniproc_executor.py renamed
+9 -3
@@ -10,7 +10,7 @@ from vllm.v1.worker.gpu_worker import Worker
10 logger = init_logger(__name__)
11
12
13 -class GPUExecutor:
13 +class UniprocExecutor:
14
15 def __init__(self, vllm_config: VllmConfig) -> None:
16 self.vllm_config = vllm_config
@@ -54,7 +54,7 @@ class GPUExecutor:
54 """
55 return self.worker.determine_num_available_blocks()
56
57 - def initialize_cache(self, num_gpu_blocks: int) -> None:
57 + def initialize(self, num_gpu_blocks: int) -> None:
58 """Initialize the KV cache by invoking the underlying worker.
59 """
60 # NOTE: This is logged in the executor because there can be >1 worker
@@ -71,7 +71,13 @@ class GPUExecutor:
71 output = self.worker.execute_model(scheduler_output)
72 return output
73
74 + def profile(self, is_start: bool = True):
75 + self.worker.profile(is_start)
76 +
77 + def shutdown(self):
78 + self.worker = None
79 +
80 def check_health(self) -> None:
75 - # GPUExecutor will always be healthy as long as
81 + # UniprocExecutor will always be healthy as long as
82 # it's running.
83 return
vllm/v1/outputs.py
+4 -2
@@ -8,7 +8,7 @@ import torch
8 class SamplerOutput:
9
10 # [num_reqs]
11 - sampled_token_ids: torch.Tensor
11 + sampled_token_ids: List[int]
12
13 # [num_reqs, max_num_logprobs + 1]
14 logprob_token_ids: Optional[torch.Tensor]
@@ -20,6 +20,8 @@ class SamplerOutput:
20 prompt_logprobs: Optional[torch.Tensor]
21
22
23 +# ModelRunnerOutput is serialized and sent to the scheduler process.
24 +# This is expensive for torch.Tensor so prefer to use List instead.
25 @dataclass
26 class ModelRunnerOutput:
27
@@ -29,7 +31,7 @@ class ModelRunnerOutput:
31 req_id_to_index: Dict[str, int]
32
33 # [num_reqs]
32 - sampled_token_ids_cpu: torch.Tensor
34 + sampled_token_ids: List[int]
35
36 # [num_reqs, max_num_logprobs + 1]
37 logprob_token_ids_cpu: Optional[torch.Tensor]
vllm/v1/sample/sampler.py
+2 -1
@@ -37,8 +37,9 @@ class Sampler(nn.Module):
37 topk_logprobs = None
38 topk_indices = None
39
40 + # NOTE: CPU-GPU synchronization happens here.
41 sampler_output = SamplerOutput(
41 - sampled_token_ids=sampled,
42 + sampled_token_ids=sampled.tolist(),
43 logprob_token_ids=topk_indices,
44 logprobs=topk_logprobs,
45 prompt_logprob_token_ids=None,
vllm/v1/utils.py
+32 -1
@@ -1,4 +1,11 @@
1 -from typing import Generic, List, TypeVar, overload
1 +from contextlib import contextmanager
2 +from typing import Any, Generic, Iterator, List, TypeVar, overload
3 +
4 +import zmq
5 +
6 +from vllm.logger import init_logger
7 +
8 +logger = init_logger(__name__)
9
10 T = TypeVar("T")
11
@@ -62,3 +69,27 @@ class ConstantList(Generic[T]):
69
70 def __len__(self):
71 return len(self._x)
72 +
73 +
74 +@contextmanager
75 +def make_zmq_socket(path: str, type: Any) -> Iterator[zmq.Socket]:
76 + """Context manager for a ZMQ socket"""
77 +
78 + ctx = zmq.Context()
79 + try:
80 + socket = ctx.socket(type)
81 +
82 + if type == zmq.constants.PULL:
83 + socket.connect(path)
84 + elif type == zmq.constants.PUSH:
85 + socket.bind(path)
86 + else:
87 + raise ValueError(f"Unknown Socket Type: {type}")
88 +
89 + yield socket
90 +
91 + except KeyboardInterrupt:
92 + logger.debug("Worker had Keyboard Interrupt.")
93 +
94 + finally:
95 + ctx.destroy(linger=0)
vllm/v1/worker/gpu_model_runner.py
+5 -7
@@ -34,6 +34,7 @@ class GPUModelRunner:
34 def __init__(
35 self,
36 vllm_config: VllmConfig,
37 + device: torch.device,
38 input_registry: InputRegistry = INPUT_REGISTRY,
39 ):
40 self.vllm_config = vllm_config
@@ -43,7 +44,6 @@ class GPUModelRunner:
44 self.load_config = vllm_config.load_config
45 self.parallel_config = vllm_config.parallel_config
46 self.scheduler_config = vllm_config.scheduler_config
46 - self.device_config = vllm_config.device_config
47 self.speculative_config = vllm_config.speculative_config
48 self.prompt_adapter_config = vllm_config.prompt_adapter_config
49 self.observability_config = vllm_config.observability_config
@@ -52,7 +52,7 @@ class GPUModelRunner:
52 cache_config = self.cache_config
53 scheduler_config = self.scheduler_config
54 parallel_config = self.parallel_config
55 - self.device = self.device_config.device
55 + self.device = device
56 self.pin_memory = is_pin_memory_available()
57 self.dtype = self.model_config.dtype
58 if cache_config.cache_dtype == "auto":
@@ -477,9 +477,7 @@ class GPUModelRunner:
477 sampling_metadata=sampling_metadata,
478 )
479
480 - # NOTE: CPU-GPU synchronization happens here.
481 - sampled_token_ids = sampler_output.sampled_token_ids.cpu()
482 - sampled_token_ids_list = sampled_token_ids.tolist()
480 + sampled_token_ids = sampler_output.sampled_token_ids
481 # TODO(woosuk): The following loop can be slow since it iterates over
482 # the requests one by one. Optimize.
483 num_reqs = self.input_batch.num_reqs
@@ -490,7 +488,7 @@ class GPUModelRunner:
488 assert seq_len <= req_state.num_tokens
489 if seq_len == req_state.num_tokens:
490 # Append the sampled token to the output token ids.
493 - token_id = sampled_token_ids_list[i]
491 + token_id = sampled_token_ids[i]
492 self.input_batch.token_ids_cpu[i, seq_len] = token_id
493 req_state.output_token_ids.append(token_id)
494 else:
@@ -512,7 +510,7 @@ class GPUModelRunner:
510 model_runner_output = ModelRunnerOutput(
511 req_ids=self.input_batch.req_ids[:num_reqs],
512 req_id_to_index=self.input_batch.req_id_to_index,
515 - sampled_token_ids_cpu=sampled_token_ids,
513 + sampled_token_ids=sampled_token_ids,
514 logprob_token_ids_cpu=logprob_token_ids,
515 logprobs_cpu=logprobs,
516 )
vllm/v1/worker/gpu_worker.py
+9 -2
@@ -15,6 +15,7 @@ from vllm.logger import init_logger
15 from vllm.model_executor import set_random_seed
16 from vllm.platforms import current_platform
17 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
18 +from vllm.v1.core.scheduler import SchedulerOutput
19 from vllm.v1.outputs import ModelRunnerOutput
20 from vllm.v1.worker.gpu_model_runner import GPUModelRunner
21
@@ -56,7 +57,6 @@ class Worker:
57 from vllm.utils import init_cached_hf_modules
58 init_cached_hf_modules()
59
59 - self.model_runner = GPUModelRunner(vllm_config)
60 # Torch profiler. Enabled and configured through env vars:
61 # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
62 if envs.VLLM_TORCH_PROFILER_DIR:
@@ -103,6 +103,9 @@ class Worker:
103 # Set random seed.
104 set_random_seed(self.model_config.seed)
105
106 + # Construct the model runner
107 + self.model_runner = GPUModelRunner(self.vllm_config, self.device)
108 +
109 def load_model(self) -> None:
110 self.model_runner.load_model()
111
@@ -198,7 +201,7 @@ class Worker:
201 scheduler_output: "SchedulerOutput",
202 ) -> ModelRunnerOutput:
203 output = self.model_runner.execute_model(scheduler_output)
201 - # TODO(woosuk): Send the output to the engine process.
204 + return output if self.rank == 0 else None
205 return output
206
207 def profile(self, is_start=True):
@@ -209,6 +212,10 @@ class Worker:
212 else:
213 self.profiler.stop()
214
215 + def check_health(self) -> None:
216 + # worker will always be healthy as long as it's running.
217 + return
218 +
219
220 def init_worker_distributed_environment(
221 parallel_config: ParallelConfig,